Ubuntu
422 lines
15 KiB
Python
422 lines
15 KiB
Python
"""
|
||
微信公众号文章采集模块
|
||
专门用于采集微信公众号文章，独立于通用采集模块
|
||
todo: 后续可扩展批量采集公众号历史文章
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import uuid
|
||
from datetime import datetime
|
||
from urllib.parse import urlparse, urljoin
|
||
from typing import List, Optional
|
||
|
||
import requests
|
||
from fastapi import APIRouter, Depends, HTTPException
|
||
from pydantic import BaseModel
|
||
from sqlalchemy.orm import Session
|
||
|
||
from backend.database import get_db
|
||
from backend.models import Image
|
||
from backend.routers.auth import get_current_admin
|
||
|
||
router = APIRouter(prefix="/wechat-crawl", tags=["wechat-crawl"])
|
||
|
||
# ========== Playwright 检测 ==========
|
||
try:
|
||
from playwright.async_api import async_playwright
|
||
PLAYWRIGHT_AVAILABLE = True
|
||
except ImportError:
|
||
PLAYWRIGHT_AVAILABLE = False
|
||
print("警告: Playwright未安装，微信公众号采集功能不可用")
|
||
|
||
|
||
# ========== Pydantic 模型 ==========
|
||
class WechatCrawlRequest(BaseModel):
|
||
url: str
|
||
|
||
|
||
class CrawlImageInfo(BaseModel):
|
||
original_url: str
|
||
local_path: Optional[str] = None
|
||
filename: Optional[str] = None
|
||
success: bool = False
|
||
id: Optional[int] = None
|
||
|
||
|
||
class WechatCrawlResponse(BaseModel):
|
||
success: bool
|
||
title: Optional[str] = None
|
||
content: Optional[str] = None
|
||
summary: Optional[str] = None
|
||
images: List[CrawlImageInfo] = []
|
||
error: Optional[str] = None
|
||
|
||
|
||
# ========== 辅助函数 ==========
|
||
|
||
def get_image_dimensions(filepath: str):
|
||
"""获取图片尺寸"""
|
||
try:
|
||
from PIL import Image as PILImage
|
||
with PILImage.open(filepath) as img:
|
||
return img.width, img.height
|
||
except Exception as e:
|
||
print(f"获取图片尺寸失败: {e}")
|
||
return None, None
|
||
|
||
|
||
def generate_thumbnail(source_path: str, thumb_path: str, max_size: tuple = (400, 300)) -> bool:
|
||
"""生成缩略图"""
|
||
try:
|
||
from PIL import Image as PILImage
|
||
with PILImage.open(source_path) as img:
|
||
img.thumbnail(max_size)
|
||
img.save(thumb_path, 'JPEG', quality=85)
|
||
return True
|
||
except Exception as e:
|
||
print(f"生成缩略图失败: {e}")
|
||
return False
|
||
|
||
|
||
def download_image(url: str, temp_dir: str) -> tuple:
|
||
"""下载图片到临时目录"""
|
||
try:
|
||
headers = {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||
'Referer': 'https://mp.weixin.qq.com/'
|
||
}
|
||
|
||
response = requests.get(url, headers=headers, timeout=30, stream=True)
|
||
response.raise_for_status()
|
||
|
||
content_type = response.headers.get('content-type', '')
|
||
ext = '.jpg'
|
||
if 'png' in content_type:
|
||
ext = '.png'
|
||
elif 'gif' in content_type:
|
||
ext = '.gif'
|
||
elif 'webp' in content_type:
|
||
ext = '.webp'
|
||
else:
|
||
parsed = urlparse(url)
|
||
path_ext = os.path.splitext(parsed.path)[1]
|
||
if path_ext:
|
||
ext = path_ext
|
||
|
||
filename = f"wechat_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}"
|
||
filepath = os.path.join(temp_dir, filename)
|
||
|
||
with open(filepath, 'wb') as f:
|
||
for chunk in response.iter_content(chunk_size=8192):
|
||
f.write(chunk)
|
||
|
||
return filename, filepath, True
|
||
except Exception as e:
|
||
print(f"下载图片失败 {url}: {e}")
|
||
return None, None, False
|
||
|
||
|
||
def convert_wechat_html_to_markdown(html: str, images: List[CrawlImageInfo], base_url: str) -> str:
|
||
"""将微信文章HTML转为Markdown"""
|
||
text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'\s+style="[^"]*"', '', text, flags=re.IGNORECASE)
|
||
text = re.sub(r'\s+class="[^"]*"', '', text, flags=re.IGNORECASE)
|
||
parsed_base = urlparse(base_url)
|
||
|
||
# 微信文章特殊处理：图片使用 data-src
|
||
text = re.sub(
|
||
r'<img[^>]+data-src=["\']([^"\']+)["\'][^>]*>',
|
||
lambda m: f'})',
|
||
text, flags=re.IGNORECASE
|
||
)
|
||
|
||
# 匹配普通 img 标签
|
||
def img_to_markdown(match):
|
||
src = match.group(1)
|
||
alt = match.group(2) if match.lastindex and match.lastindex >= 2 else ""
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/'):
|
||
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
|
||
return f''
|
||
|
||
text = re.sub(
|
||
r'<img[^>]+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>',
|
||
img_to_markdown, text, flags=re.IGNORECASE
|
||
)
|
||
|
||
def img_to_markdown_no_alt(match):
|
||
src = match.group(1)
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/'):
|
||
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
|
||
return f''
|
||
|
||
text = re.sub(
|
||
r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>',
|
||
img_to_markdown_no_alt,
|
||
text, flags=re.IGNORECASE
|
||
)
|
||
|
||
# 处理代码块
|
||
def convert_code_block(match):
|
||
code_content = match.group(1)
|
||
lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE)
|
||
lang = lang_match.group(1) if lang_match else ''
|
||
return f'\n```{lang}\n{code_content}\n```\n'
|
||
|
||
text = re.sub(r'<pre[^>]*>\s*<code[^>]*>(.*?)</code>\s*</pre>', convert_code_block,
|
||
text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<pre[^>]*>(.*?)</pre>', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<code[^>]*>(.*?)</code>', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
# 转换其他标签
|
||
text = re.sub(r'<h1[^>]*>(.*?)</h1>', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h2[^>]*>(.*?)</h2>', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h3[^>]*>(.*?)</h3>', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h4[^>]*>(.*?)</h4>', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<p[^>]*>(.*?)</p>', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
|
||
text = re.sub(r'<strong[^>]*>(.*?)</strong>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<b[^>]*>(.*?)</b>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<em[^>]*>(.*?)</em>', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<li[^>]*>(.*?)</li>', r'- \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
|
||
# 解码HTML实体
|
||
text = text.replace(' ', ' ').replace('<', '<').replace('>', '>')
|
||
text = text.replace('&', '&').replace('"', '"')
|
||
|
||
# 清理空行
|
||
lines = [line for line in text.split('\n')]
|
||
cleaned = []
|
||
prev_empty = False
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if stripped == '':
|
||
if not prev_empty:
|
||
cleaned.append('')
|
||
prev_empty = True
|
||
else:
|
||
cleaned.append(line)
|
||
prev_empty = False
|
||
|
||
return '\n'.join(cleaned).strip()
|
||
|
||
|
||
async def crawl_wechat_with_playwright(url: str) -> dict:
|
||
"""使用Playwright采集微信公众号文章"""
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
raise Exception("Playwright未安装，无法采集微信公众号文章")
|
||
|
||
async with async_playwright() as p:
|
||
browser = await p.chromium.launch(headless=True)
|
||
page = await browser.new_page()
|
||
|
||
# 模拟微信内打开环境
|
||
await page.set_extra_http_headers({
|
||
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36 MicroMessenger/8.0.0',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||
'Referer': 'https://mp.weixin.qq.com/'
|
||
})
|
||
|
||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||
await page.wait_for_timeout(5000)
|
||
|
||
# 提取标题
|
||
title = ""
|
||
try:
|
||
if await page.locator('#activity_name').count() > 0:
|
||
title = await page.locator('#activity_name').first.inner_text()
|
||
title = title.strip()
|
||
except:
|
||
pass
|
||
if not title:
|
||
title = await page.title()
|
||
|
||
# 提取正文
|
||
content_html = ""
|
||
try:
|
||
if await page.locator('#js_content').count() > 0:
|
||
content_html = await page.locator('#js_content').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
# 提取图片
|
||
images = []
|
||
try:
|
||
img_elements = await page.locator('#js_content img').all()
|
||
for img in img_elements[:10]:
|
||
try:
|
||
src = await img.get_attribute('data-src') or await img.get_attribute('src')
|
||
alt = await img.get_attribute('alt') or ''
|
||
if src and src.startswith('http'):
|
||
images.append(CrawlImageInfo(original_url=src))
|
||
except:
|
||
pass
|
||
except:
|
||
pass
|
||
|
||
# 尝试提取公众号名称
|
||
nickname = ""
|
||
try:
|
||
if await page.locator('#js_name').count() > 0:
|
||
nickname = await page.locator('#js_name').first.inner_text()
|
||
nickname = nickname.strip()
|
||
except:
|
||
pass
|
||
|
||
await browser.close()
|
||
|
||
return {
|
||
'title': title if title else "未命名文章",
|
||
'content_html': content_html,
|
||
'images': images,
|
||
'nickname': nickname
|
||
}
|
||
|
||
|
||
# ========== API 端点 ==========
|
||
|
||
@router.post("")
|
||
async def crawl_wechat_article(request: WechatCrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)):
|
||
"""
|
||
采集微信公众号文章（专用接口）
|
||
todo: 后续支持批量采集公众号历史文章
|
||
"""
|
||
try:
|
||
if not request.url.startswith(('http://', 'https://')):
|
||
raise HTTPException(status_code=400, detail="无效的URL")
|
||
|
||
if 'mp.weixin.qq.com' not in request.url:
|
||
return WechatCrawlResponse(
|
||
success=False,
|
||
error="该接口仅支持微信公众号文章（mp.weixin.qq.com）"
|
||
)
|
||
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
return WechatCrawlResponse(
|
||
success=False,
|
||
error="微信公众号文章需要Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium"
|
||
)
|
||
|
||
print(f"[公众号采集] 开始采集: {request.url}")
|
||
|
||
# 使用Playwright采集
|
||
result = await crawl_wechat_with_playwright(request.url)
|
||
|
||
title = result['title']
|
||
images = result['images']
|
||
content_html = result['content_html']
|
||
nickname = result.get('nickname', '')
|
||
|
||
# 转换为Markdown
|
||
markdown_content = convert_wechat_html_to_markdown(content_html, images, request.url)
|
||
|
||
# 创建临时目录
|
||
temp_dir = "/tmp/blog_wechat_crawl"
|
||
os.makedirs(temp_dir, exist_ok=True)
|
||
|
||
# uploads 目录配置
|
||
backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
upload_dir = os.path.join(backend_dir, "uploads")
|
||
thumb_dir = os.path.join(upload_dir, "thumbnails")
|
||
os.makedirs(upload_dir, exist_ok=True)
|
||
os.makedirs(thumb_dir, exist_ok=True)
|
||
|
||
# 下载图片（最多5张）
|
||
downloaded_images = []
|
||
for i, img in enumerate(images[:5]):
|
||
filename, filepath, success = download_image(img.original_url, temp_dir)
|
||
if success:
|
||
try:
|
||
ext = os.path.splitext(filename)[1].lower()
|
||
if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']:
|
||
ext = '.jpg'
|
||
new_filename = f"wechat_{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}"
|
||
|
||
new_filepath = os.path.join(upload_dir, new_filename)
|
||
with open(filepath, 'rb') as src:
|
||
with open(new_filepath, 'wb') as dst:
|
||
dst.write(src.read())
|
||
|
||
os.remove(filepath)
|
||
|
||
thumb_filename = f"thumb_{new_filename}"
|
||
thumb_filepath = os.path.join(thumb_dir, thumb_filename)
|
||
thumb_path_db = None
|
||
if generate_thumbnail(new_filepath, thumb_filepath):
|
||
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
|
||
|
||
width, height = get_image_dimensions(new_filepath)
|
||
file_size = os.path.getsize(new_filepath)
|
||
|
||
image_record = Image(
|
||
filename=new_filename,
|
||
original_name=filename,
|
||
file_path=f"/uploads/{new_filename}",
|
||
thumb_path=thumb_path_db,
|
||
description=f"公众号采集自 {request.url}" + (f" ({nickname})" if nickname else ""),
|
||
file_size=file_size,
|
||
width=width,
|
||
height=height
|
||
)
|
||
db.add(image_record)
|
||
db.commit()
|
||
db.refresh(image_record)
|
||
|
||
img.local_path = f"/uploads/{new_filename}"
|
||
img.filename = new_filename
|
||
img.success = True
|
||
img.id = image_record.id
|
||
downloaded_images.append(img)
|
||
|
||
if img.original_url in markdown_content:
|
||
markdown_content = markdown_content.replace(
|
||
img.original_url,
|
||
f"/uploads/{new_filename}"
|
||
)
|
||
print(f"[公众号采集] 图片替换: {img.original_url[:60]}... -> /uploads/{new_filename}")
|
||
|
||
print(f"[公众号采集] 图片已保存: id={image_record.id}, filename={new_filename}")
|
||
|
||
except Exception as e:
|
||
print(f"[公众号采集] 保存图片失败: {e}")
|
||
img.local_path = filepath
|
||
img.filename = filename
|
||
img.success = True
|
||
downloaded_images.append(img)
|
||
|
||
# 生成摘要
|
||
summary = markdown_content[:200].replace('#', '').replace('*', '').strip()
|
||
|
||
# 在标题前添加公众号名称
|
||
if nickname and nickname not in title:
|
||
title_prefix = f"【{nickname}】"
|
||
else:
|
||
title_prefix = ""
|
||
|
||
print(f"[公众号采集] 完成: {title}")
|
||
|
||
return WechatCrawlResponse(
|
||
success=True,
|
||
title=title_prefix + title,
|
||
content=markdown_content,
|
||
summary=summary,
|
||
images=downloaded_images
|
||
)
|
||
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
import traceback
|
||
print(f"[公众号采集] 失败: {e}")
|
||
print(traceback.format_exc())
|
||
return WechatCrawlResponse(
|
||
success=False,
|
||
error=f"公众号采集失败: {str(e)}"
|
||
)
|