Ubuntu
531 lines
18 KiB
Python
531 lines
18 KiB
Python
"""
|
||
微信公众号文章采集路由
|
||
使用 Playwright + Stealth 采集微信公众号文章内容
|
||
"""
|
||
import os
|
||
import re
|
||
import uuid
|
||
import json
|
||
import asyncio
|
||
import requests
|
||
from datetime import datetime
|
||
from typing import Optional, List
|
||
from urllib.parse import urljoin, urlparse
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException, Body
|
||
from pydantic import BaseModel, Field
|
||
from sqlalchemy.orm import Session
|
||
from bs4 import BeautifulSoup, NavigableString
|
||
|
||
# 异步 Playwright 支持
|
||
try:
|
||
import nest_asyncio
|
||
nest_asyncio.apply()
|
||
except ImportError:
|
||
pass
|
||
|
||
try:
|
||
from playwright.async_api import async_playwright
|
||
PLAYWRIGHT_AVAILABLE = True
|
||
except ImportError:
|
||
PLAYWRIGHT_AVAILABLE = False
|
||
|
||
try:
|
||
import markdownify
|
||
MARKDOWNIFY_AVAILABLE = True
|
||
except ImportError:
|
||
MARKDOWNIFY_AVAILABLE = False
|
||
|
||
# 尝试导入 Stealth 插件
|
||
try:
|
||
from playwright_stealth import stealth_async
|
||
STEALTH_AVAILABLE = True
|
||
except ImportError:
|
||
STEALTH_AVAILABLE = False
|
||
|
||
from app.database import SessionLocal, get_db
|
||
from app.models.article import Article
|
||
from app.models.column import ColumnModel
|
||
from app.models.image import Image
|
||
from app.routers.auth import get_current_admin
|
||
from app.config import get_settings
|
||
|
||
settings = get_settings()
|
||
router = APIRouter(tags=["公众号采集"])
|
||
|
||
# ========== Pydantic 模型 ==========
|
||
|
||
class WechatPreviewRequest(BaseModel):
|
||
url: str = Field(..., min_length=1, description="微信公众号文章URL（支持 mp.weixin.qq.com 和 weixin.sogou.com）")
|
||
|
||
class WechatPreviewResponse(BaseModel):
|
||
code: int = 200
|
||
message: str = "success"
|
||
data: Optional[dict] = None
|
||
|
||
class WechatSaveRequest(BaseModel):
|
||
url: str = Field(..., description="原始URL")
|
||
title: str = Field(..., min_length=1, max_length=200, description="标题")
|
||
author: Optional[str] = Field(None, max_length=50, description="作者")
|
||
summary: Optional[str] = Field(None, max_length=500, description="摘要")
|
||
cover_image: Optional[str] = Field(None, description="封面图片URL")
|
||
content: str = Field(..., description="Markdown内容")
|
||
images: List[str] = Field(default=[], description="文章中的图片URL列表")
|
||
column_id: int = Field(..., description="所属栏目ID")
|
||
source_url: Optional[str] = Field(None, description="来源URL（用于去重）")
|
||
|
||
class WechatSaveResponse(BaseModel):
|
||
code: int = 200
|
||
message: str = "success"
|
||
data: Optional[dict] = None
|
||
|
||
# ========== 核心工具函数 ==========
|
||
|
||
def fix_wechat_html(html: str) -> str:
|
||
"""修复微信公众号文章的HTML"""
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
# 修复图片：微信用 data-src 存真实URL，src 常为占位图
|
||
for img in soup.find_all('img'):
|
||
data_src = img.get('data-src', '')
|
||
src = img.get('src', '')
|
||
# 只要 data-src 是真实http图片URL，且 src 不是有效http图片，就替换
|
||
if data_src and data_src.startswith('http') and (
|
||
not src or src.startswith('data:image') or not src.startswith('http')
|
||
):
|
||
img['src'] = data_src
|
||
if 'data-src' in img.attrs:
|
||
del img['data-src']
|
||
|
||
# 修复微信代码块：<pre class="code-snippet__js"> 内多个 span 行
|
||
for pre in soup.find_all('pre', class_=re.compile('code-snippet')):
|
||
code_tag = pre.find('code')
|
||
if code_tag:
|
||
lines = []
|
||
for span in code_tag.find_all('span', class_='code-snippet__line'):
|
||
line_text = span.get_text(separator='', strip=False)
|
||
if line_text:
|
||
lines.append(line_text)
|
||
if lines:
|
||
code_tag.clear()
|
||
for line in lines:
|
||
code_tag.append(NavigableString(line + '\n'))
|
||
|
||
# 移除空标签（保留包含图片的标签）
|
||
for tag in soup.find_all(['p', 'div', 'section']):
|
||
if tag.get_text(strip=True) == '' and not tag.find_all('img'):
|
||
tag.decompose()
|
||
|
||
return str(soup)
|
||
|
||
|
||
def html_to_markdown(html: str) -> str:
|
||
"""将微信HTML转换为Markdown"""
|
||
if not MARKDOWNIFY_AVAILABLE:
|
||
return html
|
||
try:
|
||
md = markdownify.markdownify(
|
||
html,
|
||
heading_style="ATX",
|
||
bullets="-",
|
||
strip=["script", "style"],
|
||
code_language="",
|
||
)
|
||
# 清理多余空行
|
||
md = re.sub(r'\n{3,}', '\n\n', md)
|
||
return md.strip()
|
||
except Exception:
|
||
return html
|
||
|
||
|
||
async def crawl_wechat_article(url: str) -> dict:
|
||
"""使用 Playwright 采集微信公众号文章"""
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
raise HTTPException(status_code=400, detail="Playwright 未安装，无法采集公众号文章")
|
||
|
||
result = {"title": "", "content": "", "author": "", "summary": ""}
|
||
result["images"] = []
|
||
result["cover_image"] = ""
|
||
|
||
async with async_playwright() as p:
|
||
browser = await p.chromium.launch(
|
||
headless=True,
|
||
args=[
|
||
"--no-sandbox",
|
||
"--disable-setuid-sandbox",
|
||
"--disable-blink-features=AutomationControlled",
|
||
"--disable-web-security",
|
||
"--disable-features=IsolateOrigins,site-per-process",
|
||
],
|
||
)
|
||
context = await browser.new_context(
|
||
viewport={"width": 1920, "height": 1080},
|
||
user_agent=(
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/120.0.0.0 Safari/537.36"
|
||
),
|
||
locale="zh-CN",
|
||
)
|
||
|
||
# 尝试应用 Stealth
|
||
try:
|
||
page = await context.new_page()
|
||
if STEALTH_AVAILABLE:
|
||
await stealth_async(page)
|
||
except Exception:
|
||
page = await context.new_page()
|
||
|
||
try:
|
||
# 搜狗链接需要先激活会话（搜索一次获取 cookies），否则会被反爬拦截
|
||
is_sogou = "weixin.sogou.com" in url
|
||
if is_sogou:
|
||
await page.goto(
|
||
"https://weixin.sogou.com/weixin?type=2&query=test",
|
||
wait_until="networkidle", timeout=30000,
|
||
)
|
||
await asyncio.sleep(2)
|
||
|
||
await page.goto(url, wait_until="networkidle", timeout=30000)
|
||
await asyncio.sleep(3)
|
||
|
||
# 微信文章常有的弹窗
|
||
try:
|
||
await page.wait_for_selector("#js_pc_qr_code_img", timeout=5000)
|
||
# 有二维码，尝试取消
|
||
close_btn = await page.query_selector(".js_pc_qr_code_close_btn, .weui-desktop-media__tips-close")
|
||
if close_btn:
|
||
await close_btn.click()
|
||
await asyncio.sleep(2)
|
||
except Exception:
|
||
pass
|
||
|
||
# 获取页面标题
|
||
title = await page.title()
|
||
if title:
|
||
title = title.strip()
|
||
# 处理微信的 "页面标题 - 文章" 格式
|
||
if " - " in title:
|
||
title = title.split(" - ")[0]
|
||
result["title"] = title
|
||
|
||
# 尝试获取 HTML 内容
|
||
html = await page.content()
|
||
|
||
# 提取微信公众号的正文
|
||
content_html = ""
|
||
# 新版微信文章
|
||
content_elem = await page.query_selector("#js_content")
|
||
if not content_elem:
|
||
content_elem = await page.query_selector(".rich_media_content")
|
||
if not content_elem:
|
||
content_elem = await page.query_selector("#js_article")
|
||
if not content_elem:
|
||
# 通用：取 body 内容
|
||
body = await page.query_selector("body")
|
||
if body:
|
||
content_html = await body.inner_html()
|
||
else:
|
||
content_html = await content_elem.inner_html()
|
||
|
||
if content_html:
|
||
content_html = fix_wechat_html(content_html)
|
||
result["content"] = html_to_markdown(content_html)
|
||
result["raw_html"] = content_html
|
||
|
||
# 提取作者
|
||
try:
|
||
author_elem = await page.query_selector(
|
||
"#js_name, .rich_media_meta_nickname, .profile_nickname, .rich_media_meta_text"
|
||
)
|
||
if not author_elem:
|
||
author_elem = await page.query_selector(
|
||
# 备用选择器
|
||
"[id$='_name'], [class*='_name'], .wx_article_author"
|
||
)
|
||
if author_elem:
|
||
author = await author_elem.inner_text()
|
||
result["author"] = author.strip()
|
||
except Exception:
|
||
pass
|
||
|
||
# 提取摘要/描述
|
||
try:
|
||
desc_elem = await page.query_selector(
|
||
"#js_description, .rich_media_meta_desc, [class*='_desc'], meta[name='description']"
|
||
)
|
||
if desc_elem:
|
||
desc = await desc_elem.inner_text()
|
||
result["summary"] = desc.strip()
|
||
except Exception:
|
||
pass
|
||
|
||
# 提取封面图片
|
||
try:
|
||
# 尝试从 meta 标签获取
|
||
cover_meta = await page.query_selector('meta[property="og:image"]')
|
||
if cover_meta:
|
||
cover_url = await cover_meta.get_attribute("content")
|
||
if cover_url:
|
||
result["cover_image"] = cover_url
|
||
if not result["cover_image"]:
|
||
cover_img = await page.query_selector(
|
||
"#js_cover, .rich_media_cover img, .cover_img, img.cover"
|
||
)
|
||
if cover_img:
|
||
cover_url = await cover_img.get_attribute("src") or await cover_img.get_attribute("data-src")
|
||
if cover_url:
|
||
result["cover_image"] = cover_url
|
||
except Exception:
|
||
pass
|
||
|
||
# 提取文章中所有图片
|
||
try:
|
||
img_tags = await page.query_selector_all("#js_content img, .rich_media_content img")
|
||
img_urls = set()
|
||
for img in img_tags:
|
||
src = await img.get_attribute("src") or await img.get_attribute("data-src")
|
||
if src and ("mmbiz.qpic.cn" in src or src.startswith("http")):
|
||
img_urls.add(src)
|
||
result["images"] = list(img_urls)
|
||
except Exception:
|
||
pass
|
||
|
||
await browser.close()
|
||
return result
|
||
|
||
except Exception as e:
|
||
await browser.close()
|
||
raise HTTPException(
|
||
status_code=500,
|
||
detail=f"微信文章采集失败: {str(e)}"
|
||
)
|
||
|
||
|
||
def download_image(url: str, save_dir: str) -> Optional[str]:
|
||
"""下载图片到本地"""
|
||
try:
|
||
if not url or not url.startswith("http"):
|
||
return url
|
||
|
||
# 生成唯一文件名
|
||
ext = ".jpg"
|
||
url_lower = url.lower()
|
||
if ".png" in url_lower:
|
||
ext = ".png"
|
||
elif ".gif" in url_lower:
|
||
ext = ".gif"
|
||
elif ".webp" in url_lower:
|
||
ext = ".webp"
|
||
elif ".jpeg" in url_lower or ".jpg" in url_lower:
|
||
ext = ".jpg"
|
||
|
||
filename = f"{uuid.uuid4().hex}{ext}"
|
||
filepath = os.path.join(save_dir, filename)
|
||
|
||
# 创建目录
|
||
os.makedirs(save_dir, exist_ok=True)
|
||
|
||
# 下载
|
||
headers = {
|
||
"User-Agent": (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/120.0.0.0 Safari/537.36"
|
||
),
|
||
"Referer": "https://mp.weixin.qq.com/",
|
||
}
|
||
resp = requests.get(url, headers=headers, timeout=30, stream=True)
|
||
if resp.status_code == 200:
|
||
with open(filepath, "wb") as f:
|
||
for chunk in resp.iter_content(chunk_size=8192):
|
||
f.write(chunk)
|
||
return filename
|
||
return None
|
||
except Exception as e:
|
||
print(f"下载图片失败 {url}: {e}")
|
||
return None
|
||
|
||
|
||
# ========== API 端点 ==========
|
||
|
||
@router.post("/wechat-crawl/preview", response_model=WechatPreviewResponse)
|
||
async def wechat_preview(
|
||
request: WechatPreviewRequest,
|
||
admin: dict = Depends(get_current_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""预览微信公众号文章（支持 mp.weixin.qq.com 和 weixin.sogou.com 链接）"""
|
||
if not (request.url.startswith("https://mp.weixin.qq.com/") or
|
||
request.url.startswith("https://weixin.sogou.com/")):
|
||
raise HTTPException(status_code=400, detail="请输入正确的微信公众号文章链接")
|
||
|
||
result = await crawl_wechat_article(request.url)
|
||
|
||
# 生成摘要（取内容前200个字符）
|
||
if not result.get("summary") and result.get("content"):
|
||
content_text = re.sub(r'[#*`\[\]()>\-\n]', '', result["content"])
|
||
result["summary"] = content_text[:200].strip()
|
||
|
||
# 获取栏目列表供前端选择
|
||
columns = db.query(ColumnModel).filter(
|
||
ColumnModel.is_enabled == True
|
||
).order_by(ColumnModel.sort_order).all()
|
||
|
||
return WechatPreviewResponse(
|
||
code=200,
|
||
message="success",
|
||
data={
|
||
"title": result.get("title", ""),
|
||
"author": result.get("author", ""),
|
||
"summary": result.get("summary", ""),
|
||
"cover_image": result.get("cover_image", ""),
|
||
"content": result.get("content", ""),
|
||
"images": result.get("images", []),
|
||
"columns": [c.to_dict() for c in columns],
|
||
}
|
||
)
|
||
|
||
|
||
@router.post("/wechat-crawl/save", response_model=WechatSaveResponse)
|
||
async def wechat_save(
|
||
request: WechatSaveRequest,
|
||
admin: dict = Depends(get_current_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""保存采集的微信公众号文章"""
|
||
# 验证栏目
|
||
column = db.query(ColumnModel).filter(
|
||
ColumnModel.id == request.column_id,
|
||
ColumnModel.is_enabled == True
|
||
).first()
|
||
if not column:
|
||
raise HTTPException(status_code=400, detail="所选栏目不存在或已禁用")
|
||
|
||
# 准备图片存储目录（与 crawl 模块保持一致，使用项目根 storage/admin/images/YYYYMM）
|
||
settings_obj = get_settings()
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
upload_dir = os.path.join(base_dir, settings_obj.UPLOAD_DIR)
|
||
# 按年月创建子目录
|
||
year_month = datetime.now().strftime("%Y%m")
|
||
upload_dir = os.path.join(upload_dir, year_month)
|
||
os.makedirs(upload_dir, exist_ok=True)
|
||
|
||
# 下载封面图
|
||
local_cover = None
|
||
if request.cover_image:
|
||
filename = download_image(request.cover_image, upload_dir)
|
||
if filename:
|
||
local_cover = f"/storage/admin/images/{year_month}/{filename}"
|
||
# 添加到 Image 表
|
||
try:
|
||
img = Image(
|
||
filename=filename,
|
||
original_name=filename,
|
||
file_path=f"/storage/admin/images/{year_month}/{filename}",
|
||
module="wechat",
|
||
)
|
||
db.add(img)
|
||
db.flush()
|
||
except Exception:
|
||
pass
|
||
|
||
# 下载内容中的图片并替换URL
|
||
content = request.content
|
||
image_urls_to_download = set()
|
||
|
||
# 从 markdown 中找到所有图片 URL
|
||
img_pattern = re.compile(r'!\[.*?\]\((https?://[^\s)]+)\)')
|
||
for match in img_pattern.finditer(content):
|
||
img_url = match.group(1)
|
||
image_urls_to_download.add(img_url)
|
||
|
||
# 也包括请求中提供的图片列表
|
||
for img_url in request.images:
|
||
if img_url.startswith("http"):
|
||
image_urls_to_download.add(img_url)
|
||
|
||
# 下载并替换
|
||
url_map = {}
|
||
for img_url in image_urls_to_download:
|
||
filename = download_image(img_url, upload_dir)
|
||
if filename:
|
||
local_path = f"/storage/admin/images/{year_month}/{filename}"
|
||
url_map[img_url] = local_path
|
||
# 添加到 Image 表
|
||
try:
|
||
img = Image(
|
||
filename=filename,
|
||
original_name=filename,
|
||
file_path=f"/storage/admin/images/{year_month}/{filename}",
|
||
module="wechat",
|
||
)
|
||
db.add(img)
|
||
db.flush()
|
||
except Exception:
|
||
pass
|
||
|
||
# 替换 Markdown 中的图片URL
|
||
for old_url, local_path in url_map.items():
|
||
content = content.replace(
|
||
f"]({old_url})",
|
||
f"]({local_path})"
|
||
)
|
||
|
||
# 生成 slug
|
||
def generate_slug(text: str) -> str:
|
||
try:
|
||
from pypinyin import lazy_pinyin
|
||
py = lazy_pinyin(text, style=0)
|
||
slug = "-".join(py)
|
||
slug = re.sub(r"[^a-zA-Z0-9-]", "", slug).lower()
|
||
slug = re.sub(r"-+", "-", slug).strip("-")
|
||
return slug or f"article-{int(datetime.now().timestamp())}"
|
||
except ImportError:
|
||
import hashlib
|
||
return hashlib.md5(text.encode()).hexdigest()[:12]
|
||
|
||
slug = generate_slug(request.title)
|
||
# 检查 slug 唯一性
|
||
existing = db.query(Article).filter(Article.slug == slug).first()
|
||
if existing:
|
||
slug = f"{slug}-{int(datetime.now().timestamp())}"
|
||
|
||
# 创建文章
|
||
article = Article(
|
||
title=request.title,
|
||
slug=slug,
|
||
content=content,
|
||
content_type="markdown",
|
||
summary=request.summary or "",
|
||
cover_image=local_cover,
|
||
column_id=request.column_id,
|
||
author=request.author or "",
|
||
status="draft",
|
||
source_url=request.source_url or request.url,
|
||
)
|
||
db.add(article)
|
||
db.commit()
|
||
db.refresh(article)
|
||
|
||
return WechatSaveResponse(
|
||
code=200,
|
||
message="公众号文章采集并保存成功",
|
||
data=article.to_dict(include_content=True)
|
||
)
|
||
|
||
|
||
@router.post("/wechat-crawl/columns")
|
||
async def get_columns(
|
||
admin: dict = Depends(get_current_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""获取栏目列表（供前端下拉选择）"""
|
||
columns = db.query(ColumnModel).filter(
|
||
ColumnModel.is_enabled == True
|
||
).order_by(ColumnModel.sort_order).all()
|
||
return {
|
||
"code": 200,
|
||
"data": [c.to_dict() for c in columns]
|
||
}
|