a
This commit is contained in:
@@ -0,0 +1,530 @@
|
||||
"""
|
||||
微信公众号文章采集路由
|
||||
使用 Playwright + Stealth 采集微信公众号文章内容
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
import json
|
||||
import asyncio
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Body
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
from bs4 import BeautifulSoup, NavigableString
|
||||
|
||||
# 异步 Playwright 支持
|
||||
try:
|
||||
import nest_asyncio
|
||||
nest_asyncio.apply()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
PLAYWRIGHT_AVAILABLE = True
|
||||
except ImportError:
|
||||
PLAYWRIGHT_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import markdownify
|
||||
MARKDOWNIFY_AVAILABLE = True
|
||||
except ImportError:
|
||||
MARKDOWNIFY_AVAILABLE = False
|
||||
|
||||
# 尝试导入 Stealth 插件
|
||||
try:
|
||||
from playwright_stealth import stealth_async
|
||||
STEALTH_AVAILABLE = True
|
||||
except ImportError:
|
||||
STEALTH_AVAILABLE = False
|
||||
|
||||
from app.database import SessionLocal, get_db
|
||||
from app.models.article import Article
|
||||
from app.models.column import ColumnModel
|
||||
from app.models.image import Image
|
||||
from app.routers.auth import get_current_admin
|
||||
from app.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
router = APIRouter(tags=["公众号采集"])
|
||||
|
||||
# ========== Pydantic 模型 ==========
|
||||
|
||||
class WechatPreviewRequest(BaseModel):
|
||||
url: str = Field(..., min_length=1, description="微信公众号文章URL(支持 mp.weixin.qq.com 和 weixin.sogou.com)")
|
||||
|
||||
class WechatPreviewResponse(BaseModel):
|
||||
code: int = 200
|
||||
message: str = "success"
|
||||
data: Optional[dict] = None
|
||||
|
||||
class WechatSaveRequest(BaseModel):
|
||||
url: str = Field(..., description="原始URL")
|
||||
title: str = Field(..., min_length=1, max_length=200, description="标题")
|
||||
author: Optional[str] = Field(None, max_length=50, description="作者")
|
||||
summary: Optional[str] = Field(None, max_length=500, description="摘要")
|
||||
cover_image: Optional[str] = Field(None, description="封面图片URL")
|
||||
content: str = Field(..., description="Markdown内容")
|
||||
images: List[str] = Field(default=[], description="文章中的图片URL列表")
|
||||
column_id: int = Field(..., description="所属栏目ID")
|
||||
source_url: Optional[str] = Field(None, description="来源URL(用于去重)")
|
||||
|
||||
class WechatSaveResponse(BaseModel):
|
||||
code: int = 200
|
||||
message: str = "success"
|
||||
data: Optional[dict] = None
|
||||
|
||||
# ========== 核心工具函数 ==========
|
||||
|
||||
def fix_wechat_html(html: str) -> str:
|
||||
"""修复微信公众号文章的HTML"""
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# 修复图片:微信用 data-src 存真实URL,src 常为占位图
|
||||
for img in soup.find_all('img'):
|
||||
data_src = img.get('data-src', '')
|
||||
src = img.get('src', '')
|
||||
# 只要 data-src 是真实http图片URL,且 src 不是有效http图片,就替换
|
||||
if data_src and data_src.startswith('http') and (
|
||||
not src or src.startswith('data:image') or not src.startswith('http')
|
||||
):
|
||||
img['src'] = data_src
|
||||
if 'data-src' in img.attrs:
|
||||
del img['data-src']
|
||||
|
||||
# 修复微信代码块:<pre class="code-snippet__js"> 内多个 span 行
|
||||
for pre in soup.find_all('pre', class_=re.compile('code-snippet')):
|
||||
code_tag = pre.find('code')
|
||||
if code_tag:
|
||||
lines = []
|
||||
for span in code_tag.find_all('span', class_='code-snippet__line'):
|
||||
line_text = span.get_text(separator='', strip=False)
|
||||
if line_text:
|
||||
lines.append(line_text)
|
||||
if lines:
|
||||
code_tag.clear()
|
||||
for line in lines:
|
||||
code_tag.append(NavigableString(line + '\n'))
|
||||
|
||||
# 移除空标签(保留包含图片的标签)
|
||||
for tag in soup.find_all(['p', 'div', 'section']):
|
||||
if tag.get_text(strip=True) == '' and not tag.find_all('img'):
|
||||
tag.decompose()
|
||||
|
||||
return str(soup)
|
||||
|
||||
|
||||
def html_to_markdown(html: str) -> str:
|
||||
"""将微信HTML转换为Markdown"""
|
||||
if not MARKDOWNIFY_AVAILABLE:
|
||||
return html
|
||||
try:
|
||||
md = markdownify.markdownify(
|
||||
html,
|
||||
heading_style="ATX",
|
||||
bullets="-",
|
||||
strip=["script", "style"],
|
||||
code_language="",
|
||||
)
|
||||
# 清理多余空行
|
||||
md = re.sub(r'\n{3,}', '\n\n', md)
|
||||
return md.strip()
|
||||
except Exception:
|
||||
return html
|
||||
|
||||
|
||||
async def crawl_wechat_article(url: str) -> dict:
|
||||
"""使用 Playwright 采集微信公众号文章"""
|
||||
if not PLAYWRIGHT_AVAILABLE:
|
||||
raise HTTPException(status_code=400, detail="Playwright 未安装,无法采集公众号文章")
|
||||
|
||||
result = {"title": "", "content": "", "author": "", "summary": ""}
|
||||
result["images"] = []
|
||||
result["cover_image"] = ""
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True,
|
||||
args=[
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-web-security",
|
||||
"--disable-features=IsolateOrigins,site-per-process",
|
||||
],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
user_agent=(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
locale="zh-CN",
|
||||
)
|
||||
|
||||
# 尝试应用 Stealth
|
||||
try:
|
||||
page = await context.new_page()
|
||||
if STEALTH_AVAILABLE:
|
||||
await stealth_async(page)
|
||||
except Exception:
|
||||
page = await context.new_page()
|
||||
|
||||
try:
|
||||
# 搜狗链接需要先激活会话(搜索一次获取 cookies),否则会被反爬拦截
|
||||
is_sogou = "weixin.sogou.com" in url
|
||||
if is_sogou:
|
||||
await page.goto(
|
||||
"https://weixin.sogou.com/weixin?type=2&query=test",
|
||||
wait_until="networkidle", timeout=30000,
|
||||
)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
await page.goto(url, wait_until="networkidle", timeout=30000)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 微信文章常有的弹窗
|
||||
try:
|
||||
await page.wait_for_selector("#js_pc_qr_code_img", timeout=5000)
|
||||
# 有二维码,尝试取消
|
||||
close_btn = await page.query_selector(".js_pc_qr_code_close_btn, .weui-desktop-media__tips-close")
|
||||
if close_btn:
|
||||
await close_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 获取页面标题
|
||||
title = await page.title()
|
||||
if title:
|
||||
title = title.strip()
|
||||
# 处理微信的 "页面标题 - 文章" 格式
|
||||
if " - " in title:
|
||||
title = title.split(" - ")[0]
|
||||
result["title"] = title
|
||||
|
||||
# 尝试获取 HTML 内容
|
||||
html = await page.content()
|
||||
|
||||
# 提取微信公众号的正文
|
||||
content_html = ""
|
||||
# 新版微信文章
|
||||
content_elem = await page.query_selector("#js_content")
|
||||
if not content_elem:
|
||||
content_elem = await page.query_selector(".rich_media_content")
|
||||
if not content_elem:
|
||||
content_elem = await page.query_selector("#js_article")
|
||||
if not content_elem:
|
||||
# 通用:取 body 内容
|
||||
body = await page.query_selector("body")
|
||||
if body:
|
||||
content_html = await body.inner_html()
|
||||
else:
|
||||
content_html = await content_elem.inner_html()
|
||||
|
||||
if content_html:
|
||||
content_html = fix_wechat_html(content_html)
|
||||
result["content"] = html_to_markdown(content_html)
|
||||
result["raw_html"] = content_html
|
||||
|
||||
# 提取作者
|
||||
try:
|
||||
author_elem = await page.query_selector(
|
||||
"#js_name, .rich_media_meta_nickname, .profile_nickname, .rich_media_meta_text"
|
||||
)
|
||||
if not author_elem:
|
||||
author_elem = await page.query_selector(
|
||||
# 备用选择器
|
||||
"[id$='_name'], [class*='_name'], .wx_article_author"
|
||||
)
|
||||
if author_elem:
|
||||
author = await author_elem.inner_text()
|
||||
result["author"] = author.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 提取摘要/描述
|
||||
try:
|
||||
desc_elem = await page.query_selector(
|
||||
"#js_description, .rich_media_meta_desc, [class*='_desc'], meta[name='description']"
|
||||
)
|
||||
if desc_elem:
|
||||
desc = await desc_elem.inner_text()
|
||||
result["summary"] = desc.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 提取封面图片
|
||||
try:
|
||||
# 尝试从 meta 标签获取
|
||||
cover_meta = await page.query_selector('meta[property="og:image"]')
|
||||
if cover_meta:
|
||||
cover_url = await cover_meta.get_attribute("content")
|
||||
if cover_url:
|
||||
result["cover_image"] = cover_url
|
||||
if not result["cover_image"]:
|
||||
cover_img = await page.query_selector(
|
||||
"#js_cover, .rich_media_cover img, .cover_img, img.cover"
|
||||
)
|
||||
if cover_img:
|
||||
cover_url = await cover_img.get_attribute("src") or await cover_img.get_attribute("data-src")
|
||||
if cover_url:
|
||||
result["cover_image"] = cover_url
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 提取文章中所有图片
|
||||
try:
|
||||
img_tags = await page.query_selector_all("#js_content img, .rich_media_content img")
|
||||
img_urls = set()
|
||||
for img in img_tags:
|
||||
src = await img.get_attribute("src") or await img.get_attribute("data-src")
|
||||
if src and ("mmbiz.qpic.cn" in src or src.startswith("http")):
|
||||
img_urls.add(src)
|
||||
result["images"] = list(img_urls)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await browser.close()
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
await browser.close()
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"微信文章采集失败: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def download_image(url: str, save_dir: str) -> Optional[str]:
|
||||
"""下载图片到本地"""
|
||||
try:
|
||||
if not url or not url.startswith("http"):
|
||||
return url
|
||||
|
||||
# 生成唯一文件名
|
||||
ext = ".jpg"
|
||||
url_lower = url.lower()
|
||||
if ".png" in url_lower:
|
||||
ext = ".png"
|
||||
elif ".gif" in url_lower:
|
||||
ext = ".gif"
|
||||
elif ".webp" in url_lower:
|
||||
ext = ".webp"
|
||||
elif ".jpeg" in url_lower or ".jpg" in url_lower:
|
||||
ext = ".jpg"
|
||||
|
||||
filename = f"{uuid.uuid4().hex}{ext}"
|
||||
filepath = os.path.join(save_dir, filename)
|
||||
|
||||
# 创建目录
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
|
||||
# 下载
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": "https://mp.weixin.qq.com/",
|
||||
}
|
||||
resp = requests.get(url, headers=headers, timeout=30, stream=True)
|
||||
if resp.status_code == 200:
|
||||
with open(filepath, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
return filename
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"下载图片失败 {url}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ========== API 端点 ==========
|
||||
|
||||
@router.post("/wechat-crawl/preview", response_model=WechatPreviewResponse)
|
||||
async def wechat_preview(
|
||||
request: WechatPreviewRequest,
|
||||
admin: dict = Depends(get_current_admin),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""预览微信公众号文章(支持 mp.weixin.qq.com 和 weixin.sogou.com 链接)"""
|
||||
if not (request.url.startswith("https://mp.weixin.qq.com/") or
|
||||
request.url.startswith("https://weixin.sogou.com/")):
|
||||
raise HTTPException(status_code=400, detail="请输入正确的微信公众号文章链接")
|
||||
|
||||
result = await crawl_wechat_article(request.url)
|
||||
|
||||
# 生成摘要(取内容前200个字符)
|
||||
if not result.get("summary") and result.get("content"):
|
||||
content_text = re.sub(r'[#*`\[\]()>\-\n]', '', result["content"])
|
||||
result["summary"] = content_text[:200].strip()
|
||||
|
||||
# 获取栏目列表供前端选择
|
||||
columns = db.query(ColumnModel).filter(
|
||||
ColumnModel.is_enabled == True
|
||||
).order_by(ColumnModel.sort_order).all()
|
||||
|
||||
return WechatPreviewResponse(
|
||||
code=200,
|
||||
message="success",
|
||||
data={
|
||||
"title": result.get("title", ""),
|
||||
"author": result.get("author", ""),
|
||||
"summary": result.get("summary", ""),
|
||||
"cover_image": result.get("cover_image", ""),
|
||||
"content": result.get("content", ""),
|
||||
"images": result.get("images", []),
|
||||
"columns": [c.to_dict() for c in columns],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/wechat-crawl/save", response_model=WechatSaveResponse)
|
||||
async def wechat_save(
|
||||
request: WechatSaveRequest,
|
||||
admin: dict = Depends(get_current_admin),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""保存采集的微信公众号文章"""
|
||||
# 验证栏目
|
||||
column = db.query(ColumnModel).filter(
|
||||
ColumnModel.id == request.column_id,
|
||||
ColumnModel.is_enabled == True
|
||||
).first()
|
||||
if not column:
|
||||
raise HTTPException(status_code=400, detail="所选栏目不存在或已禁用")
|
||||
|
||||
# 准备图片存储目录(与 crawl 模块保持一致,使用项目根 storage/admin/images/YYYYMM)
|
||||
settings_obj = get_settings()
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
upload_dir = os.path.join(base_dir, settings_obj.UPLOAD_DIR)
|
||||
# 按年月创建子目录
|
||||
year_month = datetime.now().strftime("%Y%m")
|
||||
upload_dir = os.path.join(upload_dir, year_month)
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
|
||||
# 下载封面图
|
||||
local_cover = None
|
||||
if request.cover_image:
|
||||
filename = download_image(request.cover_image, upload_dir)
|
||||
if filename:
|
||||
local_cover = f"/storage/admin/images/{year_month}/{filename}"
|
||||
# 添加到 Image 表
|
||||
try:
|
||||
img = Image(
|
||||
filename=filename,
|
||||
original_name=filename,
|
||||
file_path=f"/storage/admin/images/{year_month}/{filename}",
|
||||
module="wechat",
|
||||
)
|
||||
db.add(img)
|
||||
db.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 下载内容中的图片并替换URL
|
||||
content = request.content
|
||||
image_urls_to_download = set()
|
||||
|
||||
# 从 markdown 中找到所有图片 URL
|
||||
img_pattern = re.compile(r'!\[.*?\]\((https?://[^\s)]+)\)')
|
||||
for match in img_pattern.finditer(content):
|
||||
img_url = match.group(1)
|
||||
image_urls_to_download.add(img_url)
|
||||
|
||||
# 也包括请求中提供的图片列表
|
||||
for img_url in request.images:
|
||||
if img_url.startswith("http"):
|
||||
image_urls_to_download.add(img_url)
|
||||
|
||||
# 下载并替换
|
||||
url_map = {}
|
||||
for img_url in image_urls_to_download:
|
||||
filename = download_image(img_url, upload_dir)
|
||||
if filename:
|
||||
local_path = f"/storage/admin/images/{year_month}/{filename}"
|
||||
url_map[img_url] = local_path
|
||||
# 添加到 Image 表
|
||||
try:
|
||||
img = Image(
|
||||
filename=filename,
|
||||
original_name=filename,
|
||||
file_path=f"/storage/admin/images/{year_month}/{filename}",
|
||||
module="wechat",
|
||||
)
|
||||
db.add(img)
|
||||
db.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 替换 Markdown 中的图片URL
|
||||
for old_url, local_path in url_map.items():
|
||||
content = content.replace(
|
||||
f"]({old_url})",
|
||||
f"]({local_path})"
|
||||
)
|
||||
|
||||
# 生成 slug
|
||||
def generate_slug(text: str) -> str:
|
||||
try:
|
||||
from pypinyin import lazy_pinyin
|
||||
py = lazy_pinyin(text, style=0)
|
||||
slug = "-".join(py)
|
||||
slug = re.sub(r"[^a-zA-Z0-9-]", "", slug).lower()
|
||||
slug = re.sub(r"-+", "-", slug).strip("-")
|
||||
return slug or f"article-{int(datetime.now().timestamp())}"
|
||||
except ImportError:
|
||||
import hashlib
|
||||
return hashlib.md5(text.encode()).hexdigest()[:12]
|
||||
|
||||
slug = generate_slug(request.title)
|
||||
# 检查 slug 唯一性
|
||||
existing = db.query(Article).filter(Article.slug == slug).first()
|
||||
if existing:
|
||||
slug = f"{slug}-{int(datetime.now().timestamp())}"
|
||||
|
||||
# 创建文章
|
||||
article = Article(
|
||||
title=request.title,
|
||||
slug=slug,
|
||||
content=content,
|
||||
content_type="markdown",
|
||||
summary=request.summary or "",
|
||||
cover_image=local_cover,
|
||||
column_id=request.column_id,
|
||||
author=request.author or "",
|
||||
status="draft",
|
||||
source_url=request.source_url or request.url,
|
||||
)
|
||||
db.add(article)
|
||||
db.commit()
|
||||
db.refresh(article)
|
||||
|
||||
return WechatSaveResponse(
|
||||
code=200,
|
||||
message="公众号文章采集并保存成功",
|
||||
data=article.to_dict(include_content=True)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/wechat-crawl/columns")
|
||||
async def get_columns(
|
||||
admin: dict = Depends(get_current_admin),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""获取栏目列表(供前端下拉选择)"""
|
||||
columns = db.query(ColumnModel).filter(
|
||||
ColumnModel.is_enabled == True
|
||||
).order_by(ColumnModel.sort_order).all()
|
||||
return {
|
||||
"code": 200,
|
||||
"data": [c.to_dict() for c in columns]
|
||||
}
|
||||
Reference in New Issue
Block a user