""" 微信公众号文章采集模块 专门用于采集微信公众号文章,独立于通用采集模块 todo: 后续可扩展批量采集公众号历史文章 """ import os import re import uuid from datetime import datetime from urllib.parse import urlparse, urljoin from typing import List, Optional import requests from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.orm import Session from backend.database import get_db from backend.models import Image from backend.routers.auth import get_current_admin router = APIRouter(prefix="/wechat-crawl", tags=["wechat-crawl"]) # ========== Playwright 检测 ========== try: from playwright.async_api import async_playwright PLAYWRIGHT_AVAILABLE = True except ImportError: PLAYWRIGHT_AVAILABLE = False print("警告: Playwright未安装,微信公众号采集功能不可用") # ========== Pydantic 模型 ========== class WechatCrawlRequest(BaseModel): url: str class CrawlImageInfo(BaseModel): original_url: str local_path: Optional[str] = None filename: Optional[str] = None success: bool = False id: Optional[int] = None class WechatCrawlResponse(BaseModel): success: bool title: Optional[str] = None content: Optional[str] = None summary: Optional[str] = None images: List[CrawlImageInfo] = [] error: Optional[str] = None # ========== 辅助函数 ========== def get_image_dimensions(filepath: str): """获取图片尺寸""" try: from PIL import Image as PILImage with PILImage.open(filepath) as img: return img.width, img.height except Exception as e: print(f"获取图片尺寸失败: {e}") return None, None def generate_thumbnail(source_path: str, thumb_path: str, max_size: tuple = (400, 300)) -> bool: """生成缩略图""" try: from PIL import Image as PILImage with PILImage.open(source_path) as img: img.thumbnail(max_size) img.save(thumb_path, 'JPEG', quality=85) return True except Exception as e: print(f"生成缩略图失败: {e}") return False def download_image(url: str, temp_dir: str) -> tuple: """下载图片到临时目录""" try: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', 'Referer': 'https://mp.weixin.qq.com/' } response = requests.get(url, headers=headers, timeout=30, stream=True) response.raise_for_status() content_type = response.headers.get('content-type', '') ext = '.jpg' if 'png' in content_type: ext = '.png' elif 'gif' in content_type: ext = '.gif' elif 'webp' in content_type: ext = '.webp' else: parsed = urlparse(url) path_ext = os.path.splitext(parsed.path)[1] if path_ext: ext = path_ext filename = f"wechat_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}" filepath = os.path.join(temp_dir, filename) with open(filepath, 'wb') as f: for chunk in response.iter_content(chunk_size=8192): f.write(chunk) return filename, filepath, True except Exception as e: print(f"下载图片失败 {url}: {e}") return None, None, False def convert_wechat_html_to_markdown(html: str, images: List[CrawlImageInfo], base_url: str) -> str: """将微信文章HTML转为Markdown""" text = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>.*?', '', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'\s+style="[^"]*"', '', text, flags=re.IGNORECASE) text = re.sub(r'\s+class="[^"]*"', '', text, flags=re.IGNORECASE) parsed_base = urlparse(base_url) # 微信文章特殊处理:图片使用 data-src text = re.sub( r']+data-src=["\']([^"\']+)["\'][^>]*>', lambda m: f'![图片]({m.group(1)})', text, flags=re.IGNORECASE ) # 匹配普通 img 标签 def img_to_markdown(match): src = match.group(1) alt = match.group(2) if match.lastindex and match.lastindex >= 2 else "" if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" return f'![{alt}]({src})' text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>', img_to_markdown, text, flags=re.IGNORECASE ) def img_to_markdown_no_alt(match): src = match.group(1) if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" return f'![图片]({src})' text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*>', img_to_markdown_no_alt, text, flags=re.IGNORECASE ) # 处理代码块 def convert_code_block(match): code_content = match.group(1) lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE) lang = lang_match.group(1) if lang_match else '' return f'\n```{lang}\n{code_content}\n```\n' text = re.sub(r']*>\s*]*>(.*?)\s*', convert_code_block, text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE) # 转换其他标签 text = re.sub(r']*>(.*?)', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)

', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'', '\n', text, flags=re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'- \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ').replace('<', '<').replace('>', '>') text = text.replace('&', '&').replace('"', '"') # 清理空行 lines = [line for line in text.split('\n')] cleaned = [] prev_empty = False for line in lines: stripped = line.strip() if stripped == '': if not prev_empty: cleaned.append('') prev_empty = True else: cleaned.append(line) prev_empty = False return '\n'.join(cleaned).strip() async def crawl_wechat_with_playwright(url: str) -> dict: """使用Playwright采集微信公众号文章""" if not PLAYWRIGHT_AVAILABLE: raise Exception("Playwright未安装,无法采集微信公众号文章") async with async_playwright() as p: browser = await p.chromium.launch(headless=True) page = await browser.new_page() # 模拟微信内打开环境 await page.set_extra_http_headers({ 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36 MicroMessenger/8.0.0', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Referer': 'https://mp.weixin.qq.com/' }) await page.goto(url, wait_until='domcontentloaded', timeout=30000) await page.wait_for_timeout(5000) # 提取标题 title = "" try: if await page.locator('#activity_name').count() > 0: title = await page.locator('#activity_name').first.inner_text() title = title.strip() except: pass if not title: title = await page.title() # 提取正文 content_html = "" try: if await page.locator('#js_content').count() > 0: content_html = await page.locator('#js_content').first.inner_html() except: pass # 提取图片 images = [] try: img_elements = await page.locator('#js_content img').all() for img in img_elements[:10]: try: src = await img.get_attribute('data-src') or await img.get_attribute('src') alt = await img.get_attribute('alt') or '' if src and src.startswith('http'): images.append(CrawlImageInfo(original_url=src)) except: pass except: pass # 尝试提取公众号名称 nickname = "" try: if await page.locator('#js_name').count() > 0: nickname = await page.locator('#js_name').first.inner_text() nickname = nickname.strip() except: pass await browser.close() return { 'title': title if title else "未命名文章", 'content_html': content_html, 'images': images, 'nickname': nickname } # ========== API 端点 ========== @router.post("") async def crawl_wechat_article(request: WechatCrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)): """ 采集微信公众号文章(专用接口) todo: 后续支持批量采集公众号历史文章 """ try: if not request.url.startswith(('http://', 'https://')): raise HTTPException(status_code=400, detail="无效的URL") if 'mp.weixin.qq.com' not in request.url: return WechatCrawlResponse( success=False, error="该接口仅支持微信公众号文章(mp.weixin.qq.com)" ) if not PLAYWRIGHT_AVAILABLE: return WechatCrawlResponse( success=False, error="微信公众号文章需要Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium" ) print(f"[公众号采集] 开始采集: {request.url}") # 使用Playwright采集 result = await crawl_wechat_with_playwright(request.url) title = result['title'] images = result['images'] content_html = result['content_html'] nickname = result.get('nickname', '') # 转换为Markdown markdown_content = convert_wechat_html_to_markdown(content_html, images, request.url) # 创建临时目录 temp_dir = "/tmp/blog_wechat_crawl" os.makedirs(temp_dir, exist_ok=True) # uploads 目录配置 backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) upload_dir = os.path.join(backend_dir, "uploads") thumb_dir = os.path.join(upload_dir, "thumbnails") os.makedirs(upload_dir, exist_ok=True) os.makedirs(thumb_dir, exist_ok=True) # 下载图片(最多5张) downloaded_images = [] for i, img in enumerate(images[:5]): filename, filepath, success = download_image(img.original_url, temp_dir) if success: try: ext = os.path.splitext(filename)[1].lower() if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']: ext = '.jpg' new_filename = f"wechat_{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}" new_filepath = os.path.join(upload_dir, new_filename) with open(filepath, 'rb') as src: with open(new_filepath, 'wb') as dst: dst.write(src.read()) os.remove(filepath) thumb_filename = f"thumb_{new_filename}" thumb_filepath = os.path.join(thumb_dir, thumb_filename) thumb_path_db = None if generate_thumbnail(new_filepath, thumb_filepath): thumb_path_db = f"/uploads/thumbnails/{thumb_filename}" width, height = get_image_dimensions(new_filepath) file_size = os.path.getsize(new_filepath) image_record = Image( filename=new_filename, original_name=filename, file_path=f"/uploads/{new_filename}", thumb_path=thumb_path_db, description=f"公众号采集自 {request.url}" + (f" ({nickname})" if nickname else ""), file_size=file_size, width=width, height=height ) db.add(image_record) db.commit() db.refresh(image_record) img.local_path = f"/uploads/{new_filename}" img.filename = new_filename img.success = True img.id = image_record.id downloaded_images.append(img) if img.original_url in markdown_content: markdown_content = markdown_content.replace( img.original_url, f"/uploads/{new_filename}" ) print(f"[公众号采集] 图片替换: {img.original_url[:60]}... -> /uploads/{new_filename}") print(f"[公众号采集] 图片已保存: id={image_record.id}, filename={new_filename}") except Exception as e: print(f"[公众号采集] 保存图片失败: {e}") img.local_path = filepath img.filename = filename img.success = True downloaded_images.append(img) # 生成摘要 summary = markdown_content[:200].replace('#', '').replace('*', '').strip() # 在标题前添加公众号名称 if nickname and nickname not in title: title_prefix = f"【{nickname}】" else: title_prefix = "" print(f"[公众号采集] 完成: {title}") return WechatCrawlResponse( success=True, title=title_prefix + title, content=markdown_content, summary=summary, images=downloaded_images ) except HTTPException: raise except Exception as e: import traceback print(f"[公众号采集] 失败: {e}") print(traceback.format_exc()) return WechatCrawlResponse( success=False, error=f"公众号采集失败: {str(e)}" )