""" 文章采集路由 - 从网页URL采集内容 支持SPA网站采集 """ import base64 import os import re import uuid import requests from datetime import datetime from typing import Optional, List from urllib.parse import urljoin, urlparse try: from bs4 import BeautifulSoup BS4_AVAILABLE = True except ImportError: BS4_AVAILABLE = False # Playwright用于SPA采集 try: from playwright.async_api import async_playwright PLAYWRIGHT_AVAILABLE = True except ImportError: PLAYWRIGHT_AVAILABLE = False # Playwright Stealth 用于绕过WAF try: from playwright_stealth import stealth_async STEALTH_AVAILABLE = True except ImportError: STEALTH_AVAILABLE = False from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import JSONResponse from pydantic import BaseModel from sqlalchemy.orm import Session import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from database import get_db from models import Image from routers.auth import get_current_admin from routers.images import generate_thumbnail, get_image_dimensions router = APIRouter(prefix="/api/crawl", tags=["文章采集"]) def fix_encoding(text: str) -> str: """ 修复常见的编码问题 处理UTF-8内容被错误解码为Latin-1的情况 支持双重编码错误（UTF-8 -> Latin-1 -> UTF-8） """ if not text: return text # 检测是否包含乱码特征（UTF-8被错误解码为Latin-1） # 常见的乱码模式：é, è, Ã, Â, Ã¥ 等 garbled_patterns = ['é', 'è', 'Ã', 'Â', 'Ã¥', 'ü', 'ö', 'ä', 'ß', 'â€', 'æ', 'å', 'è', '§'] # 检查是否包含乱码特征 has_garbled = any(pattern in text for pattern in garbled_patterns) if has_garbled: try: # 尝试单层修复：文本以latin1存储但实际是utf-8内容 fixed = text.encode('latin1').decode('utf-8') # 检查是否还有乱码（双重编码错误） still_garbled = any(p in fixed for p in garbled_patterns) if still_garbled: try: # 双重修复 fixed = fixed.encode('latin1').decode('utf-8') except: pass return fixed except (UnicodeEncodeError, UnicodeDecodeError): # 如果修复失败，返回原文本 pass return text class CrawlRequest(BaseModel): url: str class CrawlImage(BaseModel): id: Optional[int] = None # 数据库ID original_url: str local_path: Optional[str] = None filename: Optional[str] = None success: bool = False class CrawlResponse(BaseModel): success: bool title: Optional[str] = None content: Optional[str] = None # Markdown内容 summary: Optional[str] = None images: List[CrawlImage] = [] error: Optional[str] = None def clean_html_tags(text: str) -> str: """移除文本中的HTML标签""" # 移除所有HTML标签 text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ') text = text.replace('<', '<') text = text.replace('>', '>') text = text.replace('&', '&') text = text.replace('"', '"') text = text.replace(''', "'") return text.strip() def fix_math_formulas(text: str) -> str: """修复数学公式格式，处理混合格式问题""" import re # 1. 清理已渲染的数学符号（保留LaTeX代码） # 移除 Unicode 数学运算符，但保留后面的LaTeX text = re.sub(r'[⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿⁱ]+', '', text) # 2. 规范化反斜杠（双重转义变单重） text = text.replace('\\\\', '\\') # 3. 检测并包裹行内公式 # 匹配常见的行内数学模式：\mathbf{x}, \alpha, x_{i}, x^{2}, \sum 等 inline_patterns = [ r'\\[a-zA-Z]+\{[^}]*\}', # \mathbf{x}, \frac{a}{b} r'\\[a-zA-Z]+', # \alpha, \beta r'[a-zA-Z]_[a-zA-Z0-9]+', # x_i, y_j r'[a-zA-Z]\^[a-zA-Z0-9]+', # x^2, y^n ] # 4. 检测块级公式（多行或包含 array, matrix 等） block_keywords = ['\\begin{array}', '\\begin{matrix}', '\\begin{equation}', '\\sum_', '\\int_', '\\prod_', '\\lim_'] lines = text.split('\n') result_lines = [] in_block = False block_content = [] for line in lines: stripped = line.strip() # 检测块级公式开始 if any(kw in stripped for kw in block_keywords): if not in_block: in_block = True block_content = [] block_content.append(line) continue # 如果在块内，继续收集 if in_block: block_content.append(line) # 检测块结束 if '\\end{array}' in stripped or '\\end{matrix}' in stripped or '\\end{equation}' in stripped: # 包裹整个块 block_text = '\n'.join(block_content) if not block_text.startswith('$$') and not block_text.startswith('\\['): result_lines.append('$$' + block_text + '$$') else: result_lines.append(block_text) in_block = False block_content = [] continue # 处理行内公式 # 如果一行中有明显的LaTeX命令，但还没有被包裹 if re.search(r'\\[a-zA-Z]+', line) and not re.search(r'\\\(|\\\)|\\\[|\\\]', line): # 简单处理：如果包含复杂结构，作为块级；否则作为行内 if re.search(r'\\(frac|sum|int|prod|lim)\{', line): line = '$$' + line + '$$' else: line = '\\(' + line + '\\)' result_lines.append(line) # 处理未闭合的块 if in_block and block_content: block_text = '\n'.join(block_content) result_lines.append('$$' + block_text + '$$') return '\n'.join(result_lines) def extract_title_from_html(html: str) -> str: """从HTML中提取标题 - 优先提取文章主标题""" # 1. 尝试提取 Open Graph title (og:title) - 通常是文章主标题 og_title_match = re.search( r']+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE ) if not og_title_match: og_title_match = re.search( r']+content=["\']([^"\']+)["\'][^>]+property=["\']og:title["\']', html, re.IGNORECASE ) if og_title_match: title = clean_html_tags(og_title_match.group(1)) if len(title) > 5: # 确保不是太短的无意义标题 # 清理常见后缀 title = re.sub(r'\s+[–—-]\s+.*$', '', title).strip() return title # 1.5 博客园/CSDN等特殊平台处理（放在h1之前，因为这些平台的h1可能是博客名而非文章标题） # 博客园：h1.postTitle 或 a#cb_post_title_url cnblog_title = re.search(r']+class=["\'][^"\']*postTitle[^"\']*["\'][^>]*>(.*?)', html, re.IGNORECASE | re.DOTALL) if cnblog_title: title = clean_html_tags(cnblog_title.group(1)) if len(title) > 5: return title cnblog_title2 = re.search(r']+id=["\']cb_post_title_url["\'][^>]*>(.*?)', html, re.IGNORECASE | re.DOTALL) if cnblog_title2: title = clean_html_tags(cnblog_title2.group(1)) if len(title) > 5: return title # 1.6 人人都是产品经理 (woshipm.com) woshipm_title = re.search(r'<[^>]+class=["\'][^"\']*article--title[^"\']*["\'][^>]*>(.*?)]+>', html, re.IGNORECASE | re.DOTALL) if woshipm_title: title = clean_html_tags(woshipm_title.group(1)) if len(title) > 5: return title # 1.7 菜鸟教程：从article-intro内的h1提取标题，或使用title标签 runoob_title = re.search(r']*class=["\']article-intro["\'][^>]*>.*?]*>(.*?)', html, re.IGNORECASE | re.DOTALL) if runoob_title: title = clean_html_tags(runoob_title.group(1)) if len(title) > 5 and "菜鸟教程" not in title: # 排除网站名 return title # 从title标签提取（菜鸟教程等网站） title_match = re.search(r']*>(.*?)', html, re.IGNORECASE | re.DOTALL) if title_match: title = clean_html_tags(title_match.group(1)) # 移除常见的分隔符和后缀（但保留技术词汇中的连字符如Objective-C） # 使用更严格的匹配，确保是标题分隔符而不是单词中的连字符 title = re.sub(r'\s+[|│┃]\s+.*$', '', title).strip() # | 分隔符 title = re.sub(r'\s+[-–—]\s+[^\s]*教程.*$', '', title).strip() # 匹配 " - 菜鸟教程" title = re.sub(r'\s+[-–—]\s+[^\s]*博客.*$', '', title).strip() # 匹配 " - XXX博客" if len(title) > 5: return title # 2. 尝试提取 h1 标题（文章主标题通常是页面第一个h1） # 先清理HTML获取纯文本 h1_matches = re.findall(r']*>(.*?)', html, re.IGNORECASE | re.DOTALL) for h1 in h1_matches: title = clean_html_tags(h1) # 过滤掉太短的标题 if len(title) > 5: return title # 3. 尝试从 article/main 内容区域提取 h2 或 h3（作为备选） content_patterns = [ r']*>.*?]*>(.*?).*?', r']*>.*?]*>(.*?).*?', r']*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>.*?]*>(.*?)', ] for pattern in content_patterns: match = re.search(pattern, html, re.IGNORECASE | re.DOTALL) if match: title = clean_html_tags(match.group(1)) if len(title) > 10: # 内容区标题通常较长 return title # 4. 尝试提取 Twitter Card title twitter_title_match = re.search( r']+name=["\']twitter:title["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE ) if twitter_title_match: title = clean_html_tags(twitter_title_match.group(1)) if len(title) > 5: return title # 5. 最后回退到 标签 title_match = re.search(r'<title[^>]*>([^<]+)', html, re.IGNORECASE) if title_match: title = clean_html_tags(title_match.group(1)) # 清理常见后缀（网站名、分类等） # 处理各种分隔符: | - – — :: · • title = re.sub(r'\s*[\|\-–—:·•]\s*.+$', '', title) # 清理特定后缀词 title = re.sub(r'\s*[-–]\s*(Home|首页|主页|官网|官方网站|新闻|News|Press Release).*?$', '', title, flags=re.IGNORECASE) if title: return title return "未命名文章" def extract_main_content(html: str) -> str: """从HTML中提取主要内容（简化版）""" # 使用 BeautifulSoup 如果可用，更精确可靠 if BS4_AVAILABLE: try: soup = BeautifulSoup(html, 'html.parser') # 移除 script 和 style for tag in soup(['script', 'style']): tag.decompose() # 按优先级尝试不同的内容选择器 content_selectors = [ # 人人都是产品经理 '.article--content.grap', # 博客园 '#cnblogs_post_body', # 菜鸟教程 '.article-intro', # 掘金 'article', # 通用 'article', 'main', '.post-content', '.article-content', '.entry-content', '#content', '#main-content', ] for selector in content_selectors: elem = soup.select_one(selector) if elem: # 返回元素的HTML内容 return str(elem) # 如果都没找到，返回body内容 body = soup.find('body') if body: return str(body) except Exception as e: print(f"BeautifulSoup提取失败，回退到正则: {e}") # 回退到正则表达式方式 # 移除 script 和 style html = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) html = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) # 尝试找到主要内容区域 content = "" # 常见的文章容器 patterns = [ # 人人都是产品经理 (woshipm.com) r']*class=["\']article--content grap["\'][^>]*>(.*?)\s*\s*]*class=["\']article--footer', r']*class=["\']article--content grap["\'][^>]*>(.*?)\s*\s*]*class=["\']article-bottom-meta', r']*class=["\']article--content grap["\'][^>]*>(.*?)\s*]*class=["\']u-marginTop30', # 菜鸟教程 r']*class=["\']article-intro["\'][^>]*>(.*?)]*class=["\'](?:google-auto-placed|pre|next)', r']*class=["\']article-intro["\'][^>]*>(.*?)\s*(?:]*class=["\']article-intro["\'][^>]*>(.*?)\s*$', r']*class=["\']article-intro["\'][^>]*>(.*)', # 博客园 r']+id=["\']cnblogs_post_body["\'][^>]*>(.*?)\s*]+id=["\']blog_post_info_wrap["\']', r']+id=["\']cnblogs_post_body["\'][^>]*>(.*?)', # 通用 r']*>(.*?)', r']*>(.*?)', r']*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)', r']*id=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)', ] for pattern in patterns: match = re.search(pattern, html, re.DOTALL | re.IGNORECASE) if match: content = match.group(1) break # 如果没找到，尝试提取 body if not content: body_match = re.search(r']*>(.*?)', html, re.DOTALL | re.IGNORECASE) if body_match: content = body_match.group(1) return content if content else html def html_to_markdown(html: str, base_url: str = "") -> str: """HTML转Markdown，保留图片在原始位置""" # 修复编码问题 html = fix_encoding(html) # 检查base_url是否为HTTPS parsed_base = urlparse(base_url) if base_url else None base_is_https = parsed_base and parsed_base.scheme == 'https' def make_absolute_url(src: str) -> str: """将相对URL转换为绝对URL""" if not src: return src if src.startswith('//'): src = 'https:' + src elif src.startswith('/') and base_url: src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" elif not src.startswith(('http://', 'https://')) and base_url: src = urljoin(base_url, src) # 修复混合内容：页面HTTPS时图片也用HTTPS if base_is_https and src.startswith('http://'): src = 'https://' + src[7:] return src if BS4_AVAILABLE: # 使用BeautifulSoup更精确地处理 soup = BeautifulSoup(html, 'html.parser') # 处理所有图片，保留在原位置 for img in soup.find_all('img'): src = img.get('src', '') alt = img.get('alt', '') if src and not src.startswith('data:'): src = make_absolute_url(src) # 创建Markdown图片标记 md_img = f'![{alt}]({src})' img.replace_with(md_img) elif src.startswith('data:'): # 移除 data URL 图片 img.decompose() # 处理标题 for tag in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']): level = int(tag.name[1]) tag.replace_with(f'\n{"#" * level} {tag.get_text().strip()}\n') # 处理段落 for p in soup.find_all('p'): text = p.get_text() # 保留段落中的Markdown图片标记 for content in p.contents: if isinstance(content, str) and content.startswith('!['): text = text.replace(content, content) p.replace_with(f'\n{text}\n') # 处理换行 for br in soup.find_all('br'): br.replace_with('\n') # 处理代码块 
...
for pre in soup.find_all('pre'): code = pre.find('code') if code: # 获取代码内容，保留原始格式 code_text = code.get_text() # 检测语言（从class属性） lang = '' code_classes = code.get('class', []) for cls in code_classes: if 'language-' in cls or 'lang-' in cls: lang = cls.replace('language-', '').replace('lang-', '') break # 创建Markdown代码块 md_code = f'\n```{lang}\n{code_text}\n```\n' pre.replace_with(md_code) else: # 纯pre标签 pre_text = pre.get_text() md_code = f'\n```\n{pre_text}\n```\n' pre.replace_with(md_code) # 处理行内代码 ... for code in soup.find_all('code'): code_text = code.get_text() # 检查是否在pre内（已处理过） if not code.find_parent('pre'): code.replace_with(f'`{code_text}`') # 处理粗体 for tag in soup.find_all(['strong', 'b']): tag.replace_with(f'**{tag.get_text()}**') # 处理斜体 for tag in soup.find_all(['em', 'i']): tag.replace_with(f'*{tag.get_text()}*') # 获取文本 text = soup.get_text() else: # 回退到正则方式 def convert_img_to_markdown(match): src = match.group(1) alt = match.group(2) if match.lastindex and match.lastindex >= 2 else "" src = make_absolute_url(src) return f'![{alt}]({src})' text = html # 先处理图片 text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>', convert_img_to_markdown, text, flags=re.IGNORECASE ) text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*>', lambda m: f'![图片]({make_absolute_url(m.group(1))})', text, flags=re.IGNORECASE ) # 处理其他标签 text = re.sub(r']*>(.*?)', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)

', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'', '\n', text, flags=re.IGNORECASE) # 处理代码块 
...
def convert_code_block(match): code_content = match.group(1) # 提取语言 lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE) lang = lang_match.group(1) if lang_match else '' return f'\n```{lang}\n{code_content}\n```\n' text = re.sub(r']*>\s*]*>(.*?)\s*', convert_code_block, text, flags=re.DOTALL | re.IGNORECASE) # 处理纯pre标签 text = re.sub(r']*>(.*?)', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE) # 处理行内代码 text = re.sub(r']*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ') text = text.replace('<', '<') text = text.replace('>', '>') text = text.replace('&', '&') text = text.replace('"', '"') # 修复数学公式格式 text = fix_math_formulas(text) # 清理多余空白行 lines = text.split('\n') cleaned_lines = [] prev_empty = False for line in lines: stripped = line.strip() if stripped == '': if not prev_empty: cleaned_lines.append('') prev_empty = True else: cleaned_lines.append(line) prev_empty = False return '\n'.join(cleaned_lines).strip() def extract_images(html: str, base_url: str) -> List[CrawlImage]: """从HTML中提取图片URL""" images = [] parsed_base = urlparse(base_url) base_is_https = parsed_base.scheme == 'https' # 匹配 img_pattern = r']+src=["\']([^"\']+)["\'][^>]*>' for match in re.finditer(img_pattern, html, re.IGNORECASE): img_url = match.group(1) # 跳过 data URL if img_url.startswith('data:'): continue # 转换为绝对URL if img_url.startswith('//'): img_url = 'https:' + img_url elif img_url.startswith('/'): img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}" elif not img_url.startswith(('http://', 'https://')): img_url = urljoin(base_url, img_url) # 修复混合内容问题：如果页面是HTTPS，图片也要用HTTPS if base_is_https and img_url.startswith('http://'): img_url = 'https://' + img_url[7:] images.append(CrawlImage(original_url=img_url)) return images def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[CrawlImage]: """从Markdown内容中提取图片URL（确保只提取实际在文章中的图片）""" images = [] parsed_base = urlparse(base_url) base_is_https = parsed_base.scheme == 'https' # 匹配Markdown图片语法: ![alt](url) img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)' for match in re.finditer(img_pattern, markdown_content): img_url = match.group(2) # 跳过已经是本地路径的图片 if img_url.startswith('/uploads/'): continue # 转换为绝对URL if img_url.startswith('//'): img_url = 'https:' + img_url elif img_url.startswith('/'): img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}" elif not img_url.startswith(('http://', 'https://')): img_url = urljoin(base_url, img_url) # 修复混合内容问题 if base_is_https and img_url.startswith('http://'): img_url = 'https://' + img_url[7:] images.append(CrawlImage(original_url=img_url)) return images def download_image(url: str, temp_dir: str) -> tuple: """下载图片到临时目录""" try: # 跳过 data URL（base64 内嵌图片） if url.startswith('data:'): return None, None, False # 跳过 Google favicon 服务等外部小图标（国内服务器无法访问） if 'google.com/s2/favicons' in url or 'google.com/favicon' in url: return None, None, False headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', } # 微信图片需要 referer if 'mmbiz.qpic.cn' in url or 'mmbiz.qlogo.cn' in url: headers['Referer'] = 'https://mp.weixin.qq.com/' response = requests.get(url, headers=headers, timeout=10, stream=True) response.raise_for_status() # 获取文件扩展名 content_type = response.headers.get('content-type', '') ext = '.jpg' if 'png' in content_type: ext = '.png' elif 'gif' in content_type: ext = '.gif' elif 'webp' in content_type: ext = '.webp' else: # 从URL获取扩展名 parsed = urlparse(url) path_ext = os.path.splitext(parsed.path)[1] if path_ext: ext = path_ext # 生成文件名 filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}" filepath = os.path.join(temp_dir, filename) # 保存文件 with open(filepath, 'wb') as f: for chunk in response.iter_content(chunk_size=8192): f.write(chunk) # 检查文件大小，微信防盗链返回的占位图通常 < 2KB file_size = os.path.getsize(filepath) if file_size < 2048 and 'mmbiz.qpic.cn' in url: print(f"图片疑似防盗链占位图，跳过: {url} (大小: {file_size} bytes)") os.remove(filepath) return None, None, False return filename, filepath, True except Exception as e: print(f"下载图片失败 {url}: {e}") return None, None, False def is_spa_site(html: str) -> bool: """检测是否为SPA网站""" # 检测标志 spa_indicators = [ # React/Vue/Angular常见标记 '
', '
', '
', # Nuxt.js SSR:
or
'__nuxt', 'You need to enable JavaScript', # 加载中标记 'Please wait', 'Loading...', # 常见SPA框架 'react-root', 'ng-app', 'data-reactroot', # 动态加载标记 'window.__INITIAL_STATE__', 'window.__DATA__', ] html_lower = html.lower() for indicator in spa_indicators: if indicator.lower() in html_lower: return True # 检查内容是否极少（只有JS没有实际内容） # 先剔除 script 和 style 标签及其内容，因为 Next.js RSC 等框架会在 # script 中塞入大量序列化数据，干扰文字量判断 html_for_check = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) html_for_check = re.sub(r']*>.*?', '', html_for_check, flags=re.DOTALL | re.IGNORECASE) text_content = re.sub(r'<[^>]+>', '', html_for_check) text_content = re.sub(r'\s+', '', text_content) if len(text_content) < 200: return True return False async def crawl_with_playwright(url: str) -> dict: """使用Playwright异步采集SPA页面""" if not PLAYWRIGHT_AVAILABLE: raise Exception("Playwright未安装，无法采集SPA网站") # 检测平台类型 is_wechat = 'mp.weixin.qq.com' in url is_tencent_cloud = 'cloud.tencent.com' in url is_lark_wiki = 'larkoffice.com/wiki' in url or 'feishu.cn' in url is_yiigle = 'yiigle.com' in url async with async_playwright() as p: # 使用 stealth 参数启动浏览器，绕过WAF browser = await p.chromium.launch( headless=True, args=[ '--disable-blink-features=AutomationControlled', '--disable-web-security', '--disable-features=IsolateOrigins,site-per-process', '--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox', '--disable-accelerated-2d-canvas', '--disable-gpu', '--window-size=1920,1080', ] ) # 创建新页面并设置视口和更多参数 context = await browser.new_context( viewport={'width': 1920, 'height': 1080}, user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', locale='zh-CN', timezone_id='Asia/Shanghai', permissions=['geolocation'], java_script_enabled=True, ) # 添加额外脚本隐藏 webdriver 属性 await context.add_init_script(""" Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); window.chrome = { runtime: {} }; """) page = await context.new_page() # 应用 stealth 模式（绕过反爬虫检测） if STEALTH_AVAILABLE: await stealth_async(page) print("[Playwright] Stealth 模式已启用") # 打开页面 # 优先使用 networkidle 等 JS 渲染完成，但很多现代站点有长连接会导致超时 # 超时后回退到 domcontentloaded + 额外等待 try: if is_tencent_cloud: # 腾讯云有大量广告/跟踪脚本，networkidle 容易超时，直接 domcontentloaded await page.goto(url, wait_until='domcontentloaded', timeout=30000) else: await page.goto(url, wait_until='networkidle', timeout=15000) except Exception as e: print(f"[Playwright] networkidle 超时，回退到 domcontentloaded: {e}") await page.goto(url, wait_until='domcontentloaded', timeout=15000) # 微信文章特殊处理 if is_wechat: await page.wait_for_timeout(5000) # 微信需要更长等待时间 # 提取标题 - 微信文章标题在 #activity_name title = "" try: if await page.locator('#activity_name').count() > 0: title = await page.locator('#activity_name').first.inner_text() title = title.strip() except: pass if not title: title = await page.title() # 提取正文 - 微信文章正文在 #js_content content_html = "" try: if await page.locator('#js_content').count() > 0: content_html = await page.locator('#js_content').first.inner_html() except: pass # 提取图片 - 微信图片用data-src，优先取浏览器渲染后的实际URL images = [] try: img_elements = await page.locator('#js_content img').all() for img in img_elements[:30]: try: # 优先级：dataset.src > currentSrc > data-src > src src = None # 1. 尝试通过 JS 获取 dataset.src（微信懒加载后的真实地址） try: src = await img.evaluate('el => el.dataset?.src || ""') except: pass # 2. 尝试 currentSrc（浏览器实际加载的地址） if not src or src.startswith('data:'): try: src = await img.evaluate('el => el.currentSrc || ""') except: pass # 3. 回退到 data-src 属性 if not src or src.startswith('data:'): src = await img.get_attribute('data-src') # 4. 最后回退到 src 属性 if not src or src.startswith('data:'): src = await img.get_attribute('src') alt = await img.get_attribute('alt') or '' # 过滤 data URL 和无效地址 if src and src.startswith('http') and not src.startswith('data:'): images.append(CrawlImage(original_url=src)) except: pass except: pass # 微信图片防盗链：用浏览器 fetch 下载（携带 cookies） temp_dir = os.environ.get('TEMP_DIR') or '/tmp' downloaded_images = [] for crawl_img in images[:15]: try: img_url = crawl_img.original_url # 在浏览器里 fetch 下载，携带微信 cookies js_code = f""" async () => {{ try {{ const resp = await fetch("{img_url}", {{ referrer: "https://mp.weixin.qq.com/" }}); if (!resp.ok) return null; const blob = await resp.blob(); return new Promise((resolve) => {{ const reader = new FileReader(); reader.onloadend = () => resolve(reader.result.split(',')[1]); reader.readAsDataURL(blob); }}); }} catch (e) {{ return null; }} }} """ b64_data = await page.evaluate(js_code) if b64_data: # 确定扩展名 ext = os.path.splitext(urlparse(img_url).path)[1].lower() or '.jpg' if not ext or ext == '.': ext = '.jpg' filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}" filepath = os.path.join(temp_dir, filename) with open(filepath, 'wb') as f: f.write(base64.b64decode(b64_data)) # 检查文件大小，排除防盗链占位图 file_size = os.path.getsize(filepath) if file_size < 2048: os.remove(filepath) print(f"微信图片疑似防盗链占位图，跳过: {img_url} ({file_size} bytes)") downloaded_images.append(crawl_img) else: crawl_img.local_path = filepath crawl_img.filename = filename crawl_img.success = True downloaded_images.append(crawl_img) print(f"微信图片下载成功: {filename} ({file_size} bytes)") else: downloaded_images.append(crawl_img) except Exception as e: print(f"浏览器下载微信图片失败: {e}") downloaded_images.append(crawl_img) images = downloaded_images await context.close() await browser.close() return { 'title': title if title else "未命名文章", 'content_html': content_html, 'images': images } # 普通SPA网站处理 # 检测平台（is_tencent_cloud 已在函数开头定义） is_juejin = 'juejin.cn' in url is_segmentfault = 'segmentfault.com' in url # 等待内容加载 if is_juejin: # 掘金需要等待article标签 try: await page.wait_for_selector('article', timeout=15000) except: pass await page.wait_for_timeout(3000) elif is_tencent_cloud: # 腾讯云开发者社区：.mod-content__markdown 一开始存在但为空，需等待 JS 填充 try: await page.wait_for_function( """() => { const el = document.querySelector('.mod-content__markdown'); return el && el.innerHTML.length > 500; }""", timeout=15000 ) except: pass await page.wait_for_timeout(2000) elif is_segmentfault: # SegmentFault 等待文章正文 print("[Playwright] 等待 SegmentFault 页面加载...") try: await page.wait_for_load_state('networkidle', timeout=20000) except: pass await page.wait_for_timeout(8000) # 给JS更多时间渲染 print("[Playwright] 等待完成，开始提取") else: try: await page.wait_for_selector('h1, .article-title, .content, article, main', timeout=10000) except: pass await page.wait_for_timeout(5000) # 提取标题 title = "" if is_juejin: # 掘金标题选择器 for selector in ['article h1', 'h1']: try: if await page.locator(selector).count() > 0: title = await page.locator(selector).first.inner_text() if title.strip(): break except: continue elif is_segmentfault: # SegmentFault 标题选择器 sf_title_selectors = ['h1.article-title', 'h1', '.article-title', '[class*="title"]', 'title'] for selector in sf_title_selectors: try: count = await page.locator(selector).count() if count > 0: t = await page.locator(selector).first.inner_text() print(f"[Playwright] 标题选择器 '{selector}': '{t[:30] if t else '空'}...'") if t and t.strip(): title = t.strip() break except Exception as e: print(f"[Playwright] 标题选择器 '{selector}' 失败: {e}") elif is_tencent_cloud: # 腾讯云开发者社区标题 for selector in ['h1', '.article-title', '[class*="title"]']: try: if await page.locator(selector).count() > 0: title = await page.locator(selector).first.inner_text() if title.strip(): break except: continue elif is_lark_wiki: # 飞书/Lark Wiki 标题：page.title() 格式为 "文章标题 - 飞书云文档" page_title = await page.title() if ' - ' in page_title: title = page_title.rsplit(' - ', 1)[0].strip() # 如果拆分失败，尝试 h1 的第二个（第一个通常是"飞书云文档"网站名） if not title: try: h1_count = await page.locator('h1').count() if h1_count >= 2: title = await page.locator('h1').nth(1).inner_text() title = title.strip() except: pass elif is_yiigle: # 中华医学期刊网(yiigle.com)标题：page.title() 格式为 "文章标题 - 期刊名" page_title = await page.title() if ' - ' in page_title: title = page_title.rsplit(' - ', 1)[0].strip() # 备选：从 og:title meta 提取 if not title: try: og_title = await page.locator('meta[property="og:title"]').get_attribute('content') if og_title: title = og_title.strip() except: pass print(f"[Playwright] yiigle 标题: {title}") if not title: for selector in ['h1', '.article-title', '[class*="title"]']: try: if await page.locator(selector).count() > 0: title = await page.locator(selector).first.inner_text() if title.strip(): break except: continue if not title: title = await page.title() # 提取正文HTML content_html = "" print(f"[Playwright] 开始提取内容，URL: {url}") print(f"[Playwright] is_segmentfault: {is_segmentfault}") if is_juejin: # 掘金内容在 .markdown-body 中 try: print(f"[Playwright] 掘金: 检查 .markdown-body") mb_count = await page.locator('.markdown-body').count() print(f"[Playwright] 掘金: .markdown-body 数量: {mb_count}") if mb_count > 0: content_html = await page.locator('.markdown-body').first.inner_html() print(f"[Playwright] 掘金: 从 .markdown-body 提取内容，长度 {len(content_html)}") elif await page.locator('article').count() > 0: content_html = await page.locator('article').first.inner_html() print(f"[Playwright] 掘金: 从 article 提取内容，长度 {len(content_html)}") else: # 诊断：截图和保存页面源码 print(f"[Playwright] 掘金: 未找到内容区域，保存诊断信息") try: await page.screenshot(path='/tmp/juejin_debug.png', full_page=True) page_source = await page.content() with open('/tmp/juejin_debug.html', 'w', encoding='utf-8') as f: f.write(page_source[:50000]) # 保存前50KB print(f"[Playwright] 掘金: 诊断信息已保存到 /tmp/juejin_debug.*") except Exception as e: print(f"[Playwright] 掘金: 保存诊断信息失败: {e}") except Exception as e: print(f"[Playwright] 掘金: 提取内容异常: {e}") # 清理掘金内容：移除 style 和 script 标签（掘金有大量内嵌CSS） if content_html: import re # 移除 style 标签及其内容 content_html = re.sub(r']*>.*?', '', content_html, flags=re.DOTALL | re.IGNORECASE) # 移除 script 标签及其内容 content_html = re.sub(r']*>.*?', '', content_html, flags=re.DOTALL | re.IGNORECASE) # 移除所有元素的 style 属性 content_html = re.sub(r'\s+style="[^"]*"', '', content_html, flags=re.IGNORECASE) # 移除 class 属性（减少无用属性） content_html = re.sub(r'\s+class="[^"]*"', '', content_html, flags=re.IGNORECASE) elif is_segmentfault: # SegmentFault 内容提取 - 尝试多种选择器 sf_selectors = ['.article-content', '.markdown-body', 'article', '.post-content', '[data-id]', 'main', '#content', '.content'] for selector in sf_selectors: try: count = await page.locator(selector).count() print(f"[Playwright] 检查选择器 '{selector}': {count} 个匹配") if count > 0: html = await page.locator(selector).first.inner_html() if len(html) > 200: # 确保内容足够长 content_html = html print(f"[Playwright] 使用 '{selector}' 提取，长度: {len(content_html)}") break except Exception as e: print(f"[Playwright] 选择器 '{selector}' 失败: {e}") if not content_html: print(f"[Playwright] 未找到 SegmentFault 内容，尝试获取整个body") try: content_html = await page.locator('body').first.inner_html() print(f"[Playwright] 使用 body，长度: {len(content_html)}") except Exception as e: print(f"[Playwright] 获取 body 失败: {e}") # 清理 SegmentFault 内容 if content_html: import re # 移除 style 和 script 标签 content_html = re.sub(r']*>.*?', '', content_html, flags=re.DOTALL | re.IGNORECASE) content_html = re.sub(r']*>.*?', '', content_html, flags=re.DOTALL | re.IGNORECASE) # 移除广告相关元素 content_html = re.sub(r']*class="[^"]*ad[^"]*"[^>]*>.*?
', '', content_html, flags=re.DOTALL | re.IGNORECASE) elif is_tencent_cloud: # 腾讯云开发者社区内容 try: # 先尝试 .mod-content__markdown（JS 渲染后会有内容） if await page.locator('.mod-content__markdown').count() > 0: html = await page.locator('.mod-content__markdown').first.inner_html() if len(html) > 500: content_html = html # 如果为空，回退到 .mod-article-content if not content_html or len(content_html) < 500: if await page.locator('.mod-article-content').count() > 0: html = await page.locator('.mod-article-content').first.inner_html() if len(html) > 500: content_html = html except Exception as e: print(f"[Playwright] 腾讯云内容提取异常: {e}") elif is_lark_wiki: # 飞书/Lark Wiki 内容提取 # 飞书文档使用 block-based 编辑器 + 虚拟列表懒加载 # DOM 只渲染可见区域(~30 blocks)，完整数据在 window.DATA.clientVars.data.block_map print("[Playwright] 飞书Wiki: 从 JS 数据提取完整内容...") try: # 等待 page-block-children 出现确保 JS 初始化完成 await page.wait_for_selector('.page-block-children', timeout=15000) except: pass await page.wait_for_timeout(3000) # 优先从 JS 数据提取（完整 229 blocks vs DOM 的 ~30 blocks） lark_markdown = await page.evaluate("""() => { const data = window.DATA?.clientVars?.data; if (!data) return null; const block_map = data.block_map; const block_sequence = data.block_sequence || []; function extractText(blockData) { const textObj = blockData.text; if (!textObj) return ''; const iat = textObj.initialAttributedTexts; if (!iat) return ''; const texts = iat.text; if (!texts) return ''; let parts = []; for (let key of Object.keys(texts).sort((a,b) => Number(a)-Number(b))) { parts.push(texts[key]); } return parts.join(''); } function blockToMarkdown(blockId) { const block = block_map[blockId]; if (!block) return ''; const bd = block.data || block; const type = bd.type || 'unknown'; const text = extractText(bd); // Skip root page itself, process its children if (type === 'page' && !bd.parent_id) { let md = ''; for (const childId of (bd.children || [])) { md += blockToMarkdown(childId); } return md; } let md = ''; switch(type) { case 'heading1': md += '\\n# ' + text + '\\n\\n'; break; case 'heading2': md += '\\n## ' + text + '\\n\\n'; break; case 'heading3': md += '\\n### ' + text + '\\n\\n'; break; case 'heading4': md += '\\n#### ' + text + '\\n\\n'; break; case 'heading5': md += '\\n##### ' + text + '\\n\\n'; break; case 'text': if (text.trim()) md += text + '\\n\\n'; else md += '\\n'; break; case 'callout': if (text.trim()) md += '> 📌 ' + text + '\\n\\n'; break; case 'ordered': if (text.trim()) md += '1. ' + text + '\\n'; break; case 'bullet': if (text.trim()) md += '- ' + text + '\\n'; break; case 'code': md += '```\\n' + text + '\\n```\\n\\n'; break; case 'divider': md += '\\n---\\n\\n'; break; case 'quote': if (text.trim()) md += '> ' + text + '\\n\\n'; break; default: if (text.trim()) md += text + '\\n\\n'; } for (const childId of (bd.children || [])) { md += blockToMarkdown(childId); } return md; } const rootBlockId = block_sequence[0]; let md = blockToMarkdown(rootBlockId); md = md.replace(/\\n{4,}/g, '\\n\\n\\n'); return md; }""") if lark_markdown and len(lark_markdown) > 500: print(f"[Playwright] 飞书Wiki: JS数据提取成功，Markdown 长度 {len(lark_markdown)}") content_html = lark_markdown # 已经是 Markdown _is_lark_markdown = True else: # 回退到 DOM 提取 print("[Playwright] 飞书Wiki: JS数据为空，回退到 DOM 提取") try: if await page.locator('.page-block-children').count() > 0: html = await page.locator('.page-block-children').first.inner_html() if len(html) > 500: content_html = html print(f"[Playwright] 飞书Wiki: DOM 提取内容，长度 {len(content_html)}") except Exception as e: print(f"[Playwright] 飞书Wiki DOM 提取异常: {e}") elif is_yiigle: # 中华医学期刊网(yiigle.com)：文章全文需登录，仅提取可见的摘要和元数据 print("[Playwright] yiigle: 提取摘要和元数据...") try: # 等待摘要区域渲染完成 await page.wait_for_selector('#abstract_sec_main', timeout=15000) except: pass await page.wait_for_timeout(3000) # 提取摘要HTML abstract_html = "" if await page.locator('#abstract_sec_main').count() > 0: abstract_html = await page.locator('#abstract_sec_main').first.inner_html() print(f"[Playwright] yiigle: 摘要区域长度 {len(abstract_html)}") # 从 meta 标签提取元数据 meta_info = await page.evaluate("""() => { const metas = document.querySelectorAll('meta[name]'); const info = {}; for (const m of metas) { const name = m.getAttribute('name'); const content = m.getAttribute('content'); if (name && content) { if (name.startsWith('eprints.') || name.startsWith('DC.') || name.startsWith('citation_')) { info[name] = content; } } } return info; }""") # 构建内容HTML：摘要 + 元数据 parts = [] if abstract_html: parts.append(abstract_html) # 添加元数据（作者、期刊、DOI等） meta_lines = [] # 作者 authors = meta_info.get('eprints.creators_name') or meta_info.get('citation_author') or '' if authors: meta_lines.append(f'

作者：{authors}

') # 期刊 journal = meta_info.get('eprints.publication') or meta_info.get('citation_journal_title') or '' if journal: meta_lines.append(f'

期刊：{journal}

') # DOI doi = meta_info.get('eprints.doi') or meta_info.get('citation_doi') or '' if doi: meta_lines.append(f'

DOI：{doi}

') # 出版日期 pub_date = meta_info.get('eprints.date') or meta_info.get('citation_publication_date') or '' if pub_date: meta_lines.append(f'

出版日期：{pub_date}

') # 卷期页码 vol = meta_info.get('eprints.volume') or meta_info.get('citation_volume') or '' issue = meta_info.get('eprints.number') or meta_info.get('citation_issue') or '' pages = meta_info.get('eprints.pagerange') or '' if vol or issue or pages: vol_info = f'第{vol}卷' if vol else '' issue_info = f'第{issue}期' if issue else '' page_info = f'第{pages}页' if pages else '' meta_lines.append(f'

卷期页码：{vol_info} {issue_info} {page_info}

'.strip()) # 全文提示 meta_lines.append('

（全文需登录机构账户或个人账户后获取）

') meta_html = '\n'.join(meta_lines) content_html = f'
{abstract_html}\n{meta_html}
' print(f"[Playwright] yiigle: 构建内容长度 {len(content_html)}") if not content_html: selectors = ['article', '.article-content', '.content', '.post-content', '.detail-content', '.news-content', 'main', '[class*="content"]'] for selector in selectors: try: if await page.locator(selector).count() > 0: content_html = await page.locator(selector).first.inner_html() if len(content_html) > 500: break except: continue # 提取图片 images = [] parsed_url = urlparse(url) url_is_https = parsed_url.scheme == 'https' try: img_elements = await page.locator('img').all() for img in img_elements[:10]: try: src = await img.get_attribute('src') alt = await img.get_attribute('alt') or '' if src and not src.startswith('data:') and len(src) > 5: # 转为绝对URL if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_url.scheme}://{parsed_url.netloc}{src}" elif not src.startswith(('http://', 'https://')): src = urljoin(url, src) # 修复混合内容：页面HTTPS时图片也用HTTPS if url_is_https and src.startswith('http://'): src = 'https://' + src[7:] images.append(CrawlImage(original_url=src)) except: pass except: pass await context.close() await browser.close() # 修复编码问题 print(f"[Playwright] 编码修复前标题: {repr(title[:50] if title else '空')}") print(f"[Playwright] 编码修复前内容前200: {repr(content_html[:200] if content_html else '空')}") title = fix_encoding(title) if title else title content_html = fix_encoding(content_html) if content_html else content_html print(f"[Playwright] 编码修复后标题: {repr(title[:50] if title else '空')}") print(f"[Playwright] 返回结果 - 内容长度: {len(content_html) if content_html else 0}") return { 'title': title.strip() if title else "未命名文章", 'content_html': content_html, 'images': images, 'is_markdown': is_lark_wiki and len(content_html) > 500 if content_html else False # 飞书JS提取的已是Markdown } def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url: str) -> str: """将SPA采集的HTML转为Markdown""" # 修复编码问题 html = fix_encoding(html) # 先清理 style 和 script 标签 text = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>.*?', '', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'\s+style="[^"]*"', '', text, flags=re.IGNORECASE) text = re.sub(r'\s+class="[^"]*"', '', text, flags=re.IGNORECASE) parsed_base = urlparse(base_url) base_is_https = parsed_base.scheme == 'https' is_wechat = 'mp.weixin.qq.com' in base_url # 首先将img标签转为Markdown def img_to_markdown(match): src = match.group(1) alt = match.group(2) if match.lastindex and match.lastindex >= 2 else "" # 过滤 data URL if src.startswith('data:'): return '' # 处理URL if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" # 修复混合内容：页面HTTPS时图片也用HTTPS if base_is_https and src.startswith('http://'): src = 'https://' + src[7:] return f'![{alt}]({src})' # 微信文章特殊处理：图片使用data-src，过滤data URL if is_wechat: def wechat_img_replace(m): src = m.group(1) if src.startswith('data:'): return '' return f'![图片]({src})' text = re.sub( r']+data-src=["\']([^"\']+)["\'][^>]*>', wechat_img_replace, text, flags=re.IGNORECASE ) # 匹配普通img标签 text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>', img_to_markdown, text, flags=re.IGNORECASE ) # 匹配没有alt的img标签 - 需要处理URL转换！ def img_to_markdown_no_alt(match): src = match.group(1) # 过滤 data URL if src.startswith('data:'): return '' # 处理URL（与img_to_markdown保持一致） if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" # 修复混合内容 if base_is_https and src.startswith('http://'): src = 'https://' + src[7:] return f'![图片]({src})' text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*>', img_to_markdown_no_alt, text, flags=re.IGNORECASE ) # 处理代码块 
...
- 必须在其他标签之前处理 def convert_code_block(match): code_content = match.group(1) # 提取语言 lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE) lang = lang_match.group(1) if lang_match else '' return f'\n```{lang}\n{code_content}\n```\n' text = re.sub(r']*>\s*]*>(.*?)\s*', convert_code_block, text, flags=re.DOTALL | re.IGNORECASE) # 处理纯pre标签 text = re.sub(r']*>(.*?)', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE) # 处理行内代码 text = re.sub(r']*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE) # 转换其他标签 text = re.sub(r']*>(.*?)', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)

', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'', '\n', text, flags=re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'- \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ').replace('<', '<').replace('>', '>') text = text.replace('&', '&').replace('"', '"') # 修复数学公式格式（掘金等平台） text = fix_math_formulas(text) # 清理空行 lines = [line for line in text.split('\n')] cleaned = [] prev_empty = False for line in lines: stripped = line.strip() if stripped == '': if not prev_empty: cleaned.append('') prev_empty = True else: cleaned.append(line) prev_empty = False return '\n'.join(cleaned).strip() @router.post("") async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)): """ 采集文章（自动检测SPA并使用Playwright） """ # 设置响应头，防止缓存和确保正确编码 response_headers = { "Cache-Control": "no-cache, no-store, must-revalidate", "Pragma": "no-cache", "Expires": "0", "Content-Type": "application/json; charset=utf-8" } try: # 验证URL if not request.url.startswith(('http://', 'https://')): raise HTTPException(status_code=400, detail="无效的URL") # 微信文章特殊处理：直接用Playwright（requests会被拦截） is_wechat = 'mp.weixin.qq.com' in request.url # 掘金文章特殊处理：requests返回的是混淆代码，必须用Playwright is_juejin = 'juejin.cn' in request.url if is_wechat or is_juejin: site_name = "微信文章" if is_wechat else "掘金文章" print(f"检测到{site_name}，使用Playwright采集: {request.url}") if not PLAYWRIGHT_AVAILABLE: error_msg = f"{site_name}需要Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium" return JSONResponse( content={"success": False, "error": error_msg}, headers=response_headers ) # 使用Playwright采集 spa_result = await crawl_with_playwright(request.url) title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] # 转换为Markdown markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) else: # 第一步：尝试普通requests采集 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', 'Accept-Encoding': 'gzip, deflate', 'Cache-Control': 'max-age=0', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', } try: response = requests.get(request.url, headers=headers, timeout=30) response.raise_for_status() except requests.exceptions.HTTPError as e: # 遇到反爬虫拦截(468/403/429)，尝试使用Playwright if response.status_code in (468, 403, 429, 503) and PLAYWRIGHT_AVAILABLE: print(f"Requests被拦截({response.status_code})，尝试Playwright: {request.url}") spa_result = await crawl_with_playwright(request.url) title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) return JSONResponse( content={ "success": True, "title": title, "url": request.url, "content": markdown_content, "images": [{"original_url": img.original_url, "local_path": img.local_path, "filename": img.filename} for img in images] }, headers=response_headers ) else: raise # 智能编码处理：优先使用声明的编码，而不是猜测 # 1. 先检查HTTP响应头中的编码 declared_encoding = None if response.encoding: declared_encoding = response.encoding # 2. 如果响应头没有明确编码，从HTML meta标签中提取 if not declared_encoding or declared_encoding.lower() in ['iso-8859-1', 'latin-1']: # 尝试从HTML中提取charset raw_content = response.content charset_match = re.search(rb']+charset=["\']?([^"\'>\s]+)', raw_content, re.IGNORECASE) if charset_match: declared_encoding = charset_match.group(1).decode('ascii', errors='ignore') # 3. 使用声明的编码，如果没有才用 apparent_encoding 猜测 if declared_encoding and declared_encoding.lower() not in ['iso-8859-1', 'latin-1']: response.encoding = declared_encoding else: response.encoding = response.apparent_encoding or 'utf-8' html = response.text # 检测是否为SPA if is_spa_site(html): print(f"检测到SPA网站，使用Playwright采集: {request.url}") if not PLAYWRIGHT_AVAILABLE: return JSONResponse( content={"success": False, "error": "该网站为SPA单页应用，需要安装Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium"}, headers=response_headers ) # 使用Playwright采集 spa_result = await crawl_with_playwright(request.url) title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] # 转换HTML为Markdown（飞书JS提取的已是Markdown，跳过转换） if spa_result.get('is_markdown'): markdown_content = content_html print(f"飞书Wiki: 内容已是Markdown，跳过HTML转换") else: markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) else: # 普通网站，使用原有逻辑 print(f"普通网站，使用requests采集: {request.url}") # 提取标题 title = extract_title_from_html(html) # 提取主要内容 main_content = extract_main_content(html) # 转换HTML为Markdown（必须先执行，用于图片提取） markdown_content = html_to_markdown(main_content, request.url) # 从Markdown内容中提取图片（确保只提取实际在文章中的图片） images = extract_images_from_markdown(markdown_content, request.url) # 安全网：如果提取的内容过短（< 300 chars），可能是 Next.js RSC 等 # JS 渲染型页面，自动回退到 Playwright if len(markdown_content.strip()) < 300 and PLAYWRIGHT_AVAILABLE: print(f"requests提取内容过短({len(markdown_content.strip())} chars)，疑似JS渲染页面，回退Playwright") spa_result = await crawl_with_playwright(request.url) title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) # 创建临时目录 temp_dir = "/tmp/blog_crawl" os.makedirs(temp_dir, exist_ok=True) # uploads 目录配置 backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) upload_dir = os.path.join(backend_dir, "uploads") thumb_dir = os.path.join(upload_dir, "thumbnails") os.makedirs(upload_dir, exist_ok=True) os.makedirs(thumb_dir, exist_ok=True) # 下载图片并保存到数据库（最多30张） downloaded_images = [] for i, img in enumerate(images[:30]): # 如果已经在 Playwright 里下载好了（如微信文章浏览器 fetch），直接复用 if img.success and img.local_path and os.path.exists(img.local_path): filename = img.filename or os.path.basename(img.local_path) filepath = img.local_path success = True print(f"复用已下载的图片: {filename}") else: filename, filepath, success = download_image(img.original_url, temp_dir) if success: try: # 生成新的文件名（避免冲突） ext = os.path.splitext(filename)[1].lower() if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']: ext = '.jpg' new_filename = f"crawl_{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}" # 移动文件到 uploads 目录 new_filepath = os.path.join(upload_dir, new_filename) with open(filepath, 'rb') as src: with open(new_filepath, 'wb') as dst: dst.write(src.read()) # 删除临时文件 os.remove(filepath) # 生成缩略图 thumb_filename = f"thumb_{new_filename}" thumb_filepath = os.path.join(thumb_dir, thumb_filename) thumb_path_db = None if generate_thumbnail(new_filepath, thumb_filepath): thumb_path_db = f"/uploads/thumbnails/{thumb_filename}" # 获取图片尺寸 width, height = get_image_dimensions(new_filepath) # 获取文件大小 file_size = os.path.getsize(new_filepath) # 保存到数据库 images 表 image_record = Image( filename=new_filename, original_name=filename, file_path=f"/uploads/{new_filename}", thumb_path=thumb_path_db, description=f"采集自 {request.url}", file_size=file_size, width=width, height=height ) db.add(image_record) db.commit() db.refresh(image_record) # 更新返回的图片信息 img.local_path = f"/uploads/{new_filename}" img.filename = new_filename img.success = True img.id = image_record.id # 保存数据库ID供前端使用 downloaded_images.append(img) # 将 content 中的原始URL替换为本地路径 # 微信图片URL可能带 #imgIndex=xx 锚点，去掉后再匹配 url_for_match = img.original_url.split('#')[0] if url_for_match in markdown_content: markdown_content = markdown_content.replace( url_for_match, f"/uploads/{new_filename}" ) print(f"✅ 已将内容中的图片URL替换: {url_for_match[:60]}... -> /uploads/{new_filename}") print(f"图片已保存到数据库: id={image_record.id}, filename={new_filename}") except Exception as e: print(f"保存图片到数据库失败: {e}") # 如果数据库保存失败，仍然保留临时文件信息 img.local_path = filepath img.filename = filename img.success = True downloaded_images.append(img) # 生成摘要（前200字符） summary = markdown_content[:200].replace('#', '').replace('*', '').strip() # 序列化图片数据 images_data = [] for img in downloaded_images: img_dict = {"original_url": img.original_url} if hasattr(img, 'local_path') and img.local_path: img_dict["local_path"] = img.local_path if hasattr(img, 'filename') and img.filename: img_dict["filename"] = img.filename if hasattr(img, 'id') and img.id: img_dict["id"] = img.id images_data.append(img_dict) return JSONResponse( content={ "success": True, "title": title, "content": markdown_content, "summary": summary, "images": images_data }, headers=response_headers ) except requests.RequestException as e: return JSONResponse( content={"success": False, "error": f"网络请求失败: {str(e)}"}, headers=response_headers ) except Exception as e: import traceback print(f"采集失败: {e}") print(traceback.format_exc()) return JSONResponse( content={"success": False, "error": f"采集失败: {str(e)}"}, headers=response_headers )