""" 文章采集路由 - 从网页URL采集内容 支持SPA网站采集 """ import os import re import uuid import requests from datetime import datetime from typing import Optional, List from urllib.parse import urljoin, urlparse try: from bs4 import BeautifulSoup BS4_AVAILABLE = True except ImportError: BS4_AVAILABLE = False # Playwright用于SPA采集 try: from playwright.async_api import async_playwright PLAYWRIGHT_AVAILABLE = True except ImportError: PLAYWRIGHT_AVAILABLE = False from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.orm import Session import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from database import get_db from models import Image from routers.auth import get_current_admin from routers.images import generate_thumbnail, get_image_dimensions router = APIRouter(prefix="/api/crawl", tags=["文章采集"]) class CrawlRequest(BaseModel): url: str class CrawlImage(BaseModel): id: Optional[int] = None # 数据库ID original_url: str local_path: Optional[str] = None filename: Optional[str] = None success: bool = False class CrawlResponse(BaseModel): success: bool title: Optional[str] = None content: Optional[str] = None # Markdown内容 summary: Optional[str] = None images: List[CrawlImage] = [] error: Optional[str] = None def clean_html_tags(text: str) -> str: """移除文本中的HTML标签""" # 移除所有HTML标签 text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ') text = text.replace('<', '<') text = text.replace('>', '>') text = text.replace('&', '&') text = text.replace('"', '"') text = text.replace(''', "'") return text.strip() def fix_math_formulas(text: str) -> str: """修复数学公式格式,处理混合格式问题""" import re # 1. 清理已渲染的数学符号(保留LaTeX代码) # 移除 Unicode 数学运算符,但保留后面的LaTeX text = re.sub(r'[⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿⁱ]+', '', text) # 2. 规范化反斜杠(双重转义变单重) text = text.replace('\\\\', '\\') # 3. 检测并包裹行内公式 # 匹配常见的行内数学模式:\mathbf{x}, \alpha, x_{i}, x^{2}, \sum 等 inline_patterns = [ r'\\[a-zA-Z]+\{[^}]*\}', # \mathbf{x}, \frac{a}{b} r'\\[a-zA-Z]+', # \alpha, \beta r'[a-zA-Z]_[a-zA-Z0-9]+', # x_i, y_j r'[a-zA-Z]\^[a-zA-Z0-9]+', # x^2, y^n ] # 4. 检测块级公式(多行或包含 array, matrix 等) block_keywords = ['\\begin{array}', '\\begin{matrix}', '\\begin{equation}', '\\sum_', '\\int_', '\\prod_', '\\lim_'] lines = text.split('\n') result_lines = [] in_block = False block_content = [] for line in lines: stripped = line.strip() # 检测块级公式开始 if any(kw in stripped for kw in block_keywords): if not in_block: in_block = True block_content = [] block_content.append(line) continue # 如果在块内,继续收集 if in_block: block_content.append(line) # 检测块结束 if '\\end{array}' in stripped or '\\end{matrix}' in stripped or '\\end{equation}' in stripped: # 包裹整个块 block_text = '\n'.join(block_content) if not block_text.startswith('$$') and not block_text.startswith('\\['): result_lines.append('$$' + block_text + '$$') else: result_lines.append(block_text) in_block = False block_content = [] continue # 处理行内公式 # 如果一行中有明显的LaTeX命令,但还没有被包裹 if re.search(r'\\[a-zA-Z]+', line) and not re.search(r'\\\(|\\\)|\\\[|\\\]', line): # 简单处理:如果包含复杂结构,作为块级;否则作为行内 if re.search(r'\\(frac|sum|int|prod|lim)\{', line): line = '$$' + line + '$$' else: line = '\\(' + line + '\\)' result_lines.append(line) # 处理未闭合的块 if in_block and block_content: block_text = '\n'.join(block_content) result_lines.append('$$' + block_text + '$$') return '\n'.join(result_lines) def extract_title_from_html(html: str) -> str: """从HTML中提取标题 - 优先提取文章主标题""" # 1. 尝试提取 Open Graph title (og:title) - 通常是文章主标题 og_title_match = re.search( r']+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE ) if not og_title_match: og_title_match = re.search( r']+content=["\']([^"\']+)["\'][^>]+property=["\']og:title["\']', html, re.IGNORECASE ) if og_title_match: title = clean_html_tags(og_title_match.group(1)) if len(title) > 5: # 确保不是太短的无意义标题 # 清理常见后缀 title = re.sub(r'\s+[–—-]\s+.*$', '', title).strip() return title # 1.5 博客园/CSDN等特殊平台处理(放在h1之前,因为这些平台的h1可能是博客名而非文章标题) # 博客园:h1.postTitle 或 a#cb_post_title_url cnblog_title = re.search(r']+class=["\'][^"\']*postTitle[^"\']*["\'][^>]*>(.*?)', html, re.IGNORECASE | re.DOTALL) if cnblog_title: title = clean_html_tags(cnblog_title.group(1)) if len(title) > 5: return title cnblog_title2 = re.search(r']+id=["\']cb_post_title_url["\'][^>]*>(.*?)', html, re.IGNORECASE | re.DOTALL) if cnblog_title2: title = clean_html_tags(cnblog_title2.group(1)) if len(title) > 5: return title # 1.6 人人都是产品经理 (woshipm.com) woshipm_title = re.search(r'<[^>]+class=["\'][^"\']*article--title[^"\']*["\'][^>]*>(.*?)]+>', html, re.IGNORECASE | re.DOTALL) if woshipm_title: title = clean_html_tags(woshipm_title.group(1)) if len(title) > 5: return title # 1.7 菜鸟教程:从article-intro内的h1提取标题,或使用title标签 runoob_title = re.search(r']*class=["\']article-intro["\'][^>]*>.*?]*>(.*?)', html, re.IGNORECASE | re.DOTALL) if runoob_title: title = clean_html_tags(runoob_title.group(1)) if len(title) > 5 and "菜鸟教程" not in title: # 排除网站名 return title # 从title标签提取(菜鸟教程等网站) title_match = re.search(r']*>(.*?)', html, re.IGNORECASE | re.DOTALL) if title_match: title = clean_html_tags(title_match.group(1)) # 移除常见的分隔符和后缀(但保留技术词汇中的连字符如Objective-C) # 使用更严格的匹配,确保是标题分隔符而不是单词中的连字符 title = re.sub(r'\s+[|│┃]\s+.*$', '', title).strip() # | 分隔符 title = re.sub(r'\s+[-–—]\s+[^\s]*教程.*$', '', title).strip() # 匹配 " - 菜鸟教程" title = re.sub(r'\s+[-–—]\s+[^\s]*博客.*$', '', title).strip() # 匹配 " - XXX博客" if len(title) > 5: return title # 2. 尝试提取 h1 标题(文章主标题通常是页面第一个h1) # 先清理HTML获取纯文本 h1_matches = re.findall(r']*>(.*?)', html, re.IGNORECASE | re.DOTALL) for h1 in h1_matches: title = clean_html_tags(h1) # 过滤掉太短的标题 if len(title) > 5: return title # 3. 尝试从 article/main 内容区域提取 h2 或 h3(作为备选) content_patterns = [ r']*>.*?]*>(.*?).*?', r']*>.*?]*>(.*?).*?', r']*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>.*?]*>(.*?)', ] for pattern in content_patterns: match = re.search(pattern, html, re.IGNORECASE | re.DOTALL) if match: title = clean_html_tags(match.group(1)) if len(title) > 10: # 内容区标题通常较长 return title # 4. 尝试提取 Twitter Card title twitter_title_match = re.search( r']+name=["\']twitter:title["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE ) if twitter_title_match: title = clean_html_tags(twitter_title_match.group(1)) if len(title) > 5: return title # 5. 最后回退到 标签 title_match = re.search(r'<title[^>]*>([^<]+)', html, re.IGNORECASE) if title_match: title = clean_html_tags(title_match.group(1)) # 清理常见后缀(网站名、分类等) # 处理各种分隔符: | - – — :: · • title = re.sub(r'\s*[\|\-–—:·•]\s*.+$', '', title) # 清理特定后缀词 title = re.sub(r'\s*[-–]\s*(Home|首页|主页|官网|官方网站|新闻|News|Press Release).*?$', '', title, flags=re.IGNORECASE) if title: return title return "未命名文章" def extract_main_content(html: str) -> str: """从HTML中提取主要内容(简化版)""" # 使用 BeautifulSoup 如果可用,更精确可靠 if BS4_AVAILABLE: try: soup = BeautifulSoup(html, 'html.parser') # 移除 script 和 style for tag in soup(['script', 'style']): tag.decompose() # 按优先级尝试不同的内容选择器 content_selectors = [ # 人人都是产品经理 '.article--content.grap', # 博客园 '#cnblogs_post_body', # 菜鸟教程 '.article-intro', # 掘金 'article', # 通用 'article', 'main', '.post-content', '.article-content', '.entry-content', '#content', '#main-content', ] for selector in content_selectors: elem = soup.select_one(selector) if elem: # 返回元素的HTML内容 return str(elem) # 如果都没找到,返回body内容 body = soup.find('body') if body: return str(body) except Exception as e: print(f"BeautifulSoup提取失败,回退到正则: {e}") # 回退到正则表达式方式 # 移除 script 和 style html = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) html = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) # 尝试找到主要内容区域 content = "" # 常见的文章容器 patterns = [ # 人人都是产品经理 (woshipm.com) r']*class=["\']article--content grap["\'][^>]*>(.*?)\s*\s*]*class=["\']article--footer', r']*class=["\']article--content grap["\'][^>]*>(.*?)\s*\s*]*class=["\']article-bottom-meta', r']*class=["\']article--content grap["\'][^>]*>(.*?)\s*]*class=["\']u-marginTop30', # 菜鸟教程 r']*class=["\']article-intro["\'][^>]*>(.*?)]*class=["\'](?:google-auto-placed|pre|next)', r']*class=["\']article-intro["\'][^>]*>(.*?)\s*(?:]*class=["\']article-intro["\'][^>]*>(.*?)\s*$', r']*class=["\']article-intro["\'][^>]*>(.*)', # 博客园 r']+id=["\']cnblogs_post_body["\'][^>]*>(.*?)\s*]+id=["\']blog_post_info_wrap["\']', r']+id=["\']cnblogs_post_body["\'][^>]*>(.*?)', # 通用 r']*>(.*?)', r']*>(.*?)', r']*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)', r']*id=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)', ] for pattern in patterns: match = re.search(pattern, html, re.DOTALL | re.IGNORECASE) if match: content = match.group(1) break # 如果没找到,尝试提取 body if not content: body_match = re.search(r']*>(.*?)', html, re.DOTALL | re.IGNORECASE) if body_match: content = body_match.group(1) return content if content else html def html_to_markdown(html: str, base_url: str = "") -> str: """HTML转Markdown,保留图片在原始位置""" # 检查base_url是否为HTTPS parsed_base = urlparse(base_url) if base_url else None base_is_https = parsed_base and parsed_base.scheme == 'https' def make_absolute_url(src: str) -> str: """将相对URL转换为绝对URL""" if not src: return src if src.startswith('//'): src = 'https:' + src elif src.startswith('/') and base_url: src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" elif not src.startswith(('http://', 'https://')) and base_url: src = urljoin(base_url, src) # 修复混合内容:页面HTTPS时图片也用HTTPS if base_is_https and src.startswith('http://'): src = 'https://' + src[7:] return src if BS4_AVAILABLE: # 使用BeautifulSoup更精确地处理 soup = BeautifulSoup(html, 'html.parser') # 处理所有图片,保留在原位置 for img in soup.find_all('img'): src = img.get('src', '') alt = img.get('alt', '') if src: src = make_absolute_url(src) # 创建Markdown图片标记 md_img = f'![{alt}]({src})' img.replace_with(md_img) # 处理标题 for tag in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']): level = int(tag.name[1]) tag.replace_with(f'\n{"#" * level} {tag.get_text().strip()}\n') # 处理段落 for p in soup.find_all('p'): text = p.get_text() # 保留段落中的Markdown图片标记 for content in p.contents: if isinstance(content, str) and content.startswith('!['): text = text.replace(content, content) p.replace_with(f'\n{text}\n') # 处理换行 for br in soup.find_all('br'): br.replace_with('\n') # 处理代码块
...
for pre in soup.find_all('pre'): code = pre.find('code') if code: # 获取代码内容,保留原始格式 code_text = code.get_text() # 检测语言(从class属性) lang = '' code_classes = code.get('class', []) for cls in code_classes: if 'language-' in cls or 'lang-' in cls: lang = cls.replace('language-', '').replace('lang-', '') break # 创建Markdown代码块 md_code = f'\n```{lang}\n{code_text}\n```\n' pre.replace_with(md_code) else: # 纯pre标签 pre_text = pre.get_text() md_code = f'\n```\n{pre_text}\n```\n' pre.replace_with(md_code) # 处理行内代码 ... for code in soup.find_all('code'): code_text = code.get_text() # 检查是否在pre内(已处理过) if not code.find_parent('pre'): code.replace_with(f'`{code_text}`') # 处理粗体 for tag in soup.find_all(['strong', 'b']): tag.replace_with(f'**{tag.get_text()}**') # 处理斜体 for tag in soup.find_all(['em', 'i']): tag.replace_with(f'*{tag.get_text()}*') # 获取文本 text = soup.get_text() else: # 回退到正则方式 def convert_img_to_markdown(match): src = match.group(1) alt = match.group(2) if match.lastindex and match.lastindex >= 2 else "" src = make_absolute_url(src) return f'![{alt}]({src})' text = html # 先处理图片 text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>', convert_img_to_markdown, text, flags=re.IGNORECASE ) text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*>', lambda m: f'![图片]({make_absolute_url(m.group(1))})', text, flags=re.IGNORECASE ) # 处理其他标签 text = re.sub(r']*>(.*?)', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)

', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'', '\n', text, flags=re.IGNORECASE) # 处理代码块
...
def convert_code_block(match): code_content = match.group(1) # 提取语言 lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE) lang = lang_match.group(1) if lang_match else '' return f'\n```{lang}\n{code_content}\n```\n' text = re.sub(r']*>\s*]*>(.*?)\s*', convert_code_block, text, flags=re.DOTALL | re.IGNORECASE) # 处理纯pre标签 text = re.sub(r']*>(.*?)', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE) # 处理行内代码 text = re.sub(r']*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ') text = text.replace('<', '<') text = text.replace('>', '>') text = text.replace('&', '&') text = text.replace('"', '"') # 修复数学公式格式 text = fix_math_formulas(text) # 清理多余空白行 lines = text.split('\n') cleaned_lines = [] prev_empty = False for line in lines: stripped = line.strip() if stripped == '': if not prev_empty: cleaned_lines.append('') prev_empty = True else: cleaned_lines.append(line) prev_empty = False return '\n'.join(cleaned_lines).strip() def extract_images(html: str, base_url: str) -> List[CrawlImage]: """从HTML中提取图片URL""" images = [] parsed_base = urlparse(base_url) base_is_https = parsed_base.scheme == 'https' # 匹配 img_pattern = r']+src=["\']([^"\']+)["\'][^>]*>' for match in re.finditer(img_pattern, html, re.IGNORECASE): img_url = match.group(1) # 转换为绝对URL if img_url.startswith('//'): img_url = 'https:' + img_url elif img_url.startswith('/'): img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}" elif not img_url.startswith(('http://', 'https://')): img_url = urljoin(base_url, img_url) # 修复混合内容问题:如果页面是HTTPS,图片也要用HTTPS if base_is_https and img_url.startswith('http://'): img_url = 'https://' + img_url[7:] images.append(CrawlImage(original_url=img_url)) return images def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[CrawlImage]: """从Markdown内容中提取图片URL(确保只提取实际在文章中的图片)""" images = [] parsed_base = urlparse(base_url) base_is_https = parsed_base.scheme == 'https' # 匹配Markdown图片语法: ![alt](url) img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)' for match in re.finditer(img_pattern, markdown_content): img_url = match.group(2) # 跳过已经是本地路径的图片 if img_url.startswith('/uploads/'): continue # 转换为绝对URL if img_url.startswith('//'): img_url = 'https:' + img_url elif img_url.startswith('/'): img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}" elif not img_url.startswith(('http://', 'https://')): img_url = urljoin(base_url, img_url) # 修复混合内容问题 if base_is_https and img_url.startswith('http://'): img_url = 'https://' + img_url[7:] images.append(CrawlImage(original_url=img_url)) return images def download_image(url: str, temp_dir: str) -> tuple: """下载图片到临时目录""" try: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' } # 微信图片需要referer if 'mmbiz.qpic.cn' in url or 'mmbiz.qlogo.cn' in url: headers['Referer'] = 'https://mp.weixin.qq.com/' response = requests.get(url, headers=headers, timeout=30, stream=True) response.raise_for_status() # 获取文件扩展名 content_type = response.headers.get('content-type', '') ext = '.jpg' if 'png' in content_type: ext = '.png' elif 'gif' in content_type: ext = '.gif' elif 'webp' in content_type: ext = '.webp' else: # 从URL获取扩展名 parsed = urlparse(url) path_ext = os.path.splitext(parsed.path)[1] if path_ext: ext = path_ext # 生成文件名 filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}" filepath = os.path.join(temp_dir, filename) # 保存文件 with open(filepath, 'wb') as f: for chunk in response.iter_content(chunk_size=8192): f.write(chunk) return filename, filepath, True except Exception as e: print(f"下载图片失败 {url}: {e}") return None, None, False def is_spa_site(html: str) -> bool: """检测是否为SPA网站""" # 检测标志 spa_indicators = [ # React/Vue/Angular常见标记 '
', '
', '
', '
', 'You need to enable JavaScript', # 加载中标记 'Please wait', 'Loading...', # 常见SPA框架 'react-root', 'ng-app', 'data-reactroot', # 动态加载标记 'window.__INITIAL_STATE__', 'window.__DATA__', ] html_lower = html.lower() for indicator in spa_indicators: if indicator.lower() in html_lower: return True # 检查内容是否极少(只有JS没有实际内容) text_content = re.sub(r'<[^>]+>', '', html) text_content = re.sub(r'\s+', '', text_content) if len(text_content) < 200: return True return False async def crawl_with_playwright(url: str) -> dict: """使用Playwright异步采集SPA页面""" if not PLAYWRIGHT_AVAILABLE: raise Exception("Playwright未安装,无法采集SPA网站") # 检测是否为微信文章 is_wechat = 'mp.weixin.qq.com' in url async with async_playwright() as p: browser = await p.chromium.launch(headless=True) page = await browser.new_page() # 打开页面 await page.goto(url, wait_until='domcontentloaded', timeout=30000) # 微信文章特殊处理 if is_wechat: await page.wait_for_timeout(5000) # 微信需要更长等待时间 # 提取标题 - 微信文章标题在 #activity_name title = "" try: if await page.locator('#activity_name').count() > 0: title = await page.locator('#activity_name').first.inner_text() title = title.strip() except: pass if not title: title = await page.title() # 提取正文 - 微信文章正文在 #js_content content_html = "" try: if await page.locator('#js_content').count() > 0: content_html = await page.locator('#js_content').first.inner_html() except: pass # 提取图片 - 微信图片用data-src images = [] try: img_elements = await page.locator('#js_content img').all() for img in img_elements[:10]: try: src = await img.get_attribute('data-src') or await img.get_attribute('src') alt = await img.get_attribute('alt') or '' if src and src.startswith('http'): images.append(CrawlImage(original_url=src)) except: pass except: pass await browser.close() return { 'title': title if title else "未命名文章", 'content_html': content_html, 'images': images } # 普通SPA网站处理 # 检测平台 is_juejin = 'juejin.cn' in url is_tencent_cloud = 'cloud.tencent.com' in url # 等待内容加载 if is_juejin: # 掘金需要等待article标签 try: await page.wait_for_selector('article', timeout=15000) except: pass await page.wait_for_timeout(3000) elif is_tencent_cloud: # 腾讯云开发者社区等待markdown内容 try: await page.wait_for_selector('[class*="markdown"]', timeout=15000) except: pass await page.wait_for_timeout(3000) else: try: await page.wait_for_selector('h1, .article-title, .content, article, main', timeout=10000) except: pass await page.wait_for_timeout(5000) # 提取标题 title = "" if is_juejin: # 掘金标题选择器 for selector in ['article h1', 'h1']: try: if await page.locator(selector).count() > 0: title = await page.locator(selector).first.inner_text() if title.strip(): break except: continue elif is_tencent_cloud: # 腾讯云开发者社区标题 for selector in ['h1', '.article-title', '[class*="title"]']: try: if await page.locator(selector).count() > 0: title = await page.locator(selector).first.inner_text() if title.strip(): break except: continue if not title: for selector in ['h1', '.article-title', '[class*="title"]']: try: if await page.locator(selector).count() > 0: title = await page.locator(selector).first.inner_text() if title.strip(): break except: continue if not title: title = await page.title() # 提取正文HTML content_html = "" if is_juejin: # 掘金内容在 .markdown-body 中 try: if await page.locator('.markdown-body').count() > 0: content_html = await page.locator('.markdown-body').first.inner_html() elif await page.locator('article').count() > 0: content_html = await page.locator('article').first.inner_html() except: pass # 清理掘金内容:移除 style 和 script 标签(掘金有大量内嵌CSS) if content_html: import re # 移除 style 标签及其内容 content_html = re.sub(r']*>.*?', '', content_html, flags=re.DOTALL | re.IGNORECASE) # 移除 script 标签及其内容 content_html = re.sub(r']*>.*?', '', content_html, flags=re.DOTALL | re.IGNORECASE) # 移除所有元素的 style 属性 content_html = re.sub(r'\s+style="[^"]*"', '', content_html, flags=re.IGNORECASE) # 移除 class 属性(减少无用属性) content_html = re.sub(r'\s+class="[^"]*"', '', content_html, flags=re.IGNORECASE) elif is_tencent_cloud: # 腾讯云开发者社区内容 try: if await page.locator('.mod-content__markdown').count() > 0: content_html = await page.locator('.mod-content__markdown').first.inner_html() except: pass if not content_html: selectors = ['article', '.article-content', '.content', '.post-content', '.detail-content', '.news-content', 'main', '[class*="content"]'] for selector in selectors: try: if await page.locator(selector).count() > 0: content_html = await page.locator(selector).first.inner_html() if len(content_html) > 500: break except: continue # 提取图片 images = [] parsed_url = urlparse(url) url_is_https = parsed_url.scheme == 'https' try: img_elements = await page.locator('img').all() for img in img_elements[:10]: try: src = await img.get_attribute('src') alt = await img.get_attribute('alt') or '' if src and not src.startswith('data:') and len(src) > 5: # 转为绝对URL if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_url.scheme}://{parsed_url.netloc}{src}" elif not src.startswith(('http://', 'https://')): src = urljoin(url, src) # 修复混合内容:页面HTTPS时图片也用HTTPS if url_is_https and src.startswith('http://'): src = 'https://' + src[7:] images.append(CrawlImage(original_url=src)) except: pass except: pass await browser.close() return { 'title': title.strip() if title else "未命名文章", 'content_html': content_html, 'images': images } def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url: str) -> str: """将SPA采集的HTML转为Markdown""" # 先清理 style 和 script 标签 text = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>.*?', '', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'\s+style="[^"]*"', '', text, flags=re.IGNORECASE) text = re.sub(r'\s+class="[^"]*"', '', text, flags=re.IGNORECASE) parsed_base = urlparse(base_url) base_is_https = parsed_base.scheme == 'https' is_wechat = 'mp.weixin.qq.com' in base_url # 首先将img标签转为Markdown def img_to_markdown(match): src = match.group(1) alt = match.group(2) if match.lastindex and match.lastindex >= 2 else "" # 处理URL if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" # 修复混合内容:页面HTTPS时图片也用HTTPS if base_is_https and src.startswith('http://'): src = 'https://' + src[7:] return f'![{alt}]({src})' # 微信文章特殊处理:图片使用data-src if is_wechat: text = re.sub( r']+data-src=["\']([^"\']+)["\'][^>]*>', lambda m: f'![图片]({m.group(1)})', text, flags=re.IGNORECASE ) # 匹配普通img标签 text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>', img_to_markdown, text, flags=re.IGNORECASE ) # 匹配没有alt的img标签 - 需要处理URL转换! def img_to_markdown_no_alt(match): src = match.group(1) # 处理URL(与img_to_markdown保持一致) if src.startswith('//'): src = 'https:' + src elif src.startswith('/'): src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}" # 修复混合内容 if base_is_https and src.startswith('http://'): src = 'https://' + src[7:] return f'![图片]({src})' text = re.sub( r']+src=["\']([^"\']+)["\'][^>]*>', img_to_markdown_no_alt, text, flags=re.IGNORECASE ) # 处理代码块
...
- 必须在其他标签之前处理 def convert_code_block(match): code_content = match.group(1) # 提取语言 lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE) lang = lang_match.group(1) if lang_match else '' return f'\n```{lang}\n{code_content}\n```\n' text = re.sub(r']*>\s*]*>(.*?)\s*', convert_code_block, text, flags=re.DOTALL | re.IGNORECASE) # 处理纯pre标签 text = re.sub(r']*>(.*?)', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE) # 处理行内代码 text = re.sub(r']*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE) # 转换其他标签 text = re.sub(r']*>(.*?)', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)

', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'', '\n', text, flags=re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>(.*?)', r'- \1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ').replace('<', '<').replace('>', '>') text = text.replace('&', '&').replace('"', '"') # 修复数学公式格式(掘金等平台) text = fix_math_formulas(text) # 清理空行 lines = [line for line in text.split('\n')] cleaned = [] prev_empty = False for line in lines: stripped = line.strip() if stripped == '': if not prev_empty: cleaned.append('') prev_empty = True else: cleaned.append(line) prev_empty = False return '\n'.join(cleaned).strip() @router.post("") async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)): """ 采集文章(自动检测SPA并使用Playwright) """ try: # 验证URL if not request.url.startswith(('http://', 'https://')): raise HTTPException(status_code=400, detail="无效的URL") # 微信文章特殊处理:直接用Playwright(requests会被拦截) is_wechat = 'mp.weixin.qq.com' in request.url if is_wechat: print(f"检测到微信文章,使用Playwright采集: {request.url}") if not PLAYWRIGHT_AVAILABLE: return CrawlResponse( success=False, error="微信文章需要Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium" ) # 使用Playwright采集微信文章 spa_result = await crawl_with_playwright(request.url) title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] # 微信文章使用专门的转换 markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) else: # 第一步:尝试普通requests采集 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', } response = requests.get(request.url, headers=headers, timeout=30) response.raise_for_status() # 智能编码处理:优先使用声明的编码,而不是猜测 # 1. 先检查HTTP响应头中的编码 declared_encoding = None if response.encoding: declared_encoding = response.encoding # 2. 如果响应头没有明确编码,从HTML meta标签中提取 if not declared_encoding or declared_encoding.lower() in ['iso-8859-1', 'latin-1']: # 尝试从HTML中提取charset raw_content = response.content charset_match = re.search(rb']+charset=["\']?([^"\'>\s]+)', raw_content, re.IGNORECASE) if charset_match: declared_encoding = charset_match.group(1).decode('ascii', errors='ignore') # 3. 使用声明的编码,如果没有才用 apparent_encoding 猜测 if declared_encoding and declared_encoding.lower() not in ['iso-8859-1', 'latin-1']: response.encoding = declared_encoding else: response.encoding = response.apparent_encoding or 'utf-8' html = response.text # 检测是否为SPA if is_spa_site(html): print(f"检测到SPA网站,使用Playwright采集: {request.url}") if not PLAYWRIGHT_AVAILABLE: return CrawlResponse( success=False, error="该网站为SPA单页应用,需要安装Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium" ) # 使用Playwright采集 spa_result = await crawl_with_playwright(request.url) title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] # 转换HTML为Markdown markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) else: # 普通网站,使用原有逻辑 print(f"普通网站,使用requests采集: {request.url}") # 提取标题 title = extract_title_from_html(html) # 提取主要内容 main_content = extract_main_content(html) # 转换HTML为Markdown(必须先执行,用于图片提取) markdown_content = html_to_markdown(main_content, request.url) # 从Markdown内容中提取图片(确保只提取实际在文章中的图片) images = extract_images_from_markdown(markdown_content, request.url) # 创建临时目录 temp_dir = "/tmp/blog_crawl" os.makedirs(temp_dir, exist_ok=True) # uploads 目录配置 backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) upload_dir = os.path.join(backend_dir, "uploads") thumb_dir = os.path.join(upload_dir, "thumbnails") os.makedirs(upload_dir, exist_ok=True) os.makedirs(thumb_dir, exist_ok=True) # 下载图片并保存到数据库(最多5张) downloaded_images = [] for i, img in enumerate(images[:5]): filename, filepath, success = download_image(img.original_url, temp_dir) if success: try: # 生成新的文件名(避免冲突) ext = os.path.splitext(filename)[1].lower() if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']: ext = '.jpg' new_filename = f"crawl_{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}" # 移动文件到 uploads 目录 new_filepath = os.path.join(upload_dir, new_filename) with open(filepath, 'rb') as src: with open(new_filepath, 'wb') as dst: dst.write(src.read()) # 删除临时文件 os.remove(filepath) # 生成缩略图 thumb_filename = f"thumb_{new_filename}" thumb_filepath = os.path.join(thumb_dir, thumb_filename) thumb_path_db = None if generate_thumbnail(new_filepath, thumb_filepath): thumb_path_db = f"/uploads/thumbnails/{thumb_filename}" # 获取图片尺寸 width, height = get_image_dimensions(new_filepath) # 获取文件大小 file_size = os.path.getsize(new_filepath) # 保存到数据库 images 表 image_record = Image( filename=new_filename, original_name=filename, file_path=f"/uploads/{new_filename}", thumb_path=thumb_path_db, description=f"采集自 {request.url}", file_size=file_size, width=width, height=height ) db.add(image_record) db.commit() db.refresh(image_record) # 更新返回的图片信息 img.local_path = f"/uploads/{new_filename}" img.filename = new_filename img.success = True img.id = image_record.id # 保存数据库ID供前端使用 downloaded_images.append(img) # 将 content 中的原始URL替换为本地路径 if img.original_url in markdown_content: markdown_content = markdown_content.replace( img.original_url, f"/uploads/{new_filename}" ) print(f"✅ 已将内容中的图片URL替换: {img.original_url[:60]}... -> /uploads/{new_filename}") print(f"图片已保存到数据库: id={image_record.id}, filename={new_filename}") except Exception as e: print(f"保存图片到数据库失败: {e}") # 如果数据库保存失败,仍然保留临时文件信息 img.local_path = filepath img.filename = filename img.success = True downloaded_images.append(img) # 生成摘要(前200字符) summary = markdown_content[:200].replace('#', '').replace('*', '').strip() return CrawlResponse( success=True, title=title, content=markdown_content, summary=summary, images=downloaded_images ) except requests.RequestException as e: return CrawlResponse( success=False, error=f"网络请求失败: {str(e)}" ) except Exception as e: import traceback print(f"采集失败: {e}") print(traceback.format_exc()) return CrawlResponse( success=False, error=f"采集失败: {str(e)}" )