""" 文章采集路由 - 从网页URL采集内容 支持SPA网站采集 """ import os import re import uuid import requests from datetime import datetime from typing import Optional, List from urllib.parse import urljoin, urlparse try: from bs4 import BeautifulSoup BS4_AVAILABLE = True except ImportError: BS4_AVAILABLE = False # Playwright用于SPA采集 try: from playwright.async_api import async_playwright PLAYWRIGHT_AVAILABLE = True except ImportError: PLAYWRIGHT_AVAILABLE = False from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.orm import Session import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from database import get_db from models import Image from routers.auth import get_current_admin from routers.images import generate_thumbnail, get_image_dimensions router = APIRouter(prefix="/api/crawl", tags=["文章采集"]) class CrawlRequest(BaseModel): url: str class CrawlImage(BaseModel): id: Optional[int] = None # 数据库ID original_url: str local_path: Optional[str] = None filename: Optional[str] = None success: bool = False class CrawlResponse(BaseModel): success: bool title: Optional[str] = None content: Optional[str] = None # Markdown内容 summary: Optional[str] = None images: List[CrawlImage] = [] error: Optional[str] = None def clean_html_tags(text: str) -> str: """移除文本中的HTML标签""" # 移除所有HTML标签 text = re.sub(r'<[^>]+>', '', text) # 解码HTML实体 text = text.replace(' ', ' ') text = text.replace('<', '<') text = text.replace('>', '>') text = text.replace('&', '&') text = text.replace('"', '"') text = text.replace(''', "'") return text.strip() def fix_math_formulas(text: str) -> str: """修复数学公式格式,处理混合格式问题""" import re # 1. 清理已渲染的数学符号(保留LaTeX代码) # 移除 Unicode 数学运算符,但保留后面的LaTeX text = re.sub(r'[⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿⁱ]+', '', text) # 2. 规范化反斜杠(双重转义变单重) text = text.replace('\\\\', '\\') # 3. 检测并包裹行内公式 # 匹配常见的行内数学模式:\mathbf{x}, \alpha, x_{i}, x^{2}, \sum 等 inline_patterns = [ r'\\[a-zA-Z]+\{[^}]*\}', # \mathbf{x}, \frac{a}{b} r'\\[a-zA-Z]+', # \alpha, \beta r'[a-zA-Z]_[a-zA-Z0-9]+', # x_i, y_j r'[a-zA-Z]\^[a-zA-Z0-9]+', # x^2, y^n ] # 4. 检测块级公式(多行或包含 array, matrix 等) block_keywords = ['\\begin{array}', '\\begin{matrix}', '\\begin{equation}', '\\sum_', '\\int_', '\\prod_', '\\lim_'] lines = text.split('\n') result_lines = [] in_block = False block_content = [] for line in lines: stripped = line.strip() # 检测块级公式开始 if any(kw in stripped for kw in block_keywords): if not in_block: in_block = True block_content = [] block_content.append(line) continue # 如果在块内,继续收集 if in_block: block_content.append(line) # 检测块结束 if '\\end{array}' in stripped or '\\end{matrix}' in stripped or '\\end{equation}' in stripped: # 包裹整个块 block_text = '\n'.join(block_content) if not block_text.startswith('$$') and not block_text.startswith('\\['): result_lines.append('$$' + block_text + '$$') else: result_lines.append(block_text) in_block = False block_content = [] continue # 处理行内公式 # 如果一行中有明显的LaTeX命令,但还没有被包裹 if re.search(r'\\[a-zA-Z]+', line) and not re.search(r'\\\(|\\\)|\\\[|\\\]', line): # 简单处理:如果包含复杂结构,作为块级;否则作为行内 if re.search(r'\\(frac|sum|int|prod|lim)\{', line): line = '$$' + line + '$$' else: line = '\\(' + line + '\\)' result_lines.append(line) # 处理未闭合的块 if in_block and block_content: block_text = '\n'.join(block_content) result_lines.append('$$' + block_text + '$$') return '\n'.join(result_lines) def extract_title_from_html(html: str) -> str: """从HTML中提取标题 - 优先提取文章主标题""" # 1. 尝试提取 Open Graph title (og:title) - 通常是文章主标题 og_title_match = re.search( r']+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE ) if not og_title_match: og_title_match = re.search( r']+content=["\']([^"\']+)["\'][^>]+property=["\']og:title["\']', html, re.IGNORECASE ) if og_title_match: title = clean_html_tags(og_title_match.group(1)) if len(title) > 5: # 确保不是太短的无意义标题 # 清理常见后缀 title = re.sub(r'\s+[–—-]\s+.*$', '', title).strip() return title # 1.5 博客园/CSDN等特殊平台处理(放在h1之前,因为这些平台的h1可能是博客名而非文章标题) # 博客园:h1.postTitle 或 a#cb_post_title_url cnblog_title = re.search(r'
...
for pre in soup.find_all('pre'):
code = pre.find('code')
if code:
# 获取代码内容,保留原始格式
code_text = code.get_text()
# 检测语言(从class属性)
lang = ''
code_classes = code.get('class', [])
for cls in code_classes:
if 'language-' in cls or 'lang-' in cls:
lang = cls.replace('language-', '').replace('lang-', '')
break
# 创建Markdown代码块
md_code = f'\n```{lang}\n{code_text}\n```\n'
pre.replace_with(md_code)
else:
# 纯pre标签
pre_text = pre.get_text()
md_code = f'\n```\n{pre_text}\n```\n'
pre.replace_with(md_code)
# 处理行内代码 ...
for code in soup.find_all('code'):
code_text = code.get_text()
# 检查是否在pre内(已处理过)
if not code.find_parent('pre'):
code.replace_with(f'`{code_text}`')
# 处理粗体
for tag in soup.find_all(['strong', 'b']):
tag.replace_with(f'**{tag.get_text()}**')
# 处理斜体
for tag in soup.find_all(['em', 'i']):
tag.replace_with(f'*{tag.get_text()}*')
# 获取文本
text = soup.get_text()
else:
# 回退到正则方式
def convert_img_to_markdown(match):
src = match.group(1)
alt = match.group(2) if match.lastindex and match.lastindex >= 2 else ""
src = make_absolute_url(src)
return f''
text = html
# 先处理图片
text = re.sub(
r']*>(.*?)
', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'...
def convert_code_block(match):
code_content = match.group(1)
# 提取语言
lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE)
lang = lang_match.group(1) if lang_match else ''
return f'\n```{lang}\n{code_content}\n```\n'
text = re.sub(r']*>\s*]*>(.*?)\s*', convert_code_block,
text, flags=re.DOTALL | re.IGNORECASE)
# 处理纯pre标签
text = re.sub(r']*>(.*?)', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE) # 处理行内代码 text = re.sub(r'
]*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r']*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r']*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'<[^>]+>', '', text)
# 解码HTML实体
text = text.replace(' ', ' ')
text = text.replace('<', '<')
text = text.replace('>', '>')
text = text.replace('&', '&')
text = text.replace('"', '"')
# 修复数学公式格式
text = fix_math_formulas(text)
# 清理多余空白行
lines = text.split('\n')
cleaned_lines = []
prev_empty = False
for line in lines:
stripped = line.strip()
if stripped == '':
if not prev_empty:
cleaned_lines.append('')
prev_empty = True
else:
cleaned_lines.append(line)
prev_empty = False
return '\n'.join(cleaned_lines).strip()
def extract_images(html: str, base_url: str) -> List[CrawlImage]:
"""从HTML中提取图片URL"""
images = []
parsed_base = urlparse(base_url)
base_is_https = parsed_base.scheme == 'https'
# 匹配 ... - 必须在其他标签之前处理
def convert_code_block(match):
code_content = match.group(1)
# 提取语言
lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE)
lang = lang_match.group(1) if lang_match else ''
return f'\n```{lang}\n{code_content}\n```\n'
text = re.sub(r']*>\s*]*>(.*?)\s*', convert_code_block,
text, flags=re.DOTALL | re.IGNORECASE)
# 处理纯pre标签
text = re.sub(r']*>(.*?)', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE) # 处理行内代码 text = re.sub(r'
]*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
# 转换其他标签
text = re.sub(r']*>(.*?)
', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r'