Ubuntu
1032 lines
41 KiB
Python
1032 lines
41 KiB
Python
"""
|
||
文章采集路由 - 从网页URL采集内容
|
||
支持SPA网站采集
|
||
"""
|
||
import os
|
||
import re
|
||
import uuid
|
||
import requests
|
||
from datetime import datetime
|
||
from typing import Optional, List
|
||
from urllib.parse import urljoin, urlparse
|
||
|
||
try:
|
||
from bs4 import BeautifulSoup
|
||
BS4_AVAILABLE = True
|
||
except ImportError:
|
||
BS4_AVAILABLE = False
|
||
|
||
# Playwright用于SPA采集
|
||
try:
|
||
from playwright.sync_api import sync_playwright
|
||
PLAYWRIGHT_AVAILABLE = True
|
||
except ImportError:
|
||
PLAYWRIGHT_AVAILABLE = False
|
||
|
||
# Playwright Stealth 用于绕过WAF
|
||
try:
|
||
from playwright_stealth import stealth_sync
|
||
STEALTH_AVAILABLE = True
|
||
except ImportError:
|
||
STEALTH_AVAILABLE = False
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException
|
||
from fastapi.responses import JSONResponse
|
||
from pydantic import BaseModel
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.database import get_db
|
||
from app.routers.auth import get_current_admin
|
||
from PIL import Image as PILImage
|
||
|
||
router = APIRouter(prefix="/crawl", tags=["文章采集"])
|
||
|
||
# 上传配置
|
||
from app.config import get_settings
|
||
settings = get_settings()
|
||
|
||
|
||
def get_upload_dir():
|
||
"""获取上传目录路径 - 扁平存储（统一指向项目根目录 storage）"""
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
||
upload_dir = os.path.join(base_dir, settings.UPLOAD_DIR)
|
||
os.makedirs(upload_dir, exist_ok=True)
|
||
# 缩略图目录
|
||
thumbnail_dir = os.path.join(upload_dir, "thumbnails")
|
||
os.makedirs(thumbnail_dir, exist_ok=True)
|
||
return upload_dir
|
||
|
||
|
||
def get_crawl_temp_dir():
|
||
"""获取采集临时目录 - 跨平台（统一指向项目根目录 storage/tmp/crawl）"""
|
||
if os.name == 'nt': # Windows
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
||
temp_dir = os.path.join(base_dir, '..', 'storage', 'tmp', 'crawl')
|
||
else: # Linux/macOS
|
||
temp_dir = '/tmp/news_crawl'
|
||
os.makedirs(temp_dir, exist_ok=True)
|
||
# 缩略图临时目录
|
||
thumb_dir = os.path.join(temp_dir, 'thumbnails')
|
||
os.makedirs(thumb_dir, exist_ok=True)
|
||
return temp_dir
|
||
|
||
|
||
def generate_unique_filename(original_filename: str) -> str:
|
||
"""生成唯一的文件名"""
|
||
ext = os.path.splitext(original_filename)[1].lower()
|
||
if not ext:
|
||
ext = '.jpg'
|
||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||
unique_id = uuid.uuid4().hex[:8]
|
||
return f"{timestamp}_{unique_id}{ext}"
|
||
|
||
|
||
class CrawlRequest(BaseModel):
|
||
url: str
|
||
use_playwright: bool = False
|
||
|
||
|
||
class CrawlImage(BaseModel):
|
||
original_url: str
|
||
local_path: Optional[str] = None
|
||
filename: Optional[str] = None
|
||
success: bool = False
|
||
|
||
|
||
class CrawlResponse(BaseModel):
|
||
success: bool
|
||
title: Optional[str] = None
|
||
content: Optional[str] = None
|
||
summary: Optional[str] = None
|
||
images: List[dict] = []
|
||
error: Optional[str] = None
|
||
|
||
|
||
def fix_encoding(text: str) -> str:
|
||
"""修复UTF-8编码问题"""
|
||
if not text:
|
||
return text
|
||
garbled_patterns = ['é', 'è', 'Ã', 'Â', 'Ã¥', 'ü', 'ö', 'ä', 'ß', 'â€', 'æ', 'å', 'è', '§']
|
||
has_garbled = any(pattern in text for pattern in garbled_patterns)
|
||
if has_garbled:
|
||
try:
|
||
fixed = text.encode('latin1').decode('utf-8')
|
||
still_garbled = any(p in fixed for p in garbled_patterns)
|
||
if still_garbled:
|
||
try:
|
||
fixed = fixed.encode('latin1').decode('utf-8')
|
||
except:
|
||
pass
|
||
return fixed
|
||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||
pass
|
||
return text
|
||
|
||
|
||
def clean_html_tags(text: str) -> str:
|
||
"""移除文本中的HTML标签"""
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
text = text.replace(' ', ' ')
|
||
text = text.replace('<', '<')
|
||
text = text.replace('>', '>')
|
||
text = text.replace('&', '&')
|
||
text = text.replace('"', '"')
|
||
text = text.replace(''', "'")
|
||
return text.strip()
|
||
|
||
|
||
def fix_math_formulas(text: str) -> str:
|
||
"""修复数学公式格式，处理混合格式问题（从 personal-blog 移植）"""
|
||
# 1. 清理已渲染的数学符号（保留LaTeX代码）
|
||
text = re.sub(r'[⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿⁱ]+', '', text)
|
||
|
||
# 2. 规范化反斜杠（双重转义变单重）
|
||
text = text.replace('\\\\', '\\')
|
||
|
||
# 3. 检测块级公式（多行或包含 array, matrix 等）
|
||
block_keywords = ['\\begin{array}', '\\begin{matrix}', '\\begin{equation}',
|
||
'\\sum_', '\\int_', '\\prod_', '\\lim_']
|
||
|
||
lines = text.split('\n')
|
||
result_lines = []
|
||
in_block = False
|
||
block_content = []
|
||
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
|
||
# 检测块级公式开始
|
||
if any(kw in stripped for kw in block_keywords):
|
||
if not in_block:
|
||
in_block = True
|
||
block_content = []
|
||
block_content.append(line)
|
||
continue
|
||
|
||
# 如果在块内，继续收集
|
||
if in_block:
|
||
block_content.append(line)
|
||
# 检测块结束
|
||
if '\\end{array}' in stripped or '\\end{matrix}' in stripped or '\\end{equation}' in stripped:
|
||
block_text = '\n'.join(block_content)
|
||
if not block_text.startswith('$$') and not block_text.startswith('\\['):
|
||
result_lines.append('$$' + block_text + '$$')
|
||
else:
|
||
result_lines.append(block_text)
|
||
in_block = False
|
||
block_content = []
|
||
continue
|
||
|
||
# 处理行内公式
|
||
# 如果一行中有明显的LaTeX命令，但还没有被包裹
|
||
if re.search(r'\\[a-zA-Z]+', line) and not re.search(r'\\\(|\\\)|\\\[|\\\]', line):
|
||
# 简单处理：如果包含复杂结构，作为块级；否则作为行内
|
||
if re.search(r'\\(frac|sum|int|prod|lim)\{', line):
|
||
line = '$$' + line + '$$'
|
||
else:
|
||
line = '\\(' + line + '\\)'
|
||
|
||
result_lines.append(line)
|
||
|
||
# 处理未闭合的块
|
||
if in_block and block_content:
|
||
block_text = '\n'.join(block_content)
|
||
result_lines.append('$$' + block_text + '$$')
|
||
|
||
return '\n'.join(result_lines)
|
||
|
||
|
||
def extract_title_from_html(html: str) -> str:
|
||
"""从HTML中提取标题"""
|
||
# Open Graph title
|
||
og_title_match = re.search(
|
||
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||
html, re.IGNORECASE
|
||
)
|
||
if not og_title_match:
|
||
og_title_match = re.search(
|
||
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:title["\']',
|
||
html, re.IGNORECASE
|
||
)
|
||
if og_title_match:
|
||
title = clean_html_tags(og_title_match.group(1))
|
||
if len(title) > 5:
|
||
title = re.sub(r'\s+[–—-]\s+.*$', '', title).strip()
|
||
return title
|
||
|
||
# 博客园
|
||
cnblog_title = re.search(r'<h1[^>]+class=["\'][^"\']*postTitle[^"\']*["\'][^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
|
||
if cnblog_title:
|
||
title = clean_html_tags(cnblog_title.group(1))
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# 人人都是产品经理 (woshipm.com)
|
||
woshipm_title = re.search(r'<[^>]+class=["\'][^"\']*article--title[^"\']*["\'][^>]*>(.*?)</[^>]+>', html, re.IGNORECASE | re.DOTALL)
|
||
if woshipm_title:
|
||
title = clean_html_tags(woshipm_title.group(1))
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# 菜鸟教程
|
||
runoob_title = re.search(r'<div[^>]*class=["\']article-intro["\'][^>]*>.*?<h1[^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
|
||
if runoob_title:
|
||
title = clean_html_tags(runoob_title.group(1))
|
||
if len(title) > 5 and "菜鸟教程" not in title:
|
||
return title
|
||
|
||
# title标签
|
||
title_match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||
if title_match:
|
||
title = clean_html_tags(title_match.group(1))
|
||
title = re.sub(r'\s+[|│┃]\s+.*$', '', title).strip()
|
||
title = re.sub(r'\s+[-–—]\s+[^\s]*教程.*$', '', title).strip()
|
||
title = re.sub(r'\s+[-–—]\s+[^\s]*博客.*$', '', title).strip()
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# h1标题
|
||
h1_matches = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
|
||
for h1 in h1_matches:
|
||
title = clean_html_tags(h1)
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# Twitter Card title
|
||
twitter_title_match = re.search(
|
||
r'<meta[^>]+name=["\']twitter:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||
html, re.IGNORECASE
|
||
)
|
||
if twitter_title_match:
|
||
title = clean_html_tags(twitter_title_match.group(1))
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
return "未命名文章"
|
||
|
||
|
||
def extract_main_content(html: str) -> str:
|
||
"""提取主要内容"""
|
||
if BS4_AVAILABLE:
|
||
try:
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
for tag in soup(['script', 'style']):
|
||
tag.decompose()
|
||
|
||
selectors = [
|
||
'.article--content.grap',
|
||
'#cnblogs_post_body',
|
||
'.article-intro',
|
||
'article', 'main',
|
||
'.post-content', '.article-content', '.entry-content',
|
||
'#content', '#main-content',
|
||
]
|
||
for selector in selectors:
|
||
elem = soup.select_one(selector)
|
||
if elem:
|
||
return str(elem)
|
||
|
||
body = soup.find('body')
|
||
if body:
|
||
return str(body)
|
||
except Exception as e:
|
||
print(f"BeautifulSoup失败: {e}")
|
||
|
||
# 正则回退
|
||
html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||
html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
patterns = [
|
||
r'<div[^>]*class=["\']article--content grap["\'][^>]*>(.*?)</div>\s*</div>\s*<div[^>]*class=["\']article--footer',
|
||
r'<div[^>]*class=["\']article--content grap["\'][^>]*>(.*?)</div>\s*</div>\s*<div[^>]*class=["\']article-bottom-meta',
|
||
r'<div[^>]*class=["\']article-intro["\'][^>]*>(.*?)<div[^>]*class=["\'](?:google-auto-placed|pre|next)',
|
||
r'<div[^>]+id=["\']cnblogs_post_body["\'][^>]*>(.*?)</div>\s*<div[^>]+id=["\']blog_post_info_wrap["\']',
|
||
r'<div[^>]+id=["\']cnblogs_post_body["\'][^>]*>(.*?)</div>',
|
||
r'<article[^>]*>(.*?)</article>',
|
||
r'<main[^>]*>(.*?)</main>',
|
||
r'<div[^>]*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)</div>',
|
||
]
|
||
for pattern in patterns:
|
||
match = re.search(pattern, html, re.DOTALL | re.IGNORECASE)
|
||
if match:
|
||
return match.group(1)
|
||
|
||
return html
|
||
|
||
|
||
def html_to_markdown(html: str, base_url: str = "") -> str:
|
||
"""HTML转Markdown"""
|
||
html = fix_encoding(html)
|
||
parsed_base = urlparse(base_url) if base_url else None
|
||
base_is_https = parsed_base and parsed_base.scheme == 'https'
|
||
|
||
def make_absolute_url(src: str) -> str:
|
||
if not src:
|
||
return src
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/') and base_url:
|
||
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
|
||
elif not src.startswith(('http://', 'https://')) and base_url:
|
||
src = urljoin(base_url, src)
|
||
if base_is_https and src.startswith('http://'):
|
||
src = 'https://' + src[7:]
|
||
return src
|
||
|
||
if BS4_AVAILABLE:
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
for img in soup.find_all('img'):
|
||
src = img.get('src', '')
|
||
alt = img.get('alt', '')
|
||
if src:
|
||
src = make_absolute_url(src)
|
||
md_img = f''
|
||
img.replace_with(md_img)
|
||
|
||
for tag in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
|
||
level = int(tag.name[1])
|
||
tag.replace_with(f'\n{"#" * level} {tag.get_text().strip()}\n')
|
||
|
||
for p in soup.find_all('p'):
|
||
p.replace_with(f'\n{p.get_text()}\n')
|
||
|
||
for br in soup.find_all('br'):
|
||
br.replace_with('\n')
|
||
|
||
for pre in soup.find_all('pre'):
|
||
code = pre.find('code')
|
||
if code:
|
||
code_text = code.get_text()
|
||
lang = ''
|
||
code_classes = code.get('class', [])
|
||
for cls in code_classes:
|
||
if 'language-' in cls or 'lang-' in cls:
|
||
lang = cls.replace('language-', '').replace('lang-', '')
|
||
break
|
||
md_code = f'\n```{lang}\n{code_text}\n```\n'
|
||
pre.replace_with(md_code)
|
||
else:
|
||
pre_text = pre.get_text()
|
||
md_code = f'\n```\n{pre_text}\n```\n'
|
||
pre.replace_with(md_code)
|
||
|
||
for code in soup.find_all('code'):
|
||
if not code.find_parent('pre'):
|
||
code.replace_with(f'`{code.get_text()}`')
|
||
|
||
for tag in soup.find_all(['strong', 'b']):
|
||
tag.replace_with(f'**{tag.get_text()}**')
|
||
|
||
for tag in soup.find_all(['em', 'i']):
|
||
tag.replace_with(f'*{tag.get_text()}*')
|
||
|
||
text = soup.get_text()
|
||
else:
|
||
text = html
|
||
text = re.sub(
|
||
r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>',
|
||
lambda m: f')})',
|
||
text, flags=re.IGNORECASE
|
||
)
|
||
text = re.sub(r'<h1[^>]*>(.*?)</h1>', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h2[^>]*>(.*?)</h2>', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h3[^>]*>(.*?)</h3>', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<p[^>]*>(.*?)</p>', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
|
||
text = text.replace(' ', ' ').replace('<', '<').replace('>', '>')
|
||
text = text.replace('&', '&').replace('"', '"')
|
||
|
||
lines = text.split('\n')
|
||
cleaned = []
|
||
prev_empty = False
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if stripped == '':
|
||
if not prev_empty:
|
||
cleaned.append('')
|
||
prev_empty = True
|
||
else:
|
||
cleaned.append(line)
|
||
prev_empty = False
|
||
|
||
text = '\n'.join(cleaned).strip()
|
||
|
||
# 修复数学公式格式
|
||
text = fix_math_formulas(text)
|
||
|
||
return text
|
||
|
||
|
||
def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[CrawlImage]:
|
||
"""从Markdown提取图片"""
|
||
images = []
|
||
parsed_base = urlparse(base_url)
|
||
base_is_https = parsed_base.scheme == 'https'
|
||
|
||
img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)'
|
||
for match in re.finditer(img_pattern, markdown_content):
|
||
img_url = match.group(2)
|
||
if img_url.startswith('/uploads/') or img_url.startswith('/storage/'):
|
||
continue
|
||
if img_url.startswith('//'):
|
||
img_url = 'https:' + img_url
|
||
elif img_url.startswith('/'):
|
||
img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}"
|
||
elif not img_url.startswith(('http://', 'https://')):
|
||
img_url = urljoin(base_url, img_url)
|
||
if base_is_https and img_url.startswith('http://'):
|
||
img_url = 'https://' + img_url[7:]
|
||
|
||
images.append(CrawlImage(original_url=img_url))
|
||
|
||
return images
|
||
|
||
|
||
def is_spa_site(html: str) -> bool:
|
||
"""检测是否为SPA"""
|
||
indicators = [
|
||
'<div id="root"></div>', '<div id="app"></div>',
|
||
'You need to enable JavaScript', 'Loading...',
|
||
'data-reactroot', 'window.__INITIAL_STATE__'
|
||
]
|
||
html_lower = html.lower()
|
||
for indicator in indicators:
|
||
if indicator.lower() in html_lower:
|
||
return True
|
||
|
||
text_content = re.sub(r'<[^>]+>', '', html)
|
||
text_content = re.sub(r'\s+', '', text_content)
|
||
if len(text_content) < 200:
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
def crawl_with_playwright(url: str) -> dict:
|
||
"""使用Playwright采集SPA（同步API，避免Windows asyncio子进程问题）"""
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
raise Exception("Playwright未安装")
|
||
|
||
is_wechat = 'mp.weixin.qq.com' in url
|
||
is_juejin = 'juejin.cn' in url
|
||
is_aliyun = 'developer.aliyun.com' in url
|
||
is_segmentfault = 'segmentfault.com' in url
|
||
is_tencent_cloud = 'cloud.tencent.com' in url
|
||
|
||
with sync_playwright() as p:
|
||
browser = p.chromium.launch(
|
||
headless=True,
|
||
args=[
|
||
'--disable-blink-features=AutomationControlled',
|
||
'--disable-web-security',
|
||
'--disable-features=IsolateOrigins,site-per-process',
|
||
'--disable-dev-shm-usage',
|
||
'--no-sandbox',
|
||
'--disable-setuid-sandbox',
|
||
'--disable-accelerated-2d-canvas',
|
||
'--disable-gpu',
|
||
'--window-size=1920,1080',
|
||
]
|
||
)
|
||
|
||
context = browser.new_context(
|
||
viewport={'width': 1920, 'height': 1080},
|
||
user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
locale='zh-CN',
|
||
timezone_id='Asia/Shanghai',
|
||
permissions=['geolocation'],
|
||
java_script_enabled=True,
|
||
)
|
||
|
||
context.add_init_script("""
|
||
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
||
Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
|
||
window.chrome = { runtime: {} };
|
||
""")
|
||
|
||
page = context.new_page()
|
||
|
||
if STEALTH_AVAILABLE:
|
||
stealth_sync(page)
|
||
|
||
page.goto(url, wait_until='networkidle', timeout=30000)
|
||
|
||
if is_wechat:
|
||
page.wait_for_timeout(5000)
|
||
title = ""
|
||
try:
|
||
if page.locator('#activity_name').count() > 0:
|
||
title = page.locator('#activity_name').first.inner_text()
|
||
title = title.strip()
|
||
except:
|
||
pass
|
||
if not title:
|
||
title = page.title()
|
||
|
||
content_html = ""
|
||
try:
|
||
if page.locator('#js_content').count() > 0:
|
||
content_html = page.locator('#js_content').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
images = []
|
||
try:
|
||
img_elements = page.locator('#js_content img').all()
|
||
for img in img_elements[:10]:
|
||
try:
|
||
src = img.get_attribute('data-src') or img.get_attribute('src')
|
||
if src and src.startswith('http'):
|
||
images.append(CrawlImage(original_url=src))
|
||
except:
|
||
pass
|
||
except:
|
||
pass
|
||
|
||
context.close()
|
||
browser.close()
|
||
return {'title': title or "未命名文章", 'content_html': content_html, 'images': images}
|
||
|
||
# 普通SPA处理
|
||
if is_juejin:
|
||
try:
|
||
page.wait_for_selector('article', timeout=15000)
|
||
except:
|
||
pass
|
||
page.wait_for_timeout(3000)
|
||
elif is_aliyun:
|
||
# 阿里云开发者社区：等待语雀编辑器内容加载
|
||
try:
|
||
page.wait_for_selector('.article-content', timeout=15000)
|
||
except:
|
||
pass
|
||
# 语雀编辑器内容懒加载，需要滚动触发
|
||
page.wait_for_timeout(2000)
|
||
for _ in range(5):
|
||
page.evaluate('window.scrollBy(0, 800)')
|
||
page.wait_for_timeout(500)
|
||
page.wait_for_timeout(2000)
|
||
elif is_segmentfault:
|
||
# SegmentFault 需要更长的等待时间
|
||
print("[Playwright] 等待 SegmentFault 页面加载...")
|
||
try:
|
||
page.wait_for_load_state('networkidle', timeout=20000)
|
||
except:
|
||
pass
|
||
page.wait_for_timeout(8000)
|
||
print("[Playwright] 等待完成，开始提取")
|
||
elif is_tencent_cloud:
|
||
# 腾讯云开发者社区
|
||
try:
|
||
page.wait_for_selector('[class*="markdown"]', timeout=15000)
|
||
except:
|
||
pass
|
||
page.wait_for_timeout(3000)
|
||
|
||
title = ""
|
||
if is_juejin:
|
||
for selector in ['article h1', 'h1']:
|
||
try:
|
||
if page.locator(selector).count() > 0:
|
||
title = page.locator(selector).first.inner_text()
|
||
if title.strip():
|
||
break
|
||
except:
|
||
continue
|
||
elif is_segmentfault:
|
||
# SegmentFault 标题选择器
|
||
sf_title_selectors = ['h1.article-title', 'h1', '.article-title', '[class*="title"]']
|
||
for selector in sf_title_selectors:
|
||
try:
|
||
count = page.locator(selector).count()
|
||
if count > 0:
|
||
t = page.locator(selector).first.inner_text()
|
||
print(f"[Playwright] 标题选择器 '{selector}': '{t[:30] if t else '空'}...'")
|
||
if t and t.strip():
|
||
title = t.strip()
|
||
break
|
||
except Exception as e:
|
||
print(f"[Playwright] 标题选择器 '{selector}' 失败: {e}")
|
||
elif is_tencent_cloud:
|
||
for selector in ['h1', '.article-title', '[class*="title"]']:
|
||
try:
|
||
if page.locator(selector).count() > 0:
|
||
title = page.locator(selector).first.inner_text()
|
||
if title.strip():
|
||
break
|
||
except:
|
||
continue
|
||
|
||
if not title:
|
||
for selector in ['article h1', 'h1', '.article-title']:
|
||
try:
|
||
if page.locator(selector).count() > 0:
|
||
title = page.locator(selector).first.inner_text()
|
||
if title.strip():
|
||
break
|
||
except:
|
||
continue
|
||
if not title:
|
||
title = page.title()
|
||
|
||
content_html = ""
|
||
if is_juejin:
|
||
try:
|
||
if page.locator('.markdown-body').count() > 0:
|
||
content_html = page.locator('.markdown-body').first.inner_html()
|
||
elif page.locator('article').count() > 0:
|
||
content_html = page.locator('article').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
if content_html:
|
||
content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
content_html = re.sub(r'\s+style="[^"]*"', '', content_html, flags=re.IGNORECASE)
|
||
content_html = re.sub(r'\s+class="[^"]*"', '', content_html, flags=re.IGNORECASE)
|
||
elif is_aliyun:
|
||
# 阿里云开发者社区：语雀编辑器内容
|
||
try:
|
||
if page.locator('.article-content').count() > 0:
|
||
content_html = page.locator('.article-content').first.inner_html()
|
||
elif page.locator('.article-detail').count() > 0:
|
||
content_html = page.locator('.article-detail').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
if content_html:
|
||
content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
elif is_segmentfault:
|
||
# SegmentFault 内容提取
|
||
sf_selectors = ['.article-content', '.markdown-body', 'article', '.post-content',
|
||
'[data-id]', 'main', '#content', '.content']
|
||
for selector in sf_selectors:
|
||
try:
|
||
count = page.locator(selector).count()
|
||
print(f"[Playwright] 检查选择器 '{selector}': {count} 个匹配")
|
||
if count > 0:
|
||
html = page.locator(selector).first.inner_html()
|
||
if len(html) > 200:
|
||
content_html = html
|
||
print(f"[Playwright] 使用 '{selector}' 提取，长度: {len(content_html)}")
|
||
break
|
||
except Exception as e:
|
||
print(f"[Playwright] 选择器 '{selector}' 失败: {e}")
|
||
if not content_html:
|
||
try:
|
||
content_html = page.locator('body').first.inner_html()
|
||
print(f"[Playwright] 使用 body，长度: {len(content_html)}")
|
||
except Exception as e:
|
||
print(f"[Playwright] 获取 body 失败: {e}")
|
||
if content_html:
|
||
content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
content_html = re.sub(r'<div[^>]*class="[^"]*ad[^"]*"[^>]*>.*?</div>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
elif is_tencent_cloud:
|
||
# 腾讯云开发者社区
|
||
try:
|
||
if page.locator('.mod-content__markdown').count() > 0:
|
||
content_html = page.locator('.mod-content__markdown').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
if not content_html:
|
||
selectors = ['article', '.article-content', '.content', '.post-content', 'main']
|
||
for selector in selectors:
|
||
try:
|
||
elements = page.locator(selector).all()
|
||
if elements:
|
||
parts = []
|
||
for el in elements:
|
||
try:
|
||
html = el.inner_html().strip()
|
||
if html and len(html) > 20:
|
||
parts.append(html)
|
||
except:
|
||
pass
|
||
if parts:
|
||
combined = '\n'.join(parts)
|
||
if len(combined) > 500:
|
||
content_html = combined
|
||
break
|
||
except:
|
||
continue
|
||
|
||
# 最终回退：所有选择器都失败时，用页面可见文本
|
||
if not content_html:
|
||
try:
|
||
body_text = page.locator('body').first.inner_text()
|
||
if body_text and len(body_text.strip()) > 50:
|
||
lines = [l for l in body_text.split('\n') if l.strip()]
|
||
content_html = '<body><pre>' + '\n'.join(lines) + '</pre></body>'
|
||
except:
|
||
pass
|
||
|
||
images = []
|
||
parsed_url = urlparse(url)
|
||
url_is_https = parsed_url.scheme == 'https'
|
||
try:
|
||
img_elements = page.locator('img').all()
|
||
for img in img_elements[:10]:
|
||
try:
|
||
src = img.get_attribute('src')
|
||
if src and not src.startswith('data:') and len(src) > 5:
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/'):
|
||
src = f"{parsed_url.scheme}://{parsed_url.netloc}{src}"
|
||
elif not src.startswith(('http://', 'https://')):
|
||
src = urljoin(url, src)
|
||
if url_is_https and src.startswith('http://'):
|
||
src = 'https://' + src[7:]
|
||
images.append(CrawlImage(original_url=src))
|
||
except:
|
||
pass
|
||
except:
|
||
pass
|
||
|
||
context.close()
|
||
browser.close()
|
||
|
||
title = fix_encoding(title) if title else title
|
||
content_html = fix_encoding(content_html) if content_html else content_html
|
||
|
||
return {'title': title.strip() if title else "未命名文章", 'content_html': content_html, 'images': images}
|
||
|
||
|
||
def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url: str) -> str:
|
||
"""SPA HTML转Markdown"""
|
||
html = fix_encoding(html)
|
||
text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
parsed_base = urlparse(base_url)
|
||
base_is_https = parsed_base.scheme == 'https'
|
||
|
||
def img_to_markdown(match):
|
||
src = match.group(1)
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/'):
|
||
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
|
||
if base_is_https and src.startswith('http://'):
|
||
src = 'https://' + src[7:]
|
||
return f''
|
||
|
||
text = re.sub(r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>', img_to_markdown, text, flags=re.IGNORECASE)
|
||
text = re.sub(r'<pre[^>]*>\s*<code[^>]*>(.*?)</code>\s*</pre>', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<code[^>]*>(.*?)</code>', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h1[^>]*>(.*?)</h1>', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h2[^>]*>(.*?)</h2>', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h3[^>]*>(.*?)</h3>', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<p[^>]*>(.*?)</p>', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
|
||
text = re.sub(r'<strong[^>]*>(.*?)</strong>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
|
||
text = text.replace(' ', ' ').replace('<', '<').replace('>', '>')
|
||
text = text.replace('&', '&').replace('"', '"')
|
||
|
||
lines = text.split('\n')
|
||
cleaned = []
|
||
prev_empty = False
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if stripped == '':
|
||
if not prev_empty:
|
||
cleaned.append('')
|
||
prev_empty = True
|
||
else:
|
||
cleaned.append(line)
|
||
prev_empty = False
|
||
|
||
text = '\n'.join(cleaned).strip()
|
||
|
||
# 修复数学公式格式（掘金等平台）
|
||
text = fix_math_formulas(text)
|
||
|
||
return text
|
||
|
||
|
||
@router.post("")
|
||
async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)):
|
||
"""采集文章"""
|
||
response_headers = {
|
||
"Cache-Control": "no-cache, no-store, must-revalidate",
|
||
"Content-Type": "application/json; charset=utf-8"
|
||
}
|
||
|
||
try:
|
||
if not request.url.startswith(('http://', 'https://')):
|
||
raise HTTPException(status_code=400, detail="无效的URL")
|
||
|
||
is_wechat = 'mp.weixin.qq.com' in request.url
|
||
is_juejin = 'juejin.cn' in request.url
|
||
is_aliyun = 'developer.aliyun.com' in request.url
|
||
|
||
if is_wechat or is_juejin or is_aliyun:
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
site_name = '微信' if is_wechat else ('掘金' if is_juejin else '阿里云')
|
||
return JSONResponse(
|
||
content={"success": False, "error": f"{site_name}文章需要Playwright"},
|
||
headers=response_headers
|
||
)
|
||
|
||
import asyncio
|
||
spa_result = await asyncio.to_thread(crawl_with_playwright, request.url)
|
||
title = spa_result['title']
|
||
images = spa_result['images']
|
||
content_html = spa_result['content_html']
|
||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||
else:
|
||
headers = {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
'Accept-Encoding': 'gzip, deflate, br',
|
||
'Cache-Control': 'max-age=0',
|
||
'Connection': 'keep-alive',
|
||
'Upgrade-Insecure-Requests': '1',
|
||
'Sec-Fetch-Dest': 'document',
|
||
'Sec-Fetch-Mode': 'navigate',
|
||
'Sec-Fetch-Site': 'none',
|
||
'Sec-Fetch-User': '?1',
|
||
}
|
||
|
||
try:
|
||
response = requests.get(request.url, headers=headers, timeout=30)
|
||
response.raise_for_status()
|
||
except requests.exceptions.HTTPError as e:
|
||
if response.status_code in (468, 403, 429, 503) and PLAYWRIGHT_AVAILABLE:
|
||
import asyncio
|
||
spa_result = await asyncio.to_thread(crawl_with_playwright, request.url)
|
||
title = spa_result['title']
|
||
images = spa_result['images']
|
||
content_html = spa_result['content_html']
|
||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||
|
||
return JSONResponse(
|
||
content={
|
||
"success": True,
|
||
"title": title,
|
||
"url": request.url,
|
||
"content": markdown_content,
|
||
"images": [{"original_url": img.original_url} for img in images]
|
||
},
|
||
headers=response_headers
|
||
)
|
||
else:
|
||
raise
|
||
|
||
declared_encoding = None
|
||
if response.encoding:
|
||
declared_encoding = response.encoding
|
||
|
||
if not declared_encoding or declared_encoding.lower() in ['iso-8859-1', 'latin-1']:
|
||
raw_content = response.content
|
||
charset_match = re.search(rb'<meta[^>]+charset=["\']?([^"\'>\s]+)', raw_content, re.IGNORECASE)
|
||
if charset_match:
|
||
declared_encoding = charset_match.group(1).decode('ascii', errors='ignore')
|
||
|
||
if declared_encoding and declared_encoding.lower() not in ['iso-8859-1', 'latin-1']:
|
||
response.encoding = declared_encoding
|
||
else:
|
||
response.encoding = response.apparent_encoding or 'utf-8'
|
||
|
||
html = response.text
|
||
|
||
if is_spa_site(html):
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
return JSONResponse(
|
||
content={"success": False, "error": "SPA网站需要Playwright"},
|
||
headers=response_headers
|
||
)
|
||
|
||
import asyncio
|
||
spa_result = await asyncio.to_thread(crawl_with_playwright, request.url)
|
||
title = spa_result['title']
|
||
images = spa_result['images']
|
||
content_html = spa_result['content_html']
|
||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||
else:
|
||
title = extract_title_from_html(html)
|
||
main_content = extract_main_content(html)
|
||
markdown_content = html_to_markdown(main_content, request.url)
|
||
images = extract_images_from_markdown(markdown_content, request.url)
|
||
|
||
# 下载图片到临时目录
|
||
temp_dir = get_crawl_temp_dir()
|
||
upload_dir = get_upload_dir()
|
||
|
||
downloaded_images = []
|
||
for i, img in enumerate(images[:5]):
|
||
try:
|
||
img_headers = {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||
'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
|
||
'Referer': request.url,
|
||
}
|
||
# 微信公众号图片需要特殊 Referer
|
||
if 'mmbiz.qpic.cn' in img.original_url:
|
||
img_headers['Referer'] = 'https://mp.weixin.qq.com/'
|
||
|
||
img_response = requests.get(img.original_url, headers=img_headers, timeout=30, stream=True)
|
||
img_response.raise_for_status()
|
||
|
||
content_type = img_response.headers.get('content-type', '')
|
||
ext = '.jpg'
|
||
if 'png' in content_type:
|
||
ext = '.png'
|
||
elif 'gif' in content_type:
|
||
ext = '.gif'
|
||
elif 'webp' in content_type:
|
||
ext = '.webp'
|
||
elif 'svg' in content_type:
|
||
ext = '.svg'
|
||
|
||
filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}"
|
||
filepath = os.path.join(temp_dir, filename)
|
||
|
||
with open(filepath, 'wb') as f:
|
||
for chunk in img_response.iter_content(chunk_size=8192):
|
||
f.write(chunk)
|
||
|
||
# 检查文件大小，过滤防盗链占位图（小于1KB视为失败）
|
||
file_size = os.path.getsize(filepath)
|
||
if file_size < 1024:
|
||
os.remove(filepath)
|
||
print(f"图片太小（{file_size} 字节），可能是防盗链占位图: {img.original_url}")
|
||
continue
|
||
|
||
# 验证文件是否为有效图片（SVG 除外，浏览器直接支持）
|
||
is_svg = ext == '.svg'
|
||
if not is_svg:
|
||
try:
|
||
with PILImage.open(filepath) as pil_img:
|
||
pil_img.verify() # 验证图片完整性
|
||
except Exception:
|
||
os.remove(filepath)
|
||
print(f"文件不是有效图片，已删除: {img.original_url}")
|
||
continue
|
||
|
||
# 生成缩略图 - 放在临时目录的 thumbnails 子目录下
|
||
thumb_filename = f"{os.path.splitext(filename)[0]}.jpg" # 缩略图统一为 .jpg
|
||
thumb_filepath = os.path.join(temp_dir, "thumbnails", thumb_filename)
|
||
thumb_path = None
|
||
|
||
if not is_svg:
|
||
try:
|
||
with PILImage.open(filepath) as pil_img:
|
||
# 转换为 RGB（处理透明图片）
|
||
if pil_img.mode in ('RGBA', 'P'):
|
||
pil_img = pil_img.convert('RGB')
|
||
pil_img.thumbnail((300, 300))
|
||
pil_img.save(thumb_filepath, "JPEG", quality=85)
|
||
thumb_path = f"/storage/tmp/crawl/thumbnails/{thumb_filename}"
|
||
except Exception as thumb_err:
|
||
print(f"缩略图生成失败 {filename}: {thumb_err}")
|
||
|
||
local_path = f"/storage/tmp/crawl/{filename}"
|
||
img.local_path = local_path
|
||
img.filename = filename
|
||
img.success = True
|
||
downloaded_images.append({
|
||
"original_url": img.original_url,
|
||
"local_path": local_path,
|
||
"thumb_path": thumb_path,
|
||
"filename": filename
|
||
})
|
||
|
||
# 替换内容中的URL为临时路径
|
||
if img.original_url in markdown_content:
|
||
markdown_content = markdown_content.replace(img.original_url, local_path)
|
||
|
||
except Exception as e:
|
||
print(f"下载图片失败 {img.original_url}: {e}")
|
||
|
||
summary = markdown_content[:200].replace('#', '').replace('*', '').strip()
|
||
|
||
return JSONResponse(
|
||
content={
|
||
"success": True,
|
||
"title": title,
|
||
"content": markdown_content,
|
||
"summary": summary,
|
||
"images": downloaded_images
|
||
},
|
||
headers=response_headers
|
||
)
|
||
|
||
except Exception as e:
|
||
import traceback
|
||
print(f"采集失败: {e}")
|
||
print(traceback.format_exc())
|
||
return JSONResponse(
|
||
content={"success": False, "error": f"采集失败: {str(e)}"},
|
||
headers=response_headers
|
||
)
|