Auto backup \2026-\04-\20
This commit is contained in:
Ubuntu
@@ -0,0 +1,212 @@
|
||||
"""
|
||||
站点提取策略化重构（v2）
|
||||
========================
|
||||
将散落在 crawl.py / universal_crawl.py 中的站点特判逻辑收敛为「策略注册表」，
|
||||
接口设计借鉴 crawl4ai 的 ContentScrapingStrategy(ABC) 思想。
|
||||
|
||||
双轨调度：
|
||||
- 轨A（Legacy）：调用方现有的 if-else 站点特判，逐行保留，不删除
|
||||
- 轨B（Strategy）：本文件中的策略链，每个策略类内部逻辑 = 现有代码「原样搬迁」
|
||||
- 优先级由 STRATEGY_FIRST_DOMAINS 配置决定：
|
||||
- 默认空集：全部走旧逻辑（轨A），策略链仅作未来兜底
|
||||
- 白名单命中：策略链优先（轨B），质量不达标自动回退旧逻辑（轨A）
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .crawl import (
|
||||
extract_title_from_html,
|
||||
html_to_markdown,
|
||||
is_spa_site,
|
||||
convert_spa_html_to_markdown,
|
||||
)
|
||||
|
||||
# 试点白名单：命中这些域名的站点「新策略优先」，质量不达标回退旧逻辑
|
||||
STRATEGY_FIRST_DOMAINS = set() # 例：{"juejin.cn", "csdn.net"}
|
||||
|
||||
# 质量门控：Markdown 最短长度（复用现有「内容过短」经验值 300）
|
||||
MIN_CONTENT_CHARS = 300
|
||||
FALLBACK_TITLE = "未获取到标题"
|
||||
|
||||
|
||||
class SiteExtractor(ABC):
|
||||
"""站点提取策略基类。每个策略类内部逻辑 = 现有代码原样搬迁，行为零改动。"""
|
||||
|
||||
name: str = "generic"
|
||||
|
||||
@abstractmethod
|
||||
def match(self, url: str) -> bool:
|
||||
"""URL 是否命中该站点"""
|
||||
|
||||
def needs_browser(self, url: str) -> bool:
|
||||
"""是否需要浏览器渲染（Playwright）"""
|
||||
return False
|
||||
|
||||
def extract_title(self, html: str, url: str) -> Optional[str]:
|
||||
"""站点专用标题提取；返回 None 时由 run_extractor 回退通用提取"""
|
||||
return None
|
||||
|
||||
def extract_content_html(self, html: str, url: str) -> str:
|
||||
"""提取正文容器 HTML；未命中选择器时返回原 HTML"""
|
||||
return html
|
||||
|
||||
def filter_images(self, images: list, url: str) -> list:
|
||||
"""过滤站点 UI 图标等噪音图片"""
|
||||
return images
|
||||
|
||||
def to_markdown(self, content_html: str, url: str, images: list) -> str:
|
||||
"""将正文容器 HTML 转为 Markdown"""
|
||||
return html_to_markdown(content_html, url)
|
||||
|
||||
|
||||
class JuejinExtractor(SiteExtractor):
|
||||
"""掘金：requests 返回混淆代码必须浏览器渲染；正文 article.article"""
|
||||
name = "juejin"
|
||||
|
||||
def match(self, url: str) -> bool:
|
||||
return 'juejin.cn' in url
|
||||
|
||||
def needs_browser(self, url: str) -> bool:
|
||||
return True
|
||||
|
||||
def extract_title(self, html: str, url: str) -> Optional[str]:
|
||||
# 原样搬迁 universal_crawl._extract_juejin_title
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# 去掉 " - 掘金" 后缀
|
||||
title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
def extract_content_html(self, html: str, url: str) -> str:
|
||||
# 原样搬迁 universal_crawl._extract_juejin_content
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
# 优先使用 article.article
|
||||
container = soup.select_one('article.article')
|
||||
if not container:
|
||||
# fallback 到 .article-viewer
|
||||
container = soup.select_one('.article-viewer')
|
||||
if not container:
|
||||
# 再 fallback 到 #article-root
|
||||
container = soup.select_one('#article-root')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
|
||||
class CsdnExtractor(SiteExtractor):
|
||||
"""CSDN：正文 #content_views / .article_content；过滤 csdnimg UI 图标"""
|
||||
name = "csdn"
|
||||
|
||||
def match(self, url: str) -> bool:
|
||||
return 'csdn.net' in url or 'csdn.com' in url
|
||||
|
||||
def extract_title(self, html: str, url: str) -> Optional[str]:
|
||||
# 原样搬迁 universal_crawl._extract_csdn_title
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# 去掉 " - CSDN博客" / "-CSDN博客" 后缀
|
||||
title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
def extract_content_html(self, html: str, url: str) -> str:
|
||||
# 原样搬迁 universal_crawl._extract_csdn_content
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
# 优先使用 #content_views（CSDN 正文主容器）
|
||||
container = soup.select_one('#content_views')
|
||||
if not container:
|
||||
# fallback 到 .article_content
|
||||
container = soup.select_one('.article_content')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
def filter_images(self, images: list, url: str) -> list:
|
||||
# 原样搬迁 universal_crawl._filter_csdn_images
|
||||
filtered = []
|
||||
skip_patterns = [
|
||||
'newHeart2023',
|
||||
'tobarCollect',
|
||||
'tobarCollection',
|
||||
'copyright',
|
||||
'avatar',
|
||||
'blogv2/dist/pc/img',
|
||||
]
|
||||
for img in images:
|
||||
img_url = img.original_url.lower()
|
||||
if any(p in img_url for p in skip_patterns):
|
||||
continue
|
||||
filtered.append(img)
|
||||
return filtered
|
||||
|
||||
|
||||
class GenericExtractor(SiteExtractor):
|
||||
"""通用兜底：不做站点特判，复刻旧逻辑的通用分支（含 SPA 判断）"""
|
||||
name = "generic"
|
||||
|
||||
def match(self, url: str) -> bool:
|
||||
return True
|
||||
|
||||
def to_markdown(self, content_html: str, url: str, images: list) -> str:
|
||||
if is_spa_site(content_html):
|
||||
return convert_spa_html_to_markdown(content_html, images, url)
|
||||
return html_to_markdown(content_html, url)
|
||||
|
||||
|
||||
# 策略注册表：有序列表，先匹配先命中；GenericExtractor 永远兜底
|
||||
SITE_EXTRACTORS = [
|
||||
JuejinExtractor(),
|
||||
CsdnExtractor(),
|
||||
GenericExtractor(),
|
||||
]
|
||||
|
||||
|
||||
def get_extractor_for_url(url: str) -> Optional[SiteExtractor]:
|
||||
"""返回第一个匹配的策略；GenericExtractor 恒匹配（兜底）"""
|
||||
for extractor in SITE_EXTRACTORS:
|
||||
if extractor.match(url):
|
||||
return extractor
|
||||
return None
|
||||
|
||||
|
||||
def is_strategy_first(url: str) -> bool:
|
||||
"""url 是否命中试点白名单（新策略优先，失败回退旧逻辑）"""
|
||||
return any(domain in url for domain in STRATEGY_FIRST_DOMAINS)
|
||||
|
||||
|
||||
class ExtractorResult:
|
||||
"""策略链输出，与旧逻辑 _legacy_extract 返回 (title, markdown, images) 同构"""
|
||||
|
||||
__slots__ = ("title", "markdown", "images")
|
||||
|
||||
def __init__(self, title: str, markdown: str, images: list):
|
||||
self.title = title
|
||||
self.markdown = markdown
|
||||
self.images = images
|
||||
|
||||
|
||||
def run_extractor(extractor: SiteExtractor, url: str, html: str, images: list) -> ExtractorResult:
|
||||
"""执行策略链：专用标题 → 通用标题降级 → 图片过滤 → 正文容器 → Markdown"""
|
||||
title = extractor.extract_title(html, url) or extract_title_from_html(html) or FALLBACK_TITLE
|
||||
filtered_images = extractor.filter_images(images, url)
|
||||
content_html = extractor.extract_content_html(html, url)
|
||||
markdown = extractor.to_markdown(content_html, url, filtered_images)
|
||||
return ExtractorResult(title=title, markdown=markdown, images=filtered_images)
|
||||
|
||||
|
||||
def is_strategy_success(result: ExtractorResult, min_chars: int = MIN_CONTENT_CHARS) -> bool:
|
||||
"""质量门控：标题有效 + Markdown 长度达标（复用现有「内容过短」经验值）"""
|
||||
if not result.title or result.title == FALLBACK_TITLE:
|
||||
return False
|
||||
if len((result.markdown or "").strip()) < min_chars:
|
||||
return False
|
||||
return True
|
||||
@@ -23,6 +23,14 @@ from bs4 import BeautifulSoup
|
||||
# Scrapling 引擎
|
||||
from scrapling import Fetcher, StealthyFetcher
|
||||
|
||||
# 策略化重构（v2）：站点提取策略注册表 + 双轨调度
|
||||
from .crawl_strategies import (
|
||||
get_extractor_for_url,
|
||||
is_strategy_first,
|
||||
run_extractor,
|
||||
is_strategy_success,
|
||||
)
|
||||
|
||||
# 复用 crawl.py 的通用工具函数
|
||||
from .crawl import (
|
||||
fix_encoding,
|
||||
@@ -375,6 +383,70 @@ def _filter_51cto_images(images: list) -> list:
|
||||
return filtered
|
||||
|
||||
|
||||
def _legacy_extract(url: str, html: str, images: Optional[list] = None):
|
||||
"""旧逻辑（轨A）：现有站点特判提取，逐行保留原实现，不删除。
|
||||
|
||||
返回 (title, markdown_content, images)，与策略链 ExtractorResult 同构。
|
||||
images 为 None 时内部自行调用 extract_images；否则复用调用方已提取的结果。
|
||||
原第 6 步的 is_wechat 变量从未被使用，搬迁时省略。
|
||||
"""
|
||||
# 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||||
is_juejin = 'juejin.cn' in url
|
||||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||||
is_cnblogs = 'cnblogs.com' in url
|
||||
is_51cto = '51cto.com' in url
|
||||
if is_infoq:
|
||||
title = _extract_infoq_title(html)
|
||||
elif is_juejin:
|
||||
title = _extract_juejin_title(html)
|
||||
elif is_csdn:
|
||||
title = _extract_csdn_title(html)
|
||||
elif is_cnblogs:
|
||||
title = _extract_cnblogs_title(html)
|
||||
elif is_51cto:
|
||||
title = _extract_51cto_title(html)
|
||||
else:
|
||||
title = None
|
||||
if not title:
|
||||
title = extract_title_from_html(html)
|
||||
if not title:
|
||||
title = "未获取到标题"
|
||||
|
||||
# 6. 提取正文和图片
|
||||
if images is None:
|
||||
images = extract_images(html, url)
|
||||
if is_csdn:
|
||||
images = _filter_csdn_images(images)
|
||||
elif is_cnblogs:
|
||||
images = _filter_cnblogs_images(images)
|
||||
elif is_51cto:
|
||||
images = _filter_51cto_images(images)
|
||||
|
||||
# 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
if is_infoq:
|
||||
infoq_html = _extract_infoq_content(html)
|
||||
markdown_content = html_to_markdown(infoq_html, url)
|
||||
elif is_juejin:
|
||||
juejin_html = _extract_juejin_content(html)
|
||||
markdown_content = html_to_markdown(juejin_html, url)
|
||||
elif is_csdn:
|
||||
csdn_html = _extract_csdn_content(html)
|
||||
markdown_content = html_to_markdown(csdn_html, url)
|
||||
elif is_cnblogs:
|
||||
cnblogs_html = _extract_cnblogs_content(html)
|
||||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||||
elif is_51cto:
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
|
||||
return title, markdown_content, images
|
||||
|
||||
|
||||
@router.post("/fetch", response_model=CrawlResponse)
|
||||
async def universal_fetch(request: UniversalCrawlRequest):
|
||||
"""通用采集：使用 Scrapling 引擎获取文章"""
|
||||
@@ -402,64 +474,24 @@ async def universal_fetch(request: UniversalCrawlRequest):
|
||||
# 4. 微信图片防盗链处理
|
||||
html = _bypass_wechat_images(html, url)
|
||||
|
||||
# 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||||
is_juejin = 'juejin.cn' in url
|
||||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||||
is_cnblogs = 'cnblogs.com' in url
|
||||
is_51cto = '51cto.com' in url
|
||||
if is_infoq:
|
||||
title = _extract_infoq_title(html)
|
||||
elif is_juejin:
|
||||
title = _extract_juejin_title(html)
|
||||
elif is_csdn:
|
||||
title = _extract_csdn_title(html)
|
||||
elif is_cnblogs:
|
||||
title = _extract_cnblogs_title(html)
|
||||
elif is_51cto:
|
||||
title = _extract_51cto_title(html)
|
||||
# 5. 提取标题和正文（双轨调度 v2：默认旧逻辑优先；白名单站点新策略优先，质量不达标回退旧逻辑）
|
||||
extractor = get_extractor_for_url(url)
|
||||
if extractor and is_strategy_first(url):
|
||||
images = extract_images(html, url)
|
||||
result = run_extractor(extractor, url, html, images)
|
||||
if is_strategy_success(result):
|
||||
title, markdown_content, images = result.title, result.markdown, result.images
|
||||
print(f"[Universal Crawl] 策略链成功: {extractor.name}")
|
||||
else:
|
||||
print(f"[Universal Crawl] 策略链质量不达标({extractor.name})，回退旧逻辑")
|
||||
title, markdown_content, images = _legacy_extract(url, html, images)
|
||||
else:
|
||||
title = None
|
||||
if not title:
|
||||
title = extract_title_from_html(html)
|
||||
if not title:
|
||||
title = "未获取到标题"
|
||||
title, markdown_content, images = _legacy_extract(url, html)
|
||||
|
||||
# 6. 提取正文和图片
|
||||
is_wechat = 'mp.weixin.qq.com' in url
|
||||
images = extract_images(html, url)
|
||||
if is_csdn:
|
||||
images = _filter_csdn_images(images)
|
||||
elif is_cnblogs:
|
||||
images = _filter_cnblogs_images(images)
|
||||
elif is_51cto:
|
||||
images = _filter_51cto_images(images)
|
||||
|
||||
# 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
if is_infoq:
|
||||
infoq_html = _extract_infoq_content(html)
|
||||
markdown_content = html_to_markdown(infoq_html, url)
|
||||
elif is_juejin:
|
||||
juejin_html = _extract_juejin_content(html)
|
||||
markdown_content = html_to_markdown(juejin_html, url)
|
||||
elif is_csdn:
|
||||
csdn_html = _extract_csdn_content(html)
|
||||
markdown_content = html_to_markdown(csdn_html, url)
|
||||
elif is_cnblogs:
|
||||
cnblogs_html = _extract_cnblogs_content(html)
|
||||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||||
elif is_51cto:
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
|
||||
# 8. 数学公式修复
|
||||
# 6. 数学公式修复
|
||||
markdown_content = fix_math_formulas(markdown_content)
|
||||
|
||||
# 9. 清理 Markdown 中的 HTML 标签
|
||||
# 7. 清理 Markdown 中的 HTML 标签
|
||||
markdown_content = re.sub(r'<[^>]+>', '', markdown_content)
|
||||
markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
|
||||
markdown_content = markdown_content.strip()
|
||||
|
||||
Reference in New Issue
Block a user