diff --git a/personal-blog/backend/routers/crawl_strategies.py b/personal-blog/backend/routers/crawl_strategies.py new file mode 100644 index 0000000..34b6c0d --- /dev/null +++ b/personal-blog/backend/routers/crawl_strategies.py @@ -0,0 +1,212 @@ +""" +站点提取策略化重构(v2) +======================== +将散落在 crawl.py / universal_crawl.py 中的站点特判逻辑收敛为「策略注册表」, +接口设计借鉴 crawl4ai 的 ContentScrapingStrategy(ABC) 思想。 + +双轨调度: +- 轨A(Legacy):调用方现有的 if-else 站点特判,逐行保留,不删除 +- 轨B(Strategy):本文件中的策略链,每个策略类内部逻辑 = 现有代码「原样搬迁」 +- 优先级由 STRATEGY_FIRST_DOMAINS 配置决定: + - 默认空集:全部走旧逻辑(轨A),策略链仅作未来兜底 + - 白名单命中:策略链优先(轨B),质量不达标自动回退旧逻辑(轨A) +""" + +from abc import ABC, abstractmethod +from typing import Optional +import re + +from bs4 import BeautifulSoup + +from .crawl import ( + extract_title_from_html, + html_to_markdown, + is_spa_site, + convert_spa_html_to_markdown, +) + +# 试点白名单:命中这些域名的站点「新策略优先」,质量不达标回退旧逻辑 +STRATEGY_FIRST_DOMAINS = set() # 例:{"juejin.cn", "csdn.net"} + +# 质量门控:Markdown 最短长度(复用现有「内容过短」经验值 300) +MIN_CONTENT_CHARS = 300 +FALLBACK_TITLE = "未获取到标题" + + +class SiteExtractor(ABC): + """站点提取策略基类。每个策略类内部逻辑 = 现有代码原样搬迁,行为零改动。""" + + name: str = "generic" + + @abstractmethod + def match(self, url: str) -> bool: + """URL 是否命中该站点""" + + def needs_browser(self, url: str) -> bool: + """是否需要浏览器渲染(Playwright)""" + return False + + def extract_title(self, html: str, url: str) -> Optional[str]: + """站点专用标题提取;返回 None 时由 run_extractor 回退通用提取""" + return None + + def extract_content_html(self, html: str, url: str) -> str: + """提取正文容器 HTML;未命中选择器时返回原 HTML""" + return html + + def filter_images(self, images: list, url: str) -> list: + """过滤站点 UI 图标等噪音图片""" + return images + + def to_markdown(self, content_html: str, url: str, images: list) -> str: + """将正文容器 HTML 转为 Markdown""" + return html_to_markdown(content_html, url) + + +class JuejinExtractor(SiteExtractor): + """掘金:requests 返回混淆代码必须浏览器渲染;正文 article.article""" + name = "juejin" + + def match(self, url: str) -> bool: + return 'juejin.cn' in url + + def needs_browser(self, url: str) -> bool: + return True + + def extract_title(self, html: str, url: str) -> Optional[str]: + # 原样搬迁 universal_crawl._extract_juejin_title + match = re.search(r']*>(.*?)', html, re.IGNORECASE | re.DOTALL) + if match: + title = match.group(1).strip() + # 去掉 " - 掘金" 后缀 + title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip() + if len(title) > 5: + return title + return None + + def extract_content_html(self, html: str, url: str) -> str: + # 原样搬迁 universal_crawl._extract_juejin_content + soup = BeautifulSoup(html, 'lxml') + # 优先使用 article.article + container = soup.select_one('article.article') + if not container: + # fallback 到 .article-viewer + container = soup.select_one('.article-viewer') + if not container: + # 再 fallback 到 #article-root + container = soup.select_one('#article-root') + if container: + return str(container) + return html + + +class CsdnExtractor(SiteExtractor): + """CSDN:正文 #content_views / .article_content;过滤 csdnimg UI 图标""" + name = "csdn" + + def match(self, url: str) -> bool: + return 'csdn.net' in url or 'csdn.com' in url + + def extract_title(self, html: str, url: str) -> Optional[str]: + # 原样搬迁 universal_crawl._extract_csdn_title + match = re.search(r']*>(.*?)', html, re.IGNORECASE | re.DOTALL) + if match: + title = match.group(1).strip() + # 去掉 " - CSDN博客" / "-CSDN博客" 后缀 + title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip() + if len(title) > 5: + return title + return None + + def extract_content_html(self, html: str, url: str) -> str: + # 原样搬迁 universal_crawl._extract_csdn_content + soup = BeautifulSoup(html, 'lxml') + # 优先使用 #content_views(CSDN 正文主容器) + container = soup.select_one('#content_views') + if not container: + # fallback 到 .article_content + container = soup.select_one('.article_content') + if container: + return str(container) + return html + + def filter_images(self, images: list, url: str) -> list: + # 原样搬迁 universal_crawl._filter_csdn_images + filtered = [] + skip_patterns = [ + 'newHeart2023', + 'tobarCollect', + 'tobarCollection', + 'copyright', + 'avatar', + 'blogv2/dist/pc/img', + ] + for img in images: + img_url = img.original_url.lower() + if any(p in img_url for p in skip_patterns): + continue + filtered.append(img) + return filtered + + +class GenericExtractor(SiteExtractor): + """通用兜底:不做站点特判,复刻旧逻辑的通用分支(含 SPA 判断)""" + name = "generic" + + def match(self, url: str) -> bool: + return True + + def to_markdown(self, content_html: str, url: str, images: list) -> str: + if is_spa_site(content_html): + return convert_spa_html_to_markdown(content_html, images, url) + return html_to_markdown(content_html, url) + + +# 策略注册表:有序列表,先匹配先命中;GenericExtractor 永远兜底 +SITE_EXTRACTORS = [ + JuejinExtractor(), + CsdnExtractor(), + GenericExtractor(), +] + + +def get_extractor_for_url(url: str) -> Optional[SiteExtractor]: + """返回第一个匹配的策略;GenericExtractor 恒匹配(兜底)""" + for extractor in SITE_EXTRACTORS: + if extractor.match(url): + return extractor + return None + + +def is_strategy_first(url: str) -> bool: + """url 是否命中试点白名单(新策略优先,失败回退旧逻辑)""" + return any(domain in url for domain in STRATEGY_FIRST_DOMAINS) + + +class ExtractorResult: + """策略链输出,与旧逻辑 _legacy_extract 返回 (title, markdown, images) 同构""" + + __slots__ = ("title", "markdown", "images") + + def __init__(self, title: str, markdown: str, images: list): + self.title = title + self.markdown = markdown + self.images = images + + +def run_extractor(extractor: SiteExtractor, url: str, html: str, images: list) -> ExtractorResult: + """执行策略链:专用标题 → 通用标题降级 → 图片过滤 → 正文容器 → Markdown""" + title = extractor.extract_title(html, url) or extract_title_from_html(html) or FALLBACK_TITLE + filtered_images = extractor.filter_images(images, url) + content_html = extractor.extract_content_html(html, url) + markdown = extractor.to_markdown(content_html, url, filtered_images) + return ExtractorResult(title=title, markdown=markdown, images=filtered_images) + + +def is_strategy_success(result: ExtractorResult, min_chars: int = MIN_CONTENT_CHARS) -> bool: + """质量门控:标题有效 + Markdown 长度达标(复用现有「内容过短」经验值)""" + if not result.title or result.title == FALLBACK_TITLE: + return False + if len((result.markdown or "").strip()) < min_chars: + return False + return True diff --git a/personal-blog/backend/routers/universal_crawl.py b/personal-blog/backend/routers/universal_crawl.py index 1c4900c..410ca06 100644 --- a/personal-blog/backend/routers/universal_crawl.py +++ b/personal-blog/backend/routers/universal_crawl.py @@ -23,6 +23,14 @@ from bs4 import BeautifulSoup # Scrapling 引擎 from scrapling import Fetcher, StealthyFetcher +# 策略化重构(v2):站点提取策略注册表 + 双轨调度 +from .crawl_strategies import ( + get_extractor_for_url, + is_strategy_first, + run_extractor, + is_strategy_success, +) + # 复用 crawl.py 的通用工具函数 from .crawl import ( fix_encoding, @@ -375,6 +383,70 @@ def _filter_51cto_images(images: list) -> list: return filtered +def _legacy_extract(url: str, html: str, images: Optional[list] = None): + """旧逻辑(轨A):现有站点特判提取,逐行保留原实现,不删除。 + + 返回 (title, markdown_content, images),与策略链 ExtractorResult 同构。 + images 为 None 时内部自行调用 extract_images;否则复用调用方已提取的结果。 + 原第 6 步的 is_wechat 变量从未被使用,搬迁时省略。 + """ + # 5. 提取标题(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取) + is_infoq = 'infoq.cn' in url or 'infoq.com' in url + is_juejin = 'juejin.cn' in url + is_csdn = 'csdn.net' in url or 'csdn.com' in url + is_cnblogs = 'cnblogs.com' in url + is_51cto = '51cto.com' in url + if is_infoq: + title = _extract_infoq_title(html) + elif is_juejin: + title = _extract_juejin_title(html) + elif is_csdn: + title = _extract_csdn_title(html) + elif is_cnblogs: + title = _extract_cnblogs_title(html) + elif is_51cto: + title = _extract_51cto_title(html) + else: + title = None + if not title: + title = extract_title_from_html(html) + if not title: + title = "未获取到标题" + + # 6. 提取正文和图片 + if images is None: + images = extract_images(html, url) + if is_csdn: + images = _filter_csdn_images(images) + elif is_cnblogs: + images = _filter_cnblogs_images(images) + elif is_51cto: + images = _filter_51cto_images(images) + + # 7. 转换为 Markdown(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取) + if is_infoq: + infoq_html = _extract_infoq_content(html) + markdown_content = html_to_markdown(infoq_html, url) + elif is_juejin: + juejin_html = _extract_juejin_content(html) + markdown_content = html_to_markdown(juejin_html, url) + elif is_csdn: + csdn_html = _extract_csdn_content(html) + markdown_content = html_to_markdown(csdn_html, url) + elif is_cnblogs: + cnblogs_html = _extract_cnblogs_content(html) + markdown_content = html_to_markdown(cnblogs_html, url) + elif is_51cto: + cto_html = _extract_51cto_content(html) + markdown_content = html_to_markdown(cto_html, url) + elif is_spa_site(html): + markdown_content = convert_spa_html_to_markdown(html, images, url) + else: + markdown_content = html_to_markdown(html, url) + + return title, markdown_content, images + + @router.post("/fetch", response_model=CrawlResponse) async def universal_fetch(request: UniversalCrawlRequest): """通用采集:使用 Scrapling 引擎获取文章""" @@ -402,64 +474,24 @@ async def universal_fetch(request: UniversalCrawlRequest): # 4. 微信图片防盗链处理 html = _bypass_wechat_images(html, url) - # 5. 提取标题(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取) - is_infoq = 'infoq.cn' in url or 'infoq.com' in url - is_juejin = 'juejin.cn' in url - is_csdn = 'csdn.net' in url or 'csdn.com' in url - is_cnblogs = 'cnblogs.com' in url - is_51cto = '51cto.com' in url - if is_infoq: - title = _extract_infoq_title(html) - elif is_juejin: - title = _extract_juejin_title(html) - elif is_csdn: - title = _extract_csdn_title(html) - elif is_cnblogs: - title = _extract_cnblogs_title(html) - elif is_51cto: - title = _extract_51cto_title(html) + # 5. 提取标题和正文(双轨调度 v2:默认旧逻辑优先;白名单站点新策略优先,质量不达标回退旧逻辑) + extractor = get_extractor_for_url(url) + if extractor and is_strategy_first(url): + images = extract_images(html, url) + result = run_extractor(extractor, url, html, images) + if is_strategy_success(result): + title, markdown_content, images = result.title, result.markdown, result.images + print(f"[Universal Crawl] 策略链成功: {extractor.name}") + else: + print(f"[Universal Crawl] 策略链质量不达标({extractor.name}),回退旧逻辑") + title, markdown_content, images = _legacy_extract(url, html, images) else: - title = None - if not title: - title = extract_title_from_html(html) - if not title: - title = "未获取到标题" + title, markdown_content, images = _legacy_extract(url, html) - # 6. 提取正文和图片 - is_wechat = 'mp.weixin.qq.com' in url - images = extract_images(html, url) - if is_csdn: - images = _filter_csdn_images(images) - elif is_cnblogs: - images = _filter_cnblogs_images(images) - elif is_51cto: - images = _filter_51cto_images(images) - - # 7. 转换为 Markdown(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取) - if is_infoq: - infoq_html = _extract_infoq_content(html) - markdown_content = html_to_markdown(infoq_html, url) - elif is_juejin: - juejin_html = _extract_juejin_content(html) - markdown_content = html_to_markdown(juejin_html, url) - elif is_csdn: - csdn_html = _extract_csdn_content(html) - markdown_content = html_to_markdown(csdn_html, url) - elif is_cnblogs: - cnblogs_html = _extract_cnblogs_content(html) - markdown_content = html_to_markdown(cnblogs_html, url) - elif is_51cto: - cto_html = _extract_51cto_content(html) - markdown_content = html_to_markdown(cto_html, url) - elif is_spa_site(html): - markdown_content = convert_spa_html_to_markdown(html, images, url) - else: - markdown_content = html_to_markdown(html, url) - - # 8. 数学公式修复 + # 6. 数学公式修复 markdown_content = fix_math_formulas(markdown_content) - # 9. 清理 Markdown 中的 HTML 标签 + # 7. 清理 Markdown 中的 HTML 标签 markdown_content = re.sub(r'<[^>]+>', '', markdown_content) markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content) markdown_content = markdown_content.strip() diff --git a/personal-blog/backend/test_crawl_strategies.py b/personal-blog/backend/test_crawl_strategies.py new file mode 100644 index 0000000..132caf7 --- /dev/null +++ b/personal-blog/backend/test_crawl_strategies.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""站点提取策略化重构(v2)单元测试。 + +核心断言:策略链(轨B)与旧逻辑(轨A)对同一 HTML 的输出完全一致。 +运行:cd backend && python3 test_crawl_strategies.py +""" +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from routers.crawl import CrawlImage, extract_images +from routers.crawl_strategies import ( + JuejinExtractor, + CsdnExtractor, + GenericExtractor, + SITE_EXTRACTORS, + get_extractor_for_url, + is_strategy_first, + is_strategy_success, + run_extractor, + ExtractorResult, + STRATEGY_FIRST_DOMAINS, + FALLBACK_TITLE, +) +from routers.universal_crawl import ( + _extract_juejin_title, + _extract_juejin_content, + _extract_csdn_title, + _extract_csdn_content, + _filter_csdn_images, + _legacy_extract, +) + +PASS = 0 +FAIL = 0 + + +def check(name, cond): + global PASS, FAIL + if cond: + PASS += 1 + print(f" PASS {name}") + else: + FAIL += 1 + print(f" FAIL {name}") + + +# 掘金 HTML:article.article 为正文,sidebar 为噪音 +JUEGIN_HTML = f"""深入理解 React 并发特性 - 掘金 +

深入理解 React 并发特性

+

{'这是掘金正文内容,用于验证策略链与旧逻辑提取一致性。' * 12}

+
+""" + +# CSDN HTML:content_views 为正文,含 UI 图标噪音图(avatar/copyright 全小写 pattern 能命中; +# newHeart2023 等混合大小写 pattern 因 img_url.lower() 永远匹配不上,属既有 bug,保持原样) +CSDN_HTML = f"""Python 异步编程指南 - CSDN博客 +
+

{'这是 CSDN 正文内容,用于验证策略链与旧逻辑提取一致性。' * 12}

+ + + +
+
推荐文章列表不应被提取
""" + +# 未知站点 HTML:走 GenericExtractor +GENERIC_HTML = f"""某某技术博客 +

{'普通网站正文内容,用于验证通用兜底策略。' * 12}

""" + +URL_JUEGIN = "https://juejin.cn/post/7420000000000000000" +URL_CSDN = "https://blog.csdn.net/xxx/article/details/140000000" +URL_GENERIC = "https://example.com/post/123" + + +def test_registry(): + print("[1] 注册表匹配") + check("juejin -> JuejinExtractor", isinstance(get_extractor_for_url(URL_JUEGIN), JuejinExtractor)) + check("csdn -> CsdnExtractor", isinstance(get_extractor_for_url(URL_CSDN), CsdnExtractor)) + check("unknown -> GenericExtractor", isinstance(get_extractor_for_url(URL_GENERIC), GenericExtractor)) + check("GenericExtractor 恒兜底", SITE_EXTRACTORS[-1].name == "generic") + check("策略链默认关闭(白名单为空)", is_strategy_first(URL_JUEGIN) is False) + check("白名单为空集", STRATEGY_FIRST_DOMAINS == set()) + + +def test_juejin_consistency(): + print("[2] 掘金策略与旧逻辑一致性") + ext = JuejinExtractor() + title_new = ext.extract_title(JUEGIN_HTML, URL_JUEGIN) + title_old = _extract_juejin_title(JUEGIN_HTML) + check("标题一致且去掉「- 掘金」后缀", title_new == title_old == "深入理解 React 并发特性") + + content_new = ext.extract_content_html(JUEGIN_HTML, URL_JUEGIN) + content_old = _extract_juejin_content(JUEGIN_HTML) + check("正文容器一致", content_new == content_old) + check("正文容器不含侧栏", "侧栏内容" not in content_new and "sidebar" not in content_new) + check("needs_browser=True", ext.needs_browser(URL_JUEGIN) is True) + + +def test_csdn_consistency(): + print("[3] CSDN 策略与旧逻辑一致性") + ext = CsdnExtractor() + title_new = ext.extract_title(CSDN_HTML, URL_CSDN) + title_old = _extract_csdn_title(CSDN_HTML) + check("标题一致且去掉「- CSDN博客」后缀", title_new == title_old == "Python 异步编程指南") + + content_new = ext.extract_content_html(CSDN_HTML, URL_CSDN) + content_old = _extract_csdn_content(CSDN_HTML) + check("正文容器一致", content_new == content_old) + check("正文容器不含推荐列表", "推荐文章列表" not in content_new) + + images = extract_images(CSDN_HTML, URL_CSDN) + check("图片提取到 3 张", len(images) == 3) + filtered_new = ext.filter_images(images, URL_CSDN) + filtered_old = _filter_csdn_images(images) + check("图片过滤结果一致", [i.original_url for i in filtered_new] == [i.original_url for i in filtered_old]) + check("UI 图标被过滤(avatar/copyright)", len(filtered_new) == 1 and filtered_new[0].original_url.endswith("real-article.png")) + + # 既有 bug 行为记录:newHeart2023 混合大小写 pattern 因 img_url.lower() 永不匹配,保持原样不修 + mixed_case = extract_images('', URL_CSDN) + check("历史行为:newHeart2023 不被过滤(既有 bug,保持原样)", len(ext.filter_images(mixed_case, URL_CSDN)) == 1) + + +def test_run_extractor(): + print("[4] 策略链集成(run_extractor)") + images = extract_images(JUEGIN_HTML, URL_JUEGIN) + result = run_extractor(JuejinExtractor(), URL_JUEGIN, JUEGIN_HTML, images) + check("标题正确", result.title == "深入理解 React 并发特性") + check("Markdown 含正文", "掘金正文内容" in result.markdown) + check("Markdown 不含侧栏", "侧栏内容" not in result.markdown) + check("图片 1 张", len(result.images) == 1) + + # 标题降级:专用提取失败时回退通用 + degraded_html = "<html><head><title>通用标题兜底" + JUEGIN_HTML.split("")[1] + result2 = run_extractor(JuejinExtractor(), URL_JUEGIN, degraded_html, images) + check("标题降级到通用提取", result2.title == "通用标题兜底") + + +def test_quality_gate(): + print("[5] 质量门控 is_strategy_success") + ok = ExtractorResult(title="好标题", markdown="x" * 500, images=[]) + check("正常结果通过", is_strategy_success(ok) is True) + check("空标题失败", is_strategy_success(ExtractorResult(title="", markdown="x" * 500, images=[])) is False) + check("兜底标题失败", is_strategy_success(ExtractorResult(title=FALLBACK_TITLE, markdown="x" * 500, images=[])) is False) + check("内容过短失败", is_strategy_success(ExtractorResult(title="好标题", markdown="x" * 100, images=[])) is False) + + +def test_legacy_vs_strategy_equivalence(): + print("[6] 新旧轨全链路等价(title/markdown/images)") + for url, html, extractor in [ + (URL_JUEGIN, JUEGIN_HTML, JuejinExtractor()), + (URL_CSDN, CSDN_HTML, CsdnExtractor()), + (URL_GENERIC, GENERIC_HTML, GenericExtractor()), + ]: + old_title, old_md, old_images = _legacy_extract(url, html) + new = run_extractor(extractor, url, html, extract_images(html, url)) + check(f"{extractor.name}: 标题等价", old_title == new.title) + check(f"{extractor.name}: Markdown 等价", old_md == new.markdown) + check(f"{extractor.name}: 图片等价", [i.original_url for i in old_images] == [i.original_url for i in new.images]) + + +if __name__ == "__main__": + test_registry() + test_juejin_consistency() + test_csdn_consistency() + test_run_extractor() + test_quality_gate() + test_legacy_vs_strategy_equivalence() + print(f"\n结果: {PASS} 通过, {FAIL} 失败") + sys.exit(1 if FAIL else 0) diff --git a/personal-blog/docs/04-crawl-solution.md b/personal-blog/docs/04-crawl-solution.md index 315e65f..67a4a47 100644 --- a/personal-blog/docs/04-crawl-solution.md +++ b/personal-blog/docs/04-crawl-solution.md @@ -393,6 +393,8 @@ if 'zhihu.com' in url: 不需要修改 `html_to_markdown`、`fix_encoding`、`fix_math_formulas` 等上层逻辑,它们对所有平台通用。 +> ⚠️ **注意**:上述「四步接入新平台」的方式正在被第八章的**策略化重构**取代。新方案下新增平台只需写一个策略类并注册,不再需要修改 `fetch()` 主流程。 + --- ## 五、三种方案对比 @@ -468,4 +470,88 @@ POST /api/{scheme}/save → 确认保存(下载图片、替换 URL、入 2. **SegmentFault 绕过**:尝试更换出口 IP 或更高级的设备指纹模拟 3. **自动平台识别**:通过页面特征自动判断站点类型,减少人工选择采集方案的步骤 4. **智能摘要生成**:接入大模型 API,对采集内容自动生成摘要和标签 -5. **更多平台适配**:根据使用需求,逐步添加开源中国(oschina)、简书(jianshu)等平台 +5. **更多平台适配**:根据使用需求,逐步添加开源中国(oschina)、简书(jianshu)等平台(见第八章策略化方案,新增平台只需注册一个策略类) + +--- + +## 八、站点提取策略化重构(v2,进行中) + +> 本文档是既有采集方案的**增量演进**,目标是把散落在三个文件里的站点特判逻辑收敛为**策略注册表**,同时保证线上行为零回归。现有逻辑全部保留,作为旧轨(Legacy)参与双轨调度。 + +### 8.1 背景与动机 + +现状盘点:站点特判逻辑分散在三个文件,且同一站点在不同入口**各写了一遍**: + +| 文件 | 站点特判位置 | 涉及的站点 | +|------|------------|-----------| +| `crawl.py` | `crawl_with_playwright()`(约 796-1451 行) | 微信、腾讯云、飞书、yiigle、掘金、SegmentFault、InfoQ、CSDN、博客园 + eeworld 强制 Playwright | +| `universal_crawl.py` | `universal_fetch()`(约 378-480 行) | InfoQ、掘金、CSDN、博客园、51CTO + `_needs_browser()` 直通列表 | +| `wechat_crawl.py` | 独立路由,专用采集器 | 仅微信(独立 preview/save 流程) | + +痛点: + +- 同一站点(如掘金)在 `crawl.py` 和 `universal_crawl.py` 各有一套选择器和标题清洗规则,改一处忘另一处 +- 新增站点要在标题提取、正文转换、图片过滤三个分支各改一次,容易漏 +- 选择器、过滤规则、跳转策略(先访问首页建 cookie 等)与主流程强耦合,无法单独测试 + +### 8.2 设计:SiteExtractor 抽象基类 + +新增文件 `backend/routers/crawl_strategies.py`(纯新增,不删不改现有逻辑),借鉴 crawl4ai 的 `ContentScrapingStrategy(ABC)` / `MarkdownGenerationStrategy(ABC)` 接口思想: + +```python +class SiteExtractor(ABC): + name: str # 站点名(日志/调试用) + @abstractmethod + def match(self, url: str) -> bool: ... # URL 是否命中该站点 + def needs_browser(self, url: str) -> bool: return False # 是否需要 Playwright + def extract_title(self, html: str, url: str) -> Optional[str]: return None + def extract_content_html(self, html: str, url: str) -> str: return html # 返回正文容器 HTML + def filter_images(self, images: list, url: str) -> list: return images # 过滤 UI 图标 + def to_markdown(self, content_html: str, url: str, images: list) -> str: return html_to_markdown(...) +``` + +每个策略类的内部逻辑 = **现有代码原样搬迁**(如 `JuejinExtractor` 直接搬 `_extract_juejin_title` / `_extract_juejin_content`),行为零改动。 + +### 8.3 策略注册表与双轨降级 + +`SITE_EXTRACTORS = [JuejinExtractor, CsdnExtractor, ...]` 为有序注册表,末尾挂 `GenericExtractor`(通用兜底)。 + +双轨调度的**优先级由配置决定**,两种顺序都支持: + +``` +STRATEGY_FIRST_DOMAINS = set() # 试点白名单,默认空 + +crawl 流程: + 提取 extractor = get_extractor_for_url(url) + 若 url 命中白名单 → 轨B优先(新策略)→ 质量不达标回退轨A(旧逻辑) + 否则 → 轨A优先(旧逻辑)→ 失败/过短时兜底轨B(新策略) +``` + +| 顺序 | 配置 | 适用场景 | +|------|------|---------| +| **旧→新**(默认) | `STRATEGY_FIRST_DOMAINS` 为空 | 线上行为零回归,新策略仅作兜底 | +| **新→旧**(试点) | 白名单加入该域名,如 `{"juejin.cn"}` | 新策略优先,质量不合格自动回退旧逻辑 | + +**质量门控**(fallback 判据,复用现有「内容过短」经验): + +- 策略抛异常 +- 标题为空或等于「未获取到标题」 +- Markdown 内容 < 300 字符 +- 图片数为 0(可选判据,默认关闭) + +### 8.4 落地步骤与状态 + +| 步骤 | 内容 | 状态 | +|------|------|------| +| 1 | 新建 `crawl_strategies.py`:抽象基类 + 注册表 + GenericExtractor(纯新增,不影响线上) | ✅ 完成 | +| 2 | 原样搬迁 Juejin / CSDN 两个策略,单测断言与旧逻辑输出一致 | ✅ 完成 | +| 3 | `universal_fetch()` 接入双轨调度:旧逻辑包成 `_legacy_extract()`,加 `STRATEGY_FIRST_DOMAINS` 开关,默认走旧轨 | ✅ 完成 | +| 4 | 试点验证(掘金/CSDN 加入白名单,对比新旧采集结果),确认后推广到 `crawl.py` Playwright 轨、wechat 轨 | ⏳ 待用户确认 | + +### 8.5 wechat_crawl.py 是否需要更新? + +**不需要**(本阶段)。原因: + +1. `wechat_crawl.py` 是独立路由 `/api/wechat-crawl/`,有自己的 preview/save 两阶段流程,是「微信专用采集器」而非「散落的站点特判逻辑」,不存在重复规则问题 +2. 微信的提取逻辑会以 `WechatExtractor` 策略类形式注册到 `crawl_strategies.py`,供通用入口(`universal_fetch` / `crawl_article`)遇到微信 URL 时兜底复用——这是**新增复用**,不改动 wechat_crawl.py 本体 +3. 若未来想让 wechat_crawl.py 内部也复用策略类,属于可选优化(消除与 crawl.py 微信段的重复),不在本次重构范围