Auto backup \2026-\04-\20
This commit is contained in:
Ubuntu
@@ -0,0 +1,212 @@
|
||||
"""
|
||||
站点提取策略化重构（v2）
|
||||
========================
|
||||
将散落在 crawl.py / universal_crawl.py 中的站点特判逻辑收敛为「策略注册表」，
|
||||
接口设计借鉴 crawl4ai 的 ContentScrapingStrategy(ABC) 思想。
|
||||
|
||||
双轨调度：
|
||||
- 轨A（Legacy）：调用方现有的 if-else 站点特判，逐行保留，不删除
|
||||
- 轨B（Strategy）：本文件中的策略链，每个策略类内部逻辑 = 现有代码「原样搬迁」
|
||||
- 优先级由 STRATEGY_FIRST_DOMAINS 配置决定：
|
||||
- 默认空集：全部走旧逻辑（轨A），策略链仅作未来兜底
|
||||
- 白名单命中：策略链优先（轨B），质量不达标自动回退旧逻辑（轨A）
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .crawl import (
|
||||
extract_title_from_html,
|
||||
html_to_markdown,
|
||||
is_spa_site,
|
||||
convert_spa_html_to_markdown,
|
||||
)
|
||||
|
||||
# 试点白名单：命中这些域名的站点「新策略优先」，质量不达标回退旧逻辑
|
||||
STRATEGY_FIRST_DOMAINS = set() # 例：{"juejin.cn", "csdn.net"}
|
||||
|
||||
# 质量门控：Markdown 最短长度（复用现有「内容过短」经验值 300）
|
||||
MIN_CONTENT_CHARS = 300
|
||||
FALLBACK_TITLE = "未获取到标题"
|
||||
|
||||
|
||||
class SiteExtractor(ABC):
|
||||
"""站点提取策略基类。每个策略类内部逻辑 = 现有代码原样搬迁，行为零改动。"""
|
||||
|
||||
name: str = "generic"
|
||||
|
||||
@abstractmethod
|
||||
def match(self, url: str) -> bool:
|
||||
"""URL 是否命中该站点"""
|
||||
|
||||
def needs_browser(self, url: str) -> bool:
|
||||
"""是否需要浏览器渲染（Playwright）"""
|
||||
return False
|
||||
|
||||
def extract_title(self, html: str, url: str) -> Optional[str]:
|
||||
"""站点专用标题提取；返回 None 时由 run_extractor 回退通用提取"""
|
||||
return None
|
||||
|
||||
def extract_content_html(self, html: str, url: str) -> str:
|
||||
"""提取正文容器 HTML；未命中选择器时返回原 HTML"""
|
||||
return html
|
||||
|
||||
def filter_images(self, images: list, url: str) -> list:
|
||||
"""过滤站点 UI 图标等噪音图片"""
|
||||
return images
|
||||
|
||||
def to_markdown(self, content_html: str, url: str, images: list) -> str:
|
||||
"""将正文容器 HTML 转为 Markdown"""
|
||||
return html_to_markdown(content_html, url)
|
||||
|
||||
|
||||
class JuejinExtractor(SiteExtractor):
|
||||
"""掘金：requests 返回混淆代码必须浏览器渲染；正文 article.article"""
|
||||
name = "juejin"
|
||||
|
||||
def match(self, url: str) -> bool:
|
||||
return 'juejin.cn' in url
|
||||
|
||||
def needs_browser(self, url: str) -> bool:
|
||||
return True
|
||||
|
||||
def extract_title(self, html: str, url: str) -> Optional[str]:
|
||||
# 原样搬迁 universal_crawl._extract_juejin_title
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# 去掉 " - 掘金" 后缀
|
||||
title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
def extract_content_html(self, html: str, url: str) -> str:
|
||||
# 原样搬迁 universal_crawl._extract_juejin_content
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
# 优先使用 article.article
|
||||
container = soup.select_one('article.article')
|
||||
if not container:
|
||||
# fallback 到 .article-viewer
|
||||
container = soup.select_one('.article-viewer')
|
||||
if not container:
|
||||
# 再 fallback 到 #article-root
|
||||
container = soup.select_one('#article-root')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
|
||||
class CsdnExtractor(SiteExtractor):
|
||||
"""CSDN：正文 #content_views / .article_content；过滤 csdnimg UI 图标"""
|
||||
name = "csdn"
|
||||
|
||||
def match(self, url: str) -> bool:
|
||||
return 'csdn.net' in url or 'csdn.com' in url
|
||||
|
||||
def extract_title(self, html: str, url: str) -> Optional[str]:
|
||||
# 原样搬迁 universal_crawl._extract_csdn_title
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# 去掉 " - CSDN博客" / "-CSDN博客" 后缀
|
||||
title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
def extract_content_html(self, html: str, url: str) -> str:
|
||||
# 原样搬迁 universal_crawl._extract_csdn_content
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
# 优先使用 #content_views（CSDN 正文主容器）
|
||||
container = soup.select_one('#content_views')
|
||||
if not container:
|
||||
# fallback 到 .article_content
|
||||
container = soup.select_one('.article_content')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
def filter_images(self, images: list, url: str) -> list:
|
||||
# 原样搬迁 universal_crawl._filter_csdn_images
|
||||
filtered = []
|
||||
skip_patterns = [
|
||||
'newHeart2023',
|
||||
'tobarCollect',
|
||||
'tobarCollection',
|
||||
'copyright',
|
||||
'avatar',
|
||||
'blogv2/dist/pc/img',
|
||||
]
|
||||
for img in images:
|
||||
img_url = img.original_url.lower()
|
||||
if any(p in img_url for p in skip_patterns):
|
||||
continue
|
||||
filtered.append(img)
|
||||
return filtered
|
||||
|
||||
|
||||
class GenericExtractor(SiteExtractor):
|
||||
"""通用兜底：不做站点特判，复刻旧逻辑的通用分支（含 SPA 判断）"""
|
||||
name = "generic"
|
||||
|
||||
def match(self, url: str) -> bool:
|
||||
return True
|
||||
|
||||
def to_markdown(self, content_html: str, url: str, images: list) -> str:
|
||||
if is_spa_site(content_html):
|
||||
return convert_spa_html_to_markdown(content_html, images, url)
|
||||
return html_to_markdown(content_html, url)
|
||||
|
||||
|
||||
# 策略注册表：有序列表，先匹配先命中；GenericExtractor 永远兜底
|
||||
SITE_EXTRACTORS = [
|
||||
JuejinExtractor(),
|
||||
CsdnExtractor(),
|
||||
GenericExtractor(),
|
||||
]
|
||||
|
||||
|
||||
def get_extractor_for_url(url: str) -> Optional[SiteExtractor]:
|
||||
"""返回第一个匹配的策略；GenericExtractor 恒匹配（兜底）"""
|
||||
for extractor in SITE_EXTRACTORS:
|
||||
if extractor.match(url):
|
||||
return extractor
|
||||
return None
|
||||
|
||||
|
||||
def is_strategy_first(url: str) -> bool:
|
||||
"""url 是否命中试点白名单（新策略优先，失败回退旧逻辑）"""
|
||||
return any(domain in url for domain in STRATEGY_FIRST_DOMAINS)
|
||||
|
||||
|
||||
class ExtractorResult:
|
||||
"""策略链输出，与旧逻辑 _legacy_extract 返回 (title, markdown, images) 同构"""
|
||||
|
||||
__slots__ = ("title", "markdown", "images")
|
||||
|
||||
def __init__(self, title: str, markdown: str, images: list):
|
||||
self.title = title
|
||||
self.markdown = markdown
|
||||
self.images = images
|
||||
|
||||
|
||||
def run_extractor(extractor: SiteExtractor, url: str, html: str, images: list) -> ExtractorResult:
|
||||
"""执行策略链：专用标题 → 通用标题降级 → 图片过滤 → 正文容器 → Markdown"""
|
||||
title = extractor.extract_title(html, url) or extract_title_from_html(html) or FALLBACK_TITLE
|
||||
filtered_images = extractor.filter_images(images, url)
|
||||
content_html = extractor.extract_content_html(html, url)
|
||||
markdown = extractor.to_markdown(content_html, url, filtered_images)
|
||||
return ExtractorResult(title=title, markdown=markdown, images=filtered_images)
|
||||
|
||||
|
||||
def is_strategy_success(result: ExtractorResult, min_chars: int = MIN_CONTENT_CHARS) -> bool:
|
||||
"""质量门控：标题有效 + Markdown 长度达标（复用现有「内容过短」经验值）"""
|
||||
if not result.title or result.title == FALLBACK_TITLE:
|
||||
return False
|
||||
if len((result.markdown or "").strip()) < min_chars:
|
||||
return False
|
||||
return True
|
||||
@@ -23,6 +23,14 @@ from bs4 import BeautifulSoup
|
||||
# Scrapling 引擎
|
||||
from scrapling import Fetcher, StealthyFetcher
|
||||
|
||||
# 策略化重构（v2）：站点提取策略注册表 + 双轨调度
|
||||
from .crawl_strategies import (
|
||||
get_extractor_for_url,
|
||||
is_strategy_first,
|
||||
run_extractor,
|
||||
is_strategy_success,
|
||||
)
|
||||
|
||||
# 复用 crawl.py 的通用工具函数
|
||||
from .crawl import (
|
||||
fix_encoding,
|
||||
@@ -375,6 +383,70 @@ def _filter_51cto_images(images: list) -> list:
|
||||
return filtered
|
||||
|
||||
|
||||
def _legacy_extract(url: str, html: str, images: Optional[list] = None):
|
||||
"""旧逻辑（轨A）：现有站点特判提取，逐行保留原实现，不删除。
|
||||
|
||||
返回 (title, markdown_content, images)，与策略链 ExtractorResult 同构。
|
||||
images 为 None 时内部自行调用 extract_images；否则复用调用方已提取的结果。
|
||||
原第 6 步的 is_wechat 变量从未被使用，搬迁时省略。
|
||||
"""
|
||||
# 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||||
is_juejin = 'juejin.cn' in url
|
||||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||||
is_cnblogs = 'cnblogs.com' in url
|
||||
is_51cto = '51cto.com' in url
|
||||
if is_infoq:
|
||||
title = _extract_infoq_title(html)
|
||||
elif is_juejin:
|
||||
title = _extract_juejin_title(html)
|
||||
elif is_csdn:
|
||||
title = _extract_csdn_title(html)
|
||||
elif is_cnblogs:
|
||||
title = _extract_cnblogs_title(html)
|
||||
elif is_51cto:
|
||||
title = _extract_51cto_title(html)
|
||||
else:
|
||||
title = None
|
||||
if not title:
|
||||
title = extract_title_from_html(html)
|
||||
if not title:
|
||||
title = "未获取到标题"
|
||||
|
||||
# 6. 提取正文和图片
|
||||
if images is None:
|
||||
images = extract_images(html, url)
|
||||
if is_csdn:
|
||||
images = _filter_csdn_images(images)
|
||||
elif is_cnblogs:
|
||||
images = _filter_cnblogs_images(images)
|
||||
elif is_51cto:
|
||||
images = _filter_51cto_images(images)
|
||||
|
||||
# 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
if is_infoq:
|
||||
infoq_html = _extract_infoq_content(html)
|
||||
markdown_content = html_to_markdown(infoq_html, url)
|
||||
elif is_juejin:
|
||||
juejin_html = _extract_juejin_content(html)
|
||||
markdown_content = html_to_markdown(juejin_html, url)
|
||||
elif is_csdn:
|
||||
csdn_html = _extract_csdn_content(html)
|
||||
markdown_content = html_to_markdown(csdn_html, url)
|
||||
elif is_cnblogs:
|
||||
cnblogs_html = _extract_cnblogs_content(html)
|
||||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||||
elif is_51cto:
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
|
||||
return title, markdown_content, images
|
||||
|
||||
|
||||
@router.post("/fetch", response_model=CrawlResponse)
|
||||
async def universal_fetch(request: UniversalCrawlRequest):
|
||||
"""通用采集：使用 Scrapling 引擎获取文章"""
|
||||
@@ -402,64 +474,24 @@ async def universal_fetch(request: UniversalCrawlRequest):
|
||||
# 4. 微信图片防盗链处理
|
||||
html = _bypass_wechat_images(html, url)
|
||||
|
||||
# 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||||
is_juejin = 'juejin.cn' in url
|
||||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||||
is_cnblogs = 'cnblogs.com' in url
|
||||
is_51cto = '51cto.com' in url
|
||||
if is_infoq:
|
||||
title = _extract_infoq_title(html)
|
||||
elif is_juejin:
|
||||
title = _extract_juejin_title(html)
|
||||
elif is_csdn:
|
||||
title = _extract_csdn_title(html)
|
||||
elif is_cnblogs:
|
||||
title = _extract_cnblogs_title(html)
|
||||
elif is_51cto:
|
||||
title = _extract_51cto_title(html)
|
||||
# 5. 提取标题和正文（双轨调度 v2：默认旧逻辑优先；白名单站点新策略优先，质量不达标回退旧逻辑）
|
||||
extractor = get_extractor_for_url(url)
|
||||
if extractor and is_strategy_first(url):
|
||||
images = extract_images(html, url)
|
||||
result = run_extractor(extractor, url, html, images)
|
||||
if is_strategy_success(result):
|
||||
title, markdown_content, images = result.title, result.markdown, result.images
|
||||
print(f"[Universal Crawl] 策略链成功: {extractor.name}")
|
||||
else:
|
||||
print(f"[Universal Crawl] 策略链质量不达标({extractor.name})，回退旧逻辑")
|
||||
title, markdown_content, images = _legacy_extract(url, html, images)
|
||||
else:
|
||||
title = None
|
||||
if not title:
|
||||
title = extract_title_from_html(html)
|
||||
if not title:
|
||||
title = "未获取到标题"
|
||||
title, markdown_content, images = _legacy_extract(url, html)
|
||||
|
||||
# 6. 提取正文和图片
|
||||
is_wechat = 'mp.weixin.qq.com' in url
|
||||
images = extract_images(html, url)
|
||||
if is_csdn:
|
||||
images = _filter_csdn_images(images)
|
||||
elif is_cnblogs:
|
||||
images = _filter_cnblogs_images(images)
|
||||
elif is_51cto:
|
||||
images = _filter_51cto_images(images)
|
||||
|
||||
# 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
if is_infoq:
|
||||
infoq_html = _extract_infoq_content(html)
|
||||
markdown_content = html_to_markdown(infoq_html, url)
|
||||
elif is_juejin:
|
||||
juejin_html = _extract_juejin_content(html)
|
||||
markdown_content = html_to_markdown(juejin_html, url)
|
||||
elif is_csdn:
|
||||
csdn_html = _extract_csdn_content(html)
|
||||
markdown_content = html_to_markdown(csdn_html, url)
|
||||
elif is_cnblogs:
|
||||
cnblogs_html = _extract_cnblogs_content(html)
|
||||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||||
elif is_51cto:
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
|
||||
# 8. 数学公式修复
|
||||
# 6. 数学公式修复
|
||||
markdown_content = fix_math_formulas(markdown_content)
|
||||
|
||||
# 9. 清理 Markdown 中的 HTML 标签
|
||||
# 7. 清理 Markdown 中的 HTML 标签
|
||||
markdown_content = re.sub(r'<[^>]+>', '', markdown_content)
|
||||
markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
|
||||
markdown_content = markdown_content.strip()
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""站点提取策略化重构（v2）单元测试。
|
||||
|
||||
核心断言：策略链（轨B）与旧逻辑（轨A）对同一 HTML 的输出完全一致。
|
||||
运行：cd backend && python3 test_crawl_strategies.py
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from routers.crawl import CrawlImage, extract_images
|
||||
from routers.crawl_strategies import (
|
||||
JuejinExtractor,
|
||||
CsdnExtractor,
|
||||
GenericExtractor,
|
||||
SITE_EXTRACTORS,
|
||||
get_extractor_for_url,
|
||||
is_strategy_first,
|
||||
is_strategy_success,
|
||||
run_extractor,
|
||||
ExtractorResult,
|
||||
STRATEGY_FIRST_DOMAINS,
|
||||
FALLBACK_TITLE,
|
||||
)
|
||||
from routers.universal_crawl import (
|
||||
_extract_juejin_title,
|
||||
_extract_juejin_content,
|
||||
_extract_csdn_title,
|
||||
_extract_csdn_content,
|
||||
_filter_csdn_images,
|
||||
_legacy_extract,
|
||||
)
|
||||
|
||||
PASS = 0
|
||||
FAIL = 0
|
||||
|
||||
|
||||
def check(name, cond):
|
||||
global PASS, FAIL
|
||||
if cond:
|
||||
PASS += 1
|
||||
print(f" PASS {name}")
|
||||
else:
|
||||
FAIL += 1
|
||||
print(f" FAIL {name}")
|
||||
|
||||
|
||||
# 掘金 HTML：article.article 为正文，sidebar 为噪音
|
||||
JUEGIN_HTML = f"""<html><head><title>深入理解 React 并发特性 - 掘金</title></head><body>
|
||||
<article class="article"><h1>深入理解 React 并发特性</h1>
|
||||
<p>{'这是掘金正文内容，用于验证策略链与旧逻辑提取一致性。' * 12}</p>
|
||||
<img src="https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/abc.webp"></article>
|
||||
<div class="sidebar">侧栏内容不应被提取</div></body></html>"""
|
||||
|
||||
# CSDN HTML：content_views 为正文，含 UI 图标噪音图（avatar/copyright 全小写 pattern 能命中；
|
||||
# newHeart2023 等混合大小写 pattern 因 img_url.lower() 永远匹配不上，属既有 bug，保持原样）
|
||||
CSDN_HTML = f"""<html><head><title>Python 异步编程指南 - CSDN博客</title></head><body>
|
||||
<div id="content_views">
|
||||
<p>{'这是 CSDN 正文内容，用于验证策略链与旧逻辑提取一致性。' * 12}</p>
|
||||
<img src="https://img-blog.csdnimg.cn/avatar-1.png">
|
||||
<img src="https://img-blog.csdnimg.cn/copyright.png">
|
||||
<img src="https://img-blog.csdnimg.cn/real-article.png">
|
||||
</div>
|
||||
<div class="recommend">推荐文章列表不应被提取</div></body></html>"""
|
||||
|
||||
# 未知站点 HTML：走 GenericExtractor
|
||||
GENERIC_HTML = f"""<html><head><title>某某技术博客</title></head><body>
|
||||
<article><p>{'普通网站正文内容，用于验证通用兜底策略。' * 12}</p></article></body></html>"""
|
||||
|
||||
URL_JUEGIN = "https://juejin.cn/post/7420000000000000000"
|
||||
URL_CSDN = "https://blog.csdn.net/xxx/article/details/140000000"
|
||||
URL_GENERIC = "https://example.com/post/123"
|
||||
|
||||
|
||||
def test_registry():
|
||||
print("[1] 注册表匹配")
|
||||
check("juejin -> JuejinExtractor", isinstance(get_extractor_for_url(URL_JUEGIN), JuejinExtractor))
|
||||
check("csdn -> CsdnExtractor", isinstance(get_extractor_for_url(URL_CSDN), CsdnExtractor))
|
||||
check("unknown -> GenericExtractor", isinstance(get_extractor_for_url(URL_GENERIC), GenericExtractor))
|
||||
check("GenericExtractor 恒兜底", SITE_EXTRACTORS[-1].name == "generic")
|
||||
check("策略链默认关闭（白名单为空）", is_strategy_first(URL_JUEGIN) is False)
|
||||
check("白名单为空集", STRATEGY_FIRST_DOMAINS == set())
|
||||
|
||||
|
||||
def test_juejin_consistency():
|
||||
print("[2] 掘金策略与旧逻辑一致性")
|
||||
ext = JuejinExtractor()
|
||||
title_new = ext.extract_title(JUEGIN_HTML, URL_JUEGIN)
|
||||
title_old = _extract_juejin_title(JUEGIN_HTML)
|
||||
check("标题一致且去掉「- 掘金」后缀", title_new == title_old == "深入理解 React 并发特性")
|
||||
|
||||
content_new = ext.extract_content_html(JUEGIN_HTML, URL_JUEGIN)
|
||||
content_old = _extract_juejin_content(JUEGIN_HTML)
|
||||
check("正文容器一致", content_new == content_old)
|
||||
check("正文容器不含侧栏", "侧栏内容" not in content_new and "sidebar" not in content_new)
|
||||
check("needs_browser=True", ext.needs_browser(URL_JUEGIN) is True)
|
||||
|
||||
|
||||
def test_csdn_consistency():
|
||||
print("[3] CSDN 策略与旧逻辑一致性")
|
||||
ext = CsdnExtractor()
|
||||
title_new = ext.extract_title(CSDN_HTML, URL_CSDN)
|
||||
title_old = _extract_csdn_title(CSDN_HTML)
|
||||
check("标题一致且去掉「- CSDN博客」后缀", title_new == title_old == "Python 异步编程指南")
|
||||
|
||||
content_new = ext.extract_content_html(CSDN_HTML, URL_CSDN)
|
||||
content_old = _extract_csdn_content(CSDN_HTML)
|
||||
check("正文容器一致", content_new == content_old)
|
||||
check("正文容器不含推荐列表", "推荐文章列表" not in content_new)
|
||||
|
||||
images = extract_images(CSDN_HTML, URL_CSDN)
|
||||
check("图片提取到 3 张", len(images) == 3)
|
||||
filtered_new = ext.filter_images(images, URL_CSDN)
|
||||
filtered_old = _filter_csdn_images(images)
|
||||
check("图片过滤结果一致", [i.original_url for i in filtered_new] == [i.original_url for i in filtered_old])
|
||||
check("UI 图标被过滤（avatar/copyright）", len(filtered_new) == 1 and filtered_new[0].original_url.endswith("real-article.png"))
|
||||
|
||||
# 既有 bug 行为记录：newHeart2023 混合大小写 pattern 因 img_url.lower() 永不匹配，保持原样不修
|
||||
mixed_case = extract_images('<img src="https://img-blog.csdnimg.cn/newHeart2023.png">', URL_CSDN)
|
||||
check("历史行为：newHeart2023 不被过滤（既有 bug，保持原样）", len(ext.filter_images(mixed_case, URL_CSDN)) == 1)
|
||||
|
||||
|
||||
def test_run_extractor():
|
||||
print("[4] 策略链集成（run_extractor）")
|
||||
images = extract_images(JUEGIN_HTML, URL_JUEGIN)
|
||||
result = run_extractor(JuejinExtractor(), URL_JUEGIN, JUEGIN_HTML, images)
|
||||
check("标题正确", result.title == "深入理解 React 并发特性")
|
||||
check("Markdown 含正文", "掘金正文内容" in result.markdown)
|
||||
check("Markdown 不含侧栏", "侧栏内容" not in result.markdown)
|
||||
check("图片 1 张", len(result.images) == 1)
|
||||
|
||||
# 标题降级：专用提取失败时回退通用 <title>
|
||||
degraded_html = "<html><head><title>通用标题兜底</title></head><body>" + JUEGIN_HTML.split("<body>")[1]
|
||||
result2 = run_extractor(JuejinExtractor(), URL_JUEGIN, degraded_html, images)
|
||||
check("标题降级到通用提取", result2.title == "通用标题兜底")
|
||||
|
||||
|
||||
def test_quality_gate():
|
||||
print("[5] 质量门控 is_strategy_success")
|
||||
ok = ExtractorResult(title="好标题", markdown="x" * 500, images=[])
|
||||
check("正常结果通过", is_strategy_success(ok) is True)
|
||||
check("空标题失败", is_strategy_success(ExtractorResult(title="", markdown="x" * 500, images=[])) is False)
|
||||
check("兜底标题失败", is_strategy_success(ExtractorResult(title=FALLBACK_TITLE, markdown="x" * 500, images=[])) is False)
|
||||
check("内容过短失败", is_strategy_success(ExtractorResult(title="好标题", markdown="x" * 100, images=[])) is False)
|
||||
|
||||
|
||||
def test_legacy_vs_strategy_equivalence():
|
||||
print("[6] 新旧轨全链路等价（title/markdown/images）")
|
||||
for url, html, extractor in [
|
||||
(URL_JUEGIN, JUEGIN_HTML, JuejinExtractor()),
|
||||
(URL_CSDN, CSDN_HTML, CsdnExtractor()),
|
||||
(URL_GENERIC, GENERIC_HTML, GenericExtractor()),
|
||||
]:
|
||||
old_title, old_md, old_images = _legacy_extract(url, html)
|
||||
new = run_extractor(extractor, url, html, extract_images(html, url))
|
||||
check(f"{extractor.name}: 标题等价", old_title == new.title)
|
||||
check(f"{extractor.name}: Markdown 等价", old_md == new.markdown)
|
||||
check(f"{extractor.name}: 图片等价", [i.original_url for i in old_images] == [i.original_url for i in new.images])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_registry()
|
||||
test_juejin_consistency()
|
||||
test_csdn_consistency()
|
||||
test_run_extractor()
|
||||
test_quality_gate()
|
||||
test_legacy_vs_strategy_equivalence()
|
||||
print(f"\n结果: {PASS} 通过, {FAIL} 失败")
|
||||
sys.exit(1 if FAIL else 0)
|
||||
Reference in New Issue
Block a user