Auto backup \2026-\04-\20
This commit is contained in:
Ubuntu
@@ -23,6 +23,14 @@ from bs4 import BeautifulSoup
|
||||
# Scrapling 引擎
|
||||
from scrapling import Fetcher, StealthyFetcher
|
||||
|
||||
# 策略化重构（v2）：站点提取策略注册表 + 双轨调度
|
||||
from .crawl_strategies import (
|
||||
get_extractor_for_url,
|
||||
is_strategy_first,
|
||||
run_extractor,
|
||||
is_strategy_success,
|
||||
)
|
||||
|
||||
# 复用 crawl.py 的通用工具函数
|
||||
from .crawl import (
|
||||
fix_encoding,
|
||||
@@ -375,6 +383,70 @@ def _filter_51cto_images(images: list) -> list:
|
||||
return filtered
|
||||
|
||||
|
||||
def _legacy_extract(url: str, html: str, images: Optional[list] = None):
|
||||
"""旧逻辑（轨A）：现有站点特判提取，逐行保留原实现，不删除。
|
||||
|
||||
返回 (title, markdown_content, images)，与策略链 ExtractorResult 同构。
|
||||
images 为 None 时内部自行调用 extract_images；否则复用调用方已提取的结果。
|
||||
原第 6 步的 is_wechat 变量从未被使用，搬迁时省略。
|
||||
"""
|
||||
# 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||||
is_juejin = 'juejin.cn' in url
|
||||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||||
is_cnblogs = 'cnblogs.com' in url
|
||||
is_51cto = '51cto.com' in url
|
||||
if is_infoq:
|
||||
title = _extract_infoq_title(html)
|
||||
elif is_juejin:
|
||||
title = _extract_juejin_title(html)
|
||||
elif is_csdn:
|
||||
title = _extract_csdn_title(html)
|
||||
elif is_cnblogs:
|
||||
title = _extract_cnblogs_title(html)
|
||||
elif is_51cto:
|
||||
title = _extract_51cto_title(html)
|
||||
else:
|
||||
title = None
|
||||
if not title:
|
||||
title = extract_title_from_html(html)
|
||||
if not title:
|
||||
title = "未获取到标题"
|
||||
|
||||
# 6. 提取正文和图片
|
||||
if images is None:
|
||||
images = extract_images(html, url)
|
||||
if is_csdn:
|
||||
images = _filter_csdn_images(images)
|
||||
elif is_cnblogs:
|
||||
images = _filter_cnblogs_images(images)
|
||||
elif is_51cto:
|
||||
images = _filter_51cto_images(images)
|
||||
|
||||
# 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
if is_infoq:
|
||||
infoq_html = _extract_infoq_content(html)
|
||||
markdown_content = html_to_markdown(infoq_html, url)
|
||||
elif is_juejin:
|
||||
juejin_html = _extract_juejin_content(html)
|
||||
markdown_content = html_to_markdown(juejin_html, url)
|
||||
elif is_csdn:
|
||||
csdn_html = _extract_csdn_content(html)
|
||||
markdown_content = html_to_markdown(csdn_html, url)
|
||||
elif is_cnblogs:
|
||||
cnblogs_html = _extract_cnblogs_content(html)
|
||||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||||
elif is_51cto:
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
|
||||
return title, markdown_content, images
|
||||
|
||||
|
||||
@router.post("/fetch", response_model=CrawlResponse)
|
||||
async def universal_fetch(request: UniversalCrawlRequest):
|
||||
"""通用采集：使用 Scrapling 引擎获取文章"""
|
||||
@@ -402,64 +474,24 @@ async def universal_fetch(request: UniversalCrawlRequest):
|
||||
# 4. 微信图片防盗链处理
|
||||
html = _bypass_wechat_images(html, url)
|
||||
|
||||
# 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||||
is_juejin = 'juejin.cn' in url
|
||||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||||
is_cnblogs = 'cnblogs.com' in url
|
||||
is_51cto = '51cto.com' in url
|
||||
if is_infoq:
|
||||
title = _extract_infoq_title(html)
|
||||
elif is_juejin:
|
||||
title = _extract_juejin_title(html)
|
||||
elif is_csdn:
|
||||
title = _extract_csdn_title(html)
|
||||
elif is_cnblogs:
|
||||
title = _extract_cnblogs_title(html)
|
||||
elif is_51cto:
|
||||
title = _extract_51cto_title(html)
|
||||
# 5. 提取标题和正文（双轨调度 v2：默认旧逻辑优先；白名单站点新策略优先，质量不达标回退旧逻辑）
|
||||
extractor = get_extractor_for_url(url)
|
||||
if extractor and is_strategy_first(url):
|
||||
images = extract_images(html, url)
|
||||
result = run_extractor(extractor, url, html, images)
|
||||
if is_strategy_success(result):
|
||||
title, markdown_content, images = result.title, result.markdown, result.images
|
||||
print(f"[Universal Crawl] 策略链成功: {extractor.name}")
|
||||
else:
|
||||
print(f"[Universal Crawl] 策略链质量不达标({extractor.name})，回退旧逻辑")
|
||||
title, markdown_content, images = _legacy_extract(url, html, images)
|
||||
else:
|
||||
title = None
|
||||
if not title:
|
||||
title = extract_title_from_html(html)
|
||||
if not title:
|
||||
title = "未获取到标题"
|
||||
title, markdown_content, images = _legacy_extract(url, html)
|
||||
|
||||
# 6. 提取正文和图片
|
||||
is_wechat = 'mp.weixin.qq.com' in url
|
||||
images = extract_images(html, url)
|
||||
if is_csdn:
|
||||
images = _filter_csdn_images(images)
|
||||
elif is_cnblogs:
|
||||
images = _filter_cnblogs_images(images)
|
||||
elif is_51cto:
|
||||
images = _filter_51cto_images(images)
|
||||
|
||||
# 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||||
if is_infoq:
|
||||
infoq_html = _extract_infoq_content(html)
|
||||
markdown_content = html_to_markdown(infoq_html, url)
|
||||
elif is_juejin:
|
||||
juejin_html = _extract_juejin_content(html)
|
||||
markdown_content = html_to_markdown(juejin_html, url)
|
||||
elif is_csdn:
|
||||
csdn_html = _extract_csdn_content(html)
|
||||
markdown_content = html_to_markdown(csdn_html, url)
|
||||
elif is_cnblogs:
|
||||
cnblogs_html = _extract_cnblogs_content(html)
|
||||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||||
elif is_51cto:
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
|
||||
# 8. 数学公式修复
|
||||
# 6. 数学公式修复
|
||||
markdown_content = fix_math_formulas(markdown_content)
|
||||
|
||||
# 9. 清理 Markdown 中的 HTML 标签
|
||||
# 7. 清理 Markdown 中的 HTML 标签
|
||||
markdown_content = re.sub(r'<[^>]+>', '', markdown_content)
|
||||
markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
|
||||
markdown_content = markdown_content.strip()
|
||||
|
||||
Reference in New Issue
Block a user