This commit is contained in:
Ubuntu
2026-08-06 16:51:39 +08:00
parent a04277e6eb
commit e20c1656b7
82 changed files with 354 additions and 8 deletions
+49 -4
View File
@@ -17,6 +17,9 @@ try:
except ImportError:
BS4_AVAILABLE = False
# 正文区 vs 噪音过滤(crawl4ai Pruning 思路),两个 HTML→Markdown 转换函数的统一入口
from .content_prune import prune_html_noise
# Playwright用于SPA采集
try:
from playwright.async_api import async_playwright
@@ -389,6 +392,8 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
"""HTML转Markdown,保留图片在原始位置"""
# 修复编码问题
html = fix_encoding(html)
# 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
html = prune_html_noise(html)
# 检查base_url是否为HTTPS
parsed_base = urlparse(base_url) if base_url else None
@@ -558,6 +563,9 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
return '\n'.join(cleaned_lines).strip()
from html import unescape
def extract_images(html: str, base_url: str) -> List[CrawlImage]:
"""从HTML中提取图片URL"""
images = []
@@ -568,6 +576,8 @@ def extract_images(html: str, base_url: str) -> List[CrawlImage]:
img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>'
for match in re.finditer(img_pattern, html, re.IGNORECASE):
img_url = match.group(1)
# HTML 实体反转义(如 &amp; -> &):regex 提取保留原样会导致预览区裂图、替换/签名校验失败
img_url = unescape(img_url)
# 跳过 data URL
if img_url.startswith('data:'):
continue
@@ -634,7 +644,10 @@ def _merge_images_from_markdown(existing_images: list, markdown_content: str, ba
def download_image(url: str, temp_dir: str) -> tuple:
"""下载图片到临时目录"""
from html import unescape
try:
# HTML 实体反转义(如 &amp; -> &),否则带签名参数的 CDN URL 会 403
url = unescape(url)
# 跳过 data URL(base64 内嵌图片)
if url.startswith('data:'):
return None, None, False
@@ -1022,6 +1035,27 @@ async def crawl_with_playwright(url: str) -> dict:
pass
await page.wait_for_timeout(5000)
# 策略链优先(与通用采集同款双轨调度):白名单站点先试策略,质量不达标回退旧逻辑
from .crawl_strategies import get_extractor_for_url, is_strategy_first, run_extractor, is_strategy_success
extractor = get_extractor_for_url(url)
if extractor and is_strategy_first(url):
page_source = await page.content()
result = run_extractor(extractor, url, page_source, extract_images(page_source, url))
if is_strategy_success(result):
print(f"[Playwright] 策略链成功: {extractor.name}")
# 只保留正文中实际出现的图片(与通用采集一致,过滤头像/侧栏等噪音图)
_md_urls = {img.original_url.replace('&amp;', '&') for img in extract_images_from_markdown(result.markdown, url)}
_images = [img for img in result.images if img.original_url.replace('&amp;', '&') in _md_urls]
await context.close()
await browser.close()
return {
'title': result.title,
'content_html': result.markdown,
'images': _images,
'is_markdown': True,
}
print(f"[Playwright] 策略链质量不达标({extractor.name}),回退旧逻辑")
# 提取标题
title = ""
if is_juejin:
@@ -1454,6 +1488,8 @@ def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url:
"""将SPA采集的HTML转为Markdown"""
# 修复编码问题
html = fix_encoding(html)
# 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
html = prune_html_noise(html)
# 先清理 style 和 script 标签
text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
@@ -1615,8 +1651,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
images = spa_result['images']
content_html = spa_result['content_html']
# 转换为Markdown
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
# 转换为Markdown(策略链结果已是Markdown时跳过转换)
if spa_result.get('is_markdown'):
markdown_content = content_html
else:
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
# 从Markdown中再提取一次图片(确保HTML中所有图片都被捕获)
images = _merge_images_from_markdown(images, markdown_content, request.url)
@@ -1648,7 +1687,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
if spa_result.get('is_markdown'):
markdown_content = content_html
else:
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
return JSONResponse(
@@ -1672,7 +1714,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
if spa_result.get('is_markdown'):
markdown_content = content_html
else:
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
return JSONResponse(