a
This commit is contained in:
@@ -17,6 +17,9 @@ try:
|
||||
except ImportError:
|
||||
BS4_AVAILABLE = False
|
||||
|
||||
# 正文区 vs 噪音过滤(crawl4ai Pruning 思路),两个 HTML→Markdown 转换函数的统一入口
|
||||
from .content_prune import prune_html_noise
|
||||
|
||||
# Playwright用于SPA采集
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
@@ -389,6 +392,8 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
|
||||
"""HTML转Markdown,保留图片在原始位置"""
|
||||
# 修复编码问题
|
||||
html = fix_encoding(html)
|
||||
# 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
|
||||
html = prune_html_noise(html)
|
||||
|
||||
# 检查base_url是否为HTTPS
|
||||
parsed_base = urlparse(base_url) if base_url else None
|
||||
@@ -558,6 +563,9 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
|
||||
return '\n'.join(cleaned_lines).strip()
|
||||
|
||||
|
||||
from html import unescape
|
||||
|
||||
|
||||
def extract_images(html: str, base_url: str) -> List[CrawlImage]:
|
||||
"""从HTML中提取图片URL"""
|
||||
images = []
|
||||
@@ -568,6 +576,8 @@ def extract_images(html: str, base_url: str) -> List[CrawlImage]:
|
||||
img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>'
|
||||
for match in re.finditer(img_pattern, html, re.IGNORECASE):
|
||||
img_url = match.group(1)
|
||||
# HTML 实体反转义(如 & -> &):regex 提取保留原样会导致预览区裂图、替换/签名校验失败
|
||||
img_url = unescape(img_url)
|
||||
# 跳过 data URL
|
||||
if img_url.startswith('data:'):
|
||||
continue
|
||||
@@ -634,7 +644,10 @@ def _merge_images_from_markdown(existing_images: list, markdown_content: str, ba
|
||||
|
||||
def download_image(url: str, temp_dir: str) -> tuple:
|
||||
"""下载图片到临时目录"""
|
||||
from html import unescape
|
||||
try:
|
||||
# HTML 实体反转义(如 & -> &),否则带签名参数的 CDN URL 会 403
|
||||
url = unescape(url)
|
||||
# 跳过 data URL(base64 内嵌图片)
|
||||
if url.startswith('data:'):
|
||||
return None, None, False
|
||||
@@ -1022,6 +1035,27 @@ async def crawl_with_playwright(url: str) -> dict:
|
||||
pass
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
# 策略链优先(与通用采集同款双轨调度):白名单站点先试策略,质量不达标回退旧逻辑
|
||||
from .crawl_strategies import get_extractor_for_url, is_strategy_first, run_extractor, is_strategy_success
|
||||
extractor = get_extractor_for_url(url)
|
||||
if extractor and is_strategy_first(url):
|
||||
page_source = await page.content()
|
||||
result = run_extractor(extractor, url, page_source, extract_images(page_source, url))
|
||||
if is_strategy_success(result):
|
||||
print(f"[Playwright] 策略链成功: {extractor.name}")
|
||||
# 只保留正文中实际出现的图片(与通用采集一致,过滤头像/侧栏等噪音图)
|
||||
_md_urls = {img.original_url.replace('&', '&') for img in extract_images_from_markdown(result.markdown, url)}
|
||||
_images = [img for img in result.images if img.original_url.replace('&', '&') in _md_urls]
|
||||
await context.close()
|
||||
await browser.close()
|
||||
return {
|
||||
'title': result.title,
|
||||
'content_html': result.markdown,
|
||||
'images': _images,
|
||||
'is_markdown': True,
|
||||
}
|
||||
print(f"[Playwright] 策略链质量不达标({extractor.name}),回退旧逻辑")
|
||||
|
||||
# 提取标题
|
||||
title = ""
|
||||
if is_juejin:
|
||||
@@ -1454,6 +1488,8 @@ def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url:
|
||||
"""将SPA采集的HTML转为Markdown"""
|
||||
# 修复编码问题
|
||||
html = fix_encoding(html)
|
||||
# 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
|
||||
html = prune_html_noise(html)
|
||||
# 先清理 style 和 script 标签
|
||||
text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||||
text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
|
||||
@@ -1615,8 +1651,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
|
||||
# 转换为Markdown
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
# 转换为Markdown(策略链结果已是Markdown时跳过转换)
|
||||
if spa_result.get('is_markdown'):
|
||||
markdown_content = content_html
|
||||
else:
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
# 从Markdown中再提取一次图片(确保HTML中所有图片都被捕获)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
@@ -1648,7 +1687,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
title = spa_result['title']
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
if spa_result.get('is_markdown'):
|
||||
markdown_content = content_html
|
||||
else:
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
return JSONResponse(
|
||||
@@ -1672,7 +1714,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
title = spa_result['title']
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
if spa_result.get('is_markdown'):
|
||||
markdown_content = content_html
|
||||
else:
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
return JSONResponse(
|
||||
|
||||
Reference in New Issue
Block a user