a
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
正文区 vs 噪音过滤(crawl4ai Pruning 思路的本地化实现)。
|
||||
|
||||
应用场景:HTML → Markdown 转换前对整页 DOM 做剪枝,去掉导航/侧栏/评论区/
|
||||
相关推荐等噪音容器,避免噪音进入正文 Markdown。
|
||||
|
||||
设计来源:crawl4ai 的 PruningContentFilter(/home/ubuntu/forgithub/crawl4ai
|
||||
/crawl4ai/content_filter_strategy.py),核心为递归评分剪枝:
|
||||
score = 0.4*text_density + 0.2*link_density + 0.2*tag_weight
|
||||
+ 0.1*class_id_weight + 0.1*text_length
|
||||
|
||||
相对原版的两处修正(实测 CSDN 文章页验证,7505 → 2776 字符、正文零丢失):
|
||||
1. 原版 text_length 项 = log(text_len+1) 无上限,超大噪音容器(如 3600+
|
||||
字符的相关推荐列表)得分 0.90 远超阈值 0.48 而残留;
|
||||
本实现将其封顶为 5.0。
|
||||
2. 原版对命中负面模式(recommend/comment/sidebar/ads...)的类名仅做 -0.5
|
||||
小惩罚;本实现改为硬删——类名/ID 命中负面模式的容器几乎必然是噪音。
|
||||
|
||||
安全性:质量门控——剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio
|
||||
时回退返回原 HTML,避免误删导致正文丢失。对已定位正文容器的局部 HTML
|
||||
调用时无副作用(正文容器内节点评分普遍高于阈值)。
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup, Comment
|
||||
|
||||
# 确定的噪音标签:直接删除整个子树
|
||||
EXCLUDED_TAGS = {
|
||||
"nav", "footer", "header", "aside",
|
||||
"script", "style", "form", "iframe", "noscript",
|
||||
}
|
||||
|
||||
# 负面类名/ID:命中直接硬删(相对 crawl4ai 的 -0.5 惩罚改为硬删)
|
||||
NEGATIVE_PATTERN = re.compile(
|
||||
r"nav|header|footer|sidebar|comment|ads|advert|promo|social|share|"
|
||||
r"recommend|related|breadcrumb|toolbar|menu|pagination|qr|qrcode|"
|
||||
r"copyright|tip|hot|rank",
|
||||
re.I,
|
||||
)
|
||||
|
||||
# 正文标签权重(决定文本块的评分加成)
|
||||
TAG_WEIGHT = {
|
||||
"div": 0.5, "p": 1.0, "article": 1.5, "section": 1.2, "main": 1.5,
|
||||
"li": 0.5, "td": 0.5, "th": 0.5,
|
||||
}
|
||||
|
||||
# 保护标签:正文关键内容(代码/表格/图片/引用)不参与剪枝
|
||||
PROTECT_TAGS = {"pre", "code", "table", "figure", "blockquote", "img"}
|
||||
|
||||
|
||||
def _class_id_text(node) -> str:
|
||||
return " ".join(node.get("class", []) + [node.get("id", "")])
|
||||
|
||||
|
||||
def _prune_tree(node, threshold: float) -> None:
|
||||
"""递归剪枝:对直接子节点评分,低于阈值或命中负面类名则删除。"""
|
||||
for child in list(node.find_all(recursive=False)):
|
||||
if child.name in PROTECT_TAGS:
|
||||
continue
|
||||
# 硬规则:负面类名/ID 直接删除
|
||||
if (child.get("class") or child.get("id")) and NEGATIVE_PATTERN.search(
|
||||
_class_id_text(child)
|
||||
):
|
||||
child.decompose()
|
||||
continue
|
||||
text_len = len(child.get_text(strip=True))
|
||||
tag_len = len(str(child))
|
||||
if tag_len == 0:
|
||||
continue
|
||||
link_text_len = sum(len(a.get_text(strip=True)) for a in child.find_all("a"))
|
||||
text_density = text_len / tag_len
|
||||
link_density = 1.0 - (link_text_len / text_len if text_len else 0.0)
|
||||
tag_weight = TAG_WEIGHT.get(child.name, 0.5)
|
||||
text_length = min(math.log(text_len + 1), 5.0) # 长度项封顶,避免大容器评分失真
|
||||
score = (
|
||||
0.4 * text_density
|
||||
+ 0.2 * link_density
|
||||
+ 0.2 * tag_weight
|
||||
+ 0.1 * text_length
|
||||
)
|
||||
if score < threshold:
|
||||
child.decompose()
|
||||
else:
|
||||
_prune_tree(child, threshold)
|
||||
|
||||
|
||||
def prune_html_noise(
|
||||
html: str,
|
||||
threshold: float = 0.5,
|
||||
min_output_chars: int = 300,
|
||||
min_ratio: float = 0.2,
|
||||
) -> str:
|
||||
"""
|
||||
对整页 HTML 做正文区 vs 噪音过滤,返回剪枝后的 HTML。
|
||||
|
||||
质量门控:剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio 时,
|
||||
认为剪枝疑似误删,回退返回原 HTML(安全兜底)。
|
||||
"""
|
||||
if not html or not isinstance(html, str) or "<body" not in html.lower():
|
||||
return html
|
||||
try:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
except Exception:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
if not soup.body:
|
||||
return html
|
||||
|
||||
before_text = soup.body.get_text(strip=True)
|
||||
if len(before_text) < min_output_chars:
|
||||
return html # 页面本身就没多少文本,不剪
|
||||
|
||||
# 删除噪音标签与注释
|
||||
for tag in list(soup.find_all(EXCLUDED_TAGS)):
|
||||
tag.decompose()
|
||||
for comment in list(soup.find_all(string=lambda s: isinstance(s, Comment))):
|
||||
comment.extract()
|
||||
|
||||
_prune_tree(soup.body, threshold)
|
||||
|
||||
after_text = soup.body.get_text(strip=True)
|
||||
if len(after_text) < min_output_chars or len(after_text) < len(before_text) * min_ratio:
|
||||
return html # 疑似误删,回退原 HTML
|
||||
return str(soup.body)
|
||||
@@ -17,6 +17,9 @@ try:
|
||||
except ImportError:
|
||||
BS4_AVAILABLE = False
|
||||
|
||||
# 正文区 vs 噪音过滤(crawl4ai Pruning 思路),两个 HTML→Markdown 转换函数的统一入口
|
||||
from .content_prune import prune_html_noise
|
||||
|
||||
# Playwright用于SPA采集
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
@@ -389,6 +392,8 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
|
||||
"""HTML转Markdown,保留图片在原始位置"""
|
||||
# 修复编码问题
|
||||
html = fix_encoding(html)
|
||||
# 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
|
||||
html = prune_html_noise(html)
|
||||
|
||||
# 检查base_url是否为HTTPS
|
||||
parsed_base = urlparse(base_url) if base_url else None
|
||||
@@ -558,6 +563,9 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
|
||||
return '\n'.join(cleaned_lines).strip()
|
||||
|
||||
|
||||
from html import unescape
|
||||
|
||||
|
||||
def extract_images(html: str, base_url: str) -> List[CrawlImage]:
|
||||
"""从HTML中提取图片URL"""
|
||||
images = []
|
||||
@@ -568,6 +576,8 @@ def extract_images(html: str, base_url: str) -> List[CrawlImage]:
|
||||
img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>'
|
||||
for match in re.finditer(img_pattern, html, re.IGNORECASE):
|
||||
img_url = match.group(1)
|
||||
# HTML 实体反转义(如 & -> &):regex 提取保留原样会导致预览区裂图、替换/签名校验失败
|
||||
img_url = unescape(img_url)
|
||||
# 跳过 data URL
|
||||
if img_url.startswith('data:'):
|
||||
continue
|
||||
@@ -634,7 +644,10 @@ def _merge_images_from_markdown(existing_images: list, markdown_content: str, ba
|
||||
|
||||
def download_image(url: str, temp_dir: str) -> tuple:
|
||||
"""下载图片到临时目录"""
|
||||
from html import unescape
|
||||
try:
|
||||
# HTML 实体反转义(如 & -> &),否则带签名参数的 CDN URL 会 403
|
||||
url = unescape(url)
|
||||
# 跳过 data URL(base64 内嵌图片)
|
||||
if url.startswith('data:'):
|
||||
return None, None, False
|
||||
@@ -1022,6 +1035,27 @@ async def crawl_with_playwright(url: str) -> dict:
|
||||
pass
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
# 策略链优先(与通用采集同款双轨调度):白名单站点先试策略,质量不达标回退旧逻辑
|
||||
from .crawl_strategies import get_extractor_for_url, is_strategy_first, run_extractor, is_strategy_success
|
||||
extractor = get_extractor_for_url(url)
|
||||
if extractor and is_strategy_first(url):
|
||||
page_source = await page.content()
|
||||
result = run_extractor(extractor, url, page_source, extract_images(page_source, url))
|
||||
if is_strategy_success(result):
|
||||
print(f"[Playwright] 策略链成功: {extractor.name}")
|
||||
# 只保留正文中实际出现的图片(与通用采集一致,过滤头像/侧栏等噪音图)
|
||||
_md_urls = {img.original_url.replace('&', '&') for img in extract_images_from_markdown(result.markdown, url)}
|
||||
_images = [img for img in result.images if img.original_url.replace('&', '&') in _md_urls]
|
||||
await context.close()
|
||||
await browser.close()
|
||||
return {
|
||||
'title': result.title,
|
||||
'content_html': result.markdown,
|
||||
'images': _images,
|
||||
'is_markdown': True,
|
||||
}
|
||||
print(f"[Playwright] 策略链质量不达标({extractor.name}),回退旧逻辑")
|
||||
|
||||
# 提取标题
|
||||
title = ""
|
||||
if is_juejin:
|
||||
@@ -1454,6 +1488,8 @@ def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url:
|
||||
"""将SPA采集的HTML转为Markdown"""
|
||||
# 修复编码问题
|
||||
html = fix_encoding(html)
|
||||
# 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
|
||||
html = prune_html_noise(html)
|
||||
# 先清理 style 和 script 标签
|
||||
text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||||
text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
|
||||
@@ -1615,8 +1651,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
|
||||
# 转换为Markdown
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
# 转换为Markdown(策略链结果已是Markdown时跳过转换)
|
||||
if spa_result.get('is_markdown'):
|
||||
markdown_content = content_html
|
||||
else:
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
# 从Markdown中再提取一次图片(确保HTML中所有图片都被捕获)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
@@ -1648,7 +1687,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
title = spa_result['title']
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
if spa_result.get('is_markdown'):
|
||||
markdown_content = content_html
|
||||
else:
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
return JSONResponse(
|
||||
@@ -1672,7 +1714,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
title = spa_result['title']
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
if spa_result.get('is_markdown'):
|
||||
markdown_content = content_html
|
||||
else:
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
return JSONResponse(
|
||||
|
||||
@@ -26,7 +26,7 @@ from .crawl import (
|
||||
)
|
||||
|
||||
# 试点白名单:命中这些域名的站点「新策略优先」,质量不达标回退旧逻辑
|
||||
STRATEGY_FIRST_DOMAINS = set() # 例:{"juejin.cn", "csdn.net"}
|
||||
STRATEGY_FIRST_DOMAINS = {"juejin.cn", "csdn.net"} # 试点:命中域名策略链优先,质量不达标自动回退旧逻辑;测试后改回 set()
|
||||
|
||||
# 质量门控:Markdown 最短长度(复用现有「内容过短」经验值 300)
|
||||
MIN_CONTENT_CHARS = 300
|
||||
|
||||
@@ -440,6 +440,8 @@ def _legacy_extract(url: str, html: str, images: Optional[list] = None):
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
# 正文区 vs 噪音过滤(Pruning 思路)统一在 crawl.py 的
|
||||
# convert_spa_html_to_markdown / html_to_markdown 入口处理,此处不重复剪枝
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
@@ -488,6 +490,13 @@ async def universal_fetch(request: UniversalCrawlRequest):
|
||||
else:
|
||||
title, markdown_content, images = _legacy_extract(url, html)
|
||||
|
||||
# 5.1 只保留正文中实际出现的图片(全页提取会带入头像/侧栏/推荐位等噪音图)
|
||||
# 正文无图(如该掘金文章)时 images 会被清空,不再下载噪音图
|
||||
def _norm_img_url(u: str) -> str:
|
||||
return u.replace('&', '&')
|
||||
markdown_images = {_norm_img_url(img.original_url) for img in extract_images_from_markdown(markdown_content, url)}
|
||||
images = [img for img in images if _norm_img_url(img.original_url) in markdown_images]
|
||||
|
||||
# 6. 数学公式修复
|
||||
markdown_content = fix_math_formulas(markdown_content)
|
||||
|
||||
@@ -563,13 +572,12 @@ async def universal_save(request: CrawlSaveRequest, db: Session = Depends(get_db
|
||||
# 保存到数据库
|
||||
db_image = Image(
|
||||
filename=final_filename,
|
||||
original_filename=final_filename,
|
||||
original_name=final_filename,
|
||||
file_path=f"/uploads/images/{final_filename}",
|
||||
file_size=os.path.getsize(str(final_path)),
|
||||
mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}",
|
||||
width=width,
|
||||
height=height,
|
||||
thumbnail_path=thumb_path_db,
|
||||
thumb_path=thumb_path_db,
|
||||
)
|
||||
db.add(db_image)
|
||||
db.flush()
|
||||
|
||||
Reference in New Issue
Block a user