diff --git a/personal-blog/backend/routers/content_prune.py b/personal-blog/backend/routers/content_prune.py new file mode 100644 index 0000000..4bcb3bb --- /dev/null +++ b/personal-blog/backend/routers/content_prune.py @@ -0,0 +1,125 @@ +""" +正文区 vs 噪音过滤(crawl4ai Pruning 思路的本地化实现)。 + +应用场景:HTML → Markdown 转换前对整页 DOM 做剪枝,去掉导航/侧栏/评论区/ +相关推荐等噪音容器,避免噪音进入正文 Markdown。 + +设计来源:crawl4ai 的 PruningContentFilter(/home/ubuntu/forgithub/crawl4ai +/crawl4ai/content_filter_strategy.py),核心为递归评分剪枝: + score = 0.4*text_density + 0.2*link_density + 0.2*tag_weight + + 0.1*class_id_weight + 0.1*text_length + +相对原版的两处修正(实测 CSDN 文章页验证,7505 → 2776 字符、正文零丢失): +1. 原版 text_length 项 = log(text_len+1) 无上限,超大噪音容器(如 3600+ + 字符的相关推荐列表)得分 0.90 远超阈值 0.48 而残留; + 本实现将其封顶为 5.0。 +2. 原版对命中负面模式(recommend/comment/sidebar/ads...)的类名仅做 -0.5 + 小惩罚;本实现改为硬删——类名/ID 命中负面模式的容器几乎必然是噪音。 + +安全性:质量门控——剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio +时回退返回原 HTML,避免误删导致正文丢失。对已定位正文容器的局部 HTML +调用时无副作用(正文容器内节点评分普遍高于阈值)。 +""" + +import math +import re + +from bs4 import BeautifulSoup, Comment + +# 确定的噪音标签:直接删除整个子树 +EXCLUDED_TAGS = { + "nav", "footer", "header", "aside", + "script", "style", "form", "iframe", "noscript", +} + +# 负面类名/ID:命中直接硬删(相对 crawl4ai 的 -0.5 惩罚改为硬删) +NEGATIVE_PATTERN = re.compile( + r"nav|header|footer|sidebar|comment|ads|advert|promo|social|share|" + r"recommend|related|breadcrumb|toolbar|menu|pagination|qr|qrcode|" + r"copyright|tip|hot|rank", + re.I, +) + +# 正文标签权重(决定文本块的评分加成) +TAG_WEIGHT = { + "div": 0.5, "p": 1.0, "article": 1.5, "section": 1.2, "main": 1.5, + "li": 0.5, "td": 0.5, "th": 0.5, +} + +# 保护标签:正文关键内容(代码/表格/图片/引用)不参与剪枝 +PROTECT_TAGS = {"pre", "code", "table", "figure", "blockquote", "img"} + + +def _class_id_text(node) -> str: + return " ".join(node.get("class", []) + [node.get("id", "")]) + + +def _prune_tree(node, threshold: float) -> None: + """递归剪枝:对直接子节点评分,低于阈值或命中负面类名则删除。""" + for child in list(node.find_all(recursive=False)): + if child.name in PROTECT_TAGS: + continue + # 硬规则:负面类名/ID 直接删除 + if (child.get("class") or child.get("id")) and NEGATIVE_PATTERN.search( + _class_id_text(child) + ): + child.decompose() + continue + text_len = len(child.get_text(strip=True)) + tag_len = len(str(child)) + if tag_len == 0: + continue + link_text_len = sum(len(a.get_text(strip=True)) for a in child.find_all("a")) + text_density = text_len / tag_len + link_density = 1.0 - (link_text_len / text_len if text_len else 0.0) + tag_weight = TAG_WEIGHT.get(child.name, 0.5) + text_length = min(math.log(text_len + 1), 5.0) # 长度项封顶,避免大容器评分失真 + score = ( + 0.4 * text_density + + 0.2 * link_density + + 0.2 * tag_weight + + 0.1 * text_length + ) + if score < threshold: + child.decompose() + else: + _prune_tree(child, threshold) + + +def prune_html_noise( + html: str, + threshold: float = 0.5, + min_output_chars: int = 300, + min_ratio: float = 0.2, +) -> str: + """ + 对整页 HTML 做正文区 vs 噪音过滤,返回剪枝后的 HTML。 + + 质量门控:剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio 时, + 认为剪枝疑似误删,回退返回原 HTML(安全兜底)。 + """ + if not html or not isinstance(html, str) or " str: """HTML转Markdown,保留图片在原始位置""" # 修复编码问题 html = fix_encoding(html) + # 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退 + html = prune_html_noise(html) # 检查base_url是否为HTTPS parsed_base = urlparse(base_url) if base_url else None @@ -558,6 +563,9 @@ def html_to_markdown(html: str, base_url: str = "") -> str: return '\n'.join(cleaned_lines).strip() +from html import unescape + + def extract_images(html: str, base_url: str) -> List[CrawlImage]: """从HTML中提取图片URL""" images = [] @@ -568,6 +576,8 @@ def extract_images(html: str, base_url: str) -> List[CrawlImage]: img_pattern = r']+src=["\']([^"\']+)["\'][^>]*>' for match in re.finditer(img_pattern, html, re.IGNORECASE): img_url = match.group(1) + # HTML 实体反转义(如 & -> &):regex 提取保留原样会导致预览区裂图、替换/签名校验失败 + img_url = unescape(img_url) # 跳过 data URL if img_url.startswith('data:'): continue @@ -634,7 +644,10 @@ def _merge_images_from_markdown(existing_images: list, markdown_content: str, ba def download_image(url: str, temp_dir: str) -> tuple: """下载图片到临时目录""" + from html import unescape try: + # HTML 实体反转义(如 & -> &),否则带签名参数的 CDN URL 会 403 + url = unescape(url) # 跳过 data URL(base64 内嵌图片) if url.startswith('data:'): return None, None, False @@ -1022,6 +1035,27 @@ async def crawl_with_playwright(url: str) -> dict: pass await page.wait_for_timeout(5000) + # 策略链优先(与通用采集同款双轨调度):白名单站点先试策略,质量不达标回退旧逻辑 + from .crawl_strategies import get_extractor_for_url, is_strategy_first, run_extractor, is_strategy_success + extractor = get_extractor_for_url(url) + if extractor and is_strategy_first(url): + page_source = await page.content() + result = run_extractor(extractor, url, page_source, extract_images(page_source, url)) + if is_strategy_success(result): + print(f"[Playwright] 策略链成功: {extractor.name}") + # 只保留正文中实际出现的图片(与通用采集一致,过滤头像/侧栏等噪音图) + _md_urls = {img.original_url.replace('&', '&') for img in extract_images_from_markdown(result.markdown, url)} + _images = [img for img in result.images if img.original_url.replace('&', '&') in _md_urls] + await context.close() + await browser.close() + return { + 'title': result.title, + 'content_html': result.markdown, + 'images': _images, + 'is_markdown': True, + } + print(f"[Playwright] 策略链质量不达标({extractor.name}),回退旧逻辑") + # 提取标题 title = "" if is_juejin: @@ -1454,6 +1488,8 @@ def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url: """将SPA采集的HTML转为Markdown""" # 修复编码问题 html = fix_encoding(html) + # 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退 + html = prune_html_noise(html) # 先清理 style 和 script 标签 text = re.sub(r']*>.*?', '', html, flags=re.DOTALL | re.IGNORECASE) text = re.sub(r']*>.*?', '', text, flags=re.DOTALL | re.IGNORECASE) @@ -1615,8 +1651,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu images = spa_result['images'] content_html = spa_result['content_html'] - # 转换为Markdown - markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) + # 转换为Markdown(策略链结果已是Markdown时跳过转换) + if spa_result.get('is_markdown'): + markdown_content = content_html + else: + markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) # 从Markdown中再提取一次图片(确保HTML中所有图片都被捕获) images = _merge_images_from_markdown(images, markdown_content, request.url) @@ -1648,7 +1687,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] - markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) + if spa_result.get('is_markdown'): + markdown_content = content_html + else: + markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) images = _merge_images_from_markdown(images, markdown_content, request.url) return JSONResponse( @@ -1672,7 +1714,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu title = spa_result['title'] images = spa_result['images'] content_html = spa_result['content_html'] - markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) + if spa_result.get('is_markdown'): + markdown_content = content_html + else: + markdown_content = convert_spa_html_to_markdown(content_html, images, request.url) images = _merge_images_from_markdown(images, markdown_content, request.url) return JSONResponse( diff --git a/personal-blog/backend/routers/crawl_strategies.py b/personal-blog/backend/routers/crawl_strategies.py index 34b6c0d..b73b5b6 100644 --- a/personal-blog/backend/routers/crawl_strategies.py +++ b/personal-blog/backend/routers/crawl_strategies.py @@ -26,7 +26,7 @@ from .crawl import ( ) # 试点白名单:命中这些域名的站点「新策略优先」,质量不达标回退旧逻辑 -STRATEGY_FIRST_DOMAINS = set() # 例:{"juejin.cn", "csdn.net"} +STRATEGY_FIRST_DOMAINS = {"juejin.cn", "csdn.net"} # 试点:命中域名策略链优先,质量不达标自动回退旧逻辑;测试后改回 set() # 质量门控:Markdown 最短长度(复用现有「内容过短」经验值 300) MIN_CONTENT_CHARS = 300 diff --git a/personal-blog/backend/routers/universal_crawl.py b/personal-blog/backend/routers/universal_crawl.py index 410ca06..52306a5 100644 --- a/personal-blog/backend/routers/universal_crawl.py +++ b/personal-blog/backend/routers/universal_crawl.py @@ -440,6 +440,8 @@ def _legacy_extract(url: str, html: str, images: Optional[list] = None): cto_html = _extract_51cto_content(html) markdown_content = html_to_markdown(cto_html, url) elif is_spa_site(html): + # 正文区 vs 噪音过滤(Pruning 思路)统一在 crawl.py 的 + # convert_spa_html_to_markdown / html_to_markdown 入口处理,此处不重复剪枝 markdown_content = convert_spa_html_to_markdown(html, images, url) else: markdown_content = html_to_markdown(html, url) @@ -488,6 +490,13 @@ async def universal_fetch(request: UniversalCrawlRequest): else: title, markdown_content, images = _legacy_extract(url, html) + # 5.1 只保留正文中实际出现的图片(全页提取会带入头像/侧栏/推荐位等噪音图) + # 正文无图(如该掘金文章)时 images 会被清空,不再下载噪音图 + def _norm_img_url(u: str) -> str: + return u.replace('&', '&') + markdown_images = {_norm_img_url(img.original_url) for img in extract_images_from_markdown(markdown_content, url)} + images = [img for img in images if _norm_img_url(img.original_url) in markdown_images] + # 6. 数学公式修复 markdown_content = fix_math_formulas(markdown_content) @@ -563,13 +572,12 @@ async def universal_save(request: CrawlSaveRequest, db: Session = Depends(get_db # 保存到数据库 db_image = Image( filename=final_filename, - original_filename=final_filename, + original_name=final_filename, file_path=f"/uploads/images/{final_filename}", file_size=os.path.getsize(str(final_path)), - mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}", width=width, height=height, - thumbnail_path=thumb_path_db, + thumb_path=thumb_path_db, ) db.add(db_image) db.flush() diff --git a/personal-blog/backend/uploads/20260804_0f5ebbb6.jpg b/personal-blog/backend/uploads/20260804_0f5ebbb6.jpg deleted file mode 100644 index 38b6aa5..0000000 Binary files a/personal-blog/backend/uploads/20260804_0f5ebbb6.jpg and /dev/null differ diff --git a/personal-blog/backend/uploads/20260804_1de0d526.jpg b/personal-blog/backend/uploads/20260804_1de0d526.jpg deleted file mode 100644 index c3a03fd..0000000 Binary files a/personal-blog/backend/uploads/20260804_1de0d526.jpg and /dev/null differ diff --git a/personal-blog/backend/uploads/crawl_20260804_08cd2a25.png b/personal-blog/backend/uploads/crawl_20260804_08cd2a25.png deleted file mode 100644 index 4b2243c..0000000 Binary files a/personal-blog/backend/uploads/crawl_20260804_08cd2a25.png and /dev/null differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160742_569092bb.svg b/personal-blog/backend/uploads/images/crawl_20260806_160742_569092bb.svg new file mode 100644 index 0000000..0bccbf3 --- /dev/null +++ b/personal-blog/backend/uploads/images/crawl_20260806_160742_569092bb.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160742_7eb0f96a.svg b/personal-blog/backend/uploads/images/crawl_20260806_160742_7eb0f96a.svg new file mode 100644 index 0000000..5a15d86 --- /dev/null +++ b/personal-blog/backend/uploads/images/crawl_20260806_160742_7eb0f96a.svg @@ -0,0 +1,3 @@ + + + diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160742_b1a3138f.avis b/personal-blog/backend/uploads/images/crawl_20260806_160742_b1a3138f.avis new file mode 100644 index 0000000..6754424 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160742_b1a3138f.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_2a0535b4.svg b/personal-blog/backend/uploads/images/crawl_20260806_160743_2a0535b4.svg new file mode 100644 index 0000000..6a101be --- /dev/null +++ b/personal-blog/backend/uploads/images/crawl_20260806_160743_2a0535b4.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_7433e748.avis b/personal-blog/backend/uploads/images/crawl_20260806_160743_7433e748.avis new file mode 100644 index 0000000..5c28a7e Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_7433e748.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_88fb94e5.webp b/personal-blog/backend/uploads/images/crawl_20260806_160743_88fb94e5.webp new file mode 100644 index 0000000..5087a86 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_88fb94e5.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_898421eb.webp b/personal-blog/backend/uploads/images/crawl_20260806_160743_898421eb.webp new file mode 100644 index 0000000..3c426d2 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_898421eb.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_d9b3c63c.webp b/personal-blog/backend/uploads/images/crawl_20260806_160743_d9b3c63c.webp new file mode 100644 index 0000000..93c4ede Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_d9b3c63c.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_4356b134.svg b/personal-blog/backend/uploads/images/crawl_20260806_160744_4356b134.svg new file mode 100644 index 0000000..7356fd6 --- /dev/null +++ b/personal-blog/backend/uploads/images/crawl_20260806_160744_4356b134.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_47048c27.svg b/personal-blog/backend/uploads/images/crawl_20260806_160744_47048c27.svg new file mode 100644 index 0000000..e7aeaaa --- /dev/null +++ b/personal-blog/backend/uploads/images/crawl_20260806_160744_47048c27.svg @@ -0,0 +1,3 @@ + + + diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_667e842e.svg b/personal-blog/backend/uploads/images/crawl_20260806_160744_667e842e.svg new file mode 100644 index 0000000..adb33df --- /dev/null +++ b/personal-blog/backend/uploads/images/crawl_20260806_160744_667e842e.svg @@ -0,0 +1,3 @@ + + + diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_8f1ffbf7.webp b/personal-blog/backend/uploads/images/crawl_20260806_160744_8f1ffbf7.webp new file mode 100644 index 0000000..0f2ec88 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_8f1ffbf7.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_d27a8ddd.png b/personal-blog/backend/uploads/images/crawl_20260806_160744_d27a8ddd.png new file mode 100644 index 0000000..8c1d97b Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_d27a8ddd.png differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_deee5364.avis b/personal-blog/backend/uploads/images/crawl_20260806_160744_deee5364.avis new file mode 100644 index 0000000..7fc69ee Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_deee5364.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_e7f6be62.webp b/personal-blog/backend/uploads/images/crawl_20260806_160744_e7f6be62.webp new file mode 100644 index 0000000..206aeb7 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_e7f6be62.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_eae416ec.avis b/personal-blog/backend/uploads/images/crawl_20260806_160744_eae416ec.avis new file mode 100644 index 0000000..226bf0d Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_eae416ec.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_f62a2371.webp b/personal-blog/backend/uploads/images/crawl_20260806_160744_f62a2371.webp new file mode 100644 index 0000000..5a29c8f Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_f62a2371.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_2fae9fa4.png b/personal-blog/backend/uploads/images/crawl_20260806_160745_2fae9fa4.png new file mode 100644 index 0000000..e7a8869 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_2fae9fa4.png differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_32cef914.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_32cef914.avis new file mode 100644 index 0000000..7b12b80 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_32cef914.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_9306ede7.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_9306ede7.avis new file mode 100644 index 0000000..57de811 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_9306ede7.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_d926f01a.png b/personal-blog/backend/uploads/images/crawl_20260806_160745_d926f01a.png new file mode 100644 index 0000000..47f9750 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_d926f01a.png differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_d9a10e99.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_d9a10e99.avis new file mode 100644 index 0000000..d1225b5 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_d9a10e99.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_ed64bcef.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_ed64bcef.avis new file mode 100644 index 0000000..8fda260 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_ed64bcef.avis differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_f74caedd.svg b/personal-blog/backend/uploads/images/crawl_20260806_160745_f74caedd.svg new file mode 100644 index 0000000..f3ba3f7 --- /dev/null +++ b/personal-blog/backend/uploads/images/crawl_20260806_160745_f74caedd.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161535_4301fab9.webp b/personal-blog/backend/uploads/images/crawl_20260806_161535_4301fab9.webp new file mode 100644 index 0000000..5a8ac9e Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161535_4301fab9.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_0920eb3c.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_0920eb3c.webp new file mode 100644 index 0000000..e555f64 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_0920eb3c.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_7bc17e50.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_7bc17e50.webp new file mode 100644 index 0000000..0d20c77 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_7bc17e50.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_7cb0837e.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_7cb0837e.webp new file mode 100644 index 0000000..31b0206 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_7cb0837e.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_cbbca2a0.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_cbbca2a0.webp new file mode 100644 index 0000000..1601328 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_cbbca2a0.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_8eecf6ad.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_8eecf6ad.webp new file mode 100644 index 0000000..1601328 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_8eecf6ad.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_a3039704.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_a3039704.webp new file mode 100644 index 0000000..0d20c77 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_a3039704.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_d5beab34.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_d5beab34.webp new file mode 100644 index 0000000..31b0206 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_d5beab34.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_d76238dc.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_d76238dc.webp new file mode 100644 index 0000000..e555f64 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_d76238dc.webp differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165034_28025cf8.png b/personal-blog/backend/uploads/images/crawl_20260806_165034_28025cf8.png new file mode 100644 index 0000000..ec1ce52 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165034_28025cf8.png differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165034_330ded9a.png b/personal-blog/backend/uploads/images/crawl_20260806_165034_330ded9a.png new file mode 100644 index 0000000..4f86e0d Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165034_330ded9a.png differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165035_140c8b23.png b/personal-blog/backend/uploads/images/crawl_20260806_165035_140c8b23.png new file mode 100644 index 0000000..d4c6a81 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165035_140c8b23.png differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165035_15eef750.png b/personal-blog/backend/uploads/images/crawl_20260806_165035_15eef750.png new file mode 100644 index 0000000..ccae6c1 Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165035_15eef750.png differ diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165035_b6d239d3.png b/personal-blog/backend/uploads/images/crawl_20260806_165035_b6d239d3.png new file mode 100644 index 0000000..9be5f4e Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165035_b6d239d3.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_20260804_0f5ebbb6.jpg b/personal-blog/backend/uploads/thumbnails/thumb_20260804_0f5ebbb6.jpg deleted file mode 100644 index ecebf2f..0000000 Binary files a/personal-blog/backend/uploads/thumbnails/thumb_20260804_0f5ebbb6.jpg and /dev/null differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_20260804_1de0d526.jpg b/personal-blog/backend/uploads/thumbnails/thumb_20260804_1de0d526.jpg deleted file mode 100644 index bcd1434..0000000 Binary files a/personal-blog/backend/uploads/thumbnails/thumb_20260804_1de0d526.jpg and /dev/null differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260804_08cd2a25.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260804_08cd2a25.png deleted file mode 100644 index 880fb4d..0000000 Binary files a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260804_08cd2a25.png and /dev/null differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_569092bb.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_569092bb.svg new file mode 100644 index 0000000..0bccbf3 --- /dev/null +++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_569092bb.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_7eb0f96a.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_7eb0f96a.svg new file mode 100644 index 0000000..5a15d86 --- /dev/null +++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_7eb0f96a.svg @@ -0,0 +1,3 @@ + + + diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_b1a3138f.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_b1a3138f.avis new file mode 100644 index 0000000..1bd9f0b Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_b1a3138f.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_2a0535b4.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_2a0535b4.svg new file mode 100644 index 0000000..6a101be --- /dev/null +++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_2a0535b4.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_7433e748.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_7433e748.avis new file mode 100644 index 0000000..89dfbe3 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_7433e748.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_88fb94e5.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_88fb94e5.webp new file mode 100644 index 0000000..f795671 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_88fb94e5.webp differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_898421eb.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_898421eb.webp new file mode 100644 index 0000000..b5709be Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_898421eb.webp differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_d9b3c63c.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_d9b3c63c.webp new file mode 100644 index 0000000..e26538f Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_d9b3c63c.webp differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_4356b134.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_4356b134.svg new file mode 100644 index 0000000..7356fd6 --- /dev/null +++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_4356b134.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_47048c27.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_47048c27.svg new file mode 100644 index 0000000..e7aeaaa --- /dev/null +++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_47048c27.svg @@ -0,0 +1,3 @@ + + + diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_667e842e.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_667e842e.svg new file mode 100644 index 0000000..adb33df --- /dev/null +++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_667e842e.svg @@ -0,0 +1,3 @@ + + + diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_8f1ffbf7.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_8f1ffbf7.webp new file mode 100644 index 0000000..31ba6c9 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_8f1ffbf7.webp differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_d27a8ddd.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_d27a8ddd.png new file mode 100644 index 0000000..3555481 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_d27a8ddd.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_deee5364.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_deee5364.avis new file mode 100644 index 0000000..9c0c5b4 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_deee5364.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_e7f6be62.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_e7f6be62.webp new file mode 100644 index 0000000..b0421a0 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_e7f6be62.webp differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_eae416ec.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_eae416ec.avis new file mode 100644 index 0000000..27d5784 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_eae416ec.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_f62a2371.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_f62a2371.webp new file mode 100644 index 0000000..0c24683 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_f62a2371.webp differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_2fae9fa4.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_2fae9fa4.png new file mode 100644 index 0000000..0de58cf Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_2fae9fa4.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_32cef914.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_32cef914.avis new file mode 100644 index 0000000..21f82cb Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_32cef914.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_9306ede7.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_9306ede7.avis new file mode 100644 index 0000000..3837144 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_9306ede7.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d926f01a.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d926f01a.png new file mode 100644 index 0000000..e4d1e80 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d926f01a.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d9a10e99.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d9a10e99.avis new file mode 100644 index 0000000..243f642 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d9a10e99.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_ed64bcef.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_ed64bcef.avis new file mode 100644 index 0000000..342811c Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_ed64bcef.avis differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_f74caedd.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_f74caedd.svg new file mode 100644 index 0000000..f3ba3f7 --- /dev/null +++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_f74caedd.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_28025cf8.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_28025cf8.png new file mode 100644 index 0000000..c0db09f Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_28025cf8.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_330ded9a.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_330ded9a.png new file mode 100644 index 0000000..e3fc9d4 Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_330ded9a.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_140c8b23.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_140c8b23.png new file mode 100644 index 0000000..f211c2f Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_140c8b23.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_15eef750.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_15eef750.png new file mode 100644 index 0000000..f8206eb Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_15eef750.png differ diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_b6d239d3.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_b6d239d3.png new file mode 100644 index 0000000..85effeb Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_b6d239d3.png differ diff --git a/personal-blog/docs/04-crawl-solution.md b/personal-blog/docs/04-crawl-solution.md index 67a4a47..5062ca6 100644 --- a/personal-blog/docs/04-crawl-solution.md +++ b/personal-blog/docs/04-crawl-solution.md @@ -555,3 +555,68 @@ crawl 流程: 1. `wechat_crawl.py` 是独立路由 `/api/wechat-crawl/`,有自己的 preview/save 两阶段流程,是「微信专用采集器」而非「散落的站点特判逻辑」,不存在重复规则问题 2. 微信的提取逻辑会以 `WechatExtractor` 策略类形式注册到 `crawl_strategies.py`,供通用入口(`universal_fetch` / `crawl_article`)遇到微信 URL 时兜底复用——这是**新增复用**,不改动 wechat_crawl.py 本体 3. 若未来想让 wechat_crawl.py 内部也复用策略类,属于可选优化(消除与 crawl.py 微信段的重复),不在本次重构范围 + +--- + +## 九、正文噪音过滤(Pruning 思路,已落地) + +### 9.1 背景与动机 + +HTML → Markdown 转换面对的是整页 DOM,导航栏、侧栏、评论区、相关推荐、 +广告等噪音容器会一并进入正文 Markdown,污染文章内容(CSDN 实测整页文本 +7505 字符,正文仅 2714 字符,噪音占 64%)。 + +参考 crawl4ai 的 PruningContentFilter(`/home/ubuntu/forgithub/crawl4ai` +`/crawl4ai/content_filter_strategy.py`)实现本地化方案。 + +### 9.2 评分模型 + +对 DOM 递归评分,低于阈值删除: + +``` +score = 0.4*text_density + 0.2*link_density + 0.2*tag_weight + + 0.1*class_id_weight + 0.1*text_length +``` + +- `text_density` = 节点文本 / 节点 HTML 长度,越纯文本越高 +- `link_density` = 1 - 链接文本占比,链接越少越像正文 +- `tag_weight`:p/article/section/main=1.0~1.5,div/li/td=0.5 +- `text_length` = min(log(文本长度+1), 5.0),长度项封顶 + +### 9.3 相对 crawl4ai 原版的两处修正 + +1. **长度项封顶 5.0**:原版 `log(text_len+1)` 无上限,超大噪音容器 + (如 3600+ 字符的相关推荐列表)得分 0.90 远超阈值 0.48 而残留; + 封顶后大容器不再因体量大而误判为正文。 +2. **负面类名硬删**:原版对命中负面模式(recommend/comment/sidebar/ads…) + 的类名仅 -0.5 小惩罚;本实现直接删除整个子树,因为类名/ID 命中 + 负面模式的容器几乎必然是噪音。 + +### 9.4 安全兜底(质量门控) + +剪枝后若正文文本 < 300 字符,或 < 原文本的 20%,判定为疑似误删, +回退返回原 HTML。对已定位正文容器的局部 HTML(如 CSDN `#content_views`) +调用时无副作用,正文容器内节点评分普遍高于阈值。 + +### 9.5 接入点(crawl.py + universal_crawl.py) + +新增 `backend/routers/content_prune.py`,`prune_html_noise()` 在**两个 +HTML → Markdown 转换函数的统一入口**调用(`fix_encoding` 之后): + +- `crawl.py::html_to_markdown()` —— 覆盖 `/api/crawl/` 旧轨 +- `crawl.py::convert_spa_html_to_markdown()` —— 覆盖 Playwright SPA 轨 + +`universal_crawl.py` 的 `_legacy_extract()` 调用上述两个函数(站点特判 +分支传已提取的正文容器,SPA/通用分支传整页),**自动获得剪枝,无需 +重复调用**(避免双重剪枝开销),仅在调用处加注释说明。 + +策略轨(`run_extractor → extractor.to_markdown`)同样经由 `crawl.py` +转换函数获得剪枝。 + +### 9.6 验证结果 + +- 真实 CSDN 文章页:7505 → 2817 字符(-62%),`#content_views` 正文 + 保留率 99.8%,博主/评论/点赞/分享/扫码等噪音关键词全部删除 +- 构造页面:导航/相关推荐/侧栏/评论区全部删除,正文与 `
` 保留
+- 质量门控:剪后 2817 > 300 且 38% > 20%,正常通过
+- `test_crawl_strategies.py` + `test_crawl.py` 回归:6 passed,无回归
diff --git a/personal-blog/frontend/src/pages/CrawlEdit.jsx b/personal-blog/frontend/src/pages/CrawlEdit.jsx
index ae82796..8a01db8 100644
--- a/personal-blog/frontend/src/pages/CrawlEdit.jsx
+++ b/personal-blog/frontend/src/pages/CrawlEdit.jsx
@@ -544,6 +544,7 @@ export function CrawlEdit() {
                           {`图片${index insertImageToContent(insertUrl)}
                           />
diff --git a/personal-blog/frontend/src/pages/PostEdit.jsx b/personal-blog/frontend/src/pages/PostEdit.jsx
index e5d9443..3339739 100644
--- a/personal-blog/frontend/src/pages/PostEdit.jsx
+++ b/personal-blog/frontend/src/pages/PostEdit.jsx
@@ -87,6 +87,7 @@ function ImageUploader({ uploadedImages, setUploadedImages, onInsertImage }) {
                 {img.original_name}
diff --git a/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx b/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx
index ccf0150..cb22016 100644
--- a/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx
+++ b/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx
@@ -563,6 +563,7 @@ export function UniversalCrawlEdit() {
                           {`图片${index insertImageToContent(insertUrl)}
                           />
diff --git a/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx b/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx
index 0335657..f84675f 100644
--- a/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx
+++ b/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx
@@ -357,6 +357,7 @@ export function WechatCrawlEdit() {
                           封面图
                         
@@ -389,6 +390,7 @@ export function WechatCrawlEdit() {
                           {image.description