diff --git a/personal-blog/backend/routers/content_prune.py b/personal-blog/backend/routers/content_prune.py
new file mode 100644
index 0000000..4bcb3bb
--- /dev/null
+++ b/personal-blog/backend/routers/content_prune.py
@@ -0,0 +1,125 @@
+"""
+正文区 vs 噪音过滤(crawl4ai Pruning 思路的本地化实现)。
+
+应用场景:HTML → Markdown 转换前对整页 DOM 做剪枝,去掉导航/侧栏/评论区/
+相关推荐等噪音容器,避免噪音进入正文 Markdown。
+
+设计来源:crawl4ai 的 PruningContentFilter(/home/ubuntu/forgithub/crawl4ai
+/crawl4ai/content_filter_strategy.py),核心为递归评分剪枝:
+ score = 0.4*text_density + 0.2*link_density + 0.2*tag_weight
+ + 0.1*class_id_weight + 0.1*text_length
+
+相对原版的两处修正(实测 CSDN 文章页验证,7505 → 2776 字符、正文零丢失):
+1. 原版 text_length 项 = log(text_len+1) 无上限,超大噪音容器(如 3600+
+ 字符的相关推荐列表)得分 0.90 远超阈值 0.48 而残留;
+ 本实现将其封顶为 5.0。
+2. 原版对命中负面模式(recommend/comment/sidebar/ads...)的类名仅做 -0.5
+ 小惩罚;本实现改为硬删——类名/ID 命中负面模式的容器几乎必然是噪音。
+
+安全性:质量门控——剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio
+时回退返回原 HTML,避免误删导致正文丢失。对已定位正文容器的局部 HTML
+调用时无副作用(正文容器内节点评分普遍高于阈值)。
+"""
+
+import math
+import re
+
+from bs4 import BeautifulSoup, Comment
+
+# 确定的噪音标签:直接删除整个子树
+EXCLUDED_TAGS = {
+ "nav", "footer", "header", "aside",
+ "script", "style", "form", "iframe", "noscript",
+}
+
+# 负面类名/ID:命中直接硬删(相对 crawl4ai 的 -0.5 惩罚改为硬删)
+NEGATIVE_PATTERN = re.compile(
+ r"nav|header|footer|sidebar|comment|ads|advert|promo|social|share|"
+ r"recommend|related|breadcrumb|toolbar|menu|pagination|qr|qrcode|"
+ r"copyright|tip|hot|rank",
+ re.I,
+)
+
+# 正文标签权重(决定文本块的评分加成)
+TAG_WEIGHT = {
+ "div": 0.5, "p": 1.0, "article": 1.5, "section": 1.2, "main": 1.5,
+ "li": 0.5, "td": 0.5, "th": 0.5,
+}
+
+# 保护标签:正文关键内容(代码/表格/图片/引用)不参与剪枝
+PROTECT_TAGS = {"pre", "code", "table", "figure", "blockquote", "img"}
+
+
+def _class_id_text(node) -> str:
+ return " ".join(node.get("class", []) + [node.get("id", "")])
+
+
+def _prune_tree(node, threshold: float) -> None:
+ """递归剪枝:对直接子节点评分,低于阈值或命中负面类名则删除。"""
+ for child in list(node.find_all(recursive=False)):
+ if child.name in PROTECT_TAGS:
+ continue
+ # 硬规则:负面类名/ID 直接删除
+ if (child.get("class") or child.get("id")) and NEGATIVE_PATTERN.search(
+ _class_id_text(child)
+ ):
+ child.decompose()
+ continue
+ text_len = len(child.get_text(strip=True))
+ tag_len = len(str(child))
+ if tag_len == 0:
+ continue
+ link_text_len = sum(len(a.get_text(strip=True)) for a in child.find_all("a"))
+ text_density = text_len / tag_len
+ link_density = 1.0 - (link_text_len / text_len if text_len else 0.0)
+ tag_weight = TAG_WEIGHT.get(child.name, 0.5)
+ text_length = min(math.log(text_len + 1), 5.0) # 长度项封顶,避免大容器评分失真
+ score = (
+ 0.4 * text_density
+ + 0.2 * link_density
+ + 0.2 * tag_weight
+ + 0.1 * text_length
+ )
+ if score < threshold:
+ child.decompose()
+ else:
+ _prune_tree(child, threshold)
+
+
+def prune_html_noise(
+ html: str,
+ threshold: float = 0.5,
+ min_output_chars: int = 300,
+ min_ratio: float = 0.2,
+) -> str:
+ """
+ 对整页 HTML 做正文区 vs 噪音过滤,返回剪枝后的 HTML。
+
+ 质量门控:剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio 时,
+ 认为剪枝疑似误删,回退返回原 HTML(安全兜底)。
+ """
+ if not html or not isinstance(html, str) or "
str:
"""HTML转Markdown,保留图片在原始位置"""
# 修复编码问题
html = fix_encoding(html)
+ # 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
+ html = prune_html_noise(html)
# 检查base_url是否为HTTPS
parsed_base = urlparse(base_url) if base_url else None
@@ -558,6 +563,9 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
return '\n'.join(cleaned_lines).strip()
+from html import unescape
+
+
def extract_images(html: str, base_url: str) -> List[CrawlImage]:
"""从HTML中提取图片URL"""
images = []
@@ -568,6 +576,8 @@ def extract_images(html: str, base_url: str) -> List[CrawlImage]:
img_pattern = r'
]+src=["\']([^"\']+)["\'][^>]*>'
for match in re.finditer(img_pattern, html, re.IGNORECASE):
img_url = match.group(1)
+ # HTML 实体反转义(如 & -> &):regex 提取保留原样会导致预览区裂图、替换/签名校验失败
+ img_url = unescape(img_url)
# 跳过 data URL
if img_url.startswith('data:'):
continue
@@ -634,7 +644,10 @@ def _merge_images_from_markdown(existing_images: list, markdown_content: str, ba
def download_image(url: str, temp_dir: str) -> tuple:
"""下载图片到临时目录"""
+ from html import unescape
try:
+ # HTML 实体反转义(如 & -> &),否则带签名参数的 CDN URL 会 403
+ url = unescape(url)
# 跳过 data URL(base64 内嵌图片)
if url.startswith('data:'):
return None, None, False
@@ -1022,6 +1035,27 @@ async def crawl_with_playwright(url: str) -> dict:
pass
await page.wait_for_timeout(5000)
+ # 策略链优先(与通用采集同款双轨调度):白名单站点先试策略,质量不达标回退旧逻辑
+ from .crawl_strategies import get_extractor_for_url, is_strategy_first, run_extractor, is_strategy_success
+ extractor = get_extractor_for_url(url)
+ if extractor and is_strategy_first(url):
+ page_source = await page.content()
+ result = run_extractor(extractor, url, page_source, extract_images(page_source, url))
+ if is_strategy_success(result):
+ print(f"[Playwright] 策略链成功: {extractor.name}")
+ # 只保留正文中实际出现的图片(与通用采集一致,过滤头像/侧栏等噪音图)
+ _md_urls = {img.original_url.replace('&', '&') for img in extract_images_from_markdown(result.markdown, url)}
+ _images = [img for img in result.images if img.original_url.replace('&', '&') in _md_urls]
+ await context.close()
+ await browser.close()
+ return {
+ 'title': result.title,
+ 'content_html': result.markdown,
+ 'images': _images,
+ 'is_markdown': True,
+ }
+ print(f"[Playwright] 策略链质量不达标({extractor.name}),回退旧逻辑")
+
# 提取标题
title = ""
if is_juejin:
@@ -1454,6 +1488,8 @@ def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url:
"""将SPA采集的HTML转为Markdown"""
# 修复编码问题
html = fix_encoding(html)
+ # 正文区 vs 噪音过滤(Pruning 思路):剪掉导航/侧栏/评论区/相关推荐,质量门控不达标自动回退
+ html = prune_html_noise(html)
# 先清理 style 和 script 标签
text = re.sub(r'', '', html, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'', '', text, flags=re.DOTALL | re.IGNORECASE)
@@ -1615,8 +1651,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
images = spa_result['images']
content_html = spa_result['content_html']
- # 转换为Markdown
- markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+ # 转换为Markdown(策略链结果已是Markdown时跳过转换)
+ if spa_result.get('is_markdown'):
+ markdown_content = content_html
+ else:
+ markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
# 从Markdown中再提取一次图片(确保HTML中所有图片都被捕获)
images = _merge_images_from_markdown(images, markdown_content, request.url)
@@ -1648,7 +1687,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
- markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+ if spa_result.get('is_markdown'):
+ markdown_content = content_html
+ else:
+ markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
return JSONResponse(
@@ -1672,7 +1714,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
- markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+ if spa_result.get('is_markdown'):
+ markdown_content = content_html
+ else:
+ markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
return JSONResponse(
diff --git a/personal-blog/backend/routers/crawl_strategies.py b/personal-blog/backend/routers/crawl_strategies.py
index 34b6c0d..b73b5b6 100644
--- a/personal-blog/backend/routers/crawl_strategies.py
+++ b/personal-blog/backend/routers/crawl_strategies.py
@@ -26,7 +26,7 @@ from .crawl import (
)
# 试点白名单:命中这些域名的站点「新策略优先」,质量不达标回退旧逻辑
-STRATEGY_FIRST_DOMAINS = set() # 例:{"juejin.cn", "csdn.net"}
+STRATEGY_FIRST_DOMAINS = {"juejin.cn", "csdn.net"} # 试点:命中域名策略链优先,质量不达标自动回退旧逻辑;测试后改回 set()
# 质量门控:Markdown 最短长度(复用现有「内容过短」经验值 300)
MIN_CONTENT_CHARS = 300
diff --git a/personal-blog/backend/routers/universal_crawl.py b/personal-blog/backend/routers/universal_crawl.py
index 410ca06..52306a5 100644
--- a/personal-blog/backend/routers/universal_crawl.py
+++ b/personal-blog/backend/routers/universal_crawl.py
@@ -440,6 +440,8 @@ def _legacy_extract(url: str, html: str, images: Optional[list] = None):
cto_html = _extract_51cto_content(html)
markdown_content = html_to_markdown(cto_html, url)
elif is_spa_site(html):
+ # 正文区 vs 噪音过滤(Pruning 思路)统一在 crawl.py 的
+ # convert_spa_html_to_markdown / html_to_markdown 入口处理,此处不重复剪枝
markdown_content = convert_spa_html_to_markdown(html, images, url)
else:
markdown_content = html_to_markdown(html, url)
@@ -488,6 +490,13 @@ async def universal_fetch(request: UniversalCrawlRequest):
else:
title, markdown_content, images = _legacy_extract(url, html)
+ # 5.1 只保留正文中实际出现的图片(全页提取会带入头像/侧栏/推荐位等噪音图)
+ # 正文无图(如该掘金文章)时 images 会被清空,不再下载噪音图
+ def _norm_img_url(u: str) -> str:
+ return u.replace('&', '&')
+ markdown_images = {_norm_img_url(img.original_url) for img in extract_images_from_markdown(markdown_content, url)}
+ images = [img for img in images if _norm_img_url(img.original_url) in markdown_images]
+
# 6. 数学公式修复
markdown_content = fix_math_formulas(markdown_content)
@@ -563,13 +572,12 @@ async def universal_save(request: CrawlSaveRequest, db: Session = Depends(get_db
# 保存到数据库
db_image = Image(
filename=final_filename,
- original_filename=final_filename,
+ original_name=final_filename,
file_path=f"/uploads/images/{final_filename}",
file_size=os.path.getsize(str(final_path)),
- mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}",
width=width,
height=height,
- thumbnail_path=thumb_path_db,
+ thumb_path=thumb_path_db,
)
db.add(db_image)
db.flush()
diff --git a/personal-blog/backend/uploads/20260804_0f5ebbb6.jpg b/personal-blog/backend/uploads/20260804_0f5ebbb6.jpg
deleted file mode 100644
index 38b6aa5..0000000
Binary files a/personal-blog/backend/uploads/20260804_0f5ebbb6.jpg and /dev/null differ
diff --git a/personal-blog/backend/uploads/20260804_1de0d526.jpg b/personal-blog/backend/uploads/20260804_1de0d526.jpg
deleted file mode 100644
index c3a03fd..0000000
Binary files a/personal-blog/backend/uploads/20260804_1de0d526.jpg and /dev/null differ
diff --git a/personal-blog/backend/uploads/crawl_20260804_08cd2a25.png b/personal-blog/backend/uploads/crawl_20260804_08cd2a25.png
deleted file mode 100644
index 4b2243c..0000000
Binary files a/personal-blog/backend/uploads/crawl_20260804_08cd2a25.png and /dev/null differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160742_569092bb.svg b/personal-blog/backend/uploads/images/crawl_20260806_160742_569092bb.svg
new file mode 100644
index 0000000..0bccbf3
--- /dev/null
+++ b/personal-blog/backend/uploads/images/crawl_20260806_160742_569092bb.svg
@@ -0,0 +1,16 @@
+
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160742_7eb0f96a.svg b/personal-blog/backend/uploads/images/crawl_20260806_160742_7eb0f96a.svg
new file mode 100644
index 0000000..5a15d86
--- /dev/null
+++ b/personal-blog/backend/uploads/images/crawl_20260806_160742_7eb0f96a.svg
@@ -0,0 +1,3 @@
+
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160742_b1a3138f.avis b/personal-blog/backend/uploads/images/crawl_20260806_160742_b1a3138f.avis
new file mode 100644
index 0000000..6754424
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160742_b1a3138f.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_2a0535b4.svg b/personal-blog/backend/uploads/images/crawl_20260806_160743_2a0535b4.svg
new file mode 100644
index 0000000..6a101be
--- /dev/null
+++ b/personal-blog/backend/uploads/images/crawl_20260806_160743_2a0535b4.svg
@@ -0,0 +1,15 @@
+
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_7433e748.avis b/personal-blog/backend/uploads/images/crawl_20260806_160743_7433e748.avis
new file mode 100644
index 0000000..5c28a7e
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_7433e748.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_88fb94e5.webp b/personal-blog/backend/uploads/images/crawl_20260806_160743_88fb94e5.webp
new file mode 100644
index 0000000..5087a86
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_88fb94e5.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_898421eb.webp b/personal-blog/backend/uploads/images/crawl_20260806_160743_898421eb.webp
new file mode 100644
index 0000000..3c426d2
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_898421eb.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160743_d9b3c63c.webp b/personal-blog/backend/uploads/images/crawl_20260806_160743_d9b3c63c.webp
new file mode 100644
index 0000000..93c4ede
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160743_d9b3c63c.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_4356b134.svg b/personal-blog/backend/uploads/images/crawl_20260806_160744_4356b134.svg
new file mode 100644
index 0000000..7356fd6
--- /dev/null
+++ b/personal-blog/backend/uploads/images/crawl_20260806_160744_4356b134.svg
@@ -0,0 +1,4 @@
+
\ No newline at end of file
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_47048c27.svg b/personal-blog/backend/uploads/images/crawl_20260806_160744_47048c27.svg
new file mode 100644
index 0000000..e7aeaaa
--- /dev/null
+++ b/personal-blog/backend/uploads/images/crawl_20260806_160744_47048c27.svg
@@ -0,0 +1,3 @@
+
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_667e842e.svg b/personal-blog/backend/uploads/images/crawl_20260806_160744_667e842e.svg
new file mode 100644
index 0000000..adb33df
--- /dev/null
+++ b/personal-blog/backend/uploads/images/crawl_20260806_160744_667e842e.svg
@@ -0,0 +1,3 @@
+
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_8f1ffbf7.webp b/personal-blog/backend/uploads/images/crawl_20260806_160744_8f1ffbf7.webp
new file mode 100644
index 0000000..0f2ec88
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_8f1ffbf7.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_d27a8ddd.png b/personal-blog/backend/uploads/images/crawl_20260806_160744_d27a8ddd.png
new file mode 100644
index 0000000..8c1d97b
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_d27a8ddd.png differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_deee5364.avis b/personal-blog/backend/uploads/images/crawl_20260806_160744_deee5364.avis
new file mode 100644
index 0000000..7fc69ee
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_deee5364.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_e7f6be62.webp b/personal-blog/backend/uploads/images/crawl_20260806_160744_e7f6be62.webp
new file mode 100644
index 0000000..206aeb7
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_e7f6be62.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_eae416ec.avis b/personal-blog/backend/uploads/images/crawl_20260806_160744_eae416ec.avis
new file mode 100644
index 0000000..226bf0d
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_eae416ec.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160744_f62a2371.webp b/personal-blog/backend/uploads/images/crawl_20260806_160744_f62a2371.webp
new file mode 100644
index 0000000..5a29c8f
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160744_f62a2371.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_2fae9fa4.png b/personal-blog/backend/uploads/images/crawl_20260806_160745_2fae9fa4.png
new file mode 100644
index 0000000..e7a8869
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_2fae9fa4.png differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_32cef914.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_32cef914.avis
new file mode 100644
index 0000000..7b12b80
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_32cef914.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_9306ede7.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_9306ede7.avis
new file mode 100644
index 0000000..57de811
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_9306ede7.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_d926f01a.png b/personal-blog/backend/uploads/images/crawl_20260806_160745_d926f01a.png
new file mode 100644
index 0000000..47f9750
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_d926f01a.png differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_d9a10e99.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_d9a10e99.avis
new file mode 100644
index 0000000..d1225b5
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_d9a10e99.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_ed64bcef.avis b/personal-blog/backend/uploads/images/crawl_20260806_160745_ed64bcef.avis
new file mode 100644
index 0000000..8fda260
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_160745_ed64bcef.avis differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_160745_f74caedd.svg b/personal-blog/backend/uploads/images/crawl_20260806_160745_f74caedd.svg
new file mode 100644
index 0000000..f3ba3f7
--- /dev/null
+++ b/personal-blog/backend/uploads/images/crawl_20260806_160745_f74caedd.svg
@@ -0,0 +1,5 @@
+
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161535_4301fab9.webp b/personal-blog/backend/uploads/images/crawl_20260806_161535_4301fab9.webp
new file mode 100644
index 0000000..5a8ac9e
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161535_4301fab9.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_0920eb3c.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_0920eb3c.webp
new file mode 100644
index 0000000..e555f64
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_0920eb3c.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_7bc17e50.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_7bc17e50.webp
new file mode 100644
index 0000000..0d20c77
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_7bc17e50.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_7cb0837e.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_7cb0837e.webp
new file mode 100644
index 0000000..31b0206
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_7cb0837e.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_161657_cbbca2a0.webp b/personal-blog/backend/uploads/images/crawl_20260806_161657_cbbca2a0.webp
new file mode 100644
index 0000000..1601328
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_161657_cbbca2a0.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_8eecf6ad.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_8eecf6ad.webp
new file mode 100644
index 0000000..1601328
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_8eecf6ad.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_a3039704.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_a3039704.webp
new file mode 100644
index 0000000..0d20c77
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_a3039704.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_d5beab34.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_d5beab34.webp
new file mode 100644
index 0000000..31b0206
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_d5beab34.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_162150_d76238dc.webp b/personal-blog/backend/uploads/images/crawl_20260806_162150_d76238dc.webp
new file mode 100644
index 0000000..e555f64
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_162150_d76238dc.webp differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165034_28025cf8.png b/personal-blog/backend/uploads/images/crawl_20260806_165034_28025cf8.png
new file mode 100644
index 0000000..ec1ce52
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165034_28025cf8.png differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165034_330ded9a.png b/personal-blog/backend/uploads/images/crawl_20260806_165034_330ded9a.png
new file mode 100644
index 0000000..4f86e0d
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165034_330ded9a.png differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165035_140c8b23.png b/personal-blog/backend/uploads/images/crawl_20260806_165035_140c8b23.png
new file mode 100644
index 0000000..d4c6a81
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165035_140c8b23.png differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165035_15eef750.png b/personal-blog/backend/uploads/images/crawl_20260806_165035_15eef750.png
new file mode 100644
index 0000000..ccae6c1
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165035_15eef750.png differ
diff --git a/personal-blog/backend/uploads/images/crawl_20260806_165035_b6d239d3.png b/personal-blog/backend/uploads/images/crawl_20260806_165035_b6d239d3.png
new file mode 100644
index 0000000..9be5f4e
Binary files /dev/null and b/personal-blog/backend/uploads/images/crawl_20260806_165035_b6d239d3.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_20260804_0f5ebbb6.jpg b/personal-blog/backend/uploads/thumbnails/thumb_20260804_0f5ebbb6.jpg
deleted file mode 100644
index ecebf2f..0000000
Binary files a/personal-blog/backend/uploads/thumbnails/thumb_20260804_0f5ebbb6.jpg and /dev/null differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_20260804_1de0d526.jpg b/personal-blog/backend/uploads/thumbnails/thumb_20260804_1de0d526.jpg
deleted file mode 100644
index bcd1434..0000000
Binary files a/personal-blog/backend/uploads/thumbnails/thumb_20260804_1de0d526.jpg and /dev/null differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260804_08cd2a25.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260804_08cd2a25.png
deleted file mode 100644
index 880fb4d..0000000
Binary files a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260804_08cd2a25.png and /dev/null differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_569092bb.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_569092bb.svg
new file mode 100644
index 0000000..0bccbf3
--- /dev/null
+++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_569092bb.svg
@@ -0,0 +1,16 @@
+
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_7eb0f96a.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_7eb0f96a.svg
new file mode 100644
index 0000000..5a15d86
--- /dev/null
+++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_7eb0f96a.svg
@@ -0,0 +1,3 @@
+
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_b1a3138f.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_b1a3138f.avis
new file mode 100644
index 0000000..1bd9f0b
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160742_b1a3138f.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_2a0535b4.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_2a0535b4.svg
new file mode 100644
index 0000000..6a101be
--- /dev/null
+++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_2a0535b4.svg
@@ -0,0 +1,15 @@
+
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_7433e748.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_7433e748.avis
new file mode 100644
index 0000000..89dfbe3
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_7433e748.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_88fb94e5.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_88fb94e5.webp
new file mode 100644
index 0000000..f795671
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_88fb94e5.webp differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_898421eb.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_898421eb.webp
new file mode 100644
index 0000000..b5709be
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_898421eb.webp differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_d9b3c63c.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_d9b3c63c.webp
new file mode 100644
index 0000000..e26538f
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160743_d9b3c63c.webp differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_4356b134.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_4356b134.svg
new file mode 100644
index 0000000..7356fd6
--- /dev/null
+++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_4356b134.svg
@@ -0,0 +1,4 @@
+
\ No newline at end of file
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_47048c27.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_47048c27.svg
new file mode 100644
index 0000000..e7aeaaa
--- /dev/null
+++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_47048c27.svg
@@ -0,0 +1,3 @@
+
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_667e842e.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_667e842e.svg
new file mode 100644
index 0000000..adb33df
--- /dev/null
+++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_667e842e.svg
@@ -0,0 +1,3 @@
+
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_8f1ffbf7.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_8f1ffbf7.webp
new file mode 100644
index 0000000..31ba6c9
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_8f1ffbf7.webp differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_d27a8ddd.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_d27a8ddd.png
new file mode 100644
index 0000000..3555481
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_d27a8ddd.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_deee5364.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_deee5364.avis
new file mode 100644
index 0000000..9c0c5b4
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_deee5364.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_e7f6be62.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_e7f6be62.webp
new file mode 100644
index 0000000..b0421a0
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_e7f6be62.webp differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_eae416ec.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_eae416ec.avis
new file mode 100644
index 0000000..27d5784
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_eae416ec.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_f62a2371.webp b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_f62a2371.webp
new file mode 100644
index 0000000..0c24683
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160744_f62a2371.webp differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_2fae9fa4.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_2fae9fa4.png
new file mode 100644
index 0000000..0de58cf
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_2fae9fa4.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_32cef914.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_32cef914.avis
new file mode 100644
index 0000000..21f82cb
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_32cef914.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_9306ede7.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_9306ede7.avis
new file mode 100644
index 0000000..3837144
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_9306ede7.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d926f01a.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d926f01a.png
new file mode 100644
index 0000000..e4d1e80
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d926f01a.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d9a10e99.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d9a10e99.avis
new file mode 100644
index 0000000..243f642
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_d9a10e99.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_ed64bcef.avis b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_ed64bcef.avis
new file mode 100644
index 0000000..342811c
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_ed64bcef.avis differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_f74caedd.svg b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_f74caedd.svg
new file mode 100644
index 0000000..f3ba3f7
--- /dev/null
+++ b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_160745_f74caedd.svg
@@ -0,0 +1,5 @@
+
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_28025cf8.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_28025cf8.png
new file mode 100644
index 0000000..c0db09f
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_28025cf8.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_330ded9a.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_330ded9a.png
new file mode 100644
index 0000000..e3fc9d4
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165034_330ded9a.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_140c8b23.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_140c8b23.png
new file mode 100644
index 0000000..f211c2f
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_140c8b23.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_15eef750.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_15eef750.png
new file mode 100644
index 0000000..f8206eb
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_15eef750.png differ
diff --git a/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_b6d239d3.png b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_b6d239d3.png
new file mode 100644
index 0000000..85effeb
Binary files /dev/null and b/personal-blog/backend/uploads/thumbnails/thumb_crawl_20260806_165035_b6d239d3.png differ
diff --git a/personal-blog/docs/04-crawl-solution.md b/personal-blog/docs/04-crawl-solution.md
index 67a4a47..5062ca6 100644
--- a/personal-blog/docs/04-crawl-solution.md
+++ b/personal-blog/docs/04-crawl-solution.md
@@ -555,3 +555,68 @@ crawl 流程:
1. `wechat_crawl.py` 是独立路由 `/api/wechat-crawl/`,有自己的 preview/save 两阶段流程,是「微信专用采集器」而非「散落的站点特判逻辑」,不存在重复规则问题
2. 微信的提取逻辑会以 `WechatExtractor` 策略类形式注册到 `crawl_strategies.py`,供通用入口(`universal_fetch` / `crawl_article`)遇到微信 URL 时兜底复用——这是**新增复用**,不改动 wechat_crawl.py 本体
3. 若未来想让 wechat_crawl.py 内部也复用策略类,属于可选优化(消除与 crawl.py 微信段的重复),不在本次重构范围
+
+---
+
+## 九、正文噪音过滤(Pruning 思路,已落地)
+
+### 9.1 背景与动机
+
+HTML → Markdown 转换面对的是整页 DOM,导航栏、侧栏、评论区、相关推荐、
+广告等噪音容器会一并进入正文 Markdown,污染文章内容(CSDN 实测整页文本
+7505 字符,正文仅 2714 字符,噪音占 64%)。
+
+参考 crawl4ai 的 PruningContentFilter(`/home/ubuntu/forgithub/crawl4ai`
+`/crawl4ai/content_filter_strategy.py`)实现本地化方案。
+
+### 9.2 评分模型
+
+对 DOM 递归评分,低于阈值删除:
+
+```
+score = 0.4*text_density + 0.2*link_density + 0.2*tag_weight
+ + 0.1*class_id_weight + 0.1*text_length
+```
+
+- `text_density` = 节点文本 / 节点 HTML 长度,越纯文本越高
+- `link_density` = 1 - 链接文本占比,链接越少越像正文
+- `tag_weight`:p/article/section/main=1.0~1.5,div/li/td=0.5
+- `text_length` = min(log(文本长度+1), 5.0),长度项封顶
+
+### 9.3 相对 crawl4ai 原版的两处修正
+
+1. **长度项封顶 5.0**:原版 `log(text_len+1)` 无上限,超大噪音容器
+ (如 3600+ 字符的相关推荐列表)得分 0.90 远超阈值 0.48 而残留;
+ 封顶后大容器不再因体量大而误判为正文。
+2. **负面类名硬删**:原版对命中负面模式(recommend/comment/sidebar/ads…)
+ 的类名仅 -0.5 小惩罚;本实现直接删除整个子树,因为类名/ID 命中
+ 负面模式的容器几乎必然是噪音。
+
+### 9.4 安全兜底(质量门控)
+
+剪枝后若正文文本 < 300 字符,或 < 原文本的 20%,判定为疑似误删,
+回退返回原 HTML。对已定位正文容器的局部 HTML(如 CSDN `#content_views`)
+调用时无副作用,正文容器内节点评分普遍高于阈值。
+
+### 9.5 接入点(crawl.py + universal_crawl.py)
+
+新增 `backend/routers/content_prune.py`,`prune_html_noise()` 在**两个
+HTML → Markdown 转换函数的统一入口**调用(`fix_encoding` 之后):
+
+- `crawl.py::html_to_markdown()` —— 覆盖 `/api/crawl/` 旧轨
+- `crawl.py::convert_spa_html_to_markdown()` —— 覆盖 Playwright SPA 轨
+
+`universal_crawl.py` 的 `_legacy_extract()` 调用上述两个函数(站点特判
+分支传已提取的正文容器,SPA/通用分支传整页),**自动获得剪枝,无需
+重复调用**(避免双重剪枝开销),仅在调用处加注释说明。
+
+策略轨(`run_extractor → extractor.to_markdown`)同样经由 `crawl.py`
+转换函数获得剪枝。
+
+### 9.6 验证结果
+
+- 真实 CSDN 文章页:7505 → 2817 字符(-62%),`#content_views` 正文
+ 保留率 99.8%,博主/评论/点赞/分享/扫码等噪音关键词全部删除
+- 构造页面:导航/相关推荐/侧栏/评论区全部删除,正文与 `` 保留
+- 质量门控:剪后 2817 > 300 且 38% > 20%,正常通过
+- `test_crawl_strategies.py` + `test_crawl.py` 回归:6 passed,无回归
diff --git a/personal-blog/frontend/src/pages/CrawlEdit.jsx b/personal-blog/frontend/src/pages/CrawlEdit.jsx
index ae82796..8a01db8 100644
--- a/personal-blog/frontend/src/pages/CrawlEdit.jsx
+++ b/personal-blog/frontend/src/pages/CrawlEdit.jsx
@@ -544,6 +544,7 @@ export function CrawlEdit() {
insertImageToContent(insertUrl)}
/>
diff --git a/personal-blog/frontend/src/pages/PostEdit.jsx b/personal-blog/frontend/src/pages/PostEdit.jsx
index e5d9443..3339739 100644
--- a/personal-blog/frontend/src/pages/PostEdit.jsx
+++ b/personal-blog/frontend/src/pages/PostEdit.jsx
@@ -87,6 +87,7 @@ function ImageUploader({ uploadedImages, setUploadedImages, onInsertImage }) {
diff --git a/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx b/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx
index ccf0150..cb22016 100644
--- a/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx
+++ b/personal-blog/frontend/src/pages/UniversalCrawlEdit.jsx
@@ -563,6 +563,7 @@ export function UniversalCrawlEdit() {
insertImageToContent(insertUrl)}
/>
diff --git a/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx b/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx
index 0335657..f84675f 100644
--- a/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx
+++ b/personal-blog/frontend/src/pages/WechatCrawlEdit.jsx
@@ -357,6 +357,7 @@ export function WechatCrawlEdit() {
@@ -389,6 +390,7 @@ export function WechatCrawlEdit() {