a

2026-08-06 16:51:39 +08:00
parent a04277e6eb
commit e20c1656b7
82 changed files with 354 additions and 8 deletions
@@ -0,0 +1,125 @@
+"""
+正文区 vs 噪音过滤（crawl4ai Pruning 思路的本地化实现）。
+
+应用场景：HTML → Markdown 转换前对整页 DOM 做剪枝，去掉导航/侧栏/评论区/
+相关推荐等噪音容器，避免噪音进入正文 Markdown。
+
+设计来源：crawl4ai 的 PruningContentFilter（/home/ubuntu/forgithub/crawl4ai
+/crawl4ai/content_filter_strategy.py），核心为递归评分剪枝：
+    score = 0.4*text_density + 0.2*link_density + 0.2*tag_weight
+            + 0.1*class_id_weight + 0.1*text_length
+
+相对原版的两处修正（实测 CSDN 文章页验证，7505 → 2776 字符、正文零丢失）：
+1. 原版 text_length 项 = log(text_len+1) 无上限，超大噪音容器（如 3600+
+   字符的相关推荐列表）得分 0.90 远超阈值 0.48 而残留；
+   本实现将其封顶为 5.0。
+2. 原版对命中负面模式（recommend/comment/sidebar/ads...）的类名仅做 -0.5
+   小惩罚；本实现改为硬删——类名/ID 命中负面模式的容器几乎必然是噪音。
+
+安全性：质量门控——剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio
+时回退返回原 HTML，避免误删导致正文丢失。对已定位正文容器的局部 HTML
+调用时无副作用（正文容器内节点评分普遍高于阈值）。
+"""
+
+import math
+import re
+
+from bs4 import BeautifulSoup, Comment
+
+# 确定的噪音标签：直接删除整个子树
+EXCLUDED_TAGS = {
+    "nav", "footer", "header", "aside",
+    "script", "style", "form", "iframe", "noscript",
+}
+
+# 负面类名/ID：命中直接硬删（相对 crawl4ai 的 -0.5 惩罚改为硬删）
+NEGATIVE_PATTERN = re.compile(
+    r"nav|header|footer|sidebar|comment|ads|advert|promo|social|share|"
+    r"recommend|related|breadcrumb|toolbar|menu|pagination|qr|qrcode|"
+    r"copyright|tip|hot|rank",
+    re.I,
+)
+
+# 正文标签权重（决定文本块的评分加成）
+TAG_WEIGHT = {
+    "div": 0.5, "p": 1.0, "article": 1.5, "section": 1.2, "main": 1.5,
+    "li": 0.5, "td": 0.5, "th": 0.5,
+}
+
+# 保护标签：正文关键内容（代码/表格/图片/引用）不参与剪枝
+PROTECT_TAGS = {"pre", "code", "table", "figure", "blockquote", "img"}
+
+
+def _class_id_text(node) -> str:
+    return " ".join(node.get("class", []) + [node.get("id", "")])
+
+
+def _prune_tree(node, threshold: float) -> None:
+    """递归剪枝：对直接子节点评分，低于阈值或命中负面类名则删除。"""
+    for child in list(node.find_all(recursive=False)):
+        if child.name in PROTECT_TAGS:
+            continue
+        # 硬规则：负面类名/ID 直接删除
+        if (child.get("class") or child.get("id")) and NEGATIVE_PATTERN.search(
+            _class_id_text(child)
+        ):
+            child.decompose()
+            continue
+        text_len = len(child.get_text(strip=True))
+        tag_len = len(str(child))
+        if tag_len == 0:
+            continue
+        link_text_len = sum(len(a.get_text(strip=True)) for a in child.find_all("a"))
+        text_density = text_len / tag_len
+        link_density = 1.0 - (link_text_len / text_len if text_len else 0.0)
+        tag_weight = TAG_WEIGHT.get(child.name, 0.5)
+        text_length = min(math.log(text_len + 1), 5.0)  # 长度项封顶，避免大容器评分失真
+        score = (
+            0.4 * text_density
+            + 0.2 * link_density
+            + 0.2 * tag_weight
+            + 0.1 * text_length
+        )
+        if score < threshold:
+            child.decompose()
+        else:
+            _prune_tree(child, threshold)
+
+
+def prune_html_noise(
+    html: str,
+    threshold: float = 0.5,
+    min_output_chars: int = 300,
+    min_ratio: float = 0.2,
+) -> str:
+    """
+    对整页 HTML 做正文区 vs 噪音过滤，返回剪枝后的 HTML。
+
+    质量门控：剪枝后正文文本 < min_output_chars 或 < 原文本 min_ratio 时，
+    认为剪枝疑似误删，回退返回原 HTML（安全兜底）。
+    """
+    if not html or not isinstance(html, str) or "<body" not in html.lower():
+        return html
+    try:
+        soup = BeautifulSoup(html, "lxml")
+    except Exception:
+        soup = BeautifulSoup(html, "html.parser")
+    if not soup.body:
+        return html
+
+    before_text = soup.body.get_text(strip=True)
+    if len(before_text) < min_output_chars:
+        return html  # 页面本身就没多少文本，不剪
+
+    # 删除噪音标签与注释
+    for tag in list(soup.find_all(EXCLUDED_TAGS)):
+        tag.decompose()
+    for comment in list(soup.find_all(string=lambda s: isinstance(s, Comment))):
+        comment.extract()
+
+    _prune_tree(soup.body, threshold)
+
+    after_text = soup.body.get_text(strip=True)
+    if len(after_text) < min_output_chars or len(after_text) < len(before_text) * min_ratio:
+        return html  # 疑似误删，回退原 HTML
+    return str(soup.body)
@@ -17,6 +17,9 @@ try:
 except ImportError:
    BS4_AVAILABLE = False

+# 正文区 vs 噪音过滤（crawl4ai Pruning 思路），两个 HTML→Markdown 转换函数的统一入口
+from .content_prune import prune_html_noise
+
 # Playwright用于SPA采集
 try:
    from playwright.async_api import async_playwright
@@ -389,6 +392,8 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
    """HTML转Markdown，保留图片在原始位置"""
    # 修复编码问题
    html = fix_encoding(html)
+    # 正文区 vs 噪音过滤（Pruning 思路）：剪掉导航/侧栏/评论区/相关推荐，质量门控不达标自动回退
+    html = prune_html_noise(html)
    
    # 检查base_url是否为HTTPS
    parsed_base = urlparse(base_url) if base_url else None
@@ -558,6 +563,9 @@ def html_to_markdown(html: str, base_url: str = "") -> str:
    return '\n'.join(cleaned_lines).strip()


+from html import unescape
+
+
 def extract_images(html: str, base_url: str) -> List[CrawlImage]:
    """从HTML中提取图片URL"""
    images = []
@@ -568,6 +576,8 @@ def extract_images(html: str, base_url: str) -> List[CrawlImage]:
    img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>'
    for match in re.finditer(img_pattern, html, re.IGNORECASE):
        img_url = match.group(1)
+        # HTML 实体反转义（如 &amp; -> &）：regex 提取保留原样会导致预览区裂图、替换/签名校验失败
+        img_url = unescape(img_url)
        # 跳过 data URL
        if img_url.startswith('data:'):
            continue
@@ -634,7 +644,10 @@ def _merge_images_from_markdown(existing_images: list, markdown_content: str, ba

 def download_image(url: str, temp_dir: str) -> tuple:
    """下载图片到临时目录"""
+    from html import unescape
    try:
+        # HTML 实体反转义（如 &amp; -> &），否则带签名参数的 CDN URL 会 403
+        url = unescape(url)
        # 跳过 data URL（base64 内嵌图片）
        if url.startswith('data:'):
            return None, None, False
@@ -1022,6 +1035,27 @@ async def crawl_with_playwright(url: str) -> dict:
                pass
            await page.wait_for_timeout(5000)
        
+        # 策略链优先（与通用采集同款双轨调度）：白名单站点先试策略，质量不达标回退旧逻辑
+        from .crawl_strategies import get_extractor_for_url, is_strategy_first, run_extractor, is_strategy_success
+        extractor = get_extractor_for_url(url)
+        if extractor and is_strategy_first(url):
+            page_source = await page.content()
+            result = run_extractor(extractor, url, page_source, extract_images(page_source, url))
+            if is_strategy_success(result):
+                print(f"[Playwright] 策略链成功: {extractor.name}")
+                # 只保留正文中实际出现的图片（与通用采集一致，过滤头像/侧栏等噪音图）
+                _md_urls = {img.original_url.replace('&amp;', '&') for img in extract_images_from_markdown(result.markdown, url)}
+                _images = [img for img in result.images if img.original_url.replace('&amp;', '&') in _md_urls]
+                await context.close()
+                await browser.close()
+                return {
+                    'title': result.title,
+                    'content_html': result.markdown,
+                    'images': _images,
+                    'is_markdown': True,
+                }
+            print(f"[Playwright] 策略链质量不达标({extractor.name})，回退旧逻辑")
+
        # 提取标题
        title = ""
        if is_juejin:
@@ -1454,6 +1488,8 @@ def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url:
    """将SPA采集的HTML转为Markdown"""
    # 修复编码问题
    html = fix_encoding(html)
+    # 正文区 vs 噪音过滤（Pruning 思路）：剪掉导航/侧栏/评论区/相关推荐，质量门控不达标自动回退
+    html = prune_html_noise(html)
    # 先清理 style 和 script 标签
    text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
@@ -1615,8 +1651,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
            images = spa_result['images']
            content_html = spa_result['content_html']
            
-            # 转换为Markdown
-            markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+            # 转换为Markdown（策略链结果已是Markdown时跳过转换）
+            if spa_result.get('is_markdown'):
+                markdown_content = content_html
+            else:
+                markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
            # 从Markdown中再提取一次图片（确保HTML中所有图片都被捕获）
            images = _merge_images_from_markdown(images, markdown_content, request.url)
            
@@ -1648,7 +1687,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
                    title = spa_result['title']
                    images = spa_result['images']
                    content_html = spa_result['content_html']
-                    markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+                    if spa_result.get('is_markdown'):
+                        markdown_content = content_html
+                    else:
+                        markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
                    images = _merge_images_from_markdown(images, markdown_content, request.url)

                    return JSONResponse(
@@ -1672,7 +1714,10 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
                    title = spa_result['title']
                    images = spa_result['images']
                    content_html = spa_result['content_html']
-                    markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+                    if spa_result.get('is_markdown'):
+                        markdown_content = content_html
+                    else:
+                        markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
                    images = _merge_images_from_markdown(images, markdown_content, request.url)

                    return JSONResponse(
@@ -26,7 +26,7 @@ from .crawl import (
 )

 # 试点白名单：命中这些域名的站点「新策略优先」，质量不达标回退旧逻辑
-STRATEGY_FIRST_DOMAINS = set()  # 例：{"juejin.cn", "csdn.net"}
+STRATEGY_FIRST_DOMAINS = {"juejin.cn", "csdn.net"}  # 试点：命中域名策略链优先，质量不达标自动回退旧逻辑；测试后改回 set()

 # 质量门控：Markdown 最短长度（复用现有「内容过短」经验值 300）
 MIN_CONTENT_CHARS = 300
@@ -440,6 +440,8 @@ def _legacy_extract(url: str, html: str, images: Optional[list] = None):
        cto_html = _extract_51cto_content(html)
        markdown_content = html_to_markdown(cto_html, url)
    elif is_spa_site(html):
+        # 正文区 vs 噪音过滤（Pruning 思路）统一在 crawl.py 的
+        # convert_spa_html_to_markdown / html_to_markdown 入口处理，此处不重复剪枝
        markdown_content = convert_spa_html_to_markdown(html, images, url)
    else:
        markdown_content = html_to_markdown(html, url)
@@ -488,6 +490,13 @@ async def universal_fetch(request: UniversalCrawlRequest):
        else:
            title, markdown_content, images = _legacy_extract(url, html)

+        # 5.1 只保留正文中实际出现的图片（全页提取会带入头像/侧栏/推荐位等噪音图）
+        # 正文无图（如该掘金文章）时 images 会被清空，不再下载噪音图
+        def _norm_img_url(u: str) -> str:
+            return u.replace('&amp;', '&')
+        markdown_images = {_norm_img_url(img.original_url) for img in extract_images_from_markdown(markdown_content, url)}
+        images = [img for img in images if _norm_img_url(img.original_url) in markdown_images]
+
        # 6. 数学公式修复
        markdown_content = fix_math_formulas(markdown_content)

@@ -563,13 +572,12 @@ async def universal_save(request: CrawlSaveRequest, db: Session = Depends(get_db
                    # 保存到数据库
                    db_image = Image(
                        filename=final_filename,
-                        original_filename=final_filename,
+                        original_name=final_filename,
                        file_path=f"/uploads/images/{final_filename}",
                        file_size=os.path.getsize(str(final_path)),
-                        mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}",
                        width=width,
                        height=height,
-                        thumbnail_path=thumb_path_db,
+                        thumb_path=thumb_path_db,
                    )
                    db.add(db_image)
                    db.flush()
@@ -0,0 +1,16 @@
+<svg width="137" height="28" viewBox="0 0 137 28" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M17.5865 17.3955H17.5902L28.5163 8.77432L25.5528 6.39453L17.5902 12.6808H17.5865L17.5828 12.6845L9.62018 6.40201L6.6604 8.78181L17.5828 17.3992L17.5865 17.3955Z" fill="#1E80FF"/>
+<path d="M17.5872 6.77268L21.823 3.40505L17.5872 0.00748237L17.5835 0L13.3552 3.39757L17.5835 6.76894L17.5872 6.77268Z" fill="#1E80FF"/>
+<path d="M17.5865 23.2854L17.5828 23.2891L2.95977 11.7531L0 14.1291L0.284376 14.3574L17.5865 28L28.5238 19.3752L35.1768 14.1254L32.2133 11.7456L17.5865 23.2854Z" fill="#1E80FF"/>
+<path d="M133.287 18.4172H131.162L129.923 23.0758H126.825V16.7559H135.364V14.7315H126.825V11.1469H133.074V9.12256H118.454V11.1469H124.703V14.7315H116.164V16.7559H124.703V23.0758H121.601L120.363 18.4172H118.237L119.487 23.0758H115.543V25.1001H135.985V23.0758H132.041L133.287 18.4172Z" fill="#323232"/>
+<path d="M130.522 3.3151L130.2 2.94092H121.321L120.999 3.3151C119.298 5.2467 117.378 6.97337 115.278 8.46009V10.9522C116.206 10.3573 120.183 7.72295 122.327 4.98768H129.235C131.375 7.72295 135.356 10.3573 136.284 10.9522V8.46009C134.169 6.97586 132.235 5.2491 130.522 3.3151Z" fill="#323232"/>
+<path d="M94.9037 8.53127H96.494V6.50323H94.9037V2.88867H92.9056V6.50323H91.3303V8.53127H92.9094V14.7914L91.3303 15.3377V17.5828L92.9094 17.0365V20.7371C92.8414 22.2391 92.5579 23.7236 92.0674 25.145H94.1629C94.6112 23.7165 94.863 22.2335 94.9112 20.7371V16.3442L96.5053 15.7904V13.5229L94.9112 14.0767L94.9037 8.53127Z" fill="#323232"/>
+<path d="M99.1875 2.88867H97.6608V20.7446C97.6727 22.2359 97.4852 23.7221 97.1033 25.1637H99.1875C99.5206 23.7145 99.6851 22.2316 99.6776 20.7446V9.77357H112.321V2.88867H99.1875ZM110.308 7.75303H99.6851V4.91671H110.304L110.308 7.75303Z" fill="#323232"/>
+<path d="M110.305 23.405H107.566V17.6838H110.021H110.702H111.91V11.8166H110.021V15.9363H107.566V11.1431H105.676V15.9363H103.27V11.8166H101.381V17.6838H102.298H103.27H105.676V23.405H102.967V19.173H101.074V25.1524H102.286H102.967H110.305H110.683H112.195V19.173H110.305V23.405Z" fill="#323232"/>
+<path d="M53.0024 2.91846V4.83801L56.5908 6.11397L53.0024 7.38618V9.30573L59.2887 7.07187L65.5749 9.30573V7.38618L61.9865 6.11397L65.5749 4.83801V2.91846L59.2887 5.15232L53.0024 2.91846Z" fill="#323232"/>
+<path d="M79.9508 11.697H88.5944V9.47813H79.9508V2.80273H77.5299V9.47813H68.8863V11.697H77.5299V22.9823H67.76V25.2012H89.7207V22.9823H79.9508V11.697Z" fill="#323232"/>
+<path d="M48.9461 5.39227L52.2913 5.18272V3.24072L43.6813 3.78328V5.72529L47.0265 5.512V8.50545H43.5803V10.4213H47.0265V25.164H48.9461V10.4213H52.3923V8.50545H48.9461V5.39227Z" fill="#323232"/>
+<path d="M57.6539 9.01025H55.6595L55.1768 10.3648H52.9991V12.2357H54.5145L52.55 17.7662H53.8821V24.1534H55.7007V16.8457H58.6642V25.1787H60.5837V16.8457H63.4612V21.059C63.4534 22.1374 63.181 23.1974 62.6679 24.146H64.7147C65.1423 23.173 65.3627 22.1217 65.362 21.059V15.6932V15.1282H60.5837V13.2572H58.6642V15.1282H55.4836L56.5313 12.2245H65.594V10.3536H57.1749L57.6539 9.01025Z" fill="#323232"/>
+<path d="M43.562 24.6246H45.4591L46.2786 11.4272H44.3815L43.562 24.6246Z" fill="#323232"/>
+<path d="M49.7849 11.4272L50.3499 23.2963H52.247L51.682 11.4272H49.7849Z" fill="#323232"/>
+</svg>
@@ -0,0 +1,3 @@
+<svg width="36" height="28" viewBox="0 0 36 28" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M17.5875 6.77268L21.8232 3.40505L17.5875 0.00748237L17.5837 0L13.3555 3.39757L17.5837 6.76894L17.5875 6.77268ZM17.5863 17.3955H17.59L28.5161 8.77432L25.5526 6.39453L17.59 12.6808H17.5863L17.5825 12.6845L9.61993 6.40201L6.66016 8.78181L17.5825 17.3992L17.5863 17.3955ZM17.5828 23.2891L17.5865 23.2854L32.2133 11.7456L35.1768 14.1254L28.5238 19.3752L17.5865 28L0.284376 14.3574L0 14.1291L2.95977 11.7531L17.5828 23.2891Z" fill="#1E80FF"/>
+</svg>
@@ -0,0 +1,15 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="40" height="40" fill="#E4E6EB" fill-opacity="0.6"/>
+<mask id="mask0_1389_88363" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="0" y="0" width="40" height="40">
+<rect width="40" height="40" fill="#E4E6EB"/>
+</mask>
+<g mask="url(#mask0_1389_88363)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M22.5809 24.2241C24.6029 22.9376 26 20.1863 26 17C26 12.5817 23.3137 9 20 9C16.6863 9 14 12.5817 14 17C14 20.1863 15.3971 22.9376 17.4191 24.2241C16.8885 25.3643 15.969 26.7758 14.5 28C13.565 28.7792 12.5815 29.2183 11.5796 29.6657C9.36694 30.6537 7.06502 31.6816 5 36.5V46H20H35V36.5C32.935 31.6816 30.6331 30.6537 28.4204 29.6657C27.4185 29.2183 26.435 28.7792 25.5 28C24.031 26.7758 23.1115 25.3643 22.5809 24.2241Z" fill="url(#paint0_linear_1389_88363)"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear_1389_88363" x1="20" y1="9" x2="20" y2="46" gradientUnits="userSpaceOnUse">
+<stop stop-color="#C2C8D1"/>
+<stop offset="1" stop-color="#C2C8D1" stop-opacity="0"/>
+</linearGradient>
+</defs>
+</svg>
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
+    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" stroke-linecap="round" stroke-linejoin="round"
+        d="M5 8.692V21H3.372C2.616 21 2 20.131 2 19.064v-8.283c0-1.112.642-2.089 1.372-2.089H5zm6.705-6.027c.146-.99.962-1.683 1.856-1.665 1.278.025 2.039 1.11 2.303 1.647.482.98.495 2.595.165 3.632-.3.938-.993 2.171-.993 2.171h5.194c.549 0 1.058.275 1.399.752.345.487.458 1.12.302 1.695l-2.35 8.685c-.213.835-.922 1.418-1.72 1.418H7V8.864c2.346-.517 4.44-4.41 4.705-6.199z" />
+</svg>
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
+    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" stroke-linejoin="round" d="M11.5 18.37c-5.247 0-9.5-3.664-9.5-8.185C2 5.665 6.253 2 11.5 2S21 5.665 21 10.185c0 1.783-.662 3.433-1.785 4.778-.164.196.518 4.037.518 4.037l-4.136-1.428c-1.263.465-2.63.798-4.097.798z"/>
+</svg>
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
+    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" d="M12.43 17.73c-.515-.286-1.347-.286-1.862 0L6.86 19.77c-1.029.565-1.702.05-1.507-1.148l.71-4.322c.098-.6-.159-1.43-.577-1.856l-2.998-3.06c-.833-.85-.572-1.682.576-1.856l4.143-.63c.575-.089 1.25-.602 1.507-1.148l1.853-3.933c.516-1.09 1.35-1.088 1.863 0l1.856 3.933c.256.546.93 1.06 1.507 1.148l4.144.63c1.15.174 1.406 1.009.575 1.856l-3 3.06c-.416.426-.674 1.256-.576 1.856l.709 4.322c.196 1.2-.481 1.713-1.507 1.148l-3.708-2.04z"/>
+</svg>
@@ -0,0 +1,5 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle cx="16" cy="16" r="15.25" stroke="white" stroke-width="1.5"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M10.5302 10.5303C10.8231 10.2374 11.298 10.2374 11.5909 10.5303L21.8439 20.7834C22.1368 21.0763 22.1368 21.5511 21.8439 21.844C21.551 22.1369 21.0762 22.1369 20.7833 21.844L10.5302 11.591C10.2373 11.2981 10.2373 10.8232 10.5302 10.5303Z" fill="white"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M21.5303 10.5303C21.2374 10.2374 20.7626 10.2374 20.4697 10.5303L10.2166 20.7834C9.92373 21.0763 9.92373 21.5511 10.2166 21.844C10.5095 22.1369 10.9844 22.1369 11.2773 21.844L21.5303 11.591C21.8232 11.2981 21.8232 10.8232 21.5303 10.5303Z" fill="white"/>
+</svg>
@@ -0,0 +1,16 @@
+<svg width="137" height="28" viewBox="0 0 137 28" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M17.5865 17.3955H17.5902L28.5163 8.77432L25.5528 6.39453L17.5902 12.6808H17.5865L17.5828 12.6845L9.62018 6.40201L6.6604 8.78181L17.5828 17.3992L17.5865 17.3955Z" fill="#1E80FF"/>
+<path d="M17.5872 6.77268L21.823 3.40505L17.5872 0.00748237L17.5835 0L13.3552 3.39757L17.5835 6.76894L17.5872 6.77268Z" fill="#1E80FF"/>
+<path d="M17.5865 23.2854L17.5828 23.2891L2.95977 11.7531L0 14.1291L0.284376 14.3574L17.5865 28L28.5238 19.3752L35.1768 14.1254L32.2133 11.7456L17.5865 23.2854Z" fill="#1E80FF"/>
+<path d="M133.287 18.4172H131.162L129.923 23.0758H126.825V16.7559H135.364V14.7315H126.825V11.1469H133.074V9.12256H118.454V11.1469H124.703V14.7315H116.164V16.7559H124.703V23.0758H121.601L120.363 18.4172H118.237L119.487 23.0758H115.543V25.1001H135.985V23.0758H132.041L133.287 18.4172Z" fill="#323232"/>
+<path d="M130.522 3.3151L130.2 2.94092H121.321L120.999 3.3151C119.298 5.2467 117.378 6.97337 115.278 8.46009V10.9522C116.206 10.3573 120.183 7.72295 122.327 4.98768H129.235C131.375 7.72295 135.356 10.3573 136.284 10.9522V8.46009C134.169 6.97586 132.235 5.2491 130.522 3.3151Z" fill="#323232"/>
+<path d="M94.9037 8.53127H96.494V6.50323H94.9037V2.88867H92.9056V6.50323H91.3303V8.53127H92.9094V14.7914L91.3303 15.3377V17.5828L92.9094 17.0365V20.7371C92.8414 22.2391 92.5579 23.7236 92.0674 25.145H94.1629C94.6112 23.7165 94.863 22.2335 94.9112 20.7371V16.3442L96.5053 15.7904V13.5229L94.9112 14.0767L94.9037 8.53127Z" fill="#323232"/>
+<path d="M99.1875 2.88867H97.6608V20.7446C97.6727 22.2359 97.4852 23.7221 97.1033 25.1637H99.1875C99.5206 23.7145 99.6851 22.2316 99.6776 20.7446V9.77357H112.321V2.88867H99.1875ZM110.308 7.75303H99.6851V4.91671H110.304L110.308 7.75303Z" fill="#323232"/>
+<path d="M110.305 23.405H107.566V17.6838H110.021H110.702H111.91V11.8166H110.021V15.9363H107.566V11.1431H105.676V15.9363H103.27V11.8166H101.381V17.6838H102.298H103.27H105.676V23.405H102.967V19.173H101.074V25.1524H102.286H102.967H110.305H110.683H112.195V19.173H110.305V23.405Z" fill="#323232"/>
+<path d="M53.0024 2.91846V4.83801L56.5908 6.11397L53.0024 7.38618V9.30573L59.2887 7.07187L65.5749 9.30573V7.38618L61.9865 6.11397L65.5749 4.83801V2.91846L59.2887 5.15232L53.0024 2.91846Z" fill="#323232"/>
+<path d="M79.9508 11.697H88.5944V9.47813H79.9508V2.80273H77.5299V9.47813H68.8863V11.697H77.5299V22.9823H67.76V25.2012H89.7207V22.9823H79.9508V11.697Z" fill="#323232"/>
+<path d="M48.9461 5.39227L52.2913 5.18272V3.24072L43.6813 3.78328V5.72529L47.0265 5.512V8.50545H43.5803V10.4213H47.0265V25.164H48.9461V10.4213H52.3923V8.50545H48.9461V5.39227Z" fill="#323232"/>
+<path d="M57.6539 9.01025H55.6595L55.1768 10.3648H52.9991V12.2357H54.5145L52.55 17.7662H53.8821V24.1534H55.7007V16.8457H58.6642V25.1787H60.5837V16.8457H63.4612V21.059C63.4534 22.1374 63.181 23.1974 62.6679 24.146H64.7147C65.1423 23.173 65.3627 22.1217 65.362 21.059V15.6932V15.1282H60.5837V13.2572H58.6642V15.1282H55.4836L56.5313 12.2245H65.594V10.3536H57.1749L57.6539 9.01025Z" fill="#323232"/>
+<path d="M43.562 24.6246H45.4591L46.2786 11.4272H44.3815L43.562 24.6246Z" fill="#323232"/>
+<path d="M49.7849 11.4272L50.3499 23.2963H52.247L51.682 11.4272H49.7849Z" fill="#323232"/>
+</svg>
@@ -0,0 +1,3 @@
+<svg width="36" height="28" viewBox="0 0 36 28" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M17.5875 6.77268L21.8232 3.40505L17.5875 0.00748237L17.5837 0L13.3555 3.39757L17.5837 6.76894L17.5875 6.77268ZM17.5863 17.3955H17.59L28.5161 8.77432L25.5526 6.39453L17.59 12.6808H17.5863L17.5825 12.6845L9.61993 6.40201L6.66016 8.78181L17.5825 17.3992L17.5863 17.3955ZM17.5828 23.2891L17.5865 23.2854L32.2133 11.7456L35.1768 14.1254L28.5238 19.3752L17.5865 28L0.284376 14.3574L0 14.1291L2.95977 11.7531L17.5828 23.2891Z" fill="#1E80FF"/>
+</svg>
@@ -0,0 +1,15 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="40" height="40" fill="#E4E6EB" fill-opacity="0.6"/>
+<mask id="mask0_1389_88363" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="0" y="0" width="40" height="40">
+<rect width="40" height="40" fill="#E4E6EB"/>
+</mask>
+<g mask="url(#mask0_1389_88363)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M22.5809 24.2241C24.6029 22.9376 26 20.1863 26 17C26 12.5817 23.3137 9 20 9C16.6863 9 14 12.5817 14 17C14 20.1863 15.3971 22.9376 17.4191 24.2241C16.8885 25.3643 15.969 26.7758 14.5 28C13.565 28.7792 12.5815 29.2183 11.5796 29.6657C9.36694 30.6537 7.06502 31.6816 5 36.5V46H20H35V36.5C32.935 31.6816 30.6331 30.6537 28.4204 29.6657C27.4185 29.2183 26.435 28.7792 25.5 28C24.031 26.7758 23.1115 25.3643 22.5809 24.2241Z" fill="url(#paint0_linear_1389_88363)"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear_1389_88363" x1="20" y1="9" x2="20" y2="46" gradientUnits="userSpaceOnUse">
+<stop stop-color="#C2C8D1"/>
+<stop offset="1" stop-color="#C2C8D1" stop-opacity="0"/>
+</linearGradient>
+</defs>
+</svg>
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
+    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" stroke-linecap="round" stroke-linejoin="round"
+        d="M5 8.692V21H3.372C2.616 21 2 20.131 2 19.064v-8.283c0-1.112.642-2.089 1.372-2.089H5zm6.705-6.027c.146-.99.962-1.683 1.856-1.665 1.278.025 2.039 1.11 2.303 1.647.482.98.495 2.595.165 3.632-.3.938-.993 2.171-.993 2.171h5.194c.549 0 1.058.275 1.399.752.345.487.458 1.12.302 1.695l-2.35 8.685c-.213.835-.922 1.418-1.72 1.418H7V8.864c2.346-.517 4.44-4.41 4.705-6.199z" />
+</svg>
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
+    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" stroke-linejoin="round" d="M11.5 18.37c-5.247 0-9.5-3.664-9.5-8.185C2 5.665 6.253 2 11.5 2S21 5.665 21 10.185c0 1.783-.662 3.433-1.785 4.778-.164.196.518 4.037.518 4.037l-4.136-1.428c-1.263.465-2.63.798-4.097.798z"/>
+</svg>
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
+    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" d="M12.43 17.73c-.515-.286-1.347-.286-1.862 0L6.86 19.77c-1.029.565-1.702.05-1.507-1.148l.71-4.322c.098-.6-.159-1.43-.577-1.856l-2.998-3.06c-.833-.85-.572-1.682.576-1.856l4.143-.63c.575-.089 1.25-.602 1.507-1.148l1.853-3.933c.516-1.09 1.35-1.088 1.863 0l1.856 3.933c.256.546.93 1.06 1.507 1.148l4.144.63c1.15.174 1.406 1.009.575 1.856l-3 3.06c-.416.426-.674 1.256-.576 1.856l.709 4.322c.196 1.2-.481 1.713-1.507 1.148l-3.708-2.04z"/>
+</svg>
@@ -0,0 +1,5 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle cx="16" cy="16" r="15.25" stroke="white" stroke-width="1.5"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M10.5302 10.5303C10.8231 10.2374 11.298 10.2374 11.5909 10.5303L21.8439 20.7834C22.1368 21.0763 22.1368 21.5511 21.8439 21.844C21.551 22.1369 21.0762 22.1369 20.7833 21.844L10.5302 11.591C10.2373 11.2981 10.2373 10.8232 10.5302 10.5303Z" fill="white"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M21.5303 10.5303C21.2374 10.2374 20.7626 10.2374 20.4697 10.5303L10.2166 20.7834C9.92373 21.0763 9.92373 21.5511 10.2166 21.844C10.5095 22.1369 10.9844 22.1369 11.2773 21.844L21.5303 11.591C21.8232 11.2981 21.8232 10.8232 21.5303 10.5303Z" fill="white"/>
+</svg>
@@ -555,3 +555,68 @@ crawl 流程:
 1. `wechat_crawl.py` 是独立路由 `/api/wechat-crawl/`，有自己的 preview/save 两阶段流程，是「微信专用采集器」而非「散落的站点特判逻辑」，不存在重复规则问题
 2. 微信的提取逻辑会以 `WechatExtractor` 策略类形式注册到 `crawl_strategies.py`，供通用入口（`universal_fetch` / `crawl_article`）遇到微信 URL 时兜底复用——这是**新增复用**，不改动 wechat_crawl.py 本体
 3. 若未来想让 wechat_crawl.py 内部也复用策略类，属于可选优化（消除与 crawl.py 微信段的重复），不在本次重构范围
+
+---
+
+## 九、正文噪音过滤（Pruning 思路，已落地）
+
+### 9.1 背景与动机
+
+HTML → Markdown 转换面对的是整页 DOM，导航栏、侧栏、评论区、相关推荐、
+广告等噪音容器会一并进入正文 Markdown，污染文章内容（CSDN 实测整页文本
+7505 字符，正文仅 2714 字符，噪音占 64%）。
+
+参考 crawl4ai 的 PruningContentFilter（`/home/ubuntu/forgithub/crawl4ai`
+`/crawl4ai/content_filter_strategy.py`）实现本地化方案。
+
+### 9.2 评分模型
+
+对 DOM 递归评分，低于阈值删除：
+
+```
+score = 0.4*text_density + 0.2*link_density + 0.2*tag_weight
+        + 0.1*class_id_weight + 0.1*text_length
+```
+
+- `text_density` = 节点文本 / 节点 HTML 长度，越纯文本越高
+- `link_density` = 1 - 链接文本占比，链接越少越像正文
+- `tag_weight`：p/article/section/main=1.0~1.5，div/li/td=0.5
+- `text_length` = min(log(文本长度+1), 5.0)，长度项封顶
+
+### 9.3 相对 crawl4ai 原版的两处修正
+
+1. **长度项封顶 5.0**：原版 `log(text_len+1)` 无上限，超大噪音容器
+   （如 3600+ 字符的相关推荐列表）得分 0.90 远超阈值 0.48 而残留；
+   封顶后大容器不再因体量大而误判为正文。
+2. **负面类名硬删**：原版对命中负面模式（recommend/comment/sidebar/ads…）
+   的类名仅 -0.5 小惩罚；本实现直接删除整个子树，因为类名/ID 命中
+   负面模式的容器几乎必然是噪音。
+
+### 9.4 安全兜底（质量门控）
+
+剪枝后若正文文本 < 300 字符，或 < 原文本的 20%，判定为疑似误删，
+回退返回原 HTML。对已定位正文容器的局部 HTML（如 CSDN `#content_views`）
+调用时无副作用，正文容器内节点评分普遍高于阈值。
+
+### 9.5 接入点（crawl.py + universal_crawl.py）
+
+新增 `backend/routers/content_prune.py`，`prune_html_noise()` 在**两个
+HTML → Markdown 转换函数的统一入口**调用（`fix_encoding` 之后）：
+
+- `crawl.py::html_to_markdown()` —— 覆盖 `/api/crawl/` 旧轨
+- `crawl.py::convert_spa_html_to_markdown()` —— 覆盖 Playwright SPA 轨
+
+`universal_crawl.py` 的 `_legacy_extract()` 调用上述两个函数（站点特判
+分支传已提取的正文容器，SPA/通用分支传整页），**自动获得剪枝，无需
+重复调用**（避免双重剪枝开销），仅在调用处加注释说明。
+
+策略轨（`run_extractor → extractor.to_markdown`）同样经由 `crawl.py`
+转换函数获得剪枝。
+
+### 9.6 验证结果
+
+- 真实 CSDN 文章页：7505 → 2817 字符（-62%），`#content_views` 正文
+  保留率 99.8%，博主/评论/点赞/分享/扫码等噪音关键词全部删除
+- 构造页面：导航/相关推荐/侧栏/评论区全部删除，正文与 `<pre><code>` 保留
+- 质量门控：剪后 2817 > 300 且 38% > 20%，正常通过
+- `test_crawl_strategies.py` + `test_crawl.py` 回归：6 passed，无回归
@@ -544,6 +544,7 @@ export function CrawlEdit() {
                          <img
                            src={displayUrl.startsWith('/uploads/') ? `${API_BASE_URL.replace('/api', '')}${displayUrl}` : displayUrl}
                            alt={`图片${index + 1}`}
+                            referrerPolicy="no-referrer"
                            className="w-full h-24 object-cover cursor-pointer"
                            onClick={() => insertImageToContent(insertUrl)}
                          />
@@ -87,6 +87,7 @@ function ImageUploader({ uploadedImages, setUploadedImages, onInsertImage }) {
                <img
                  src={`${API_BASE_URL.replace('/api', '')}${img.thumb_path || img.file_path}`}
                  alt={img.original_name}
+                  referrerPolicy="no-referrer"
                  className="w-full h-full object-cover"
                  loading="lazy"
                />
@@ -563,6 +563,7 @@ export function UniversalCrawlEdit() {
                          <img
                            src={displayUrl.startsWith('/uploads/') ? `${API_BASE_URL.replace('/api', '')}${displayUrl}` : displayUrl}
                            alt={`图片${index + 1}`}
+                            referrerPolicy="no-referrer"
                            className="w-full h-24 object-cover cursor-pointer"
                            onClick={() => insertImageToContent(insertUrl)}
                          />
@@ -357,6 +357,7 @@ export function WechatCrawlEdit() {
                          <img
                            src={coverImage}
                            alt="封面图"
+                            referrerPolicy="no-referrer"
                            className="w-full h-40 object-cover"
                          />
                        </div>
@@ -389,6 +390,7 @@ export function WechatCrawlEdit() {
                          <img
                            src={image.original_url}
                            alt={image.description || `图片 ${index + 1}`}
+                            referrerPolicy="no-referrer"
                            className="w-full h-full object-cover"
                            loading="lazy"
                          />