"""
站点提取策略化重构(v2)
========================
将散落在 crawl.py / universal_crawl.py 中的站点特判逻辑收敛为「策略注册表」,
接口设计借鉴 crawl4ai 的 ContentScrapingStrategy(ABC) 思想。
双轨调度:
- 轨A(Legacy):调用方现有的 if-else 站点特判,逐行保留,不删除
- 轨B(Strategy):本文件中的策略链,每个策略类内部逻辑 = 现有代码「原样搬迁」
- 优先级由 STRATEGY_FIRST_DOMAINS 配置决定:
- 默认空集:全部走旧逻辑(轨A),策略链仅作未来兜底
- 白名单命中:策略链优先(轨B),质量不达标自动回退旧逻辑(轨A)
"""
from abc import ABC, abstractmethod
from typing import Optional
import re
from bs4 import BeautifulSoup
from .crawl import (
extract_title_from_html,
html_to_markdown,
is_spa_site,
convert_spa_html_to_markdown,
)
# 试点白名单:命中这些域名的站点「新策略优先」,质量不达标回退旧逻辑
STRATEGY_FIRST_DOMAINS = {"juejin.cn", "csdn.net"} # 试点:命中域名策略链优先,质量不达标自动回退旧逻辑;测试后改回 set()
# 质量门控:Markdown 最短长度(复用现有「内容过短」经验值 300)
MIN_CONTENT_CHARS = 300
FALLBACK_TITLE = "未获取到标题"
class SiteExtractor(ABC):
"""站点提取策略基类。每个策略类内部逻辑 = 现有代码原样搬迁,行为零改动。"""
name: str = "generic"
@abstractmethod
def match(self, url: str) -> bool:
"""URL 是否命中该站点"""
def needs_browser(self, url: str) -> bool:
"""是否需要浏览器渲染(Playwright)"""
return False
def extract_title(self, html: str, url: str) -> Optional[str]:
"""站点专用标题提取;返回 None 时由 run_extractor 回退通用提取"""
return None
def extract_content_html(self, html: str, url: str) -> str:
"""提取正文容器 HTML;未命中选择器时返回原 HTML"""
return html
def filter_images(self, images: list, url: str) -> list:
"""过滤站点 UI 图标等噪音图片"""
return images
def to_markdown(self, content_html: str, url: str, images: list) -> str:
"""将正文容器 HTML 转为 Markdown"""
return html_to_markdown(content_html, url)
class JuejinExtractor(SiteExtractor):
"""掘金:requests 返回混淆代码必须浏览器渲染;正文 article.article"""
name = "juejin"
def match(self, url: str) -> bool:
return 'juejin.cn' in url
def needs_browser(self, url: str) -> bool:
return True
def extract_title(self, html: str, url: str) -> Optional[str]:
# 原样搬迁 universal_crawl._extract_juejin_title
match = re.search(r'
]*>(.*?)', html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
# 去掉 " - 掘金" 后缀
title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip()
if len(title) > 5:
return title
return None
def extract_content_html(self, html: str, url: str) -> str:
# 原样搬迁 universal_crawl._extract_juejin_content
soup = BeautifulSoup(html, 'lxml')
# 优先使用 article.article
container = soup.select_one('article.article')
if not container:
# fallback 到 .article-viewer
container = soup.select_one('.article-viewer')
if not container:
# 再 fallback 到 #article-root
container = soup.select_one('#article-root')
if container:
return str(container)
return html
class CsdnExtractor(SiteExtractor):
"""CSDN:正文 #content_views / .article_content;过滤 csdnimg UI 图标"""
name = "csdn"
def match(self, url: str) -> bool:
return 'csdn.net' in url or 'csdn.com' in url
def extract_title(self, html: str, url: str) -> Optional[str]:
# 原样搬迁 universal_crawl._extract_csdn_title
match = re.search(r']*>(.*?)', html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
# 去掉 " - CSDN博客" / "-CSDN博客" 后缀
title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip()
if len(title) > 5:
return title
return None
def extract_content_html(self, html: str, url: str) -> str:
# 原样搬迁 universal_crawl._extract_csdn_content
soup = BeautifulSoup(html, 'lxml')
# 优先使用 #content_views(CSDN 正文主容器)
container = soup.select_one('#content_views')
if not container:
# fallback 到 .article_content
container = soup.select_one('.article_content')
if container:
return str(container)
return html
def filter_images(self, images: list, url: str) -> list:
# 原样搬迁 universal_crawl._filter_csdn_images
filtered = []
skip_patterns = [
'newHeart2023',
'tobarCollect',
'tobarCollection',
'copyright',
'avatar',
'blogv2/dist/pc/img',
]
for img in images:
img_url = img.original_url.lower()
if any(p in img_url for p in skip_patterns):
continue
filtered.append(img)
return filtered
class GenericExtractor(SiteExtractor):
"""通用兜底:不做站点特判,复刻旧逻辑的通用分支(含 SPA 判断)"""
name = "generic"
def match(self, url: str) -> bool:
return True
def to_markdown(self, content_html: str, url: str, images: list) -> str:
if is_spa_site(content_html):
return convert_spa_html_to_markdown(content_html, images, url)
return html_to_markdown(content_html, url)
# 策略注册表:有序列表,先匹配先命中;GenericExtractor 永远兜底
SITE_EXTRACTORS = [
JuejinExtractor(),
CsdnExtractor(),
GenericExtractor(),
]
def get_extractor_for_url(url: str) -> Optional[SiteExtractor]:
"""返回第一个匹配的策略;GenericExtractor 恒匹配(兜底)"""
for extractor in SITE_EXTRACTORS:
if extractor.match(url):
return extractor
return None
def is_strategy_first(url: str) -> bool:
"""url 是否命中试点白名单(新策略优先,失败回退旧逻辑)"""
return any(domain in url for domain in STRATEGY_FIRST_DOMAINS)
class ExtractorResult:
"""策略链输出,与旧逻辑 _legacy_extract 返回 (title, markdown, images) 同构"""
__slots__ = ("title", "markdown", "images")
def __init__(self, title: str, markdown: str, images: list):
self.title = title
self.markdown = markdown
self.images = images
def run_extractor(extractor: SiteExtractor, url: str, html: str, images: list) -> ExtractorResult:
"""执行策略链:专用标题 → 通用标题降级 → 图片过滤 → 正文容器 → Markdown"""
title = extractor.extract_title(html, url) or extract_title_from_html(html) or FALLBACK_TITLE
filtered_images = extractor.filter_images(images, url)
content_html = extractor.extract_content_html(html, url)
markdown = extractor.to_markdown(content_html, url, filtered_images)
return ExtractorResult(title=title, markdown=markdown, images=filtered_images)
def is_strategy_success(result: ExtractorResult, min_chars: int = MIN_CONTENT_CHARS) -> bool:
"""质量门控:标题有效 + Markdown 长度达标(复用现有「内容过短」经验值)"""
if not result.title or result.title == FALLBACK_TITLE:
return False
if len((result.markdown or "").strip()) < min_chars:
return False
return True