',
'You need to enable JavaScript',
# 加载中标记
'Please wait',
'Loading...',
# 常见SPA框架
'react-root',
'ng-app',
'data-reactroot',
# 动态加载标记
'window.__INITIAL_STATE__',
'window.__DATA__',
]
html_lower = html.lower()
for indicator in spa_indicators:
if indicator.lower() in html_lower:
return True
# 检查内容是否极少(只有JS没有实际内容)
text_content = re.sub(r'<[^>]+>', '', html)
text_content = re.sub(r'\s+', '', text_content)
if len(text_content) < 200:
return True
return False
async def crawl_with_playwright(url: str) -> dict:
"""使用Playwright异步采集SPA页面"""
if not PLAYWRIGHT_AVAILABLE:
raise Exception("Playwright未安装,无法采集SPA网站")
# 检测平台类型
is_wechat = 'mp.weixin.qq.com' in url
is_tencent_cloud = 'cloud.tencent.com' in url
async with async_playwright() as p:
# 使用 stealth 参数启动浏览器,绕过WAF
browser = await p.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled',
'--disable-web-security',
'--disable-features=IsolateOrigins,site-per-process',
'--disable-dev-shm-usage',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--window-size=1920,1080',
]
)
# 创建新页面并设置视口和更多参数
context = await browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
locale='zh-CN',
timezone_id='Asia/Shanghai',
permissions=['geolocation'],
java_script_enabled=True,
)
# 添加额外脚本隐藏 webdriver 属性
await context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
});
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5]
});
window.chrome = { runtime: {} };
""")
page = await context.new_page()
# 应用 stealth 模式(绕过反爬虫检测)
if STEALTH_AVAILABLE:
await stealth_async(page)
print("[Playwright] Stealth 模式已启用")
# 打开页面
if is_tencent_cloud:
# 腾讯云有大量广告/跟踪脚本,networkidle 容易超时,改用 domcontentloaded
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
else:
await page.goto(url, wait_until='networkidle', timeout=30000)
# 微信文章特殊处理
if is_wechat:
await page.wait_for_timeout(5000) # 微信需要更长等待时间
# 提取标题 - 微信文章标题在 #activity_name
title = ""
try:
if await page.locator('#activity_name').count() > 0:
title = await page.locator('#activity_name').first.inner_text()
title = title.strip()
except:
pass
if not title:
title = await page.title()
# 提取正文 - 微信文章正文在 #js_content
content_html = ""
try:
if await page.locator('#js_content').count() > 0:
content_html = await page.locator('#js_content').first.inner_html()
except:
pass
# 提取图片 - 微信图片用data-src,优先取浏览器渲染后的实际URL
images = []
try:
img_elements = await page.locator('#js_content img').all()
for img in img_elements[:30]:
try:
# 优先级:dataset.src > currentSrc > data-src > src
src = None
# 1. 尝试通过 JS 获取 dataset.src(微信懒加载后的真实地址)
try:
src = await img.evaluate('el => el.dataset?.src || ""')
except:
pass
# 2. 尝试 currentSrc(浏览器实际加载的地址)
if not src or src.startswith('data:'):
try:
src = await img.evaluate('el => el.currentSrc || ""')
except:
pass
# 3. 回退到 data-src 属性
if not src or src.startswith('data:'):
src = await img.get_attribute('data-src')
# 4. 最后回退到 src 属性
if not src or src.startswith('data:'):
src = await img.get_attribute('src')
alt = await img.get_attribute('alt') or ''
# 过滤 data URL 和无效地址
if src and src.startswith('http') and not src.startswith('data:'):
images.append(CrawlImage(original_url=src))
except:
pass
except:
pass
# 微信图片防盗链:用浏览器 fetch 下载(携带 cookies)
temp_dir = os.environ.get('TEMP_DIR') or '/tmp'
downloaded_images = []
for crawl_img in images[:15]:
try:
img_url = crawl_img.original_url
# 在浏览器里 fetch 下载,携带微信 cookies
js_code = f"""
async () => {{
try {{
const resp = await fetch("{img_url}", {{ referrer: "https://mp.weixin.qq.com/" }});
if (!resp.ok) return null;
const blob = await resp.blob();
return new Promise((resolve) => {{
const reader = new FileReader();
reader.onloadend = () => resolve(reader.result.split(',')[1]);
reader.readAsDataURL(blob);
}});
}} catch (e) {{
return null;
}}
}}
"""
b64_data = await page.evaluate(js_code)
if b64_data:
# 确定扩展名
ext = os.path.splitext(urlparse(img_url).path)[1].lower() or '.jpg'
if not ext or ext == '.':
ext = '.jpg'
filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}"
filepath = os.path.join(temp_dir, filename)
with open(filepath, 'wb') as f:
f.write(base64.b64decode(b64_data))
# 检查文件大小,排除防盗链占位图
file_size = os.path.getsize(filepath)
if file_size < 2048:
os.remove(filepath)
print(f"微信图片疑似防盗链占位图,跳过: {img_url} ({file_size} bytes)")
downloaded_images.append(crawl_img)
else:
crawl_img.local_path = filepath
crawl_img.filename = filename
crawl_img.success = True
downloaded_images.append(crawl_img)
print(f"微信图片下载成功: {filename} ({file_size} bytes)")
else:
downloaded_images.append(crawl_img)
except Exception as e:
print(f"浏览器下载微信图片失败: {e}")
downloaded_images.append(crawl_img)
images = downloaded_images
await context.close()
await browser.close()
return {
'title': title if title else "未命名文章",
'content_html': content_html,
'images': images
}
# 普通SPA网站处理
# 检测平台(is_tencent_cloud 已在函数开头定义)
is_juejin = 'juejin.cn' in url
is_segmentfault = 'segmentfault.com' in url
# 等待内容加载
if is_juejin:
# 掘金需要等待article标签
try:
await page.wait_for_selector('article', timeout=15000)
except:
pass
await page.wait_for_timeout(3000)
elif is_tencent_cloud:
# 腾讯云开发者社区:.mod-content__markdown 一开始存在但为空,需等待 JS 填充
try:
await page.wait_for_function(
"""() => {
const el = document.querySelector('.mod-content__markdown');
return el && el.innerHTML.length > 500;
}""",
timeout=15000
)
except:
pass
await page.wait_for_timeout(2000)
elif is_segmentfault:
# SegmentFault 等待文章正文
print("[Playwright] 等待 SegmentFault 页面加载...")
try:
await page.wait_for_load_state('networkidle', timeout=20000)
except:
pass
await page.wait_for_timeout(8000) # 给JS更多时间渲染
print("[Playwright] 等待完成,开始提取")
else:
try:
await page.wait_for_selector('h1, .article-title, .content, article, main', timeout=10000)
except:
pass
await page.wait_for_timeout(5000)
# 提取标题
title = ""
if is_juejin:
# 掘金标题选择器
for selector in ['article h1', 'h1']:
try:
if await page.locator(selector).count() > 0:
title = await page.locator(selector).first.inner_text()
if title.strip():
break
except:
continue
elif is_segmentfault:
# SegmentFault 标题选择器
sf_title_selectors = ['h1.article-title', 'h1', '.article-title', '[class*="title"]', 'title']
for selector in sf_title_selectors:
try:
count = await page.locator(selector).count()
if count > 0:
t = await page.locator(selector).first.inner_text()
print(f"[Playwright] 标题选择器 '{selector}': '{t[:30] if t else '空'}...'")
if t and t.strip():
title = t.strip()
break
except Exception as e:
print(f"[Playwright] 标题选择器 '{selector}' 失败: {e}")
elif is_tencent_cloud:
# 腾讯云开发者社区标题
for selector in ['h1', '.article-title', '[class*="title"]']:
try:
if await page.locator(selector).count() > 0:
title = await page.locator(selector).first.inner_text()
if title.strip():
break
except:
continue
if not title:
for selector in ['h1', '.article-title', '[class*="title"]']:
try:
if await page.locator(selector).count() > 0:
title = await page.locator(selector).first.inner_text()
if title.strip():
break
except:
continue
if not title:
title = await page.title()
# 提取正文HTML
content_html = ""
print(f"[Playwright] 开始提取内容,URL: {url}")
print(f"[Playwright] is_segmentfault: {is_segmentfault}")
if is_juejin:
# 掘金内容在 .markdown-body 中
try:
print(f"[Playwright] 掘金: 检查 .markdown-body")
mb_count = await page.locator('.markdown-body').count()
print(f"[Playwright] 掘金: .markdown-body 数量: {mb_count}")
if mb_count > 0:
content_html = await page.locator('.markdown-body').first.inner_html()
print(f"[Playwright] 掘金: 从 .markdown-body 提取内容,长度 {len(content_html)}")
elif await page.locator('article').count() > 0:
content_html = await page.locator('article').first.inner_html()
print(f"[Playwright] 掘金: 从 article 提取内容,长度 {len(content_html)}")
else:
# 诊断:截图和保存页面源码
print(f"[Playwright] 掘金: 未找到内容区域,保存诊断信息")
try:
await page.screenshot(path='/tmp/juejin_debug.png', full_page=True)
page_source = await page.content()
with open('/tmp/juejin_debug.html', 'w', encoding='utf-8') as f:
f.write(page_source[:50000]) # 保存前50KB
print(f"[Playwright] 掘金: 诊断信息已保存到 /tmp/juejin_debug.*")
except Exception as e:
print(f"[Playwright] 掘金: 保存诊断信息失败: {e}")
except Exception as e:
print(f"[Playwright] 掘金: 提取内容异常: {e}")
# 清理掘金内容:移除 style 和 script 标签(掘金有大量内嵌CSS)
if content_html:
import re
# 移除 style 标签及其内容
content_html = re.sub(r'', '', content_html, flags=re.DOTALL | re.IGNORECASE)
# 移除 script 标签及其内容
content_html = re.sub(r'', '', content_html, flags=re.DOTALL | re.IGNORECASE)
# 移除所有元素的 style 属性
content_html = re.sub(r'\s+style="[^"]*"', '', content_html, flags=re.IGNORECASE)
# 移除 class 属性(减少无用属性)
content_html = re.sub(r'\s+class="[^"]*"', '', content_html, flags=re.IGNORECASE)
elif is_segmentfault:
# SegmentFault 内容提取 - 尝试多种选择器
sf_selectors = ['.article-content', '.markdown-body', 'article', '.post-content',
'[data-id]', 'main', '#content', '.content']
for selector in sf_selectors:
try:
count = await page.locator(selector).count()
print(f"[Playwright] 检查选择器 '{selector}': {count} 个匹配")
if count > 0:
html = await page.locator(selector).first.inner_html()
if len(html) > 200: # 确保内容足够长
content_html = html
print(f"[Playwright] 使用 '{selector}' 提取,长度: {len(content_html)}")
break
except Exception as e:
print(f"[Playwright] 选择器 '{selector}' 失败: {e}")
if not content_html:
print(f"[Playwright] 未找到 SegmentFault 内容,尝试获取整个body")
try:
content_html = await page.locator('body').first.inner_html()
print(f"[Playwright] 使用 body,长度: {len(content_html)}")
except Exception as e:
print(f"[Playwright] 获取 body 失败: {e}")
# 清理 SegmentFault 内容
if content_html:
import re
# 移除 style 和 script 标签
content_html = re.sub(r'', '', content_html, flags=re.DOTALL | re.IGNORECASE)
content_html = re.sub(r'', '', content_html, flags=re.DOTALL | re.IGNORECASE)
# 移除广告相关元素
content_html = re.sub(r'
]*class="[^"]*ad[^"]*"[^>]*>.*?
', '', content_html, flags=re.DOTALL | re.IGNORECASE)
elif is_tencent_cloud:
# 腾讯云开发者社区内容
try:
# 先尝试 .mod-content__markdown(JS 渲染后会有内容)
if await page.locator('.mod-content__markdown').count() > 0:
html = await page.locator('.mod-content__markdown').first.inner_html()
if len(html) > 500:
content_html = html
# 如果为空,回退到 .mod-article-content
if not content_html or len(content_html) < 500:
if await page.locator('.mod-article-content').count() > 0:
html = await page.locator('.mod-article-content').first.inner_html()
if len(html) > 500:
content_html = html
except Exception as e:
print(f"[Playwright] 腾讯云内容提取异常: {e}")
if not content_html:
selectors = ['article', '.article-content', '.content', '.post-content',
'.detail-content', '.news-content', 'main', '[class*="content"]']
for selector in selectors:
try:
if await page.locator(selector).count() > 0:
content_html = await page.locator(selector).first.inner_html()
if len(content_html) > 500:
break
except:
continue
# 提取图片
images = []
parsed_url = urlparse(url)
url_is_https = parsed_url.scheme == 'https'
try:
img_elements = await page.locator('img').all()
for img in img_elements[:10]:
try:
src = await img.get_attribute('src')
alt = await img.get_attribute('alt') or ''
if src and not src.startswith('data:') and len(src) > 5:
# 转为绝对URL
if src.startswith('//'):
src = 'https:' + src
elif src.startswith('/'):
src = f"{parsed_url.scheme}://{parsed_url.netloc}{src}"
elif not src.startswith(('http://', 'https://')):
src = urljoin(url, src)
# 修复混合内容:页面HTTPS时图片也用HTTPS
if url_is_https and src.startswith('http://'):
src = 'https://' + src[7:]
images.append(CrawlImage(original_url=src))
except:
pass
except:
pass
await context.close()
await browser.close()
# 修复编码问题
print(f"[Playwright] 编码修复前标题: {repr(title[:50] if title else '空')}")
print(f"[Playwright] 编码修复前内容前200: {repr(content_html[:200] if content_html else '空')}")
title = fix_encoding(title) if title else title
content_html = fix_encoding(content_html) if content_html else content_html
print(f"[Playwright] 编码修复后标题: {repr(title[:50] if title else '空')}")
print(f"[Playwright] 返回结果 - 内容长度: {len(content_html) if content_html else 0}")
return {
'title': title.strip() if title else "未命名文章",
'content_html': content_html,
'images': images
}
def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url: str) -> str:
"""将SPA采集的HTML转为Markdown"""
# 修复编码问题
html = fix_encoding(html)
# 先清理 style 和 script 标签
text = re.sub(r'', '', html, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'', '', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'\s+style="[^"]*"', '', text, flags=re.IGNORECASE)
text = re.sub(r'\s+class="[^"]*"', '', text, flags=re.IGNORECASE)
parsed_base = urlparse(base_url)
base_is_https = parsed_base.scheme == 'https'
is_wechat = 'mp.weixin.qq.com' in base_url
# 首先将img标签转为Markdown
def img_to_markdown(match):
src = match.group(1)
alt = match.group(2) if match.lastindex and match.lastindex >= 2 else ""
# 过滤 data URL
if src.startswith('data:'):
return ''
# 处理URL
if src.startswith('//'):
src = 'https:' + src
elif src.startswith('/'):
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
# 修复混合内容:页面HTTPS时图片也用HTTPS
if base_is_https and src.startswith('http://'):
src = 'https://' + src[7:]
return f''
# 微信文章特殊处理:图片使用data-src,过滤data URL
if is_wechat:
def wechat_img_replace(m):
src = m.group(1)
if src.startswith('data:'):
return ''
return f''
text = re.sub(
r'
![]()
]+data-src=["\']([^"\']+)["\'][^>]*>',
wechat_img_replace,
text, flags=re.IGNORECASE
)
# 匹配普通img标签
text = re.sub(
r'
![]()
]+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>',
img_to_markdown, text, flags=re.IGNORECASE
)
# 匹配没有alt的img标签 - 需要处理URL转换!
def img_to_markdown_no_alt(match):
src = match.group(1)
# 过滤 data URL
if src.startswith('data:'):
return ''
# 处理URL(与img_to_markdown保持一致)
if src.startswith('//'):
src = 'https:' + src
elif src.startswith('/'):
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
# 修复混合内容
if base_is_https and src.startswith('http://'):
src = 'https://' + src[7:]
return f''
text = re.sub(
r'
![]()
]+src=["\']([^"\']+)["\'][^>]*>',
img_to_markdown_no_alt,
text, flags=re.IGNORECASE
)
# 处理代码块
...
- 必须在其他标签之前处理
def convert_code_block(match):
code_content = match.group(1)
# 提取语言
lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE)
lang = lang_match.group(1) if lang_match else ''
return f'\n```{lang}\n{code_content}\n```\n'
text = re.sub(r'
]*>\s*]*>(.*?)\s*
', convert_code_block,
text, flags=re.DOTALL | re.IGNORECASE)
# 处理纯pre标签
text = re.sub(r'
]*>(.*?)
', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE)
# 处理行内代码
text = re.sub(r'
]*>(.*?)', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
# 转换其他标签
text = re.sub(r'
]*>(.*?)
', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
]*>(.*?)
', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
]*>(.*?)
', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
]*>(.*?)
', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
]*>(.*?)
', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
', '\n', text, flags=re.IGNORECASE)
text = re.sub(r'
]*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
]*>(.*?)', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
]*>(.*?)', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'
]*>(.*?)', r'- \1\n', text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'<[^>]+>', '', text)
# 解码HTML实体
text = text.replace(' ', ' ').replace('<', '<').replace('>', '>')
text = text.replace('&', '&').replace('"', '"')
# 修复数学公式格式(掘金等平台)
text = fix_math_formulas(text)
# 清理空行
lines = [line for line in text.split('\n')]
cleaned = []
prev_empty = False
for line in lines:
stripped = line.strip()
if stripped == '':
if not prev_empty:
cleaned.append('')
prev_empty = True
else:
cleaned.append(line)
prev_empty = False
return '\n'.join(cleaned).strip()
@router.post("")
async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)):
"""
采集文章(自动检测SPA并使用Playwright)
"""
# 设置响应头,防止缓存和确保正确编码
response_headers = {
"Cache-Control": "no-cache, no-store, must-revalidate",
"Pragma": "no-cache",
"Expires": "0",
"Content-Type": "application/json; charset=utf-8"
}
try:
# 验证URL
if not request.url.startswith(('http://', 'https://')):
raise HTTPException(status_code=400, detail="无效的URL")
# 微信文章特殊处理:直接用Playwright(requests会被拦截)
is_wechat = 'mp.weixin.qq.com' in request.url
# 掘金文章特殊处理:requests返回的是混淆代码,必须用Playwright
is_juejin = 'juejin.cn' in request.url
if is_wechat or is_juejin:
site_name = "微信文章" if is_wechat else "掘金文章"
print(f"检测到{site_name},使用Playwright采集: {request.url}")
if not PLAYWRIGHT_AVAILABLE:
error_msg = f"{site_name}需要Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium"
return JSONResponse(
content={"success": False, "error": error_msg},
headers=response_headers
)
# 使用Playwright采集
spa_result = await crawl_with_playwright(request.url)
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
# 转换为Markdown
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
else:
# 第一步:尝试普通requests采集
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
}
try:
response = requests.get(request.url, headers=headers, timeout=30)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
# 遇到反爬虫拦截(468/403/429),尝试使用Playwright
if response.status_code in (468, 403, 429, 503) and PLAYWRIGHT_AVAILABLE:
print(f"Requests被拦截({response.status_code}),尝试Playwright: {request.url}")
spa_result = await crawl_with_playwright(request.url)
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
return JSONResponse(
content={
"success": True,
"title": title,
"url": request.url,
"content": markdown_content,
"images": [{"original_url": img.original_url, "local_path": img.local_path, "filename": img.filename} for img in images]
},
headers=response_headers
)
else:
raise
# 智能编码处理:优先使用声明的编码,而不是猜测
# 1. 先检查HTTP响应头中的编码
declared_encoding = None
if response.encoding:
declared_encoding = response.encoding
# 2. 如果响应头没有明确编码,从HTML meta标签中提取
if not declared_encoding or declared_encoding.lower() in ['iso-8859-1', 'latin-1']:
# 尝试从HTML中提取charset
raw_content = response.content
charset_match = re.search(rb'
]+charset=["\']?([^"\'>\s]+)', raw_content, re.IGNORECASE)
if charset_match:
declared_encoding = charset_match.group(1).decode('ascii', errors='ignore')
# 3. 使用声明的编码,如果没有才用 apparent_encoding 猜测
if declared_encoding and declared_encoding.lower() not in ['iso-8859-1', 'latin-1']:
response.encoding = declared_encoding
else:
response.encoding = response.apparent_encoding or 'utf-8'
html = response.text
# 检测是否为SPA
if is_spa_site(html):
print(f"检测到SPA网站,使用Playwright采集: {request.url}")
if not PLAYWRIGHT_AVAILABLE:
return JSONResponse(
content={"success": False, "error": "该网站为SPA单页应用,需要安装Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium"},
headers=response_headers
)
# 使用Playwright采集
spa_result = await crawl_with_playwright(request.url)
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
# 转换HTML为Markdown
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
else:
# 普通网站,使用原有逻辑
print(f"普通网站,使用requests采集: {request.url}")
# 提取标题
title = extract_title_from_html(html)
# 提取主要内容
main_content = extract_main_content(html)
# 转换HTML为Markdown(必须先执行,用于图片提取)
markdown_content = html_to_markdown(main_content, request.url)
# 从Markdown内容中提取图片(确保只提取实际在文章中的图片)
images = extract_images_from_markdown(markdown_content, request.url)
# 创建临时目录
temp_dir = "/tmp/blog_crawl"
os.makedirs(temp_dir, exist_ok=True)
# uploads 目录配置
backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
upload_dir = os.path.join(backend_dir, "uploads")
thumb_dir = os.path.join(upload_dir, "thumbnails")
os.makedirs(upload_dir, exist_ok=True)
os.makedirs(thumb_dir, exist_ok=True)
# 下载图片并保存到数据库(最多30张)
downloaded_images = []
for i, img in enumerate(images[:30]):
# 如果已经在 Playwright 里下载好了(如微信文章浏览器 fetch),直接复用
if img.success and img.local_path and os.path.exists(img.local_path):
filename = img.filename or os.path.basename(img.local_path)
filepath = img.local_path
success = True
print(f"复用已下载的图片: {filename}")
else:
filename, filepath, success = download_image(img.original_url, temp_dir)
if success:
try:
# 生成新的文件名(避免冲突)
ext = os.path.splitext(filename)[1].lower()
if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']:
ext = '.jpg'
new_filename = f"crawl_{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}"
# 移动文件到 uploads 目录
new_filepath = os.path.join(upload_dir, new_filename)
with open(filepath, 'rb') as src:
with open(new_filepath, 'wb') as dst:
dst.write(src.read())
# 删除临时文件
os.remove(filepath)
# 生成缩略图
thumb_filename = f"thumb_{new_filename}"
thumb_filepath = os.path.join(thumb_dir, thumb_filename)
thumb_path_db = None
if generate_thumbnail(new_filepath, thumb_filepath):
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
# 获取图片尺寸
width, height = get_image_dimensions(new_filepath)
# 获取文件大小
file_size = os.path.getsize(new_filepath)
# 保存到数据库 images 表
image_record = Image(
filename=new_filename,
original_name=filename,
file_path=f"/uploads/{new_filename}",
thumb_path=thumb_path_db,
description=f"采集自 {request.url}",
file_size=file_size,
width=width,
height=height
)
db.add(image_record)
db.commit()
db.refresh(image_record)
# 更新返回的图片信息
img.local_path = f"/uploads/{new_filename}"
img.filename = new_filename
img.success = True
img.id = image_record.id # 保存数据库ID供前端使用
downloaded_images.append(img)
# 将 content 中的原始URL替换为本地路径
# 微信图片URL可能带 #imgIndex=xx 锚点,去掉后再匹配
url_for_match = img.original_url.split('#')[0]
if url_for_match in markdown_content:
markdown_content = markdown_content.replace(
url_for_match,
f"/uploads/{new_filename}"
)
print(f"✅ 已将内容中的图片URL替换: {url_for_match[:60]}... -> /uploads/{new_filename}")
print(f"图片已保存到数据库: id={image_record.id}, filename={new_filename}")
except Exception as e:
print(f"保存图片到数据库失败: {e}")
# 如果数据库保存失败,仍然保留临时文件信息
img.local_path = filepath
img.filename = filename
img.success = True
downloaded_images.append(img)
# 生成摘要(前200字符)
summary = markdown_content[:200].replace('#', '').replace('*', '').strip()
# 序列化图片数据
images_data = []
for img in downloaded_images:
img_dict = {"original_url": img.original_url}
if hasattr(img, 'local_path') and img.local_path:
img_dict["local_path"] = img.local_path
if hasattr(img, 'filename') and img.filename:
img_dict["filename"] = img.filename
if hasattr(img, 'id') and img.id:
img_dict["id"] = img.id
images_data.append(img_dict)
return JSONResponse(
content={
"success": True,
"title": title,
"content": markdown_content,
"summary": summary,
"images": images_data
},
headers=response_headers
)
except requests.RequestException as e:
return JSONResponse(
content={"success": False, "error": f"网络请求失败: {str(e)}"},
headers=response_headers
)
except Exception as e:
import traceback
print(f"采集失败: {e}")
print(traceback.format_exc())
return JSONResponse(
content={"success": False, "error": f"采集失败: {str(e)}"},
headers=response_headers
)