This commit is contained in:
Ubuntu
2026-07-08 16:22:55 +08:00
parent 065fde366a
commit 72a4318825
461 changed files with 88 additions and 101 deletions
+88 -13
View File
@@ -617,6 +617,21 @@ def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[C
return images
def _merge_images_from_markdown(existing_images: list, markdown_content: str, base_url: str) -> list:
"""从Markdown中提取图片并合并到已有列表,确保所有正文图片都被捕获"""
md_images = extract_images_from_markdown(markdown_content, base_url)
existing_urls = {img.original_url for img in existing_images}
new_count = 0
for img in md_images:
if img.original_url not in existing_urls:
existing_images.append(img)
existing_urls.add(img.original_url)
new_count += 1
if new_count > 0:
print(f"从Markdown中补充提取 {new_count} 张图片,总数: {len(existing_images)}")
return existing_images
def download_image(url: str, temp_dir: str) -> tuple:
"""下载图片到临时目录"""
try:
@@ -1366,16 +1381,15 @@ async def crawl_with_playwright(url: str) -> dict:
except:
continue
# 提取图片
# 提取图片(优先从正文区域提取,不限制数量)
images = []
parsed_url = urlparse(url)
url_is_https = parsed_url.scheme == 'https'
try:
img_elements = await page.locator('img').all()
for img in img_elements[:10]:
async def _extract_images_from_elements(elements, images_list):
for img in elements:
try:
src = await img.get_attribute('src')
alt = await img.get_attribute('alt') or ''
if src and not src.startswith('data:') and len(src) > 5:
# 转为绝对URL
if src.startswith('//'):
@@ -1387,9 +1401,31 @@ async def crawl_with_playwright(url: str) -> dict:
# 修复混合内容:页面HTTPS时图片也用HTTPS
if url_is_https and src.startswith('http://'):
src = 'https://' + src[7:]
images.append(CrawlImage(original_url=src))
images_list.append(CrawlImage(original_url=src))
except:
pass
try:
# 优先从正文区域提取图片(避免页头/页脚/广告等无关图片)
content_selectors = ['article', '.article-content', '.content', '.newstext', '.boxwrapmix', 'main']
content_container = None
for selector in content_selectors:
try:
content_container = await page.query_selector(selector)
if content_container:
break
except:
continue
if content_container:
content_imgs = await content_container.query_selector_all('img')
await _extract_images_from_elements(content_imgs, images)
else:
# 回退:从全页提取,不限制数量
img_elements = await page.locator('img').all()
await _extract_images_from_elements(img_elements, images)
print(f"从页面提取到 {len(images)} 张图片")
except:
pass
@@ -1558,9 +1594,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
is_wechat = 'mp.weixin.qq.com' in request.url
# 掘金文章特殊处理:requests返回的是混淆代码,必须用Playwright
is_juejin = 'juejin.cn' in request.url
if is_wechat or is_juejin:
site_name = "微信文章" if is_wechat else "掘金文章"
# eeworld JS WAF 防护:requests 会被直接断开连接,必须用 Playwright
is_eeworld = 'eeworld.com.cn' in request.url
if is_wechat or is_juejin or is_eeworld:
site_name = "微信文章" if is_wechat else ("掘金文章" if is_juejin else "EEPW文章")
print(f"检测到{site_name},使用Playwright采集: {request.url}")
if not PLAYWRIGHT_AVAILABLE:
@@ -1579,6 +1617,8 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
# 转换为Markdown
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
# 从Markdown中再提取一次图片(确保HTML中所有图片都被捕获)
images = _merge_images_from_markdown(images, markdown_content, request.url)
else:
# 第一步:尝试普通requests采集
@@ -1604,12 +1644,37 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
if response.status_code in (468, 403, 429, 503) and PLAYWRIGHT_AVAILABLE:
print(f"Requests被拦截({response.status_code}),尝试Playwright: {request.url}")
spa_result = await crawl_with_playwright(request.url)
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
return JSONResponse(
content={
"success": True,
"title": title,
"url": request.url,
"content": markdown_content,
"images": [{"original_url": img.original_url, "local_path": img.local_path, "filename": img.filename} for img in images]
},
headers=response_headers
)
else:
raise
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
# 连接被拒绝/超时(可能是JS WAF或反爬),回退到Playwright
if PLAYWRIGHT_AVAILABLE:
print(f"Requests连接失败({type(e).__name__}),尝试Playwright: {request.url}")
spa_result = await crawl_with_playwright(request.url)
title = spa_result['title']
images = spa_result['images']
content_html = spa_result['content_html']
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
return JSONResponse(
content={
"success": True,
@@ -1668,6 +1733,7 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
print(f"飞书Wiki: 内容已是Markdown,跳过HTML转换")
else:
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
else:
# 普通网站,使用原有逻辑
@@ -1696,6 +1762,7 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
content_html = spa_result['content_html']
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
images = _merge_images_from_markdown(images, markdown_content, request.url)
# 创建临时目录
temp_dir = "/tmp/blog_crawl"
@@ -1708,9 +1775,17 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
os.makedirs(upload_dir, exist_ok=True)
os.makedirs(thumb_dir, exist_ok=True)
# 下载图片并保存到数据库(最多30张)
# 下载图片并保存到数据库(不限数量,仅按出现顺序去重)
downloaded_images = []
for i, img in enumerate(images[:30]):
# 去重:保留首次出现的顺序
seen_urls = set()
unique_images = []
for img in images:
if img.original_url not in seen_urls:
seen_urls.add(img.original_url)
unique_images.append(img)
print(f"准备下载 {len(unique_images)} 张图片(原始 {len(images)},去重后)")
for i, img in enumerate(unique_images):
# 如果已经在 Playwright 里下载好了(如微信文章浏览器 fetch),直接复用
if img.success and img.local_path and os.path.exists(img.local_path):
filename = img.filename or os.path.basename(img.local_path)