a
This commit is contained in:
@@ -617,6 +617,21 @@ def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[C
|
||||
return images
|
||||
|
||||
|
||||
def _merge_images_from_markdown(existing_images: list, markdown_content: str, base_url: str) -> list:
|
||||
"""从Markdown中提取图片并合并到已有列表,确保所有正文图片都被捕获"""
|
||||
md_images = extract_images_from_markdown(markdown_content, base_url)
|
||||
existing_urls = {img.original_url for img in existing_images}
|
||||
new_count = 0
|
||||
for img in md_images:
|
||||
if img.original_url not in existing_urls:
|
||||
existing_images.append(img)
|
||||
existing_urls.add(img.original_url)
|
||||
new_count += 1
|
||||
if new_count > 0:
|
||||
print(f"从Markdown中补充提取 {new_count} 张图片,总数: {len(existing_images)}")
|
||||
return existing_images
|
||||
|
||||
|
||||
def download_image(url: str, temp_dir: str) -> tuple:
|
||||
"""下载图片到临时目录"""
|
||||
try:
|
||||
@@ -1366,16 +1381,15 @@ async def crawl_with_playwright(url: str) -> dict:
|
||||
except:
|
||||
continue
|
||||
|
||||
# 提取图片
|
||||
# 提取图片(优先从正文区域提取,不限制数量)
|
||||
images = []
|
||||
parsed_url = urlparse(url)
|
||||
url_is_https = parsed_url.scheme == 'https'
|
||||
try:
|
||||
img_elements = await page.locator('img').all()
|
||||
for img in img_elements[:10]:
|
||||
|
||||
async def _extract_images_from_elements(elements, images_list):
|
||||
for img in elements:
|
||||
try:
|
||||
src = await img.get_attribute('src')
|
||||
alt = await img.get_attribute('alt') or ''
|
||||
if src and not src.startswith('data:') and len(src) > 5:
|
||||
# 转为绝对URL
|
||||
if src.startswith('//'):
|
||||
@@ -1387,9 +1401,31 @@ async def crawl_with_playwright(url: str) -> dict:
|
||||
# 修复混合内容:页面HTTPS时图片也用HTTPS
|
||||
if url_is_https and src.startswith('http://'):
|
||||
src = 'https://' + src[7:]
|
||||
images.append(CrawlImage(original_url=src))
|
||||
images_list.append(CrawlImage(original_url=src))
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
# 优先从正文区域提取图片(避免页头/页脚/广告等无关图片)
|
||||
content_selectors = ['article', '.article-content', '.content', '.newstext', '.boxwrapmix', 'main']
|
||||
content_container = None
|
||||
for selector in content_selectors:
|
||||
try:
|
||||
content_container = await page.query_selector(selector)
|
||||
if content_container:
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
if content_container:
|
||||
content_imgs = await content_container.query_selector_all('img')
|
||||
await _extract_images_from_elements(content_imgs, images)
|
||||
else:
|
||||
# 回退:从全页提取,不限制数量
|
||||
img_elements = await page.locator('img').all()
|
||||
await _extract_images_from_elements(img_elements, images)
|
||||
|
||||
print(f"从页面提取到 {len(images)} 张图片")
|
||||
except:
|
||||
pass
|
||||
|
||||
@@ -1558,9 +1594,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
is_wechat = 'mp.weixin.qq.com' in request.url
|
||||
# 掘金文章特殊处理:requests返回的是混淆代码,必须用Playwright
|
||||
is_juejin = 'juejin.cn' in request.url
|
||||
|
||||
if is_wechat or is_juejin:
|
||||
site_name = "微信文章" if is_wechat else "掘金文章"
|
||||
# eeworld JS WAF 防护:requests 会被直接断开连接,必须用 Playwright
|
||||
is_eeworld = 'eeworld.com.cn' in request.url
|
||||
|
||||
if is_wechat or is_juejin or is_eeworld:
|
||||
site_name = "微信文章" if is_wechat else ("掘金文章" if is_juejin else "EEPW文章")
|
||||
print(f"检测到{site_name},使用Playwright采集: {request.url}")
|
||||
|
||||
if not PLAYWRIGHT_AVAILABLE:
|
||||
@@ -1579,6 +1617,8 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
|
||||
# 转换为Markdown
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
# 从Markdown中再提取一次图片(确保HTML中所有图片都被捕获)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
else:
|
||||
# 第一步:尝试普通requests采集
|
||||
@@ -1604,12 +1644,37 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
if response.status_code in (468, 403, 429, 503) and PLAYWRIGHT_AVAILABLE:
|
||||
print(f"Requests被拦截({response.status_code}),尝试Playwright: {request.url}")
|
||||
spa_result = await crawl_with_playwright(request.url)
|
||||
|
||||
|
||||
title = spa_result['title']
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
return JSONResponse(
|
||||
content={
|
||||
"success": True,
|
||||
"title": title,
|
||||
"url": request.url,
|
||||
"content": markdown_content,
|
||||
"images": [{"original_url": img.original_url, "local_path": img.local_path, "filename": img.filename} for img in images]
|
||||
},
|
||||
headers=response_headers
|
||||
)
|
||||
else:
|
||||
raise
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
|
||||
# 连接被拒绝/超时(可能是JS WAF或反爬),回退到Playwright
|
||||
if PLAYWRIGHT_AVAILABLE:
|
||||
print(f"Requests连接失败({type(e).__name__}),尝试Playwright: {request.url}")
|
||||
spa_result = await crawl_with_playwright(request.url)
|
||||
|
||||
title = spa_result['title']
|
||||
images = spa_result['images']
|
||||
content_html = spa_result['content_html']
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
return JSONResponse(
|
||||
content={
|
||||
"success": True,
|
||||
@@ -1668,6 +1733,7 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
print(f"飞书Wiki: 内容已是Markdown,跳过HTML转换")
|
||||
else:
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
else:
|
||||
# 普通网站,使用原有逻辑
|
||||
@@ -1696,6 +1762,7 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
content_html = spa_result['content_html']
|
||||
|
||||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||||
images = _merge_images_from_markdown(images, markdown_content, request.url)
|
||||
|
||||
# 创建临时目录
|
||||
temp_dir = "/tmp/blog_crawl"
|
||||
@@ -1708,9 +1775,17 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
os.makedirs(thumb_dir, exist_ok=True)
|
||||
|
||||
# 下载图片并保存到数据库(最多30张)
|
||||
# 下载图片并保存到数据库(不限数量,仅按出现顺序去重)
|
||||
downloaded_images = []
|
||||
for i, img in enumerate(images[:30]):
|
||||
# 去重:保留首次出现的顺序
|
||||
seen_urls = set()
|
||||
unique_images = []
|
||||
for img in images:
|
||||
if img.original_url not in seen_urls:
|
||||
seen_urls.add(img.original_url)
|
||||
unique_images.append(img)
|
||||
print(f"准备下载 {len(unique_images)} 张图片(原始 {len(images)},去重后)")
|
||||
for i, img in enumerate(unique_images):
|
||||
# 如果已经在 Playwright 里下载好了(如微信文章浏览器 fetch),直接复用
|
||||
if img.success and img.local_path and os.path.exists(img.local_path):
|
||||
filename = img.filename or os.path.basename(img.local_path)
|
||||
|
||||
Reference in New Issue
Block a user