a

2026-07-08 16:22:55 +08:00
parent 065fde366a
commit 72a4318825
461 changed files with 88 additions and 101 deletions
@@ -617,6 +617,21 @@ def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[C
    return images


+def _merge_images_from_markdown(existing_images: list, markdown_content: str, base_url: str) -> list:
+    """从Markdown中提取图片并合并到已有列表，确保所有正文图片都被捕获"""
+    md_images = extract_images_from_markdown(markdown_content, base_url)
+    existing_urls = {img.original_url for img in existing_images}
+    new_count = 0
+    for img in md_images:
+        if img.original_url not in existing_urls:
+            existing_images.append(img)
+            existing_urls.add(img.original_url)
+            new_count += 1
+    if new_count > 0:
+        print(f"从Markdown中补充提取 {new_count} 张图片，总数: {len(existing_images)}")
+    return existing_images
+
+
 def download_image(url: str, temp_dir: str) -> tuple:
    """下载图片到临时目录"""
    try:
@@ -1366,16 +1381,15 @@ async def crawl_with_playwright(url: str) -> dict:
                except:
                    continue
        
-        # 提取图片
+        # 提取图片（优先从正文区域提取，不限制数量）
        images = []
        parsed_url = urlparse(url)
        url_is_https = parsed_url.scheme == 'https'
-        try:
-            img_elements = await page.locator('img').all()
-            for img in img_elements[:10]:
+
+        async def _extract_images_from_elements(elements, images_list):
+            for img in elements:
                try:
                    src = await img.get_attribute('src')
-                    alt = await img.get_attribute('alt') or ''
                    if src and not src.startswith('data:') and len(src) > 5:
                        # 转为绝对URL
                        if src.startswith('//'):
@@ -1387,9 +1401,31 @@ async def crawl_with_playwright(url: str) -> dict:
                        # 修复混合内容：页面HTTPS时图片也用HTTPS
                        if url_is_https and src.startswith('http://'):
                            src = 'https://' + src[7:]
-                        images.append(CrawlImage(original_url=src))
+                        images_list.append(CrawlImage(original_url=src))
                except:
                    pass
+
+        try:
+            # 优先从正文区域提取图片（避免页头/页脚/广告等无关图片）
+            content_selectors = ['article', '.article-content', '.content', '.newstext', '.boxwrapmix', 'main']
+            content_container = None
+            for selector in content_selectors:
+                try:
+                    content_container = await page.query_selector(selector)
+                    if content_container:
+                        break
+                except:
+                    continue
+
+            if content_container:
+                content_imgs = await content_container.query_selector_all('img')
+                await _extract_images_from_elements(content_imgs, images)
+            else:
+                # 回退：从全页提取，不限制数量
+                img_elements = await page.locator('img').all()
+                await _extract_images_from_elements(img_elements, images)
+
+            print(f"从页面提取到 {len(images)} 张图片")
        except:
            pass
        
@@ -1558,9 +1594,11 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
        is_wechat = 'mp.weixin.qq.com' in request.url
        # 掘金文章特殊处理：requests返回的是混淆代码，必须用Playwright
        is_juejin = 'juejin.cn' in request.url
-        
-        if is_wechat or is_juejin:
-            site_name = "微信文章" if is_wechat else "掘金文章"
+        # eeworld JS WAF 防护：requests 会被直接断开连接，必须用 Playwright
+        is_eeworld = 'eeworld.com.cn' in request.url
+
+        if is_wechat or is_juejin or is_eeworld:
+            site_name = "微信文章" if is_wechat else ("掘金文章" if is_juejin else "EEPW文章")
            print(f"检测到{site_name}，使用Playwright采集: {request.url}")
            
            if not PLAYWRIGHT_AVAILABLE:
@@ -1579,6 +1617,8 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
            
            # 转换为Markdown
            markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+            # 从Markdown中再提取一次图片（确保HTML中所有图片都被捕获）
+            images = _merge_images_from_markdown(images, markdown_content, request.url)
            
        else:
            # 第一步：尝试普通requests采集
@@ -1604,12 +1644,37 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
                if response.status_code in (468, 403, 429, 503) and PLAYWRIGHT_AVAILABLE:
                    print(f"Requests被拦截({response.status_code})，尝试Playwright: {request.url}")
                    spa_result = await crawl_with_playwright(request.url)
-                    
+
                    title = spa_result['title']
                    images = spa_result['images']
                    content_html = spa_result['content_html']
                    markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
-                    
+                    images = _merge_images_from_markdown(images, markdown_content, request.url)
+
+                    return JSONResponse(
+                        content={
+                            "success": True,
+                            "title": title,
+                            "url": request.url,
+                            "content": markdown_content,
+                            "images": [{"original_url": img.original_url, "local_path": img.local_path, "filename": img.filename} for img in images]
+                        },
+                        headers=response_headers
+                    )
+                else:
+                    raise
+            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+                # 连接被拒绝/超时（可能是JS WAF或反爬），回退到Playwright
+                if PLAYWRIGHT_AVAILABLE:
+                    print(f"Requests连接失败({type(e).__name__})，尝试Playwright: {request.url}")
+                    spa_result = await crawl_with_playwright(request.url)
+
+                    title = spa_result['title']
+                    images = spa_result['images']
+                    content_html = spa_result['content_html']
+                    markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+                    images = _merge_images_from_markdown(images, markdown_content, request.url)
+
                    return JSONResponse(
                        content={
                            "success": True,
@@ -1668,6 +1733,7 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
                    print(f"飞书Wiki: 内容已是Markdown，跳过HTML转换")
                else:
                    markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+                images = _merge_images_from_markdown(images, markdown_content, request.url)
                
            else:
                # 普通网站，使用原有逻辑
@@ -1696,6 +1762,7 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
                    content_html = spa_result['content_html']
                    
                    markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
+                    images = _merge_images_from_markdown(images, markdown_content, request.url)
        
        # 创建临时目录
        temp_dir = "/tmp/blog_crawl"
@@ -1708,9 +1775,17 @@ async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), cu
        os.makedirs(upload_dir, exist_ok=True)
        os.makedirs(thumb_dir, exist_ok=True)
        
-        # 下载图片并保存到数据库（最多30张）
+        # 下载图片并保存到数据库（不限数量，仅按出现顺序去重）
        downloaded_images = []
-        for i, img in enumerate(images[:30]):
+        # 去重：保留首次出现的顺序
+        seen_urls = set()
+        unique_images = []
+        for img in images:
+            if img.original_url not in seen_urls:
+                seen_urls.add(img.original_url)
+                unique_images.append(img)
+        print(f"准备下载 {len(unique_images)} 张图片（原始 {len(images)}，去重后）")
+        for i, img in enumerate(unique_images):
            # 如果已经在 Playwright 里下载好了（如微信文章浏览器 fetch），直接复用
            if img.success and img.local_path and os.path.exists(img.local_path):
                filename = img.filename or os.path.basename(img.local_path)
@@ -1,15 +0,0 @@
-<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
-<rect width="40" height="40" fill="#E4E6EB" fill-opacity="0.6"/>
-<mask id="mask0_1389_88363" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="0" y="0" width="40" height="40">
-<rect width="40" height="40" fill="#E4E6EB"/>
-</mask>
-<g mask="url(#mask0_1389_88363)">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M22.5809 24.2241C24.6029 22.9376 26 20.1863 26 17C26 12.5817 23.3137 9 20 9C16.6863 9 14 12.5817 14 17C14 20.1863 15.3971 22.9376 17.4191 24.2241C16.8885 25.3643 15.969 26.7758 14.5 28C13.565 28.7792 12.5815 29.2183 11.5796 29.6657C9.36694 30.6537 7.06502 31.6816 5 36.5V46H20H35V36.5C32.935 31.6816 30.6331 30.6537 28.4204 29.6657C27.4185 29.2183 26.435 28.7792 25.5 28C24.031 26.7758 23.1115 25.3643 22.5809 24.2241Z" fill="url(#paint0_linear_1389_88363)"/>
-</g>
-<defs>
-<linearGradient id="paint0_linear_1389_88363" x1="20" y1="9" x2="20" y2="46" gradientUnits="userSpaceOnUse">
-<stop stop-color="#C2C8D1"/>
-<stop offset="1" stop-color="#C2C8D1" stop-opacity="0"/>
-</linearGradient>
-</defs>
-</svg>
@@ -1,3 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
-    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" d="M12.43 17.73c-.515-.286-1.347-.286-1.862 0L6.86 19.77c-1.029.565-1.702.05-1.507-1.148l.71-4.322c.098-.6-.159-1.43-.577-1.856l-2.998-3.06c-.833-.85-.572-1.682.576-1.856l4.143-.63c.575-.089 1.25-.602 1.507-1.148l1.853-3.933c.516-1.09 1.35-1.088 1.863 0l1.856 3.933c.256.546.93 1.06 1.507 1.148l4.144.63c1.15.174 1.406 1.009.575 1.856l-3 3.06c-.416.426-.674 1.256-.576 1.856l.709 4.322c.196 1.2-.481 1.713-1.507 1.148l-3.708-2.04z"/>
-</svg>
@@ -1,16 +0,0 @@
-<svg width="137" height="28" viewBox="0 0 137 28" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M17.5865 17.3955H17.5902L28.5163 8.77432L25.5528 6.39453L17.5902 12.6808H17.5865L17.5828 12.6845L9.62018 6.40201L6.6604 8.78181L17.5828 17.3992L17.5865 17.3955Z" fill="#1E80FF"/>
-<path d="M17.5872 6.77268L21.823 3.40505L17.5872 0.00748237L17.5835 0L13.3552 3.39757L17.5835 6.76894L17.5872 6.77268Z" fill="#1E80FF"/>
-<path d="M17.5865 23.2854L17.5828 23.2891L2.95977 11.7531L0 14.1291L0.284376 14.3574L17.5865 28L28.5238 19.3752L35.1768 14.1254L32.2133 11.7456L17.5865 23.2854Z" fill="#1E80FF"/>
-<path d="M133.287 18.4172H131.162L129.923 23.0758H126.825V16.7559H135.364V14.7315H126.825V11.1469H133.074V9.12256H118.454V11.1469H124.703V14.7315H116.164V16.7559H124.703V23.0758H121.601L120.363 18.4172H118.237L119.487 23.0758H115.543V25.1001H135.985V23.0758H132.041L133.287 18.4172Z" fill="#323232"/>
-<path d="M130.522 3.3151L130.2 2.94092H121.321L120.999 3.3151C119.298 5.2467 117.378 6.97337 115.278 8.46009V10.9522C116.206 10.3573 120.183 7.72295 122.327 4.98768H129.235C131.375 7.72295 135.356 10.3573 136.284 10.9522V8.46009C134.169 6.97586 132.235 5.2491 130.522 3.3151Z" fill="#323232"/>
-<path d="M94.9037 8.53127H96.494V6.50323H94.9037V2.88867H92.9056V6.50323H91.3303V8.53127H92.9094V14.7914L91.3303 15.3377V17.5828L92.9094 17.0365V20.7371C92.8414 22.2391 92.5579 23.7236 92.0674 25.145H94.1629C94.6112 23.7165 94.863 22.2335 94.9112 20.7371V16.3442L96.5053 15.7904V13.5229L94.9112 14.0767L94.9037 8.53127Z" fill="#323232"/>
-<path d="M99.1875 2.88867H97.6608V20.7446C97.6727 22.2359 97.4852 23.7221 97.1033 25.1637H99.1875C99.5206 23.7145 99.6851 22.2316 99.6776 20.7446V9.77357H112.321V2.88867H99.1875ZM110.308 7.75303H99.6851V4.91671H110.304L110.308 7.75303Z" fill="#323232"/>
-<path d="M110.305 23.405H107.566V17.6838H110.021H110.702H111.91V11.8166H110.021V15.9363H107.566V11.1431H105.676V15.9363H103.27V11.8166H101.381V17.6838H102.298H103.27H105.676V23.405H102.967V19.173H101.074V25.1524H102.286H102.967H110.305H110.683H112.195V19.173H110.305V23.405Z" fill="#323232"/>
-<path d="M53.0024 2.91846V4.83801L56.5908 6.11397L53.0024 7.38618V9.30573L59.2887 7.07187L65.5749 9.30573V7.38618L61.9865 6.11397L65.5749 4.83801V2.91846L59.2887 5.15232L53.0024 2.91846Z" fill="#323232"/>
-<path d="M79.9508 11.697H88.5944V9.47813H79.9508V2.80273H77.5299V9.47813H68.8863V11.697H77.5299V22.9823H67.76V25.2012H89.7207V22.9823H79.9508V11.697Z" fill="#323232"/>
-<path d="M48.9461 5.39227L52.2913 5.18272V3.24072L43.6813 3.78328V5.72529L47.0265 5.512V8.50545H43.5803V10.4213H47.0265V25.164H48.9461V10.4213H52.3923V8.50545H48.9461V5.39227Z" fill="#323232"/>
-<path d="M57.6539 9.01025H55.6595L55.1768 10.3648H52.9991V12.2357H54.5145L52.55 17.7662H53.8821V24.1534H55.7007V16.8457H58.6642V25.1787H60.5837V16.8457H63.4612V21.059C63.4534 22.1374 63.181 23.1974 62.6679 24.146H64.7147C65.1423 23.173 65.3627 22.1217 65.362 21.059V15.6932V15.1282H60.5837V13.2572H58.6642V15.1282H55.4836L56.5313 12.2245H65.594V10.3536H57.1749L57.6539 9.01025Z" fill="#323232"/>
-<path d="M43.562 24.6246H45.4591L46.2786 11.4272H44.3815L43.562 24.6246Z" fill="#323232"/>
-<path d="M49.7849 11.4272L50.3499 23.2963H52.247L51.682 11.4272H49.7849Z" fill="#323232"/>
-</svg>
@@ -1,3 +0,0 @@
-<svg width="36" height="28" viewBox="0 0 36 28" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M17.5875 6.77268L21.8232 3.40505L17.5875 0.00748237L17.5837 0L13.3555 3.39757L17.5837 6.76894L17.5875 6.77268ZM17.5863 17.3955H17.59L28.5161 8.77432L25.5526 6.39453L17.59 12.6808H17.5863L17.5825 12.6845L9.61993 6.40201L6.66016 8.78181L17.5825 17.3992L17.5863 17.3955ZM17.5828 23.2891L17.5865 23.2854L32.2133 11.7456L35.1768 14.1254L28.5238 19.3752L17.5865 28L0.284376 14.3574L0 14.1291L2.95977 11.7531L17.5828 23.2891Z" fill="#1E80FF"/>
-</svg>
@@ -1,3 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
-    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" stroke-linejoin="round" d="M11.5 18.37c-5.247 0-9.5-3.664-9.5-8.185C2 5.665 6.253 2 11.5 2S21 5.665 21 10.185c0 1.783-.662 3.433-1.785 4.778-.164.196.518 4.037.518 4.037l-4.136-1.428c-1.263.465-2.63.798-4.097.798z"/>
-</svg>
@@ -1,4 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="23" height="23" viewBox="0 0 23 23">
-    <path fill="none" fill-rule="evenodd" stroke="#8A93A0" stroke-linecap="round" stroke-linejoin="round"
-        d="M5 8.692V21H3.372C2.616 21 2 20.131 2 19.064v-8.283c0-1.112.642-2.089 1.372-2.089H5zm6.705-6.027c.146-.99.962-1.683 1.856-1.665 1.278.025 2.039 1.11 2.303 1.647.482.98.495 2.595.165 3.632-.3.938-.993 2.171-.993 2.171h5.194c.549 0 1.058.275 1.399.752.345.487.458 1.12.302 1.695l-2.35 8.685c-.213.835-.922 1.418-1.72 1.418H7V8.864c2.346-.517 4.44-4.41 4.705-6.199z" />
-</svg>
--- a/Show More
+++ b/Show More