"""
文章采集路由 - 从网页URL采集内容
支持SPA网站采集
"""
import os
import re
import uuid
import requests
from datetime import datetime
from typing import Optional, List
from urllib.parse import urljoin, urlparse

try:
    from bs4 import BeautifulSoup
    BS4_AVAILABLE = True
except ImportError:
    BS4_AVAILABLE = False

# Playwright用于SPA采集
try:
    from playwright.sync_api import sync_playwright
    PLAYWRIGHT_AVAILABLE = True
except ImportError:
    PLAYWRIGHT_AVAILABLE = False

# Playwright Stealth 用于绕过WAF
try:
    from playwright_stealth import stealth_sync
    STEALTH_AVAILABLE = True
except ImportError:
    STEALTH_AVAILABLE = False

from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from sqlalchemy.orm import Session

from app.database import get_db
from app.routers.auth import get_current_admin
from PIL import Image as PILImage

router = APIRouter(prefix="/crawl", tags=["文章采集"])

# 上传配置
from app.config import get_settings
settings = get_settings()


def get_upload_dir():
    """获取上传目录路径 - 扁平存储（统一指向项目根目录 storage）"""
    base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
    upload_dir = os.path.join(base_dir, settings.UPLOAD_DIR)
    os.makedirs(upload_dir, exist_ok=True)
    # 缩略图目录
    thumbnail_dir = os.path.join(upload_dir, "thumbnails")
    os.makedirs(thumbnail_dir, exist_ok=True)
    return upload_dir


def get_crawl_temp_dir():
    """获取采集临时目录 - 跨平台（统一指向项目根目录 storage/tmp/crawl）"""
    if os.name == 'nt':  # Windows
        base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
        temp_dir = os.path.join(base_dir, '..', 'storage', 'tmp', 'crawl')
    else:  # Linux/macOS
        temp_dir = '/tmp/news_crawl'
    os.makedirs(temp_dir, exist_ok=True)
    # 缩略图临时目录
    thumb_dir = os.path.join(temp_dir, 'thumbnails')
    os.makedirs(thumb_dir, exist_ok=True)
    return temp_dir


def generate_unique_filename(original_filename: str) -> str:
    """生成唯一的文件名"""
    ext = os.path.splitext(original_filename)[1].lower()
    if not ext:
        ext = '.jpg'
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    unique_id = uuid.uuid4().hex[:8]
    return f"{timestamp}_{unique_id}{ext}"


class CrawlRequest(BaseModel):
    url: str
    use_playwright: bool = False


class CrawlImage(BaseModel):
    original_url: str
    local_path: Optional[str] = None
    filename: Optional[str] = None
    success: bool = False


class CrawlResponse(BaseModel):
    success: bool
    title: Optional[str] = None
    content: Optional[str] = None
    summary: Optional[str] = None
    images: List[dict] = []
    error: Optional[str] = None


def fix_encoding(text: str) -> str:
    """修复UTF-8编码问题"""
    if not text:
        return text
    garbled_patterns = ['é', 'è', 'Ã', 'Â', 'Ã¥', 'ü', 'ö', 'ä', 'ß', 'â€', 'æ', 'å', 'è', '§']
    has_garbled = any(pattern in text for pattern in garbled_patterns)
    if has_garbled:
        try:
            fixed = text.encode('latin1').decode('utf-8')
            still_garbled = any(p in fixed for p in garbled_patterns)
            if still_garbled:
                try:
                    fixed = fixed.encode('latin1').decode('utf-8')
                except:
                    pass
            return fixed
        except (UnicodeEncodeError, UnicodeDecodeError):
            pass
    return text


def clean_html_tags(text: str) -> str:
    """移除文本中的HTML标签"""
    text = re.sub(r'<[^>]+>', '', text)
    text = text.replace('&nbsp;', ' ')
    text = text.replace('&lt;', '<')
    text = text.replace('&gt;', '>')
    text = text.replace('&amp;', '&')
    text = text.replace('&quot;', '"')
    text = text.replace('&#39;', "'")
    return text.strip()


def fix_math_formulas(text: str) -> str:
    """修复数学公式格式，处理混合格式问题（从 personal-blog 移植）"""
    # 1. 清理已渲染的数学符号（保留LaTeX代码）
    text = re.sub(r'[⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿⁱ]+', '', text)

    # 2. 规范化反斜杠（双重转义变单重）
    text = text.replace('\\\\', '\\')

    # 3. 检测块级公式（多行或包含 array, matrix 等）
    block_keywords = ['\\begin{array}', '\\begin{matrix}', '\\begin{equation}',
                      '\\sum_', '\\int_', '\\prod_', '\\lim_']

    lines = text.split('\n')
    result_lines = []
    in_block = False
    block_content = []

    for line in lines:
        stripped = line.strip()

        # 检测块级公式开始
        if any(kw in stripped for kw in block_keywords):
            if not in_block:
                in_block = True
                block_content = []
            block_content.append(line)
            continue

        # 如果在块内，继续收集
        if in_block:
            block_content.append(line)
            # 检测块结束
            if '\\end{array}' in stripped or '\\end{matrix}' in stripped or '\\end{equation}' in stripped:
                block_text = '\n'.join(block_content)
                if not block_text.startswith('$$') and not block_text.startswith('\\['):
                    result_lines.append('$$' + block_text + '$$')
                else:
                    result_lines.append(block_text)
                in_block = False
                block_content = []
            continue

        # 处理行内公式
        # 如果一行中有明显的LaTeX命令，但还没有被包裹
        if re.search(r'\\[a-zA-Z]+', line) and not re.search(r'\\\(|\\\)|\\\[|\\\]', line):
            # 简单处理：如果包含复杂结构，作为块级；否则作为行内
            if re.search(r'\\(frac|sum|int|prod|lim)\{', line):
                line = '$$' + line + '$$'
            else:
                line = '\\(' + line + '\\)'

        result_lines.append(line)

    # 处理未闭合的块
    if in_block and block_content:
        block_text = '\n'.join(block_content)
        result_lines.append('$$' + block_text + '$$')

    return '\n'.join(result_lines)


def extract_title_from_html(html: str) -> str:
    """从HTML中提取标题"""
    # Open Graph title
    og_title_match = re.search(
        r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
        html, re.IGNORECASE
    )
    if not og_title_match:
        og_title_match = re.search(
            r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:title["\']',
            html, re.IGNORECASE
        )
    if og_title_match:
        title = clean_html_tags(og_title_match.group(1))
        if len(title) > 5:
            title = re.sub(r'\s+[–—-]\s+.*$', '', title).strip()
            return title

    # 博客园
    cnblog_title = re.search(r'<h1[^>]+class=["\'][^"\']*postTitle[^"\']*["\'][^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
    if cnblog_title:
        title = clean_html_tags(cnblog_title.group(1))
        if len(title) > 5:
            return title

    # 人人都是产品经理 (woshipm.com)
    woshipm_title = re.search(r'<[^>]+class=["\'][^"\']*article--title[^"\']*["\'][^>]*>(.*?)</[^>]+>', html, re.IGNORECASE | re.DOTALL)
    if woshipm_title:
        title = clean_html_tags(woshipm_title.group(1))
        if len(title) > 5:
            return title

    # 菜鸟教程
    runoob_title = re.search(r'<div[^>]*class=["\']article-intro["\'][^>]*>.*?<h1[^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
    if runoob_title:
        title = clean_html_tags(runoob_title.group(1))
        if len(title) > 5 and "菜鸟教程" not in title:
            return title

    # title标签
    title_match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
    if title_match:
        title = clean_html_tags(title_match.group(1))
        title = re.sub(r'\s+[|│┃]\s+.*$', '', title).strip()
        title = re.sub(r'\s+[-–—]\s+[^\s]*教程.*$', '', title).strip()
        title = re.sub(r'\s+[-–—]\s+[^\s]*博客.*$', '', title).strip()
        if len(title) > 5:
            return title

    # h1标题
    h1_matches = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
    for h1 in h1_matches:
        title = clean_html_tags(h1)
        if len(title) > 5:
            return title

    # Twitter Card title
    twitter_title_match = re.search(
        r'<meta[^>]+name=["\']twitter:title["\'][^>]+content=["\']([^"\']+)["\']',
        html, re.IGNORECASE
    )
    if twitter_title_match:
        title = clean_html_tags(twitter_title_match.group(1))
        if len(title) > 5:
            return title

    return "未命名文章"


def extract_main_content(html: str) -> str:
    """提取主要内容"""
    if BS4_AVAILABLE:
        try:
            soup = BeautifulSoup(html, 'html.parser')
            for tag in soup(['script', 'style']):
                tag.decompose()

            selectors = [
                '.article--content.grap',
                '#cnblogs_post_body',
                '.article-intro',
                'article', 'main',
                '.post-content', '.article-content', '.entry-content',
                '#content', '#main-content',
            ]
            for selector in selectors:
                elem = soup.select_one(selector)
                if elem:
                    return str(elem)

            body = soup.find('body')
            if body:
                return str(body)
        except Exception as e:
            print(f"BeautifulSoup失败: {e}")

    # 正则回退
    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)

    patterns = [
        r'<div[^>]*class=["\']article--content grap["\'][^>]*>(.*?)</div>\s*</div>\s*<div[^>]*class=["\']article--footer',
        r'<div[^>]*class=["\']article--content grap["\'][^>]*>(.*?)</div>\s*</div>\s*<div[^>]*class=["\']article-bottom-meta',
        r'<div[^>]*class=["\']article-intro["\'][^>]*>(.*?)<div[^>]*class=["\'](?:google-auto-placed|pre|next)',
        r'<div[^>]+id=["\']cnblogs_post_body["\'][^>]*>(.*?)</div>\s*<div[^>]+id=["\']blog_post_info_wrap["\']',
        r'<div[^>]+id=["\']cnblogs_post_body["\'][^>]*>(.*?)</div>',
        r'<article[^>]*>(.*?)</article>',
        r'<main[^>]*>(.*?)</main>',
        r'<div[^>]*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)</div>',
    ]
    for pattern in patterns:
        match = re.search(pattern, html, re.DOTALL | re.IGNORECASE)
        if match:
            return match.group(1)

    return html


def html_to_markdown(html: str, base_url: str = "") -> str:
    """HTML转Markdown"""
    html = fix_encoding(html)
    parsed_base = urlparse(base_url) if base_url else None
    base_is_https = parsed_base and parsed_base.scheme == 'https'

    def make_absolute_url(src: str) -> str:
        if not src:
            return src
        if src.startswith('//'):
            src = 'https:' + src
        elif src.startswith('/') and base_url:
            src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
        elif not src.startswith(('http://', 'https://')) and base_url:
            src = urljoin(base_url, src)
        if base_is_https and src.startswith('http://'):
            src = 'https://' + src[7:]
        return src

    if BS4_AVAILABLE:
        soup = BeautifulSoup(html, 'html.parser')

        for img in soup.find_all('img'):
            src = img.get('src', '')
            alt = img.get('alt', '')
            if src:
                src = make_absolute_url(src)
                md_img = f'![{alt}]({src})'
                img.replace_with(md_img)

        for tag in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
            level = int(tag.name[1])
            tag.replace_with(f'\n{"#" * level} {tag.get_text().strip()}\n')

        for p in soup.find_all('p'):
            p.replace_with(f'\n{p.get_text()}\n')

        for br in soup.find_all('br'):
            br.replace_with('\n')

        for pre in soup.find_all('pre'):
            code = pre.find('code')
            if code:
                code_text = code.get_text()
                lang = ''
                code_classes = code.get('class', [])
                for cls in code_classes:
                    if 'language-' in cls or 'lang-' in cls:
                        lang = cls.replace('language-', '').replace('lang-', '')
                        break
                md_code = f'\n```{lang}\n{code_text}\n```\n'
                pre.replace_with(md_code)
            else:
                pre_text = pre.get_text()
                md_code = f'\n```\n{pre_text}\n```\n'
                pre.replace_with(md_code)

        for code in soup.find_all('code'):
            if not code.find_parent('pre'):
                code.replace_with(f'`{code.get_text()}`')

        for tag in soup.find_all(['strong', 'b']):
            tag.replace_with(f'**{tag.get_text()}**')

        for tag in soup.find_all(['em', 'i']):
            tag.replace_with(f'*{tag.get_text()}*')

        text = soup.get_text()
    else:
        text = html
        text = re.sub(
            r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>',
            lambda m: f'![图片]({make_absolute_url(m.group(1))})',
            text, flags=re.IGNORECASE
        )
        text = re.sub(r'<h1[^>]*>(.*?)</h1>', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r'<h2[^>]*>(.*?)</h2>', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r'<h3[^>]*>(.*?)</h3>', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r'<p[^>]*>(.*?)</p>', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
        text = re.sub(r'<[^>]+>', '', text)

    text = text.replace('&nbsp;', ' ').replace('&lt;', '<').replace('&gt;', '>')
    text = text.replace('&amp;', '&').replace('&quot;', '"')

    lines = text.split('\n')
    cleaned = []
    prev_empty = False
    for line in lines:
        stripped = line.strip()
        if stripped == '':
            if not prev_empty:
                cleaned.append('')
            prev_empty = True
        else:
            cleaned.append(line)
            prev_empty = False

    text = '\n'.join(cleaned).strip()

    # 修复数学公式格式
    text = fix_math_formulas(text)

    return text


def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[CrawlImage]:
    """从Markdown提取图片"""
    images = []
    parsed_base = urlparse(base_url)
    base_is_https = parsed_base.scheme == 'https'

    img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)'
    for match in re.finditer(img_pattern, markdown_content):
        img_url = match.group(2)
        if img_url.startswith('/uploads/') or img_url.startswith('/storage/'):
            continue
        if img_url.startswith('//'):
            img_url = 'https:' + img_url
        elif img_url.startswith('/'):
            img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}"
        elif not img_url.startswith(('http://', 'https://')):
            img_url = urljoin(base_url, img_url)
        if base_is_https and img_url.startswith('http://'):
            img_url = 'https://' + img_url[7:]

        images.append(CrawlImage(original_url=img_url))

    return images


def is_spa_site(html: str) -> bool:
    """检测是否为SPA"""
    indicators = [
        '<div id="root"></div>', '<div id="app"></div>',
        'You need to enable JavaScript', 'Loading...',
        'data-reactroot', 'window.__INITIAL_STATE__'
    ]
    html_lower = html.lower()
    for indicator in indicators:
        if indicator.lower() in html_lower:
            return True

    text_content = re.sub(r'<[^>]+>', '', html)
    text_content = re.sub(r'\s+', '', text_content)
    if len(text_content) < 200:
        return True

    return False


def crawl_with_playwright(url: str) -> dict:
    """使用Playwright采集SPA（同步API，避免Windows asyncio子进程问题）"""
    if not PLAYWRIGHT_AVAILABLE:
        raise Exception("Playwright未安装")

    is_wechat = 'mp.weixin.qq.com' in url
    is_juejin = 'juejin.cn' in url
    is_aliyun = 'developer.aliyun.com' in url
    is_segmentfault = 'segmentfault.com' in url
    is_tencent_cloud = 'cloud.tencent.com' in url

    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=True,
            args=[
                '--disable-blink-features=AutomationControlled',
                '--disable-web-security',
                '--disable-features=IsolateOrigins,site-per-process',
                '--disable-dev-shm-usage',
                '--no-sandbox',
                '--disable-setuid-sandbox',
                '--disable-accelerated-2d-canvas',
                '--disable-gpu',
                '--window-size=1920,1080',
            ]
        )

        context = browser.new_context(
            viewport={'width': 1920, 'height': 1080},
            user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            locale='zh-CN',
            timezone_id='Asia/Shanghai',
            permissions=['geolocation'],
            java_script_enabled=True,
        )

        context.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
            Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
            window.chrome = { runtime: {} };
        """)

        page = context.new_page()

        if STEALTH_AVAILABLE:
            stealth_sync(page)

        page.goto(url, wait_until='networkidle', timeout=30000)

        if is_wechat:
            page.wait_for_timeout(5000)
            title = ""
            try:
                if page.locator('#activity_name').count() > 0:
                    title = page.locator('#activity_name').first.inner_text()
                    title = title.strip()
            except:
                pass
            if not title:
                title = page.title()

            content_html = ""
            try:
                if page.locator('#js_content').count() > 0:
                    content_html = page.locator('#js_content').first.inner_html()
            except:
                pass

            images = []
            try:
                img_elements = page.locator('#js_content img').all()
                for img in img_elements[:10]:
                    try:
                        src = img.get_attribute('data-src') or img.get_attribute('src')
                        if src and src.startswith('http'):
                            images.append(CrawlImage(original_url=src))
                    except:
                        pass
            except:
                pass

            context.close()
            browser.close()
            return {'title': title or "未命名文章", 'content_html': content_html, 'images': images}

        # 普通SPA处理
        if is_juejin:
            try:
                page.wait_for_selector('article', timeout=15000)
            except:
                pass
            page.wait_for_timeout(3000)
        elif is_aliyun:
            # 阿里云开发者社区：等待语雀编辑器内容加载
            try:
                page.wait_for_selector('.article-content', timeout=15000)
            except:
                pass
            # 语雀编辑器内容懒加载，需要滚动触发
            page.wait_for_timeout(2000)
            for _ in range(5):
                page.evaluate('window.scrollBy(0, 800)')
                page.wait_for_timeout(500)
            page.wait_for_timeout(2000)
        elif is_segmentfault:
            # SegmentFault 需要更长的等待时间
            print("[Playwright] 等待 SegmentFault 页面加载...")
            try:
                page.wait_for_load_state('networkidle', timeout=20000)
            except:
                pass
            page.wait_for_timeout(8000)
            print("[Playwright] 等待完成，开始提取")
        elif is_tencent_cloud:
            # 腾讯云开发者社区
            try:
                page.wait_for_selector('[class*="markdown"]', timeout=15000)
            except:
                pass
            page.wait_for_timeout(3000)

        title = ""
        if is_juejin:
            for selector in ['article h1', 'h1']:
                try:
                    if page.locator(selector).count() > 0:
                        title = page.locator(selector).first.inner_text()
                        if title.strip():
                            break
                except:
                    continue
        elif is_segmentfault:
            # SegmentFault 标题选择器
            sf_title_selectors = ['h1.article-title', 'h1', '.article-title', '[class*="title"]']
            for selector in sf_title_selectors:
                try:
                    count = page.locator(selector).count()
                    if count > 0:
                        t = page.locator(selector).first.inner_text()
                        print(f"[Playwright] 标题选择器 '{selector}': '{t[:30] if t else '空'}...'")
                        if t and t.strip():
                            title = t.strip()
                            break
                except Exception as e:
                    print(f"[Playwright] 标题选择器 '{selector}' 失败: {e}")
        elif is_tencent_cloud:
            for selector in ['h1', '.article-title', '[class*="title"]']:
                try:
                    if page.locator(selector).count() > 0:
                        title = page.locator(selector).first.inner_text()
                        if title.strip():
                            break
                except:
                    continue

        if not title:
            for selector in ['article h1', 'h1', '.article-title']:
                try:
                    if page.locator(selector).count() > 0:
                        title = page.locator(selector).first.inner_text()
                        if title.strip():
                            break
                except:
                    continue
        if not title:
            title = page.title()

        content_html = ""
        if is_juejin:
            try:
                if page.locator('.markdown-body').count() > 0:
                    content_html = page.locator('.markdown-body').first.inner_html()
                elif page.locator('article').count() > 0:
                    content_html = page.locator('article').first.inner_html()
            except:
                pass

            if content_html:
                content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
                content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
                content_html = re.sub(r'\s+style="[^"]*"', '', content_html, flags=re.IGNORECASE)
                content_html = re.sub(r'\s+class="[^"]*"', '', content_html, flags=re.IGNORECASE)
        elif is_aliyun:
            # 阿里云开发者社区：语雀编辑器内容
            try:
                if page.locator('.article-content').count() > 0:
                    content_html = page.locator('.article-content').first.inner_html()
                elif page.locator('.article-detail').count() > 0:
                    content_html = page.locator('.article-detail').first.inner_html()
            except:
                pass

            if content_html:
                content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
                content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
        elif is_segmentfault:
            # SegmentFault 内容提取
            sf_selectors = ['.article-content', '.markdown-body', 'article', '.post-content',
                           '[data-id]', 'main', '#content', '.content']
            for selector in sf_selectors:
                try:
                    count = page.locator(selector).count()
                    print(f"[Playwright] 检查选择器 '{selector}': {count} 个匹配")
                    if count > 0:
                        html = page.locator(selector).first.inner_html()
                        if len(html) > 200:
                            content_html = html
                            print(f"[Playwright] 使用 '{selector}' 提取，长度: {len(content_html)}")
                            break
                except Exception as e:
                    print(f"[Playwright] 选择器 '{selector}' 失败: {e}")
            if not content_html:
                try:
                    content_html = page.locator('body').first.inner_html()
                    print(f"[Playwright] 使用 body，长度: {len(content_html)}")
                except Exception as e:
                    print(f"[Playwright] 获取 body 失败: {e}")
            if content_html:
                content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
                content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
                content_html = re.sub(r'<div[^>]*class="[^"]*ad[^"]*"[^>]*>.*?</div>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
        elif is_tencent_cloud:
            # 腾讯云开发者社区
            try:
                if page.locator('.mod-content__markdown').count() > 0:
                    content_html = page.locator('.mod-content__markdown').first.inner_html()
            except:
                pass

        if not content_html:
            selectors = ['article', '.article-content', '.content', '.post-content', 'main']
            for selector in selectors:
                try:
                    elements = page.locator(selector).all()
                    if elements:
                        parts = []
                        for el in elements:
                            try:
                                html = el.inner_html().strip()
                                if html and len(html) > 20:
                                    parts.append(html)
                            except:
                                pass
                        if parts:
                            combined = '\n'.join(parts)
                            if len(combined) > 500:
                                content_html = combined
                                break
                except:
                    continue

        # 最终回退：所有选择器都失败时，用页面可见文本
        if not content_html:
            try:
                body_text = page.locator('body').first.inner_text()
                if body_text and len(body_text.strip()) > 50:
                    lines = [l for l in body_text.split('\n') if l.strip()]
                    content_html = '<body><pre>' + '\n'.join(lines) + '</pre></body>'
            except:
                pass

        images = []
        parsed_url = urlparse(url)
        url_is_https = parsed_url.scheme == 'https'
        try:
            img_elements = page.locator('img').all()
            for img in img_elements[:10]:
                try:
                    src = img.get_attribute('src')
                    if src and not src.startswith('data:') and len(src) > 5:
                        if src.startswith('//'):
                            src = 'https:' + src
                        elif src.startswith('/'):
                            src = f"{parsed_url.scheme}://{parsed_url.netloc}{src}"
                        elif not src.startswith(('http://', 'https://')):
                            src = urljoin(url, src)
                        if url_is_https and src.startswith('http://'):
                            src = 'https://' + src[7:]
                        images.append(CrawlImage(original_url=src))
                except:
                    pass
        except:
            pass

        context.close()
        browser.close()

        title = fix_encoding(title) if title else title
        content_html = fix_encoding(content_html) if content_html else content_html

        return {'title': title.strip() if title else "未命名文章", 'content_html': content_html, 'images': images}


def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url: str) -> str:
    """SPA HTML转Markdown"""
    html = fix_encoding(html)
    text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)

    parsed_base = urlparse(base_url)
    base_is_https = parsed_base.scheme == 'https'

    def img_to_markdown(match):
        src = match.group(1)
        if src.startswith('//'):
            src = 'https:' + src
        elif src.startswith('/'):
            src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
        if base_is_https and src.startswith('http://'):
            src = 'https://' + src[7:]
        return f'![图片]({src})'

    text = re.sub(r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>', img_to_markdown, text, flags=re.IGNORECASE)
    text = re.sub(r'<pre[^>]*>\s*<code[^>]*>(.*?)</code>\s*</pre>', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<code[^>]*>(.*?)</code>', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<h1[^>]*>(.*?)</h1>', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<h2[^>]*>(.*?)</h2>', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<h3[^>]*>(.*?)</h3>', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<p[^>]*>(.*?)</p>', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'<strong[^>]*>(.*?)</strong>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r'<[^>]+>', '', text)

    text = text.replace('&nbsp;', ' ').replace('&lt;', '<').replace('&gt;', '>')
    text = text.replace('&amp;', '&').replace('&quot;', '"')

    lines = text.split('\n')
    cleaned = []
    prev_empty = False
    for line in lines:
        stripped = line.strip()
        if stripped == '':
            if not prev_empty:
                cleaned.append('')
            prev_empty = True
        else:
            cleaned.append(line)
            prev_empty = False

    text = '\n'.join(cleaned).strip()

    # 修复数学公式格式（掘金等平台）
    text = fix_math_formulas(text)

    return text


@router.post("")
async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)):
    """采集文章"""
    response_headers = {
        "Cache-Control": "no-cache, no-store, must-revalidate",
        "Content-Type": "application/json; charset=utf-8"
    }

    try:
        if not request.url.startswith(('http://', 'https://')):
            raise HTTPException(status_code=400, detail="无效的URL")

        is_wechat = 'mp.weixin.qq.com' in request.url
        is_juejin = 'juejin.cn' in request.url
        is_aliyun = 'developer.aliyun.com' in request.url

        if is_wechat or is_juejin or is_aliyun:
            if not PLAYWRIGHT_AVAILABLE:
                site_name = '微信' if is_wechat else ('掘金' if is_juejin else '阿里云')
                return JSONResponse(
                    content={"success": False, "error": f"{site_name}文章需要Playwright"},
                    headers=response_headers
                )

            import asyncio
            spa_result = await asyncio.to_thread(crawl_with_playwright, request.url)
            title = spa_result['title']
            images = spa_result['images']
            content_html = spa_result['content_html']
            markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
        else:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
                'Accept-Encoding': 'gzip, deflate, br',
                'Cache-Control': 'max-age=0',
                'Connection': 'keep-alive',
                'Upgrade-Insecure-Requests': '1',
                'Sec-Fetch-Dest': 'document',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'none',
                'Sec-Fetch-User': '?1',
            }

            try:
                response = requests.get(request.url, headers=headers, timeout=30)
                response.raise_for_status()
            except requests.exceptions.HTTPError as e:
                if response.status_code in (468, 403, 429, 503) and PLAYWRIGHT_AVAILABLE:
                    import asyncio
                    spa_result = await asyncio.to_thread(crawl_with_playwright, request.url)
                    title = spa_result['title']
                    images = spa_result['images']
                    content_html = spa_result['content_html']
                    markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)

                    return JSONResponse(
                        content={
                            "success": True,
                            "title": title,
                            "url": request.url,
                            "content": markdown_content,
                            "images": [{"original_url": img.original_url} for img in images]
                        },
                        headers=response_headers
                    )
                else:
                    raise

            declared_encoding = None
            if response.encoding:
                declared_encoding = response.encoding

            if not declared_encoding or declared_encoding.lower() in ['iso-8859-1', 'latin-1']:
                raw_content = response.content
                charset_match = re.search(rb'<meta[^>]+charset=["\']?([^"\'>\s]+)', raw_content, re.IGNORECASE)
                if charset_match:
                    declared_encoding = charset_match.group(1).decode('ascii', errors='ignore')

            if declared_encoding and declared_encoding.lower() not in ['iso-8859-1', 'latin-1']:
                response.encoding = declared_encoding
            else:
                response.encoding = response.apparent_encoding or 'utf-8'

            html = response.text

            if is_spa_site(html):
                if not PLAYWRIGHT_AVAILABLE:
                    return JSONResponse(
                        content={"success": False, "error": "SPA网站需要Playwright"},
                        headers=response_headers
                    )

                import asyncio
                spa_result = await asyncio.to_thread(crawl_with_playwright, request.url)
                title = spa_result['title']
                images = spa_result['images']
                content_html = spa_result['content_html']
                markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
            else:
                title = extract_title_from_html(html)
                main_content = extract_main_content(html)
                markdown_content = html_to_markdown(main_content, request.url)
                images = extract_images_from_markdown(markdown_content, request.url)

        # 下载图片到临时目录
        temp_dir = get_crawl_temp_dir()
        upload_dir = get_upload_dir()

        downloaded_images = []
        for i, img in enumerate(images[:5]):
            try:
                img_headers = {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
                    'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
                    'Referer': request.url,
                }
                # 微信公众号图片需要特殊 Referer
                if 'mmbiz.qpic.cn' in img.original_url:
                    img_headers['Referer'] = 'https://mp.weixin.qq.com/'

                img_response = requests.get(img.original_url, headers=img_headers, timeout=30, stream=True)
                img_response.raise_for_status()

                content_type = img_response.headers.get('content-type', '')
                ext = '.jpg'
                if 'png' in content_type:
                    ext = '.png'
                elif 'gif' in content_type:
                    ext = '.gif'
                elif 'webp' in content_type:
                    ext = '.webp'
                elif 'svg' in content_type:
                    ext = '.svg'

                filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}"
                filepath = os.path.join(temp_dir, filename)

                with open(filepath, 'wb') as f:
                    for chunk in img_response.iter_content(chunk_size=8192):
                        f.write(chunk)

                # 检查文件大小，过滤防盗链占位图（小于1KB视为失败）
                file_size = os.path.getsize(filepath)
                if file_size < 1024:
                    os.remove(filepath)
                    print(f"图片太小（{file_size} 字节），可能是防盗链占位图: {img.original_url}")
                    continue

                # 验证文件是否为有效图片（SVG 除外，浏览器直接支持）
                is_svg = ext == '.svg'
                if not is_svg:
                    try:
                        with PILImage.open(filepath) as pil_img:
                            pil_img.verify()  # 验证图片完整性
                    except Exception:
                        os.remove(filepath)
                        print(f"文件不是有效图片，已删除: {img.original_url}")
                        continue

                # 生成缩略图 - 放在临时目录的 thumbnails 子目录下
                thumb_filename = f"{os.path.splitext(filename)[0]}.jpg"  # 缩略图统一为 .jpg
                thumb_filepath = os.path.join(temp_dir, "thumbnails", thumb_filename)
                thumb_path = None

                if not is_svg:
                    try:
                        with PILImage.open(filepath) as pil_img:
                            # 转换为 RGB（处理透明图片）
                            if pil_img.mode in ('RGBA', 'P'):
                                pil_img = pil_img.convert('RGB')
                            pil_img.thumbnail((300, 300))
                            pil_img.save(thumb_filepath, "JPEG", quality=85)
                            thumb_path = f"/storage/tmp/crawl/thumbnails/{thumb_filename}"
                    except Exception as thumb_err:
                        print(f"缩略图生成失败 {filename}: {thumb_err}")

                local_path = f"/storage/tmp/crawl/{filename}"
                img.local_path = local_path
                img.filename = filename
                img.success = True
                downloaded_images.append({
                    "original_url": img.original_url,
                    "local_path": local_path,
                    "thumb_path": thumb_path,
                    "filename": filename
                })

                # 替换内容中的URL为临时路径
                if img.original_url in markdown_content:
                    markdown_content = markdown_content.replace(img.original_url, local_path)

            except Exception as e:
                print(f"下载图片失败 {img.original_url}: {e}")

        summary = markdown_content[:200].replace('#', '').replace('*', '').strip()

        return JSONResponse(
            content={
                "success": True,
                "title": title,
                "content": markdown_content,
                "summary": summary,
                "images": downloaded_images
            },
            headers=response_headers
        )

    except Exception as e:
        import traceback
        print(f"采集失败: {e}")
        print(traceback.format_exc())
        return JSONResponse(
            content={"success": False, "error": f"采集失败: {str(e)}"},
            headers=response_headers
        )