"""
通用采集路由 - 基于 Scrapling 引擎
结合上层业务逻辑：编码修复、数学公式处理、微信防盗链、图片下载等
"""

import os
import re
import sys
import uuid
import asyncio
import tempfile
import base64
from datetime import datetime
from typing import Optional, List
from urllib.parse import urljoin, urlparse
from pathlib import Path

from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
from pydantic import BaseModel, HttpUrl
from sqlalchemy.orm import Session
from bs4 import BeautifulSoup

# Scrapling 引擎
from scrapling import Fetcher, StealthyFetcher

# 复用 crawl.py 的通用工具函数
from .crawl import (
    fix_encoding,
    fix_math_formulas,
    html_to_markdown,
    extract_title_from_html,
    extract_main_content,
    download_image,
    extract_images,
    extract_images_from_markdown,
    is_spa_site,
    convert_spa_html_to_markdown,
    CrawlRequest,
    CrawlResponse,
    CrawlImage,
)

# 复用图片处理
from .images import generate_thumbnail, get_image_dimensions

# 数据库
from database import get_db
from models import Post, Category, Tag, PostTag, Image

router = APIRouter(prefix="/api/universal-crawl", tags=["通用采集"])

# 确保上传目录存在
UPLOAD_DIR = Path("uploads")
UPLOAD_DIR.mkdir(exist_ok=True)
TEMP_DIR = Path("uploads/temp_crawl")
TEMP_DIR.mkdir(exist_ok=True)


class UniversalCrawlRequest(BaseModel):
    url: str


def _needs_browser(url: str) -> bool:
    """判断是否需要使用浏览器渲染"""
    domain = urlparse(url).netloc.lower()
    browser_domains = [
        'mp.weixin.qq.com',
        'juejin.cn',
        'zhihu.com',
        'segmentfault.com',
        'infoq.cn',
        'infoq.com',
        '51cto.com',
    ]
    return any(d in domain for d in browser_domains)


def _fetch_with_scrapling(url: str) -> str:
    """使用 Scrapling Fetcher (curl_cffi) 获取页面"""
    fetcher = Fetcher()
    response = fetcher.get(url)
    if response.status != 200:
        raise Exception(f"HTTP {response.status}: {response.reason}")
    # Scrapling Response 的 text 属性返回解析后的文本，html_content 返回原始 HTML
    return response.html_content if hasattr(response, 'html_content') else response.text


async def _fetch_with_stealth(url: str) -> str:
    """使用 Scrapling StealthyFetcher 获取页面，失败时回退到原生 Playwright"""
    try:
        from scrapling import StealthyFetcher
        # Scrapling v0.3+ 推荐使用类方法直接调用，无需实例化
        response = await StealthyFetcher.async_fetch(url, wait=3000)
        if not response or response.status != 200:
            raise Exception(f"HTTP {response.status}")
        html = response.html_content if hasattr(response, 'html_content') else response.text
        if not html or len(html) < 100:
            raise Exception("Empty page")
        print(f"[Universal Crawl] StealthyFetcher success: {len(html)} bytes")
        return html
    except Exception as e:
        print(f"[Universal Crawl] StealthyFetcher failed: {e}")
        print("[Universal Crawl] Falling back to native Playwright")
        return await _fetch_with_playwright(url)


async def _fetch_with_playwright(url: str) -> str:
    """使用原生 Playwright 异步获取页面，带完善反爬配置"""
    from playwright.async_api import async_playwright

    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--disable-web-security",
                "--disable-features=IsolateOrigins,site-per-process",
                "--disable-dev-shm-usage",
                "--no-sandbox",
            ]
        )
        context = await browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
            viewport={"width": 1920, "height": 1080},
            locale="zh-CN",
            timezone_id="Asia/Shanghai",
            extra_http_headers={
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                "Accept-Encoding": "gzip, deflate, br",
                "Cache-Control": "max-age=0",
                "Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120", "Microsoft Edge";v="120"',
                "Sec-Ch-Ua-Mobile": "?0",
                "Sec-Ch-Ua-Platform": '"Windows"',
                "Sec-Fetch-Dest": "document",
                "Sec-Fetch-Mode": "navigate",
                "Sec-Fetch-Site": "none",
                "Sec-Fetch-User": "?1",
                "Upgrade-Insecure-Requests": "1",
            },
        )
        page = await context.new_page()
        try:
            # SegmentFault 需要先访问首页建立信任，再跳转目标页面
            if 'segmentfault.com' in url:
                await page.goto("https://segmentfault.com", wait_until="domcontentloaded", timeout=15000)
                await asyncio.sleep(2)
                await page.goto(url, wait_until="domcontentloaded", timeout=30000)
                await asyncio.sleep(5)
            # 知乎需要先访问首页建立 cookie，再跳转目标页面
            elif 'zhihu.com' in url:
                await page.goto("https://www.zhihu.com", wait_until="domcontentloaded", timeout=15000)
                await asyncio.sleep(2)
                await page.goto(url, wait_until="domcontentloaded", timeout=30000)
                await asyncio.sleep(3)
                # 等待正文加载（知乎专栏文章正文在 .Post-RichTextContainer 或 .RichContent-inner 里）
                try:
                    await page.wait_for_selector(".Post-RichTextContainer, .RichContent-inner", timeout=10000)
                    await asyncio.sleep(1)
                except Exception:
                    pass
            else:
                await page.goto(url, wait_until="domcontentloaded", timeout=30000)
                await asyncio.sleep(2)
            html = await page.content()
        finally:
            await context.close()
            await browser.close()
    return html


def _bypass_wechat_images(html: str, url: str) -> str:
    """
    微信图片防盗链绕过：在页面上下文中执行 fetch 获取图片 base64
    这里我们在后端模拟：如果检测到 mmbiz.qpic.cn 图片，
    将其 src 替换为 data-src（微信文章中 data-src 通常包含真实 URL）
    """
    if 'mp.weixin.qq.com' not in url:
        return html

    soup = BeautifulSoup(html, 'lxml')
    for img in soup.find_all('img'):
        data_src = img.get('data-src')
        if data_src and 'mmbiz.qpic.cn' in data_src:
            img['src'] = data_src
    return str(soup)


def _extract_infoq_content(html: str) -> str:
    """提取 InfoQ 文章正文（.ProseMirror 或 article 标签）"""
    soup = BeautifulSoup(html, 'lxml')
    # 优先使用 .ProseMirror（InfoQ 正文编辑器容器）
    container = soup.select_one('.ProseMirror')
    if not container:
        # fallback 到 article 标签
        container = soup.find('article')
    if container:
        return str(container)
    return html


def _extract_infoq_title(html: str) -> Optional[str]:
    """提取 InfoQ 文章标题，从 <title> 中去掉 ' - InfoQ' 后缀"""
    match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
    if match:
        title = match.group(1).strip()
        # 去掉 " - InfoQ" 后缀
        title = re.sub(r'\s*[-–—]\s*InfoQ\s*$', '', title, flags=re.IGNORECASE).strip()
        if len(title) > 5:
            return title
    return None


def _extract_juejin_content(html: str) -> str:
    """提取掘金文章正文（article.article 或 .article-viewer）"""
    soup = BeautifulSoup(html, 'lxml')
    # 优先使用 article.article
    container = soup.select_one('article.article')
    if not container:
        # fallback 到 .article-viewer
        container = soup.select_one('.article-viewer')
    if not container:
        # 再 fallback 到 #article-root
        container = soup.select_one('#article-root')
    if container:
        return str(container)
    return html


def _extract_juejin_title(html: str) -> Optional[str]:
    """提取掘金文章标题，从 <title> 中去掉 ' - 掘金' 后缀"""
    match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
    if match:
        title = match.group(1).strip()
        # 去掉 " - 掘金" 后缀
        title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip()
        if len(title) > 5:
            return title
    return None


def _extract_csdn_content(html: str) -> str:
    """提取 CSDN 文章正文（#content_views 或 .article_content）"""
    soup = BeautifulSoup(html, 'lxml')
    # 优先使用 #content_views（CSDN 正文主容器）
    container = soup.select_one('#content_views')
    if not container:
        # fallback 到 .article_content
        container = soup.select_one('.article_content')
    if container:
        return str(container)
    return html


def _extract_csdn_title(html: str) -> Optional[str]:
    """提取 CSDN 文章标题，从 <title> 中去掉 '-CSDN博客' 后缀"""
    match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
    if match:
        title = match.group(1).strip()
        # 去掉 " - CSDN博客" / "-CSDN博客" 后缀
        title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip()
        if len(title) > 5:
            return title
    return None


def _filter_csdn_images(images: list) -> list:
    """过滤掉 CSDN 的 UI 图标图片（csdnimg.cn 的收藏/点赞等图标）"""
    filtered = []
    skip_patterns = [
        'newHeart2023',
        'tobarCollect',
        'tobarCollection',
        'copyright',
        'avatar',
        'blogv2/dist/pc/img',
    ]
    for img in images:
        url = img.original_url.lower()
        if any(p in url for p in skip_patterns):
            continue
        filtered.append(img)
    return filtered


def _extract_cnblogs_content(html: str) -> str:
    """提取博客园文章正文（#cnblogs_post_body 或 .postBody）"""
    soup = BeautifulSoup(html, 'lxml')
    container = soup.select_one('#cnblogs_post_body')
    if not container:
        container = soup.select_one('.postBody')
    if container:
        return str(container)
    return html


def _extract_cnblogs_title(html: str) -> Optional[str]:
    """提取博客园文章标题，从 <title> 中去掉 ' - 博客园' 后缀"""
    match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
    if match:
        title = match.group(1).strip()
        title = re.sub(r'\s*[-–—]\s*博客园\s*$', '', title, flags=re.IGNORECASE).strip()
        if len(title) > 5:
            return title
    return None


def _filter_cnblogs_images(images: list) -> list:
    """过滤掉博客园的 UI 图标图片"""
    filtered = []
    skip_patterns = [
        'assets.cnblogs.com/logo',
        'assets.cnblogs.com/icons',
        'skins/custom/images/logo',
    ]
    for img in images:
        url = img.original_url.lower()
        if any(p in url for p in skip_patterns):
            continue
        filtered.append(img)
    return filtered


def _extract_51cto_content(html: str) -> str:
    """提取 51CTO 文章正文（.article-content 或 article）"""
    soup = BeautifulSoup(html, 'lxml')
    container = soup.select_one('.article-content')
    if not container:
        container = soup.find('article')
    if container:
        return str(container)
    return html


def _extract_51cto_title(html: str) -> Optional[str]:
    """提取 51CTO 文章标题，从 <title> 中去掉 '-51CTO.COM' 后缀"""
    match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
    if match:
        title = match.group(1).strip()
        title = re.sub(r'\s*[-–—]?\s*51CTO\.COM\s*$', '', title, flags=re.IGNORECASE).strip()
        if len(title) > 5:
            return title
    return None


def _filter_51cto_images(images: list) -> list:
    """过滤掉 51CTO 的 UI 图标和二维码图片"""
    filtered = []
    skip_patterns = [
        'oss/202302',     # 公众号矩阵二维码
        'oss/202408',     # 软考二维码
        'oss/202506/06',  # 顶部横幅
        'oss/202302/07',  # APP下载图
    ]
    for img in images:
        url = img.original_url.lower()
        if any(p in url for p in skip_patterns):
            continue
        filtered.append(img)
    return filtered


@router.post("/fetch", response_model=CrawlResponse)
async def universal_fetch(request: UniversalCrawlRequest):
    """通用采集：使用 Scrapling 引擎获取文章"""
    try:
        url = str(request.url)
        print(f"[Universal Crawl] Fetching: {url}")

        # 1. 判断采集策略
        use_browser = _needs_browser(url)

        # 2. 使用 Scrapling 获取页面
        if use_browser:
            print(f"[Universal Crawl] Using StealthyFetcher (browser) for {url}")
            html = await _fetch_with_stealth(url)
        else:
            print(f"[Universal Crawl] Using Fetcher (curl_cffi) for {url}")
            html = _fetch_with_scrapling(url)

        if not html or len(html) < 100:
            raise HTTPException(status_code=400, detail="无法获取页面内容，页面可能为空或需要登录")

        # 3. 编码修复
        html = fix_encoding(html)

        # 4. 微信图片防盗链处理
        html = _bypass_wechat_images(html, url)

        # 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
        is_infoq = 'infoq.cn' in url or 'infoq.com' in url
        is_juejin = 'juejin.cn' in url
        is_csdn = 'csdn.net' in url or 'csdn.com' in url
        is_cnblogs = 'cnblogs.com' in url
        is_51cto = '51cto.com' in url
        if is_infoq:
            title = _extract_infoq_title(html)
        elif is_juejin:
            title = _extract_juejin_title(html)
        elif is_csdn:
            title = _extract_csdn_title(html)
        elif is_cnblogs:
            title = _extract_cnblogs_title(html)
        elif is_51cto:
            title = _extract_51cto_title(html)
        else:
            title = None
        if not title:
            title = extract_title_from_html(html)
        if not title:
            title = "未获取到标题"

        # 6. 提取正文和图片
        is_wechat = 'mp.weixin.qq.com' in url
        images = extract_images(html, url)
        if is_csdn:
            images = _filter_csdn_images(images)
        elif is_cnblogs:
            images = _filter_cnblogs_images(images)
        elif is_51cto:
            images = _filter_51cto_images(images)

        # 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
        if is_infoq:
            infoq_html = _extract_infoq_content(html)
            markdown_content = html_to_markdown(infoq_html, url)
        elif is_juejin:
            juejin_html = _extract_juejin_content(html)
            markdown_content = html_to_markdown(juejin_html, url)
        elif is_csdn:
            csdn_html = _extract_csdn_content(html)
            markdown_content = html_to_markdown(csdn_html, url)
        elif is_cnblogs:
            cnblogs_html = _extract_cnblogs_content(html)
            markdown_content = html_to_markdown(cnblogs_html, url)
        elif is_51cto:
            cto_html = _extract_51cto_content(html)
            markdown_content = html_to_markdown(cto_html, url)
        elif is_spa_site(html):
            markdown_content = convert_spa_html_to_markdown(html, images, url)
        else:
            markdown_content = html_to_markdown(html, url)

        # 8. 数学公式修复
        markdown_content = fix_math_formulas(markdown_content)

        # 9. 清理 Markdown 中的 HTML 标签
        markdown_content = re.sub(r'<[^>]+>', '', markdown_content)
        markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
        markdown_content = markdown_content.strip()

        return CrawlResponse(
            success=True,
            title=title,
            content=markdown_content,
            images=images,
        )

    except HTTPException:
        raise
    except Exception as e:
        print(f"[Universal Crawl] Error: {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"采集失败: {str(e)}")


@router.post("/save")
async def universal_save(
    title: str,
    content: str,
    excerpt: Optional[str] = None,
    category_id: Optional[int] = None,
    source_url: Optional[str] = None,
    images: Optional[str] = None,
    db: Session = Depends(get_db)
):
    """保存通用采集的文章（复用 crawl.py 的保存逻辑）"""
    try:
        print(f"[Universal Crawl] Saving article: {title}")

        # 解析图片列表
        image_list = []
        if images:
            try:
                import json
                image_data = json.loads(images)
                for img in image_data:
                    image_list.append(CrawlImage(**img))
            except Exception as e:
                print(f"[Universal Crawl] Parse images error: {e}")

        # 创建临时目录
        temp_dir = tempfile.mkdtemp(dir=TEMP_DIR)
        final_dir = UPLOAD_DIR / "images"
        final_dir.mkdir(exist_ok=True)
        thumb_dir = UPLOAD_DIR / "thumbnails"
        thumb_dir.mkdir(exist_ok=True)

        # 下载图片
        downloaded_images = []
        for img in image_list:
            try:
                filename, filepath, success = download_image(str(img.url), temp_dir)
                if success and filepath:
                    # 移动最终文件
                    final_filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{os.path.splitext(filename)[1]}"
                    final_path = final_dir / final_filename
                    os.rename(filepath, str(final_path))

                    # 生成缩略图
                    thumb_filename = f"thumb_{final_filename}"
                    thumb_path = thumb_dir / thumb_filename
                    thumb_path_db = None
                    if generate_thumbnail(str(final_path), str(thumb_path)):
                        thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"

                    # 获取尺寸
                    width, height = get_image_dimensions(str(final_path))

                    # 保存到数据库
                    db_image = Image(
                        filename=final_filename,
                        original_filename=img.alt or final_filename,
                        file_path=f"/uploads/images/{final_filename}",
                        file_size=os.path.getsize(str(final_path)),
                        mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}",
                        width=width,
                        height=height,
                        thumbnail_path=thumb_path_db,
                    )
                    db.add(db_image)
                    db.flush()

                    # 替换内容中的图片 URL
                    original_url = str(img.url)
                    new_url = f"/uploads/images/{final_filename}"
                    content = content.replace(original_url, new_url)

                    downloaded_images.append({
                        "id": db_image.id,
                        "url": new_url,
                        "thumb": thumb_path_db,
                    })
            except Exception as e:
                print(f"[Universal Crawl] Download image error: {e}")
                continue

        # 清理 excerpt
        if excerpt:
            excerpt = re.sub(r'<[^>]+>', '', excerpt)
            excerpt = excerpt.replace('\n', ' ').strip()[:200]
        else:
            excerpt = content.replace('\n', ' ').strip()[:200]

        # 创建文章
        now = datetime.now()
        slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', title)[:50].strip('-')
        slug = f"{slug}-{now.strftime('%Y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}"

        # 生成摘要
        summary = content[:300] if len(content) > 300 else content
        summary = re.sub(r'[#*`\[\]\(\)!]', '', summary)

        # 查找或创建分类
        final_category_id = category_id
        if not final_category_id:
            default_cat = db.query(Category).filter(Category.name == "未分类").first()
            if not default_cat:
                default_cat = Category(name="未分类", slug="uncategorized")
                db.add(default_cat)
                db.flush()
            final_category_id = default_cat.id

        post = Post(
            title=title,
            slug=slug,
            content=content,
            excerpt=excerpt,
            summary=summary,
            category_id=final_category_id,
            author_id=1,
            status="draft",
            view_count=0,
            is_original=False,
            source_url=source_url,
            created_at=now,
            updated_at=now,
            published_at=None,
        )
        db.add(post)
        db.flush()

        db.commit()

        return {
            "status": "success",
            "post_id": post.id,
            "message": "文章保存成功",
            "images": downloaded_images,
        }

    except Exception as e:
        db.rollback()
        print(f"[Universal Crawl] Save error: {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")