#!/usr/bin/env python3
"""
采集静态页面到本地目录
用法: python3 crawl_static_page.py <url> <output_dir>
示例: python3 crawl_static_page.py https://mubaitea.cn/zsjm /var/www/new-media-platform/website-new/static/
"""

import os
import re
import sys
import hashlib
import requests
from datetime import datetime
from urllib.parse import urljoin, urlparse
from playwright.sync_api import sync_playwright
from bs4 import BeautifulSoup

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "*/*",
}


def slugify(url):
    """从 URL 生成文件名"""
    path = urlparse(url).path.strip('/')
    if path:
        name = path.split('/')[-1]
        if '.' in name:
            name = name.rsplit('.', 1)[0]
        return name or 'index'
    return 'index'


def download_file(url, dest_dir, skip_existing=True):
    """下载文件到指定目录，返回本地相对路径"""
    if not url or url.startswith('data:'):
        return None

    # 补全协议
    if url.startswith('//'):
        url = 'https:' + url

    try:
        # 生成文件名
        parsed = urlparse(url)
        path_parts = parsed.path.strip('/').split('/')
        filename = path_parts[-1] if path_parts else 'file'

        # 如果文件名没有扩展名或太泛，用hash重命名
        ext = os.path.splitext(filename)[1].lower()
        if not ext or len(filename) < 5:
            url_hash = hashlib.md5(url.encode()).hexdigest()[:10]
            filename = f"{url_hash}{ext or '.bin'}"

        dest_path = os.path.join(dest_dir, filename)

        if skip_existing and os.path.exists(dest_path):
            return filename

        resp = requests.get(url, headers=HEADERS, timeout=30)
        if resp.status_code != 200:
            print(f"  [下载失败] {url} HTTP {resp.status_code}")
            return None

        os.makedirs(dest_dir, exist_ok=True)
        with open(dest_path, 'wb') as f:
            f.write(resp.content)

        print(f"  [已下载] {url} -> {filename} ({len(resp.content)} bytes)")
        return filename

    except Exception as e:
        print(f"  [下载失败] {url}: {e}")
        return None


def extract_resource_urls(html_snippet, base_url):
    """从 HTML 片段中提取所有资源 URL"""
    soup = BeautifulSoup(html_snippet, 'html.parser')

    images = set()
    scripts = set()
    stylesheets = set()

    # 图片
    for tag in soup.find_all(['img', 'image']):
        for attr in ['src', '_src', 'data-src', 'data-original']:
            src = tag.get(attr)
            if src and not src.startswith('data:'):
                full = urljoin(base_url, src)
                if any(ext in full.lower() for ext in ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.bmp', '.ico']):
                    images.add(full)

    # CSS background-image (inline style)
    for tag in soup.find_all(style=True):
        style = tag['style']
        urls = re.findall(r'url\([\"\']?([^\"\'()]+)[\"\']?\)', style)
        for u in urls:
            if not u.startswith('data:'):
                full = urljoin(base_url, u)
                images.add(full)

    return {'images': list(images), 'scripts': list(scripts), 'stylesheets': list(stylesheets)}


def crawl_static_page(url, output_dir):
    """主函数：采集页面到本地目录"""
    js_dir = os.path.join(output_dir, 'js')
    images_dir = os.path.join(output_dir, 'images')
    css_dir = os.path.join(output_dir, 'css')

    page_name = slugify(url)
    html_path = os.path.join(output_dir, f'{page_name}.html')

    print(f"目标 URL: {url}")
    print(f"输出目录: {output_dir}")
    print(f"HTML 文件: {html_path}")
    print()

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page(viewport={'width': 1920, 'height': 1080})
        page.goto(url, wait_until='networkidle', timeout=30000)
        page.wait_for_timeout(5000)

        html = page.content()
        soup = BeautifulSoup(html, 'html.parser')

        # === 1. 提取正文内容（ctrl-location="main"） ===
        main_elements = soup.find_all(attrs={'ctrl-location': 'main'})
        print(f"找到 {len(main_elements)} 个正文元素 (ctrl-location=main)")

        # 取顶层 main 元素（父级不是 main 的）
        top_level_mains = []
        for el in main_elements:
            parent = el.parent
            if parent and parent.get('ctrl-location') != 'main':
                top_level_mains.append(el)

        print(f"其中顶层元素: {len(top_level_mains)} 个")

        # 收集 main 内容的 HTML
        main_html_parts = []
        for el in top_level_mains:
            main_html_parts.append(str(el))

        main_content_html = '\n'.join(main_html_parts)

        # === 2. 提取页面 CSS 引用 ===
        css_links = []
        for link in soup.find_all('link', rel='stylesheet'):
            href = link.get('href', '')
            if href:
                css_links.append(href)

        # 页面内联 style
        inline_styles = []
        for style_tag in soup.find_all('style'):
            inline_styles.append(str(style_tag))

        # === 3. 提取页面 JS 引用 ===
        script_srcs = []
        for script in soup.find_all('script', src=True):
            src = script['src']
            if src:
                script_srcs.append(src)

        # === 4. 提取 SVG symbol 图标（通常在 body 开头） ===
        svg_sprite = ''
        body = soup.find('body', attrs={'area': 'main'})
        if body:
            svg = body.find('svg', recursive=False)
            if svg:
                svg_sprite = str(svg)
                print("找到 SVG 图标精灵")

        # === 5. 提取 head 中的 meta/viewport ===
        head_meta = ''
        head = soup.find('head')
        if head:
            meta_tags = head.find_all('meta')
            for meta in meta_tags:
                head_meta += str(meta) + '\n'
            # 也保留 title
            title_tag = head.find('title')
            if title_tag:
                head_meta += str(title_tag) + '\n'

        browser.close()

    # === 6. 下载图片 ===
    print("\n=== 下载图片 ===")
    resources = extract_resource_urls(main_content_html, url)
    image_map = {}  # original_url -> local_filename

    for img_url in resources['images']:
        local_name = download_file(img_url, images_dir)
        if local_name:
            image_map[img_url] = f'images/{local_name}'

    # 也下载 main 内容中 img 标签的图片（更可靠的方式）
    main_soup = BeautifulSoup(main_content_html, 'html.parser')
    for img in main_soup.find_all('img'):
        for attr in ['src', '_src', 'data-src', 'data-original']:
            src = img.get(attr)
            if src and not src.startswith('data:'):
                full = urljoin(url, src)
                if full not in image_map:
                    local_name = download_file(full, images_dir)
                    if local_name:
                        image_map[full] = f'images/{local_name}'

    # === 7. 下载 JS 文件 ===
    print("\n=== 下载 JS 文件 ===")
    js_map = {}
    for js_url in script_srcs:
        local_name = download_file(js_url, js_dir)
        if local_name:
            js_map[js_url] = f'js/{local_name}'

    # === 8. 下载 CSS 文件 ===
    print("\n=== 下载 CSS 文件 ===")
    css_map = {}
    css_contents = []
    for css_url in css_links:
        local_name = download_file(css_url, css_dir)
        if local_name:
            css_map[css_url] = f'css/{local_name}'
            # 读取 CSS 内容以便内联 background-image 中的 URL
            css_path = os.path.join(css_dir, local_name)
            try:
                with open(css_path, 'r', encoding='utf-8', errors='replace') as f:
                    css_contents.append(f.read())
            except:
                pass

    # === 9. 替换 HTML 中的资源路径 ===
    print("\n=== 替换资源路径 ===")

    # 替换图片
    for orig_url, local_path in image_map.items():
        # 处理各种 URL 格式
        main_content_html = main_content_html.replace(orig_url, local_path)
        # 处理 protocol-relative
        if orig_url.startswith('https://'):
            proto_rel = orig_url.replace('https:', '')
            main_content_html = main_content_html.replace(proto_rel, local_path)

    # 替换 JS
    for orig_url, local_path in js_map.items():
        main_content_html = main_content_html.replace(orig_url, local_path)
        if orig_url.startswith('https://'):
            proto_rel = orig_url.replace('https:', '')
            main_content_html = main_content_html.replace(proto_rel, local_path)

    # 替换 CSS
    for orig_url, local_path in css_map.items():
        main_content_html = main_content_html.replace(orig_url, local_path)
        if orig_url.startswith('https://'):
            proto_rel = orig_url.replace('https:', '')
            main_content_html = main_content_html.replace(proto_rel, local_path)

    # 构建 CSS link 标签（使用本地路径）
    local_css_links = ''
    for orig_url, local_path in css_map.items():
        local_css_links += f'    <link rel="stylesheet" href="{local_path}">\n'

    # 构建 JS script 标签（使用本地路径）
    local_js_scripts = ''
    for orig_url, local_path in js_map.items():
        local_js_scripts += f'    <script src="{local_path}"></script>\n'

    # === 10. 构建完整 HTML ===
    full_html = f'''<!DOCTYPE html>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=1200, initial-scale=1.0">
{head_meta}
{local_css_links}
{chr(10).join(f'    <style>{s}</style>' for s in inline_styles if s.strip())}
    <style>
        /* 确保主内容区域正确显示 */
        body {{
            margin: 0;
            padding: 0;
            background: #fff;
            font-family: "Microsoft YaHei", "微软雅黑", Arial, sans-serif;
        }}
        .main-content-wrapper {{
            position: relative;
            width: 1200px;
            margin: 0 auto;
            min-height: 100vh;
        }}
    </style>
</head>
<body>
    {svg_sprite}
    <div class="main-content-wrapper">
{main_content_html}
    </div>
{local_js_scripts}
</body>
</html>'''

    # 写入文件
    with open(html_path, 'w', encoding='utf-8') as f:
        f.write(full_html)

    print(f"\n=== 完成 ===")
    print(f"HTML: {html_path}")
    print(f"图片: {len(image_map)} 个 -> {images_dir}")
    print(f"JS:   {len(js_map)} 个 -> {js_dir}")
    print(f"CSS:  {len(css_map)} 个 -> {css_dir}")
    print(f"文件大小: {len(full_html)} 字节")


if __name__ == '__main__':
    if len(sys.argv) < 3:
        print("用法: python3 crawl_static_page.py <url> <output_dir>")
        print("示例: python3 crawl_static_page.py https://mubaitea.cn/zsjm /var/www/new-media-platform/website-new/static/")
        sys.exit(1)

    target_url = sys.argv[1]
    output_dir = sys.argv[2].rstrip('/')
    crawl_static_page(target_url, output_dir)