#!/usr/bin/env python3
"""
批量采集 niaodaifu.cn 健康资讯文章
"""

import os
import re
import hashlib
import requests
import sqlite3
from datetime import datetime
from urllib.parse import urljoin, urlparse
from playwright.sync_api import sync_playwright
from bs4 import BeautifulSoup, NavigableString
from PIL import Image
import io

# ============ 配置 ============
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
UPLOAD_DIR = os.path.join(BASE_DIR, "..", "storage", "admin", "images")
DB_PATH = os.path.join(BASE_DIR, "cms.db")

MENU_TO_CATEGORY = {
    15: (2, "科普文章"),
    7:  (3, "肾友"),
    8:  (4, "糖友"),
    6:  (5, "健康"),
}

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}


# ============ 工具函数 ============
def slugify(title):
    """生成 URL 友好的 slug"""
    import unicodedata
    slug = unicodedata.normalize('NFKC', title)
    slug = re.sub(r'[^\w\s-]', '', slug).strip().lower()
    slug = re.sub(r'[-\s]+', '-', slug)
    if not slug:
        slug = 'article'
    return slug


def ensure_unique_slug(conn, slug):
    """确保 slug 唯一"""
    cursor = conn.cursor()
    original = slug
    suffix = 1
    while True:
        cursor.execute('SELECT 1 FROM news_posts WHERE slug = ?', (slug,))
        if not cursor.fetchone():
            break
        slug = f"{original}-{suffix}"
        suffix += 1
    return slug


def download_image(url, module='news'):
    """下载图片到本地存储目录，返回文件路径和数据库记录信息"""
    if not url or url.startswith('data:'):
        return None, None

    # 过滤 135editor 占位图
    if '135editor.com' in url and url.endswith('.gif'):
        return None, None

    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        if resp.status_code != 200:
            return None, None

        content = resp.content
        if len(content) < 100:
            return None, None

        # 判断扩展名
        ext = os.path.splitext(urlparse(url).path)[1].lower()
        if ext not in ('.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp'):
            # 尝试通过内容判断
            if content[:2] == b'\xff\xd8':
                ext = '.jpg'
            elif content[:8] == b'\x89PNG\r\n\x1a\n':
                ext = '.png'
            elif content[:4] == b'GIF8':
                ext = '.gif'
            elif content[:4] == b'<svg':
                ext = '.svg'
            else:
                ext = '.jpg'

        # 生成文件名
        filename = hashlib.md5(url.encode()).hexdigest() + ext
        subdir = datetime.now().strftime('%Y%m')
        dest_dir = os.path.join(UPLOAD_DIR, subdir)
        os.makedirs(dest_dir, exist_ok=True)

        filepath = os.path.join(dest_dir, filename)
        with open(filepath, 'wb') as f:
            f.write(content)

        # 生成缩略图
        thumb_path = None
        if ext in ('.jpg', '.jpeg', '.png', '.webp', '.bmp'):
            try:
                img = Image.open(io.BytesIO(content))
                img.thumbnail((300, 300))
                thumb_filename = f"thumb_{filename}"
                thumb_filepath = os.path.join(dest_dir, thumb_filename)
                if ext == '.png':
                    img.save(thumb_filepath, 'PNG')
                else:
                    img.save(thumb_filepath, 'JPEG', quality=85)
                thumb_path = f'/storage/admin/images/{subdir}/{thumb_filename}'
            except Exception:
                pass

        rel_path = f'/storage/admin/images/{subdir}/{filename}'
        file_size = len(content)

        # 获取图片尺寸
        width, height = None, None
        try:
            img = Image.open(io.BytesIO(content))
            width, height = img.size
        except Exception:
            pass

        return rel_path, {
            'filename': filename,
            'original_name': os.path.basename(urlparse(url).path) or filename,
            'file_path': rel_path,
            'thumb_path': thumb_path,
            'file_size': file_size,
            'width': width,
            'height': height,
            'module': module,
        }
    except Exception as e:
        print(f"    [图片下载失败] {url}: {e}")
        return None, None


def insert_image_to_db(conn, img_info):
    """插入图片记录到 images 表，返回 image_id"""
    if not img_info:
        return None
    cursor = conn.cursor()
    cursor.execute('''
        INSERT INTO images (filename, original_name, file_path, thumb_path, description, file_size, width, height, module, created_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    ''', (
        img_info['filename'],
        img_info['original_name'],
        img_info['file_path'],
        img_info.get('thumb_path'),
        None,
        img_info['file_size'],
        img_info.get('width'),
        img_info.get('height'),
        img_info['module'],
        datetime.now().isoformat(),
    ))
    return cursor.lastrowid


def html_to_markdown(element, base_url, downloaded_images):
    """递归将 HTML 转换为 Markdown，同时下载图片"""
    if element is None:
        return ''

    parts = []
    for child in element.children:
        if isinstance(child, NavigableString):
            text = str(child)
            # 保留换行，但压缩多余空格
            parts.append(text)
            continue

        tag = child.name
        if tag is None:
            continue

        # 跳过 135editor 装饰性元素和脚本
        if tag in ('script', 'style', 'iframe', 'video', 'audio'):
            continue

        # 跳过纯装饰性 section/div（没有文本内容且没有图片）
        inner_text = child.get_text(strip=True)
        has_img = child.find('img') is not None
        if tag in ('section', 'div') and not inner_text and not has_img:
            continue

        if tag == 'p':
            text = html_to_markdown(child, base_url, downloaded_images).strip()
            if text:
                parts.append(text + '\n\n')
        elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
            level = int(tag[1])
            text = html_to_markdown(child, base_url, downloaded_images).strip()
            if text:
                parts.append('#' * level + ' ' + text + '\n\n')
        elif tag == 'img':
            src = child.get('src') or child.get('_src') or child.get('data-src', '')
            if src and not src.endswith('.gif'):
                full_src = urljoin(base_url, src)
                # 下载图片
                if full_src not in downloaded_images:
                    rel_path, img_info = download_image(full_src)
                    if rel_path:
                        downloaded_images[full_src] = rel_path
                        # 同时记录到数据库
                        conn = sqlite3.connect(DB_PATH)
                        insert_image_to_db(conn, img_info)
                        conn.commit()
                        conn.close()
                if full_src in downloaded_images:
                    alt = child.get('alt', '图片')
                    parts.append(f'\n![{alt}]({downloaded_images[full_src]})\n')
        elif tag in ('strong', 'b'):
            text = html_to_markdown(child, base_url, downloaded_images).strip()
            if text:
                parts.append(f'**{text}**')
        elif tag in ('em', 'i'):
            text = html_to_markdown(child, base_url, downloaded_images).strip()
            if text:
                parts.append(f'*{text}*')
        elif tag == 'br':
            parts.append('\n')
        elif tag == 'a':
            href = child.get('href', '')
            text = html_to_markdown(child, base_url, downloaded_images).strip()
            if text and href:
                full_href = urljoin(base_url, href)
                parts.append(f'[{text}]({full_href})')
            elif text:
                parts.append(text)
        elif tag in ('ul', 'ol'):
            is_ordered = (tag == 'ol')
            items = []
            for li in child.find_all('li', recursive=False):
                li_text = html_to_markdown(li, base_url, downloaded_images).strip()
                if li_text:
                    prefix = f"{len(items)+1}. " if is_ordered else "- "
                    items.append(prefix + li_text)
            if items:
                parts.append('\n' + '\n'.join(items) + '\n\n')
        elif tag == 'blockquote':
            text = html_to_markdown(child, base_url, downloaded_images).strip()
            if text:
                lines = text.split('\n')
                quoted = '\n'.join('> ' + line for line in lines)
                parts.append(quoted + '\n\n')
        elif tag in ('table', 'thead', 'tbody', 'tr', 'td', 'th'):
            # 简单处理表格：提取文本
            text = html_to_markdown(child, base_url, downloaded_images).strip()
            if text:
                parts.append(text + ' ')
        elif tag == 'pre':
            code = child.get_text()
            parts.append(f'\n```\n{code}\n```\n\n')
        elif tag == 'code':
            code = child.get_text()
            parts.append(f'`{code}`')
        else:
            # 其他标签递归处理
            text = html_to_markdown(child, base_url, downloaded_images)
            if text:
                parts.append(text)

    return ''.join(parts)


def clean_markdown(md):
    """清理 Markdown 文本"""
    # 合并多个空行
    md = re.sub(r'\n{3,}', '\n\n', md)
    # 去除行首行尾空白
    md = md.strip()
    return md


def parse_articles_from_page(page):
    """从当前页面解析文章列表"""
    articles = []
    html = page.content()
    soup = BeautifulSoup(html, 'html.parser')
    medias = soup.find_all('div', class_='media')
    for media in medias:
        heading_a = media.find('a', class_='media-heading')
        if not heading_a:
            continue
        title = heading_a.get_text(strip=True)
        href = heading_a.get('href', '')
        if not href:
            continue
        m = re.search(r'article_id=(\d+)', href)
        if not m:
            continue
        article_id = int(m.group(1))
        cover_url = None
        left_a = media.find('a', class_='media-left')
        if left_a:
            img = left_a.find('img')
            if img:
                cover_url = img.get('src')
        time_str = ''
        time_el = media.find('span', class_='time')
        if time_el:
            time_str = time_el.get_text(strip=True)
        articles.append({
            'article_id': article_id,
            'title': title,
            'href': urljoin('https://niaodaifu.cn', href),
            'cover_url': cover_url,
            'time': time_str,
        })
    return articles, soup


# ============ 采集函数 ============
def get_article_list(page, menu_id):
    """获取指定栏目的所有文章列表（通过点击分页按钮翻页）"""
    base_url = f"https://niaodaifu.cn/health-information?menu_id={menu_id}"
    all_articles = []
    seen_ids = set()

    page.goto(base_url, wait_until='networkidle', timeout=30000)
    page.wait_for_timeout(3000)

    # 获取总条数
    html = page.content()
    soup = BeautifulSoup(html, 'html.parser')
    total = 0
    total_el = soup.find('span', class_='el-pagination__total')
    if total_el:
        m = re.search(r'(\d+)', total_el.get_text())
        if m:
            total = int(m.group(1))

    # 获取页码按钮总数（最后一个数字页码）
    pager_btns = page.query_selector_all('.el-pager li.number')
    total_pages = 1
    if pager_btns:
        # 最后一个页码按钮的文本就是总页数（可能被省略，需要点击更多后才知道）
        # 先查 el-pagination__total
        if total > 0:
            page_size_text = ''
            size_select = soup.find('input', {'class': 'el-input__inner'})
            page_size = 10  # 默认每页10条
            total_pages = max(1, (total + page_size - 1) // page_size)
    
    print(f"  栏目 {menu_id} 共 {total} 篇文章，预计 {total_pages} 页")

    current_page = 1
    while True:
        articles, soup = parse_articles_from_page(page)
        new_count = 0
        for art in articles:
            if art['article_id'] not in seen_ids:
                seen_ids.add(art['article_id'])
                all_articles.append(art)
                new_count += 1
        print(f"    第 {current_page} 页采集到 {len(articles)} 篇（新增 {new_count} 篇）")

        # 查找下一页按钮
        next_btn = page.query_selector('button.btn-next')
        if not next_btn:
            break
        # 检查是否禁用
        is_disabled = next_btn.get_attribute('disabled')
        if is_disabled is not None:
            break

        next_btn.click()
        page.wait_for_timeout(2000)
        current_page += 1

    return all_articles


def get_article_detail(page, url):
    """获取文章详情"""
    page.goto(url, wait_until='networkidle', timeout=30000)
    page.wait_for_timeout(3000)

    html = page.content()
    soup = BeautifulSoup(html, 'html.parser')

    # 标题：在 article > h3 中，格式 "首页 》栏目 》标题"
    title = ''
    article_el = soup.find('article')
    if article_el:
        h3 = article_el.find('h3')
        if h3:
            text = h3.get_text(strip=True)
            parts = [p.strip() for p in text.split('》')]
            # 过滤 "首页" 和栏目名，取最后的标题
            if len(parts) >= 2:
                title = parts[-1]
            else:
                title = text

    # 来源和时间
    source = ''
    time_str = ''
    if article_el:
        info_p = article_el.find('p', class_='more-info')
        if info_p:
            src_el = info_p.find('span', class_='source')
            if src_el:
                source = src_el.get_text(strip=True).replace('来源：', '')
            time_el = info_p.find('span', class_='time')
            if time_el:
                time_str = time_el.get_text(strip=True)

    # 正文 HTML
    content_div = soup.find('div', class_='article-content')
    content_html = ''
    if content_div:
        content_html = str(content_div)

    return {
        'title': title,
        'source': source,
        'time': time_str,
        'content_html': content_html,
    }


def save_article(article, detail, category_id, cover_path, content_md):
    """保存文章到数据库"""
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()

    title = detail['title'] or article['title']
    slug = ensure_unique_slug(conn, slugify(title))

    # 解析发布时间
    created_at = datetime.now().isoformat()
    if detail.get('time') or article.get('time'):
        t = detail.get('time') or article.get('time')
        try:
            created_at = datetime.strptime(t, '%Y-%m-%d %H:%M').isoformat()
        except ValueError:
            try:
                created_at = datetime.strptime(t, '%Y-%m-%d').isoformat()
            except ValueError:
                pass

    summary = title[:200]
    if detail.get('source'):
        summary = f"来源：{detail['source']} | {summary}"

    cursor.execute('''
        INSERT INTO news_posts (title, slug, content, content_type, summary, cover_image, category_id, is_published, is_draft, view_count, created_at, updated_at)
        VALUES (?, ?, ?, 'markdown', ?, ?, ?, 1, 0, 0, ?, ?)
    ''', (
        title,
        slug,
        content_md,
        summary,
        cover_path,
        category_id,
        created_at,
        created_at,
    ))

    post_id = cursor.lastrowid
    conn.commit()
    conn.close()
    return post_id


# ============ 主程序 ============
def main():
    print("=" * 60)
    print("开始批量采集 niaodaifu.cn 文章")
    print("=" * 60)

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page(viewport={'width': 1920, 'height': 1080})

        for menu_id, (category_id, cat_name) in MENU_TO_CATEGORY.items():
            print(f"\n>>> 采集栏目 [{cat_name}] menu_id={menu_id} -> category_id={category_id}")

            articles = get_article_list(page, menu_id)
            print(f"  共找到 {len(articles)} 篇文章")

            for idx, article in enumerate(articles, 1):
                print(f"  [{idx}/{len(articles)}] {article['title']}")

                try:
                    detail = get_article_detail(page, article['href'])
                except Exception as e:
                    print(f"    [详情获取失败] {e}")
                    continue

                # 下载封面图
                cover_path = None
                if article.get('cover_url'):
                    cover_path, cover_info = download_image(article['cover_url'])
                    if cover_info:
                        conn = sqlite3.connect(DB_PATH)
                        insert_image_to_db(conn, cover_info)
                        conn.commit()
                        conn.close()

                # 转换正文为 Markdown
                content_md = ''
                if detail.get('content_html'):
                    soup = BeautifulSoup(detail['content_html'], 'html.parser')
                    content_div = soup.find('div', class_='article-content')
                    if content_div:
                        downloaded_images = {}
                        content_md = html_to_markdown(content_div, article['href'], downloaded_images)
                        content_md = clean_markdown(content_md)

                if not content_md.strip():
                    print(f"    [警告] 正文为空，跳过")
                    continue

                # 保存
                try:
                    post_id = save_article(article, detail, category_id, cover_path, content_md)
                    print(f"    [已入库] post_id={post_id}")
                except Exception as e:
                    print(f"    [入库失败] {e}")

        browser.close()

    print("\n" + "=" * 60)
    print("采集完成")
    print("=" * 60)


if __name__ == '__main__':
    main()