#!/usr/bin/env python3
"""
Phase 2: 下载诗歌内容
每个诗人的诗歌直接放在诗人目录下
"""

import json
import requests
import re
import os
from datetime import datetime
from time import sleep
from pathlib import Path
import html

# 路径配置
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
INDEX_FILE = os.path.join(CONFIG_DIR, 'poets_index.json')
STORAGE_DIR = Path('/home/ubuntu/media/public-domain-poetry')
DELAY = 5  # 秒


def load_index():
    """加载索引"""
    with open(INDEX_FILE, 'r', encoding='utf-8') as f:
        return json.load(f)


def save_index(index):
    """保存索引"""
    with open(INDEX_FILE, 'w', encoding='utf-8') as f:
        json.dump(index, f, indent=2, ensure_ascii=False)


def fetch_poet_page(url):
    """获取诗人页面（诗歌列表）"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
    }
    
    response = requests.get(url, headers=headers, timeout=30)
    response.raise_for_status()
    return response.text


def parse_poems_from_html(html, poet_slug):
    """从诗人页面解析诗歌列表"""
    poems = []
    
    # 匹配诗歌链接: <a href="a-h-laidlaw/babylon-22309">Babylon. </a>
    pattern = r'<a\s+href="(' + re.escape(poet_slug) + r'/[^"]+)"[^>]*>([^<]+)</a>'
    matches = re.findall(pattern, html, re.IGNORECASE)
    
    seen = set()
    for slug, title in matches:
        slug = slug.strip()
        title = title.strip()
        
        # 去重
        if slug not in seen:
            seen.add(slug)
            poems.append({
                'slug': slug,
                'title': title,
                'url': f"https://www.public-domain-poetry.com/{slug}"
            })
    
    return poems


def fetch_poem_content(url):
    """获取诗歌内容"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
    }
    
    response = requests.get(url, headers=headers, timeout=30)
    response.raise_for_status()
    return response.text


def extract_poem_text(html, poet_name, poem_title):
    """从诗歌页面提取文本内容"""
    # 尝试多种模式提取诗歌内容
    
    # 查找标题后的内容
    # 格式通常是: Title. By Poet Name ... 诗歌内容 ...
    # 诗歌内容通常在 </center> 或 <p> 标签后
    
    # 尝试提取 <p> 标签中的内容
    patterns = [
        # 查找诗歌主体部分
        r'<p[^>]*>(.*?)</p>',
        # 查找 font 标签中的内容
        r'<font[^>]*>(.*?)</font>',
    ]
    
    lines = []
    
    # 尝试提取所有可能的诗歌行
    # 诗歌通常在页面中间，有很多 <br> 或 <p>
    poem_pattern = r'(?:<br\s*/?>\s*|</p>\s*<p[^>]*>\s*)([A-Z][^<]{10,200})(?:<br\s*/?>|</p>)'
    matches = re.findall(poem_pattern, html, re.IGNORECASE | re.DOTALL)
    
    if matches:
        for match in matches[:100]:  # 限制数量
            line = re.sub(r'<[^>]+>', '', match).strip()
            if line and len(line) > 10:
                lines.append(line)
    
    # 如果没找到，尝试更简单的提取
    if not lines:
        # 查找所有大写开头的行
        all_text = re.sub(r'<[^>]+>', ' ', html)
        all_text = html.unescape(all_text)
        
        # 查找诗歌部分（通常在 "By 诗人名" 之后）
        poet_pattern = rf'By\s+{re.escape(poet_name)}.*?(\n[A-Z].*?)(?:\n\n|\Z)'
        match = re.search(poet_pattern, all_text, re.DOTALL | re.IGNORECASE)
        
        if match:
            poem_text = match.group(1)
            lines = [line.strip() for line in poem_text.split('\n') if line.strip()]
    
    return '\n\n'.join(lines) if lines else None


def save_poem(poet_slug, poem_slug, title, content, poet_name):
    """保存诗歌到诗人目录"""
    # 创建诗人目录
    poet_dir = STORAGE_DIR / poet_slug
    poet_dir.mkdir(parents=True, exist_ok=True)
    
    # 文件名: poem-slug.txt
    poem_file = poet_dir / f"{poem_slug.split('/')[-1]}.txt"
    
    with open(poem_file, 'w', encoding='utf-8') as f:
        f.write(f"# {title}\n\n")
        f.write(f"Author: {poet_name}\n")
        f.write(f"Source: https://www.public-domain-poetry.com/{poem_slug}\n")
        f.write(f"Downloaded: {datetime.now().isoformat()}\n")
        f.write(f"\n{'='*60}\n\n")
        f.write(content)
    
    return poem_file


def download_poet(poet_info):
    """下载单个诗人的所有诗歌"""
    poet_slug = poet_info['slug']
    poet_name = poet_info['name']
    
    print(f"\n📖 诗人: {poet_name}")
    
    try:
        # 获取诗歌列表
        html = fetch_poet_page(poet_info['url'])
        poems = parse_poems_from_html(html, poet_slug)
        
        if not poems:
            print(f"  ⚠️  未找到诗歌")
            return True, 0
        
        print(f"  发现 {len(poems)} 首诗歌")
        
        downloaded_count = 0
        for i, poem in enumerate(poems, 1):
            print(f"  [{i}/{len(poems)}] {poem['title'][:40]}...", end=' ')
            
            try:
                # 获取诗歌内容
                poem_html = fetch_poem_content(poem['url'])
                content = extract_poem_text(poem_html, poet_name, poem['title'])
                
                if content:
                    save_poem(poet_slug, poem['slug'], poem['title'], content, poet_name)
                    print("✅")
                    downloaded_count += 1
                else:
                    print("⚠️  无内容")
                
                # 延迟
                if i < len(poems):
                    sleep(DELAY)
                    
            except Exception as e:
                print(f"❌ {e}")
        
        print(f"  完成: {downloaded_count}/{len(poems)} 首")
        return True, downloaded_count
        
    except Exception as e:
        print(f"  ❌ 错误: {e}")
        return False, 0


def main():
    print("="*60)
    print("📝 Public Domain Poetry - 下载器")
    print("="*60)
    
    # 加载索引
    if not os.path.exists(INDEX_FILE):
        print("❌ 索引文件不存在，请先运行 fetch_poets.py")
        return
    
    index = load_index()
    poets = index['poets']
    
    # 统计
    pending = [p for p in poets if p['status'] == 'pending']
    downloaded = [p for p in poets if p['status'] == 'downloaded']
    failed = [p for p in poets if p['status'] == 'failed']
    
    print(f"\n📊 当前状态:")
    print(f"   总计诗人: {len(poets)}")
    print(f"   待下载: {len(pending)}")
    print(f"   已完成: {len(downloaded)}")
    print(f"   失败: {len(failed)}")
    
    if not pending:
        print("\n✅ 所有诗歌已下载完成！")
        return
    
    print(f"\n⏱️  下载间隔: {DELAY} 秒")
    print(f"   预计时间: 较长（每个诗人多首诗歌）\n")
    
    print("开始下载... (按 Ctrl+C 暂停)\n")
    
    try:
        for i, poet in enumerate(pending, 1):
            print(f"[{i}/{len(pending)}] 诗人: {poet['name']}")
            
            success, count = download_poet(poet)
            
            # 更新状态
            poet['status'] = 'downloaded' if success else 'failed'
            poet['download_date'] = datetime.now().isoformat() if success else None
            poet['poems_count'] = count
            
            # 保存索引
            save_index(index)
            
            # 诗人之间延迟
            if i < len(pending):
                sleep(DELAY)
        
        print("\n" + "="*60)
        print("✅ 下载完成！")
        print("="*60)
        
    except KeyboardInterrupt:
        print("\n\n⚠️ 用户中断")
        print(f"进度已保存，下次会继续从断点下载")
        save_index(index)


if __name__ == '__main__':
    main()