nanocode/libraryofshortstories-scraper/scripts/download_stories.py

#!/usr/bin/env python3
"""
Phase 2: 下载故事内容
使用网站 API 直接获取纯文本
"""

import json
import requests
import time
import os
import re
from datetime import datetime

# 路径配置
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
INDEX_FILE = os.path.join(CONFIG_DIR, 'stories_index.json')
STORAGE_DIR = '/home/ubuntu/media/libraryofshortstories'
DELAY = 7  # 秒


def load_index():
    """加载索引"""
    with open(INDEX_FILE, 'r', encoding='utf-8') as f:
        return json.load(f)


def save_index(index):
    """保存索引"""
    with open(INDEX_FILE, 'w', encoding='utf-8') as f:
        json.dump(index, f, indent=2, ensure_ascii=False)


def fetch_story_content(code):
    """获取故事内容（纯文本）"""
    url = f"https://www.libraryofshortstories.com/storiesgzip/{code}.txt"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        'Accept-Encoding': 'gzip',
    }
    
    response = requests.get(url, headers=headers, timeout=30)
    response.raise_for_status()
    return response.text


def fetch_story_metadata(code):
    """获取故事元数据"""
    url = f"https://www.libraryofshortstories.com/storymetadata/{code}"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        'Accept-Encoding': 'gzip',
    }
    
    try:
        response = requests.get(url, headers=headers, timeout=30)
        if response.status_code == 200:
            return response.json()
    except:
        pass
    return None


def save_story(story_info, content, metadata):
    """保存故事到目录"""
    # 创建目录
    story_dir = os.path.join(STORAGE_DIR, story_info['code'])
    os.makedirs(story_dir, exist_ok=True)
    
    # 保存内容
    content_file = os.path.join(story_dir, 'content.txt')
    with open(content_file, 'w', encoding='utf-8') as f:
        f.write(f"# {story_info['title']}\n\n")
        f.write(f"Author: {story_info['author']}\n")
        if story_info.get('publish_year'):
            f.write(f"Year: {story_info['publish_year']}\n")
        if story_info.get('genres'):
            f.write(f"Genres: {', '.join(story_info['genres'])}\n")
        f.write(f"\n{'='*60}\n\n")
        f.write(content)
    
    # 保存元数据
    metadata_file = os.path.join(story_dir, 'metadata.md')
    with open(metadata_file, 'w', encoding='utf-8') as f:
        f.write(f"# {story_info['title']} - Metadata\n\n")
        f.write(f"- **Title**: {story_info['title']}\n")
        f.write(f"- **Author**: {story_info['author']}\n")
        f.write(f"- **Year**: {story_info.get('publish_year', 'Unknown')}\n")
        f.write(f"- **Words**: {story_info.get('words', 'Unknown')}\n")
        f.write(f"- **Genres**: {', '.join(story_info.get('genres', []))}\n")
        f.write(f"- **Source URL**: {story_info['url']}\n")
        f.write(f"- **Download Date**: {datetime.now().isoformat()}\n")
        
        if metadata:
            f.write(f"\n## Additional Metadata\n\n")
            for key, value in metadata.items():
                if isinstance(value, list):
                    f.write(f"- **{key}**: {', '.join(str(v) for v in value)}\n")
                else:
                    f.write(f"- **{key}**: {value}\n")
        
        if story_info.get('description'):
            f.write(f"\n## Description\n\n{story_info['description']}\n")
    
    return story_dir


def download_one_story(story_info):
    """下载单个故事"""
    try:
        print(f"  📥 下载: {story_info['title'][:50]}...")
        
        # 获取内容
        content = fetch_story_content(story_info['code'])
        
        # 获取额外元数据（可选）
        metadata = fetch_story_metadata(story_info['code'])
        
        # 保存
        story_dir = save_story(story_info, content, metadata)
        
        print(f"  ✅ 已保存: {story_dir}")
        print(f"     字数: {len(content)} 字符")
        return True, None
        
    except Exception as e:
        print(f"  ❌ 失败: {e}")
        return False, str(e)


def main():
    print("="*60)
    print("📚 Library of Short Stories - 下载器")
    print("="*60)
    
    # 加载索引
    if not os.path.exists(INDEX_FILE):
        print("❌ 索引文件不存在，请先运行: ./start.sh fetch")
        return
    
    index = load_index()
    stories = index['stories']
    
    # 统计
    pending = [s for s in stories if s['status'] == 'pending']
    downloaded = [s for s in stories if s['status'] == 'downloaded']
    failed = [s for s in stories if s['status'] == 'failed']
    
    print(f"\n📊 当前状态:")
    print(f"   总计: {len(stories)}")
    print(f"   待下载: {len(pending)}")
    print(f"   已完成: {len(downloaded)}")
    print(f"   失败: {len(failed)}")
    print(f"\n⏱️  下载间隔: {DELAY} 秒")
    print(f"   预计时间: {len(pending) * DELAY // 60} 分钟\n")
    
    if not pending:
        print("✅ 所有故事已下载完成！")
        return
    
    # 开始下载
    print("开始下载... (按 Ctrl+C 暂停)\n")
    
    try:
        for i, story in enumerate(pending, 1):
            print(f"[{i}/{len(pending)}] ID: {story['id']}")
            
            success, error = download_one_story(story)
            
            # 更新状态
            story['status'] = 'downloaded' if success else 'failed'
            story['download_date'] = datetime.now().isoformat() if success else None
            story['error'] = error
            
            # 每下载一个就保存索引（断点续传）
            save_index(index)
            
            # 延迟
            if i < len(pending):
                time.sleep(DELAY)
        
        print("\n" + "="*60)
        print("✅ 下载完成！")
        print("="*60)
        
    except KeyboardInterrupt:
        print("\n\n⚠️ 用户中断")
        print(f"进度已保存，下次会继续从断点下载")
        save_index(index)


if __name__ == '__main__':
    main()