a

2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Phase 1: 获取所有诗人列表
+从 A-Z 分页抓取所有诗人
+"""
+
+import json
+import requests
+import re
+import os
+from datetime import datetime
+from time import sleep
+
+CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
+INDEX_FILE = os.path.join(CONFIG_DIR, 'poets_index.json')
+BASE_URL = "https://www.public-domain-poetry.com"
+LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+DELAY = 5  # 秒
+
+
+def fetch_letter_page(letter):
+    """抓取单个字母页面的诗人列表"""
+    url = f"{BASE_URL}/name.php?l={letter}&s=author"
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
+    }
+    
+    print(f"  抓取字母 {letter}...")
+    response = requests.get(url, headers=headers, timeout=30)
+    response.raise_for_status()
+    return response.text
+
+
+def parse_poets_from_html(html):
+    """从 HTML 解析诗人列表"""
+    poets = []
+    
+    # 匹配诗人链接，slug 格式为小写字母和连字符，如 "a-h-laidlaw"
+    # 排除 .php 文件和 /stories/ 路径
+    pattern = r'<A\s+HREF="([a-z][a-z0-9-]*)"[^>]*>([^<0-9][^<]{2,50})</a>'
+    matches = re.findall(pattern, html, re.IGNORECASE)
+    
+    for slug, name in matches:
+        # 清理
+        slug = slug.strip().lower()
+        name = name.strip()
+        
+        # 过滤无效项
+        if len(name) < 3 or name in ['Home', 'Contact', 'Search']:
+            continue
+        
+        # 跳过已经存在的
+        if not any(p['slug'] == slug for p in poets):
+            poets.append({
+                'slug': slug,
+                'name': name,
+                'url': f"{BASE_URL}/{slug}",
+                'status': 'pending',
+                'download_date': None,
+                'poems_count': 0
+            })
+    
+    return poets
+
+
+def build_index():
+    """构建诗人索引（遍历 A-Z）"""
+    index = {
+        'fetch_date': datetime.now().isoformat(),
+        'source_url': BASE_URL,
+        'letters': LETTERS,
+        'poets': []
+    }
+    
+    all_poets = []
+    
+    for i, letter in enumerate(LETTERS, 1):
+        print(f"\n[{i}/26] 处理字母 {letter}")
+        
+        try:
+            html = fetch_letter_page(letter)
+            poets = parse_poets_from_html(html)
+            print(f"  ✅ 找到 {len(poets)} 个诗人")
+            all_poets.extend(poets)
+            
+        except Exception as e:
+            print(f"  ❌ 错误: {e}")
+        
+        # 延迟（除了最后一个）
+        if i < len(LETTERS):
+            sleep(DELAY)
+    
+    # 去重（按 slug）
+    seen = set()
+    unique_poets = []
+    for poet in all_poets:
+        if poet['slug'] not in seen:
+            seen.add(poet['slug'])
+            unique_poets.append(poet)
+    
+    index['poets'] = unique_poets
+    return index
+
+
+def save_index(index):
+    """保存索引"""
+    os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
+    
+    with open(INDEX_FILE, 'w', encoding='utf-8') as f:
+        json.dump(index, f, indent=2, ensure_ascii=False)
+    
+    print(f"\n💾 索引已保存: {INDEX_FILE}")
+    print(f"📊 总计: {len(index['poets'])} 个诗人")
+
+
+def main():
+    # 检查是否已有索引
+    if os.path.exists(INDEX_FILE):
+        print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
+        with open(INDEX_FILE, 'r', encoding='utf-8') as f:
+            existing = json.load(f)
+        print(f"   上次更新: {existing.get('fetch_date')}")
+        print(f"   诗人数量: {len(existing.get('poets', []))}")
+        
+        response = input("\n是否重新获取? (y/N): ")
+        if response.lower() != 'y':
+            print("👋 保持现有索引")
+            return
+    
+    # 获取并保存
+    print("\n🔍 开始抓取诗人列表...")
+    print(f"⏱️  预计时间: {len(LETTERS) * DELAY // 60} 分钟\n")
+    
+    index = build_index()
+    save_index(index)
+    
+    print("\n✅ Phase 1 完成")
+    print("下一步: 运行 download_poems.py 开始下载诗歌")
+
+
+if __name__ == '__main__':
+    main()