Ubuntu
143 lines
3.9 KiB
Python
Executable File
143 lines
3.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
Phase 1: 获取所有诗人列表
|
||
从 A-Z 分页抓取所有诗人
|
||
"""
|
||
|
||
import json
|
||
import requests
|
||
import re
|
||
import os
|
||
from datetime import datetime
|
||
from time import sleep
|
||
|
||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||
INDEX_FILE = os.path.join(CONFIG_DIR, 'poets_index.json')
|
||
BASE_URL = "https://www.public-domain-poetry.com"
|
||
LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||
DELAY = 5 # 秒
|
||
|
||
|
||
def fetch_letter_page(letter):
|
||
"""抓取单个字母页面的诗人列表"""
|
||
url = f"{BASE_URL}/name.php?l={letter}&s=author"
|
||
headers = {
|
||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||
}
|
||
|
||
print(f" 抓取字母 {letter}...")
|
||
response = requests.get(url, headers=headers, timeout=30)
|
||
response.raise_for_status()
|
||
return response.text
|
||
|
||
|
||
def parse_poets_from_html(html):
|
||
"""从 HTML 解析诗人列表"""
|
||
poets = []
|
||
|
||
# 匹配诗人链接，slug 格式为小写字母和连字符，如 "a-h-laidlaw"
|
||
# 排除 .php 文件和 /stories/ 路径
|
||
pattern = r'<A\s+HREF="([a-z][a-z0-9-]*)"[^>]*>([^<0-9][^<]{2,50})</a>'
|
||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||
|
||
for slug, name in matches:
|
||
# 清理
|
||
slug = slug.strip().lower()
|
||
name = name.strip()
|
||
|
||
# 过滤无效项
|
||
if len(name) < 3 or name in ['Home', 'Contact', 'Search']:
|
||
continue
|
||
|
||
# 跳过已经存在的
|
||
if not any(p['slug'] == slug for p in poets):
|
||
poets.append({
|
||
'slug': slug,
|
||
'name': name,
|
||
'url': f"{BASE_URL}/{slug}",
|
||
'status': 'pending',
|
||
'download_date': None,
|
||
'poems_count': 0
|
||
})
|
||
|
||
return poets
|
||
|
||
|
||
def build_index():
|
||
"""构建诗人索引（遍历 A-Z）"""
|
||
index = {
|
||
'fetch_date': datetime.now().isoformat(),
|
||
'source_url': BASE_URL,
|
||
'letters': LETTERS,
|
||
'poets': []
|
||
}
|
||
|
||
all_poets = []
|
||
|
||
for i, letter in enumerate(LETTERS, 1):
|
||
print(f"\n[{i}/26] 处理字母 {letter}")
|
||
|
||
try:
|
||
html = fetch_letter_page(letter)
|
||
poets = parse_poets_from_html(html)
|
||
print(f" ✅ 找到 {len(poets)} 个诗人")
|
||
all_poets.extend(poets)
|
||
|
||
except Exception as e:
|
||
print(f" ❌ 错误: {e}")
|
||
|
||
# 延迟（除了最后一个）
|
||
if i < len(LETTERS):
|
||
sleep(DELAY)
|
||
|
||
# 去重（按 slug）
|
||
seen = set()
|
||
unique_poets = []
|
||
for poet in all_poets:
|
||
if poet['slug'] not in seen:
|
||
seen.add(poet['slug'])
|
||
unique_poets.append(poet)
|
||
|
||
index['poets'] = unique_poets
|
||
return index
|
||
|
||
|
||
def save_index(index):
|
||
"""保存索引"""
|
||
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
|
||
|
||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||
|
||
print(f"\n💾 索引已保存: {INDEX_FILE}")
|
||
print(f"📊 总计: {len(index['poets'])} 个诗人")
|
||
|
||
|
||
def main():
|
||
# 检查是否已有索引
|
||
if os.path.exists(INDEX_FILE):
|
||
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
|
||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||
existing = json.load(f)
|
||
print(f" 上次更新: {existing.get('fetch_date')}")
|
||
print(f" 诗人数量: {len(existing.get('poets', []))}")
|
||
|
||
response = input("\n是否重新获取? (y/N): ")
|
||
if response.lower() != 'y':
|
||
print("👋 保持现有索引")
|
||
return
|
||
|
||
# 获取并保存
|
||
print("\n🔍 开始抓取诗人列表...")
|
||
print(f"⏱️ 预计时间: {len(LETTERS) * DELAY // 60} 分钟\n")
|
||
|
||
index = build_index()
|
||
save_index(index)
|
||
|
||
print("\n✅ Phase 1 完成")
|
||
print("下一步: 运行 download_poems.py 开始下载诗歌")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|