Ubuntu
260 lines
7.7 KiB
Python
Executable File
260 lines
7.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Phase 2: 下载诗歌内容
|
|
每个诗人的诗歌直接放在诗人目录下
|
|
"""
|
|
|
|
import json
|
|
import requests
|
|
import re
|
|
import os
|
|
from datetime import datetime
|
|
from time import sleep
|
|
from pathlib import Path
|
|
import html
|
|
|
|
# 路径配置
|
|
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
|
INDEX_FILE = os.path.join(CONFIG_DIR, 'poets_index.json')
|
|
STORAGE_DIR = Path('/home/ubuntu/media/public-domain-poetry')
|
|
DELAY = 5 # 秒
|
|
|
|
|
|
def load_index():
|
|
"""加载索引"""
|
|
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
|
|
|
|
def save_index(index):
|
|
"""保存索引"""
|
|
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
|
json.dump(index, f, indent=2, ensure_ascii=False)
|
|
|
|
|
|
def fetch_poet_page(url):
|
|
"""获取诗人页面（诗歌列表）"""
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
|
}
|
|
|
|
response = requests.get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.text
|
|
|
|
|
|
def parse_poems_from_html(html, poet_slug):
|
|
"""从诗人页面解析诗歌列表"""
|
|
poems = []
|
|
|
|
# 匹配诗歌链接: <a href="a-h-laidlaw/babylon-22309">Babylon. </a>
|
|
pattern = r'<a\s+href="(' + re.escape(poet_slug) + r'/[^"]+)"[^>]*>([^<]+)</a>'
|
|
matches = re.findall(pattern, html, re.IGNORECASE)
|
|
|
|
seen = set()
|
|
for slug, title in matches:
|
|
slug = slug.strip()
|
|
title = title.strip()
|
|
|
|
# 去重
|
|
if slug not in seen:
|
|
seen.add(slug)
|
|
poems.append({
|
|
'slug': slug,
|
|
'title': title,
|
|
'url': f"https://www.public-domain-poetry.com/{slug}"
|
|
})
|
|
|
|
return poems
|
|
|
|
|
|
def fetch_poem_content(url):
|
|
"""获取诗歌内容"""
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
|
}
|
|
|
|
response = requests.get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.text
|
|
|
|
|
|
def extract_poem_text(html, poet_name, poem_title):
|
|
"""从诗歌页面提取文本内容"""
|
|
# 尝试多种模式提取诗歌内容
|
|
|
|
# 查找标题后的内容
|
|
# 格式通常是: Title. By Poet Name ... 诗歌内容 ...
|
|
# 诗歌内容通常在 </center> 或 <p> 标签后
|
|
|
|
# 尝试提取 <p> 标签中的内容
|
|
patterns = [
|
|
# 查找诗歌主体部分
|
|
r'<p[^>]*>(.*?)</p>',
|
|
# 查找 font 标签中的内容
|
|
r'<font[^>]*>(.*?)</font>',
|
|
]
|
|
|
|
lines = []
|
|
|
|
# 尝试提取所有可能的诗歌行
|
|
# 诗歌通常在页面中间，有很多 <br> 或 <p>
|
|
poem_pattern = r'(?:<br\s*/?>\s*|</p>\s*<p[^>]*>\s*)([A-Z][^<]{10,200})(?:<br\s*/?>|</p>)'
|
|
matches = re.findall(poem_pattern, html, re.IGNORECASE | re.DOTALL)
|
|
|
|
if matches:
|
|
for match in matches[:100]: # 限制数量
|
|
line = re.sub(r'<[^>]+>', '', match).strip()
|
|
if line and len(line) > 10:
|
|
lines.append(line)
|
|
|
|
# 如果没找到，尝试更简单的提取
|
|
if not lines:
|
|
# 查找所有大写开头的行
|
|
all_text = re.sub(r'<[^>]+>', ' ', html)
|
|
all_text = html.unescape(all_text)
|
|
|
|
# 查找诗歌部分（通常在 "By 诗人名" 之后）
|
|
poet_pattern = rf'By\s+{re.escape(poet_name)}.*?(\n[A-Z].*?)(?:\n\n|\Z)'
|
|
match = re.search(poet_pattern, all_text, re.DOTALL | re.IGNORECASE)
|
|
|
|
if match:
|
|
poem_text = match.group(1)
|
|
lines = [line.strip() for line in poem_text.split('\n') if line.strip()]
|
|
|
|
return '\n\n'.join(lines) if lines else None
|
|
|
|
|
|
def save_poem(poet_slug, poem_slug, title, content, poet_name):
|
|
"""保存诗歌到诗人目录"""
|
|
# 创建诗人目录
|
|
poet_dir = STORAGE_DIR / poet_slug
|
|
poet_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 文件名: poem-slug.txt
|
|
poem_file = poet_dir / f"{poem_slug.split('/')[-1]}.txt"
|
|
|
|
with open(poem_file, 'w', encoding='utf-8') as f:
|
|
f.write(f"# {title}\n\n")
|
|
f.write(f"Author: {poet_name}\n")
|
|
f.write(f"Source: https://www.public-domain-poetry.com/{poem_slug}\n")
|
|
f.write(f"Downloaded: {datetime.now().isoformat()}\n")
|
|
f.write(f"\n{'='*60}\n\n")
|
|
f.write(content)
|
|
|
|
return poem_file
|
|
|
|
|
|
def download_poet(poet_info):
|
|
"""下载单个诗人的所有诗歌"""
|
|
poet_slug = poet_info['slug']
|
|
poet_name = poet_info['name']
|
|
|
|
print(f"\n📖 诗人: {poet_name}")
|
|
|
|
try:
|
|
# 获取诗歌列表
|
|
html = fetch_poet_page(poet_info['url'])
|
|
poems = parse_poems_from_html(html, poet_slug)
|
|
|
|
if not poems:
|
|
print(f" ⚠️ 未找到诗歌")
|
|
return True, 0
|
|
|
|
print(f" 发现 {len(poems)} 首诗歌")
|
|
|
|
downloaded_count = 0
|
|
for i, poem in enumerate(poems, 1):
|
|
print(f" [{i}/{len(poems)}] {poem['title'][:40]}...", end=' ')
|
|
|
|
try:
|
|
# 获取诗歌内容
|
|
poem_html = fetch_poem_content(poem['url'])
|
|
content = extract_poem_text(poem_html, poet_name, poem['title'])
|
|
|
|
if content:
|
|
save_poem(poet_slug, poem['slug'], poem['title'], content, poet_name)
|
|
print("✅")
|
|
downloaded_count += 1
|
|
else:
|
|
print("⚠️ 无内容")
|
|
|
|
# 延迟
|
|
if i < len(poems):
|
|
sleep(DELAY)
|
|
|
|
except Exception as e:
|
|
print(f"❌ {e}")
|
|
|
|
print(f" 完成: {downloaded_count}/{len(poems)} 首")
|
|
return True, downloaded_count
|
|
|
|
except Exception as e:
|
|
print(f" ❌ 错误: {e}")
|
|
return False, 0
|
|
|
|
|
|
def main():
|
|
print("="*60)
|
|
print("📝 Public Domain Poetry - 下载器")
|
|
print("="*60)
|
|
|
|
# 加载索引
|
|
if not os.path.exists(INDEX_FILE):
|
|
print("❌ 索引文件不存在，请先运行 fetch_poets.py")
|
|
return
|
|
|
|
index = load_index()
|
|
poets = index['poets']
|
|
|
|
# 统计
|
|
pending = [p for p in poets if p['status'] == 'pending']
|
|
downloaded = [p for p in poets if p['status'] == 'downloaded']
|
|
failed = [p for p in poets if p['status'] == 'failed']
|
|
|
|
print(f"\n📊 当前状态:")
|
|
print(f" 总计诗人: {len(poets)}")
|
|
print(f" 待下载: {len(pending)}")
|
|
print(f" 已完成: {len(downloaded)}")
|
|
print(f" 失败: {len(failed)}")
|
|
|
|
if not pending:
|
|
print("\n✅ 所有诗歌已下载完成！")
|
|
return
|
|
|
|
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
|
|
print(f" 预计时间: 较长（每个诗人多首诗歌）\n")
|
|
|
|
print("开始下载... (按 Ctrl+C 暂停)\n")
|
|
|
|
try:
|
|
for i, poet in enumerate(pending, 1):
|
|
print(f"[{i}/{len(pending)}] 诗人: {poet['name']}")
|
|
|
|
success, count = download_poet(poet)
|
|
|
|
# 更新状态
|
|
poet['status'] = 'downloaded' if success else 'failed'
|
|
poet['download_date'] = datetime.now().isoformat() if success else None
|
|
poet['poems_count'] = count
|
|
|
|
# 保存索引
|
|
save_index(index)
|
|
|
|
# 诗人之间延迟
|
|
if i < len(pending):
|
|
sleep(DELAY)
|
|
|
|
print("\n" + "="*60)
|
|
print("✅ 下载完成！")
|
|
print("="*60)
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n\n⚠️ 用户中断")
|
|
print(f"进度已保存，下次会继续从断点下载")
|
|
save_index(index)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|