a
This commit is contained in:
+259
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 2: 下载诗歌内容
|
||||
每个诗人的诗歌直接放在诗人目录下
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import re
|
||||
import os
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
from pathlib import Path
|
||||
import html
|
||||
|
||||
# 路径配置
|
||||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||||
INDEX_FILE = os.path.join(CONFIG_DIR, 'poets_index.json')
|
||||
STORAGE_DIR = Path('/home/ubuntu/media/public-domain-poetry')
|
||||
DELAY = 5 # 秒
|
||||
|
||||
|
||||
def load_index():
|
||||
"""加载索引"""
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引"""
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def fetch_poet_page(url):
|
||||
"""获取诗人页面(诗歌列表)"""
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def parse_poems_from_html(html, poet_slug):
|
||||
"""从诗人页面解析诗歌列表"""
|
||||
poems = []
|
||||
|
||||
# 匹配诗歌链接: <a href="a-h-laidlaw/babylon-22309">Babylon. </a>
|
||||
pattern = r'<a\s+href="(' + re.escape(poet_slug) + r'/[^"]+)"[^>]*>([^<]+)</a>'
|
||||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||||
|
||||
seen = set()
|
||||
for slug, title in matches:
|
||||
slug = slug.strip()
|
||||
title = title.strip()
|
||||
|
||||
# 去重
|
||||
if slug not in seen:
|
||||
seen.add(slug)
|
||||
poems.append({
|
||||
'slug': slug,
|
||||
'title': title,
|
||||
'url': f"https://www.public-domain-poetry.com/{slug}"
|
||||
})
|
||||
|
||||
return poems
|
||||
|
||||
|
||||
def fetch_poem_content(url):
|
||||
"""获取诗歌内容"""
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def extract_poem_text(html, poet_name, poem_title):
|
||||
"""从诗歌页面提取文本内容"""
|
||||
# 尝试多种模式提取诗歌内容
|
||||
|
||||
# 查找标题后的内容
|
||||
# 格式通常是: Title. By Poet Name ... 诗歌内容 ...
|
||||
# 诗歌内容通常在 </center> 或 <p> 标签后
|
||||
|
||||
# 尝试提取 <p> 标签中的内容
|
||||
patterns = [
|
||||
# 查找诗歌主体部分
|
||||
r'<p[^>]*>(.*?)</p>',
|
||||
# 查找 font 标签中的内容
|
||||
r'<font[^>]*>(.*?)</font>',
|
||||
]
|
||||
|
||||
lines = []
|
||||
|
||||
# 尝试提取所有可能的诗歌行
|
||||
# 诗歌通常在页面中间,有很多 <br> 或 <p>
|
||||
poem_pattern = r'(?:<br\s*/?>\s*|</p>\s*<p[^>]*>\s*)([A-Z][^<]{10,200})(?:<br\s*/?>|</p>)'
|
||||
matches = re.findall(poem_pattern, html, re.IGNORECASE | re.DOTALL)
|
||||
|
||||
if matches:
|
||||
for match in matches[:100]: # 限制数量
|
||||
line = re.sub(r'<[^>]+>', '', match).strip()
|
||||
if line and len(line) > 10:
|
||||
lines.append(line)
|
||||
|
||||
# 如果没找到,尝试更简单的提取
|
||||
if not lines:
|
||||
# 查找所有大写开头的行
|
||||
all_text = re.sub(r'<[^>]+>', ' ', html)
|
||||
all_text = html.unescape(all_text)
|
||||
|
||||
# 查找诗歌部分(通常在 "By 诗人名" 之后)
|
||||
poet_pattern = rf'By\s+{re.escape(poet_name)}.*?(\n[A-Z].*?)(?:\n\n|\Z)'
|
||||
match = re.search(poet_pattern, all_text, re.DOTALL | re.IGNORECASE)
|
||||
|
||||
if match:
|
||||
poem_text = match.group(1)
|
||||
lines = [line.strip() for line in poem_text.split('\n') if line.strip()]
|
||||
|
||||
return '\n\n'.join(lines) if lines else None
|
||||
|
||||
|
||||
def save_poem(poet_slug, poem_slug, title, content, poet_name):
|
||||
"""保存诗歌到诗人目录"""
|
||||
# 创建诗人目录
|
||||
poet_dir = STORAGE_DIR / poet_slug
|
||||
poet_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 文件名: poem-slug.txt
|
||||
poem_file = poet_dir / f"{poem_slug.split('/')[-1]}.txt"
|
||||
|
||||
with open(poem_file, 'w', encoding='utf-8') as f:
|
||||
f.write(f"# {title}\n\n")
|
||||
f.write(f"Author: {poet_name}\n")
|
||||
f.write(f"Source: https://www.public-domain-poetry.com/{poem_slug}\n")
|
||||
f.write(f"Downloaded: {datetime.now().isoformat()}\n")
|
||||
f.write(f"\n{'='*60}\n\n")
|
||||
f.write(content)
|
||||
|
||||
return poem_file
|
||||
|
||||
|
||||
def download_poet(poet_info):
|
||||
"""下载单个诗人的所有诗歌"""
|
||||
poet_slug = poet_info['slug']
|
||||
poet_name = poet_info['name']
|
||||
|
||||
print(f"\n📖 诗人: {poet_name}")
|
||||
|
||||
try:
|
||||
# 获取诗歌列表
|
||||
html = fetch_poet_page(poet_info['url'])
|
||||
poems = parse_poems_from_html(html, poet_slug)
|
||||
|
||||
if not poems:
|
||||
print(f" ⚠️ 未找到诗歌")
|
||||
return True, 0
|
||||
|
||||
print(f" 发现 {len(poems)} 首诗歌")
|
||||
|
||||
downloaded_count = 0
|
||||
for i, poem in enumerate(poems, 1):
|
||||
print(f" [{i}/{len(poems)}] {poem['title'][:40]}...", end=' ')
|
||||
|
||||
try:
|
||||
# 获取诗歌内容
|
||||
poem_html = fetch_poem_content(poem['url'])
|
||||
content = extract_poem_text(poem_html, poet_name, poem['title'])
|
||||
|
||||
if content:
|
||||
save_poem(poet_slug, poem['slug'], poem['title'], content, poet_name)
|
||||
print("✅")
|
||||
downloaded_count += 1
|
||||
else:
|
||||
print("⚠️ 无内容")
|
||||
|
||||
# 延迟
|
||||
if i < len(poems):
|
||||
sleep(DELAY)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
|
||||
print(f" 完成: {downloaded_count}/{len(poems)} 首")
|
||||
return True, downloaded_count
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 错误: {e}")
|
||||
return False, 0
|
||||
|
||||
|
||||
def main():
|
||||
print("="*60)
|
||||
print("📝 Public Domain Poetry - 下载器")
|
||||
print("="*60)
|
||||
|
||||
# 加载索引
|
||||
if not os.path.exists(INDEX_FILE):
|
||||
print("❌ 索引文件不存在,请先运行 fetch_poets.py")
|
||||
return
|
||||
|
||||
index = load_index()
|
||||
poets = index['poets']
|
||||
|
||||
# 统计
|
||||
pending = [p for p in poets if p['status'] == 'pending']
|
||||
downloaded = [p for p in poets if p['status'] == 'downloaded']
|
||||
failed = [p for p in poets if p['status'] == 'failed']
|
||||
|
||||
print(f"\n📊 当前状态:")
|
||||
print(f" 总计诗人: {len(poets)}")
|
||||
print(f" 待下载: {len(pending)}")
|
||||
print(f" 已完成: {len(downloaded)}")
|
||||
print(f" 失败: {len(failed)}")
|
||||
|
||||
if not pending:
|
||||
print("\n✅ 所有诗歌已下载完成!")
|
||||
return
|
||||
|
||||
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
|
||||
print(f" 预计时间: 较长(每个诗人多首诗歌)\n")
|
||||
|
||||
print("开始下载... (按 Ctrl+C 暂停)\n")
|
||||
|
||||
try:
|
||||
for i, poet in enumerate(pending, 1):
|
||||
print(f"[{i}/{len(pending)}] 诗人: {poet['name']}")
|
||||
|
||||
success, count = download_poet(poet)
|
||||
|
||||
# 更新状态
|
||||
poet['status'] = 'downloaded' if success else 'failed'
|
||||
poet['download_date'] = datetime.now().isoformat() if success else None
|
||||
poet['poems_count'] = count
|
||||
|
||||
# 保存索引
|
||||
save_index(index)
|
||||
|
||||
# 诗人之间延迟
|
||||
if i < len(pending):
|
||||
sleep(DELAY)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ 下载完成!")
|
||||
print("="*60)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ 用户中断")
|
||||
print(f"进度已保存,下次会继续从断点下载")
|
||||
save_index(index)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user