a
This commit is contained in:
Ubuntu
+259
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 2: 下载诗歌内容
|
||||
每个诗人的诗歌直接放在诗人目录下
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import re
|
||||
import os
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
from pathlib import Path
|
||||
import html
|
||||
|
||||
# 路径配置
|
||||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||||
INDEX_FILE = os.path.join(CONFIG_DIR, 'poets_index.json')
|
||||
STORAGE_DIR = Path('/home/ubuntu/media/public-domain-poetry')
|
||||
DELAY = 5 # 秒
|
||||
|
||||
|
||||
def load_index():
|
||||
"""加载索引"""
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引"""
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def fetch_poet_page(url):
|
||||
"""获取诗人页面（诗歌列表）"""
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def parse_poems_from_html(html, poet_slug):
|
||||
"""从诗人页面解析诗歌列表"""
|
||||
poems = []
|
||||
|
||||
# 匹配诗歌链接: <a href="a-h-laidlaw/babylon-22309">Babylon. </a>
|
||||
pattern = r'<a\s+href="(' + re.escape(poet_slug) + r'/[^"]+)"[^>]*>([^<]+)</a>'
|
||||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||||
|
||||
seen = set()
|
||||
for slug, title in matches:
|
||||
slug = slug.strip()
|
||||
title = title.strip()
|
||||
|
||||
# 去重
|
||||
if slug not in seen:
|
||||
seen.add(slug)
|
||||
poems.append({
|
||||
'slug': slug,
|
||||
'title': title,
|
||||
'url': f"https://www.public-domain-poetry.com/{slug}"
|
||||
})
|
||||
|
||||
return poems
|
||||
|
||||
|
||||
def fetch_poem_content(url):
|
||||
"""获取诗歌内容"""
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def extract_poem_text(html, poet_name, poem_title):
|
||||
"""从诗歌页面提取文本内容"""
|
||||
# 尝试多种模式提取诗歌内容
|
||||
|
||||
# 查找标题后的内容
|
||||
# 格式通常是: Title. By Poet Name ... 诗歌内容 ...
|
||||
# 诗歌内容通常在 </center> 或 <p> 标签后
|
||||
|
||||
# 尝试提取 <p> 标签中的内容
|
||||
patterns = [
|
||||
# 查找诗歌主体部分
|
||||
r'<p[^>]*>(.*?)</p>',
|
||||
# 查找 font 标签中的内容
|
||||
r'<font[^>]*>(.*?)</font>',
|
||||
]
|
||||
|
||||
lines = []
|
||||
|
||||
# 尝试提取所有可能的诗歌行
|
||||
# 诗歌通常在页面中间，有很多 <br> 或 <p>
|
||||
poem_pattern = r'(?:<br\s*/?>\s*|</p>\s*<p[^>]*>\s*)([A-Z][^<]{10,200})(?:<br\s*/?>|</p>)'
|
||||
matches = re.findall(poem_pattern, html, re.IGNORECASE | re.DOTALL)
|
||||
|
||||
if matches:
|
||||
for match in matches[:100]: # 限制数量
|
||||
line = re.sub(r'<[^>]+>', '', match).strip()
|
||||
if line and len(line) > 10:
|
||||
lines.append(line)
|
||||
|
||||
# 如果没找到，尝试更简单的提取
|
||||
if not lines:
|
||||
# 查找所有大写开头的行
|
||||
all_text = re.sub(r'<[^>]+>', ' ', html)
|
||||
all_text = html.unescape(all_text)
|
||||
|
||||
# 查找诗歌部分（通常在 "By 诗人名" 之后）
|
||||
poet_pattern = rf'By\s+{re.escape(poet_name)}.*?(\n[A-Z].*?)(?:\n\n|\Z)'
|
||||
match = re.search(poet_pattern, all_text, re.DOTALL | re.IGNORECASE)
|
||||
|
||||
if match:
|
||||
poem_text = match.group(1)
|
||||
lines = [line.strip() for line in poem_text.split('\n') if line.strip()]
|
||||
|
||||
return '\n\n'.join(lines) if lines else None
|
||||
|
||||
|
||||
def save_poem(poet_slug, poem_slug, title, content, poet_name):
|
||||
"""保存诗歌到诗人目录"""
|
||||
# 创建诗人目录
|
||||
poet_dir = STORAGE_DIR / poet_slug
|
||||
poet_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 文件名: poem-slug.txt
|
||||
poem_file = poet_dir / f"{poem_slug.split('/')[-1]}.txt"
|
||||
|
||||
with open(poem_file, 'w', encoding='utf-8') as f:
|
||||
f.write(f"# {title}\n\n")
|
||||
f.write(f"Author: {poet_name}\n")
|
||||
f.write(f"Source: https://www.public-domain-poetry.com/{poem_slug}\n")
|
||||
f.write(f"Downloaded: {datetime.now().isoformat()}\n")
|
||||
f.write(f"\n{'='*60}\n\n")
|
||||
f.write(content)
|
||||
|
||||
return poem_file
|
||||
|
||||
|
||||
def download_poet(poet_info):
|
||||
"""下载单个诗人的所有诗歌"""
|
||||
poet_slug = poet_info['slug']
|
||||
poet_name = poet_info['name']
|
||||
|
||||
print(f"\n📖 诗人: {poet_name}")
|
||||
|
||||
try:
|
||||
# 获取诗歌列表
|
||||
html = fetch_poet_page(poet_info['url'])
|
||||
poems = parse_poems_from_html(html, poet_slug)
|
||||
|
||||
if not poems:
|
||||
print(f" ⚠️ 未找到诗歌")
|
||||
return True, 0
|
||||
|
||||
print(f" 发现 {len(poems)} 首诗歌")
|
||||
|
||||
downloaded_count = 0
|
||||
for i, poem in enumerate(poems, 1):
|
||||
print(f" [{i}/{len(poems)}] {poem['title'][:40]}...", end=' ')
|
||||
|
||||
try:
|
||||
# 获取诗歌内容
|
||||
poem_html = fetch_poem_content(poem['url'])
|
||||
content = extract_poem_text(poem_html, poet_name, poem['title'])
|
||||
|
||||
if content:
|
||||
save_poem(poet_slug, poem['slug'], poem['title'], content, poet_name)
|
||||
print("✅")
|
||||
downloaded_count += 1
|
||||
else:
|
||||
print("⚠️ 无内容")
|
||||
|
||||
# 延迟
|
||||
if i < len(poems):
|
||||
sleep(DELAY)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
|
||||
print(f" 完成: {downloaded_count}/{len(poems)} 首")
|
||||
return True, downloaded_count
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 错误: {e}")
|
||||
return False, 0
|
||||
|
||||
|
||||
def main():
|
||||
print("="*60)
|
||||
print("📝 Public Domain Poetry - 下载器")
|
||||
print("="*60)
|
||||
|
||||
# 加载索引
|
||||
if not os.path.exists(INDEX_FILE):
|
||||
print("❌ 索引文件不存在，请先运行 fetch_poets.py")
|
||||
return
|
||||
|
||||
index = load_index()
|
||||
poets = index['poets']
|
||||
|
||||
# 统计
|
||||
pending = [p for p in poets if p['status'] == 'pending']
|
||||
downloaded = [p for p in poets if p['status'] == 'downloaded']
|
||||
failed = [p for p in poets if p['status'] == 'failed']
|
||||
|
||||
print(f"\n📊 当前状态:")
|
||||
print(f" 总计诗人: {len(poets)}")
|
||||
print(f" 待下载: {len(pending)}")
|
||||
print(f" 已完成: {len(downloaded)}")
|
||||
print(f" 失败: {len(failed)}")
|
||||
|
||||
if not pending:
|
||||
print("\n✅ 所有诗歌已下载完成！")
|
||||
return
|
||||
|
||||
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
|
||||
print(f" 预计时间: 较长（每个诗人多首诗歌）\n")
|
||||
|
||||
print("开始下载... (按 Ctrl+C 暂停)\n")
|
||||
|
||||
try:
|
||||
for i, poet in enumerate(pending, 1):
|
||||
print(f"[{i}/{len(pending)}] 诗人: {poet['name']}")
|
||||
|
||||
success, count = download_poet(poet)
|
||||
|
||||
# 更新状态
|
||||
poet['status'] = 'downloaded' if success else 'failed'
|
||||
poet['download_date'] = datetime.now().isoformat() if success else None
|
||||
poet['poems_count'] = count
|
||||
|
||||
# 保存索引
|
||||
save_index(index)
|
||||
|
||||
# 诗人之间延迟
|
||||
if i < len(pending):
|
||||
sleep(DELAY)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ 下载完成！")
|
||||
print("="*60)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ 用户中断")
|
||||
print(f"进度已保存，下次会继续从断点下载")
|
||||
save_index(index)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 1: 获取所有诗人列表
|
||||
从 A-Z 分页抓取所有诗人
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import re
|
||||
import os
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
|
||||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||||
INDEX_FILE = os.path.join(CONFIG_DIR, 'poets_index.json')
|
||||
BASE_URL = "https://www.public-domain-poetry.com"
|
||||
LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
DELAY = 5 # 秒
|
||||
|
||||
|
||||
def fetch_letter_page(letter):
|
||||
"""抓取单个字母页面的诗人列表"""
|
||||
url = f"{BASE_URL}/name.php?l={letter}&s=author"
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
print(f" 抓取字母 {letter}...")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def parse_poets_from_html(html):
|
||||
"""从 HTML 解析诗人列表"""
|
||||
poets = []
|
||||
|
||||
# 匹配诗人链接，slug 格式为小写字母和连字符，如 "a-h-laidlaw"
|
||||
# 排除 .php 文件和 /stories/ 路径
|
||||
pattern = r'<A\s+HREF="([a-z][a-z0-9-]*)"[^>]*>([^<0-9][^<]{2,50})</a>'
|
||||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||||
|
||||
for slug, name in matches:
|
||||
# 清理
|
||||
slug = slug.strip().lower()
|
||||
name = name.strip()
|
||||
|
||||
# 过滤无效项
|
||||
if len(name) < 3 or name in ['Home', 'Contact', 'Search']:
|
||||
continue
|
||||
|
||||
# 跳过已经存在的
|
||||
if not any(p['slug'] == slug for p in poets):
|
||||
poets.append({
|
||||
'slug': slug,
|
||||
'name': name,
|
||||
'url': f"{BASE_URL}/{slug}",
|
||||
'status': 'pending',
|
||||
'download_date': None,
|
||||
'poems_count': 0
|
||||
})
|
||||
|
||||
return poets
|
||||
|
||||
|
||||
def build_index():
|
||||
"""构建诗人索引（遍历 A-Z）"""
|
||||
index = {
|
||||
'fetch_date': datetime.now().isoformat(),
|
||||
'source_url': BASE_URL,
|
||||
'letters': LETTERS,
|
||||
'poets': []
|
||||
}
|
||||
|
||||
all_poets = []
|
||||
|
||||
for i, letter in enumerate(LETTERS, 1):
|
||||
print(f"\n[{i}/26] 处理字母 {letter}")
|
||||
|
||||
try:
|
||||
html = fetch_letter_page(letter)
|
||||
poets = parse_poets_from_html(html)
|
||||
print(f" ✅ 找到 {len(poets)} 个诗人")
|
||||
all_poets.extend(poets)
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 错误: {e}")
|
||||
|
||||
# 延迟（除了最后一个）
|
||||
if i < len(LETTERS):
|
||||
sleep(DELAY)
|
||||
|
||||
# 去重（按 slug）
|
||||
seen = set()
|
||||
unique_poets = []
|
||||
for poet in all_poets:
|
||||
if poet['slug'] not in seen:
|
||||
seen.add(poet['slug'])
|
||||
unique_poets.append(poet)
|
||||
|
||||
index['poets'] = unique_poets
|
||||
return index
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引"""
|
||||
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
|
||||
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n💾 索引已保存: {INDEX_FILE}")
|
||||
print(f"📊 总计: {len(index['poets'])} 个诗人")
|
||||
|
||||
|
||||
def main():
|
||||
# 检查是否已有索引
|
||||
if os.path.exists(INDEX_FILE):
|
||||
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
existing = json.load(f)
|
||||
print(f" 上次更新: {existing.get('fetch_date')}")
|
||||
print(f" 诗人数量: {len(existing.get('poets', []))}")
|
||||
|
||||
response = input("\n是否重新获取? (y/N): ")
|
||||
if response.lower() != 'y':
|
||||
print("👋 保持现有索引")
|
||||
return
|
||||
|
||||
# 获取并保存
|
||||
print("\n🔍 开始抓取诗人列表...")
|
||||
print(f"⏱️ 预计时间: {len(LETTERS) * DELAY // 60} 分钟\n")
|
||||
|
||||
index = build_index()
|
||||
save_index(index)
|
||||
|
||||
print("\n✅ Phase 1 完成")
|
||||
print("下一步: 运行 download_poems.py 开始下载诗歌")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user