a
This commit is contained in:
Ubuntu
+192
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 2: 下载故事内容
|
||||
使用网站 API 直接获取纯文本
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import time
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
# 路径配置
|
||||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||||
INDEX_FILE = os.path.join(CONFIG_DIR, 'stories_index.json')
|
||||
STORAGE_DIR = '/home/ubuntu/media/libraryofshortstories'
|
||||
DELAY = 7 # 秒
|
||||
|
||||
|
||||
def load_index():
|
||||
"""加载索引"""
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引"""
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def fetch_story_content(code):
|
||||
"""获取故事内容（纯文本）"""
|
||||
url = f"https://www.libraryofshortstories.com/storiesgzip/{code}.txt"
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept-Encoding': 'gzip',
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def fetch_story_metadata(code):
|
||||
"""获取故事元数据"""
|
||||
url = f"https://www.libraryofshortstories.com/storymetadata/{code}"
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept-Encoding': 'gzip',
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def save_story(story_info, content, metadata):
|
||||
"""保存故事到目录"""
|
||||
# 创建目录
|
||||
story_dir = os.path.join(STORAGE_DIR, story_info['code'])
|
||||
os.makedirs(story_dir, exist_ok=True)
|
||||
|
||||
# 保存内容
|
||||
content_file = os.path.join(story_dir, 'content.txt')
|
||||
with open(content_file, 'w', encoding='utf-8') as f:
|
||||
f.write(f"# {story_info['title']}\n\n")
|
||||
f.write(f"Author: {story_info['author']}\n")
|
||||
if story_info.get('publish_year'):
|
||||
f.write(f"Year: {story_info['publish_year']}\n")
|
||||
if story_info.get('genres'):
|
||||
f.write(f"Genres: {', '.join(story_info['genres'])}\n")
|
||||
f.write(f"\n{'='*60}\n\n")
|
||||
f.write(content)
|
||||
|
||||
# 保存元数据
|
||||
metadata_file = os.path.join(story_dir, 'metadata.md')
|
||||
with open(metadata_file, 'w', encoding='utf-8') as f:
|
||||
f.write(f"# {story_info['title']} - Metadata\n\n")
|
||||
f.write(f"- **Title**: {story_info['title']}\n")
|
||||
f.write(f"- **Author**: {story_info['author']}\n")
|
||||
f.write(f"- **Year**: {story_info.get('publish_year', 'Unknown')}\n")
|
||||
f.write(f"- **Words**: {story_info.get('words', 'Unknown')}\n")
|
||||
f.write(f"- **Genres**: {', '.join(story_info.get('genres', []))}\n")
|
||||
f.write(f"- **Source URL**: {story_info['url']}\n")
|
||||
f.write(f"- **Download Date**: {datetime.now().isoformat()}\n")
|
||||
|
||||
if metadata:
|
||||
f.write(f"\n## Additional Metadata\n\n")
|
||||
for key, value in metadata.items():
|
||||
if isinstance(value, list):
|
||||
f.write(f"- **{key}**: {', '.join(str(v) for v in value)}\n")
|
||||
else:
|
||||
f.write(f"- **{key}**: {value}\n")
|
||||
|
||||
if story_info.get('description'):
|
||||
f.write(f"\n## Description\n\n{story_info['description']}\n")
|
||||
|
||||
return story_dir
|
||||
|
||||
|
||||
def download_one_story(story_info):
|
||||
"""下载单个故事"""
|
||||
try:
|
||||
print(f" 📥 下载: {story_info['title'][:50]}...")
|
||||
|
||||
# 获取内容
|
||||
content = fetch_story_content(story_info['code'])
|
||||
|
||||
# 获取额外元数据（可选）
|
||||
metadata = fetch_story_metadata(story_info['code'])
|
||||
|
||||
# 保存
|
||||
story_dir = save_story(story_info, content, metadata)
|
||||
|
||||
print(f" ✅ 已保存: {story_dir}")
|
||||
print(f" 字数: {len(content)} 字符")
|
||||
return True, None
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 失败: {e}")
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def main():
|
||||
print("="*60)
|
||||
print("📚 Library of Short Stories - 下载器")
|
||||
print("="*60)
|
||||
|
||||
# 加载索引
|
||||
if not os.path.exists(INDEX_FILE):
|
||||
print("❌ 索引文件不存在，请先运行: ./start.sh fetch")
|
||||
return
|
||||
|
||||
index = load_index()
|
||||
stories = index['stories']
|
||||
|
||||
# 统计
|
||||
pending = [s for s in stories if s['status'] == 'pending']
|
||||
downloaded = [s for s in stories if s['status'] == 'downloaded']
|
||||
failed = [s for s in stories if s['status'] == 'failed']
|
||||
|
||||
print(f"\n📊 当前状态:")
|
||||
print(f" 总计: {len(stories)}")
|
||||
print(f" 待下载: {len(pending)}")
|
||||
print(f" 已完成: {len(downloaded)}")
|
||||
print(f" 失败: {len(failed)}")
|
||||
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
|
||||
print(f" 预计时间: {len(pending) * DELAY // 60} 分钟\n")
|
||||
|
||||
if not pending:
|
||||
print("✅ 所有故事已下载完成！")
|
||||
return
|
||||
|
||||
# 开始下载
|
||||
print("开始下载... (按 Ctrl+C 暂停)\n")
|
||||
|
||||
try:
|
||||
for i, story in enumerate(pending, 1):
|
||||
print(f"[{i}/{len(pending)}] ID: {story['id']}")
|
||||
|
||||
success, error = download_one_story(story)
|
||||
|
||||
# 更新状态
|
||||
story['status'] = 'downloaded' if success else 'failed'
|
||||
story['download_date'] = datetime.now().isoformat() if success else None
|
||||
story['error'] = error
|
||||
|
||||
# 每下载一个就保存索引（断点续传）
|
||||
save_index(index)
|
||||
|
||||
# 延迟
|
||||
if i < len(pending):
|
||||
time.sleep(DELAY)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ 下载完成！")
|
||||
print("="*60)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ 用户中断")
|
||||
print(f"进度已保存，下次会继续从断点下载")
|
||||
save_index(index)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 1: 获取故事索引
|
||||
从 API 获取所有故事链接并保存
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||||
INDEX_FILE = os.path.join(CONFIG_DIR, 'stories_index.json')
|
||||
|
||||
|
||||
def fetch_story_list():
|
||||
"""从 API 获取故事列表"""
|
||||
url = "https://www.libraryofshortstories.com/storylist-reduced"
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept': 'application/json',
|
||||
'Accept-Encoding': 'gzip',
|
||||
}
|
||||
|
||||
print("🔍 正在获取故事列表...")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
stories = response.json()
|
||||
print(f"✅ 获取到 {len(stories)} 个故事")
|
||||
return stories
|
||||
|
||||
|
||||
def build_index(stories):
|
||||
"""构建索引（最多1000个）"""
|
||||
index = {
|
||||
'fetch_date': datetime.now().isoformat(),
|
||||
'source_url': 'https://www.libraryofshortstories.com/storylist-reduced',
|
||||
'total_available': len(stories),
|
||||
'max_limit': 1000,
|
||||
'stories': []
|
||||
}
|
||||
|
||||
for i, story in enumerate(stories[:1000], 1):
|
||||
story_entry = {
|
||||
'id': i,
|
||||
'title': story.get('title', 'Unknown'),
|
||||
'author': story.get('author', 'Unknown'),
|
||||
'code': story.get('code'),
|
||||
'url': f"https://www.libraryofshortstories.com/onlinereader/{story.get('code')}",
|
||||
'words': story.get('words', 0),
|
||||
'publish_year': story.get('publish_year'),
|
||||
'genres': story.get('genres', []),
|
||||
'description': story.get('description', ''),
|
||||
'status': 'pending', # pending, downloaded, failed
|
||||
'download_date': None,
|
||||
'error': None
|
||||
}
|
||||
index['stories'].append(story_entry)
|
||||
|
||||
return index
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引到文件"""
|
||||
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
|
||||
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"💾 索引已保存: {INDEX_FILE}")
|
||||
print(f"📊 总计: {len(index['stories'])} 个故事")
|
||||
|
||||
|
||||
def main():
|
||||
# 检查是否已有索引
|
||||
if os.path.exists(INDEX_FILE):
|
||||
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
existing = json.load(f)
|
||||
print(f" 上次更新: {existing.get('fetch_date')}")
|
||||
print(f" 故事数量: {len(existing.get('stories', []))}")
|
||||
|
||||
response = input("\n是否重新获取? (y/N): ")
|
||||
if response.lower() != 'y':
|
||||
print("👋 保持现有索引")
|
||||
return
|
||||
|
||||
# 获取并保存
|
||||
stories = fetch_story_list()
|
||||
index = build_index(stories)
|
||||
save_index(index)
|
||||
|
||||
print("\n✅ Phase 1 完成")
|
||||
print("下一步: 运行 download_stories.py 开始下载")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user