This commit is contained in:
Ubuntu
2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
+41
View File
@@ -0,0 +1,41 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
*.egg-info/
dist/
build/
# Virtual environments
.venv/
.venv-*/
venv/
env/
# Node dependencies
node_modules/
# 运行时数据
config/stories_index.json
# 日志
logs/
*.log
# 环境变量
.env
# OS
.DS_Store
Thumbs.db
# IDE
.vscode/
.idea/
*.swp
*.swo
# 下载的故事内容(存储在 /home/ubuntu/media/)
# 这个目录在 git 外
+101
View File
@@ -0,0 +1,101 @@
# Library of Short Stories 下载器
从 https://www.libraryofshortstories.com 下载英语短篇故事。
## 功能特性
- ✅ **API 直接获取**: 无需浏览器,直接调用网站 API
- ✅ **断点续传**: 中断后自动从上次位置继续
- ✅ **后台运行**: 支持后台下载,不影响其他操作
- ✅ **7秒间隔**: 礼貌下载,避免被封
- ✅ **元数据保存**: 保存作者、年份、类型等信息
- ✅ **纯文本输出**: 去除 HTML 标签,干净阅读
## 快速开始
### 1. 获取故事索引(Phase 1)
```bash
./start.sh fetch
```
这会从 API 获取最多 1000 个故事的链接,保存到 `config/stories_index.json`
### 2. 下载故事(Phase 2)
**前台模式**(适合少量下载):
```bash
./start.sh download
```
**后台模式**(推荐,适合大量下载):
```bash
./start.sh download-bg
```
### 3. 查看状态
```bash
./start.sh status
```
### 4. 查看日志
```bash
./start.sh log # 查看最近日志
tail -f logs/download.log # 实时跟踪
```
### 5. 停止下载
```bash
./start.sh stop
```
## 存储结构
```
/home/ubuntu/media/libraryofshortstories/
├── the-adventure-of-the-red-circle/
│ ├── content.txt # 故事正文
│ └── metadata.md # 元数据
├── the-1000000-bank-note/
│ ├── content.txt
│ └── metadata.md
└── ...
```
## 索引文件格式
`config/stories_index.json`:
```json
{
"fetch_date": "2026-03-27T10:00:00",
"total_available": 1500,
"max_limit": 1000,
"stories": [
{
"id": 1,
"title": "The £1,000,000 Bank-Note",
"author": "Mark Twain",
"code": "the-1000000-bank-note",
"url": "https://www.libraryofshortstories.com/onlinereader/the-1000000-bank-note",
"status": "downloaded",
"download_date": "2026-03-27T10:05:00"
}
]
}
```
## 技术细节
- **下载间隔**: 7 秒
- **超时设置**: 30 秒
- **重试机制**: 失败标记为 failed,下次运行会重试
- **并发数**: 单线程,避免被封
## 注意事项
1. 下载约 1000 个故事需要约 **2 小时** (1000 × 7秒)
2. 建议首次使用后台模式 `./start.sh download-bg`
3. 可以安全中断 (Ctrl+C 或 `./start.sh stop`),进度会自动保存
+192
View File
@@ -0,0 +1,192 @@
#!/usr/bin/env python3
"""
Phase 2: 下载故事内容
使用网站 API 直接获取纯文本
"""
import json
import requests
import time
import os
import re
from datetime import datetime
# 路径配置
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
INDEX_FILE = os.path.join(CONFIG_DIR, 'stories_index.json')
STORAGE_DIR = '/home/ubuntu/media/libraryofshortstories'
DELAY = 7 # 秒
def load_index():
"""加载索引"""
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
return json.load(f)
def save_index(index):
"""保存索引"""
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
json.dump(index, f, indent=2, ensure_ascii=False)
def fetch_story_content(code):
"""获取故事内容(纯文本)"""
url = f"https://www.libraryofshortstories.com/storiesgzip/{code}.txt"
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Accept-Encoding': 'gzip',
}
response = requests.get(url, headers=headers, timeout=30)
response.raise_for_status()
return response.text
def fetch_story_metadata(code):
"""获取故事元数据"""
url = f"https://www.libraryofshortstories.com/storymetadata/{code}"
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Accept-Encoding': 'gzip',
}
try:
response = requests.get(url, headers=headers, timeout=30)
if response.status_code == 200:
return response.json()
except:
pass
return None
def save_story(story_info, content, metadata):
"""保存故事到目录"""
# 创建目录
story_dir = os.path.join(STORAGE_DIR, story_info['code'])
os.makedirs(story_dir, exist_ok=True)
# 保存内容
content_file = os.path.join(story_dir, 'content.txt')
with open(content_file, 'w', encoding='utf-8') as f:
f.write(f"# {story_info['title']}\n\n")
f.write(f"Author: {story_info['author']}\n")
if story_info.get('publish_year'):
f.write(f"Year: {story_info['publish_year']}\n")
if story_info.get('genres'):
f.write(f"Genres: {', '.join(story_info['genres'])}\n")
f.write(f"\n{'='*60}\n\n")
f.write(content)
# 保存元数据
metadata_file = os.path.join(story_dir, 'metadata.md')
with open(metadata_file, 'w', encoding='utf-8') as f:
f.write(f"# {story_info['title']} - Metadata\n\n")
f.write(f"- **Title**: {story_info['title']}\n")
f.write(f"- **Author**: {story_info['author']}\n")
f.write(f"- **Year**: {story_info.get('publish_year', 'Unknown')}\n")
f.write(f"- **Words**: {story_info.get('words', 'Unknown')}\n")
f.write(f"- **Genres**: {', '.join(story_info.get('genres', []))}\n")
f.write(f"- **Source URL**: {story_info['url']}\n")
f.write(f"- **Download Date**: {datetime.now().isoformat()}\n")
if metadata:
f.write(f"\n## Additional Metadata\n\n")
for key, value in metadata.items():
if isinstance(value, list):
f.write(f"- **{key}**: {', '.join(str(v) for v in value)}\n")
else:
f.write(f"- **{key}**: {value}\n")
if story_info.get('description'):
f.write(f"\n## Description\n\n{story_info['description']}\n")
return story_dir
def download_one_story(story_info):
"""下载单个故事"""
try:
print(f" 📥 下载: {story_info['title'][:50]}...")
# 获取内容
content = fetch_story_content(story_info['code'])
# 获取额外元数据(可选)
metadata = fetch_story_metadata(story_info['code'])
# 保存
story_dir = save_story(story_info, content, metadata)
print(f" ✅ 已保存: {story_dir}")
print(f" 字数: {len(content)} 字符")
return True, None
except Exception as e:
print(f" ❌ 失败: {e}")
return False, str(e)
def main():
print("="*60)
print("📚 Library of Short Stories - 下载器")
print("="*60)
# 加载索引
if not os.path.exists(INDEX_FILE):
print("❌ 索引文件不存在,请先运行: ./start.sh fetch")
return
index = load_index()
stories = index['stories']
# 统计
pending = [s for s in stories if s['status'] == 'pending']
downloaded = [s for s in stories if s['status'] == 'downloaded']
failed = [s for s in stories if s['status'] == 'failed']
print(f"\n📊 当前状态:")
print(f" 总计: {len(stories)}")
print(f" 待下载: {len(pending)}")
print(f" 已完成: {len(downloaded)}")
print(f" 失败: {len(failed)}")
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
print(f" 预计时间: {len(pending) * DELAY // 60} 分钟\n")
if not pending:
print("✅ 所有故事已下载完成!")
return
# 开始下载
print("开始下载... (按 Ctrl+C 暂停)\n")
try:
for i, story in enumerate(pending, 1):
print(f"[{i}/{len(pending)}] ID: {story['id']}")
success, error = download_one_story(story)
# 更新状态
story['status'] = 'downloaded' if success else 'failed'
story['download_date'] = datetime.now().isoformat() if success else None
story['error'] = error
# 每下载一个就保存索引(断点续传)
save_index(index)
# 延迟
if i < len(pending):
time.sleep(DELAY)
print("\n" + "="*60)
print("✅ 下载完成!")
print("="*60)
except KeyboardInterrupt:
print("\n\n⚠️ 用户中断")
print(f"进度已保存,下次会继续从断点下载")
save_index(index)
if __name__ == '__main__':
main()
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""
Phase 1: 获取故事索引
从 API 获取所有故事链接并保存
"""
import json
import requests
import os
from datetime import datetime
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
INDEX_FILE = os.path.join(CONFIG_DIR, 'stories_index.json')
def fetch_story_list():
"""从 API 获取故事列表"""
url = "https://www.libraryofshortstories.com/storylist-reduced"
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Accept': 'application/json',
'Accept-Encoding': 'gzip',
}
print("🔍 正在获取故事列表...")
response = requests.get(url, headers=headers, timeout=30)
response.raise_for_status()
stories = response.json()
print(f"✅ 获取到 {len(stories)} 个故事")
return stories
def build_index(stories):
"""构建索引(最多1000个)"""
index = {
'fetch_date': datetime.now().isoformat(),
'source_url': 'https://www.libraryofshortstories.com/storylist-reduced',
'total_available': len(stories),
'max_limit': 1000,
'stories': []
}
for i, story in enumerate(stories[:1000], 1):
story_entry = {
'id': i,
'title': story.get('title', 'Unknown'),
'author': story.get('author', 'Unknown'),
'code': story.get('code'),
'url': f"https://www.libraryofshortstories.com/onlinereader/{story.get('code')}",
'words': story.get('words', 0),
'publish_year': story.get('publish_year'),
'genres': story.get('genres', []),
'description': story.get('description', ''),
'status': 'pending', # pending, downloaded, failed
'download_date': None,
'error': None
}
index['stories'].append(story_entry)
return index
def save_index(index):
"""保存索引到文件"""
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
json.dump(index, f, indent=2, ensure_ascii=False)
print(f"💾 索引已保存: {INDEX_FILE}")
print(f"📊 总计: {len(index['stories'])} 个故事")
def main():
# 检查是否已有索引
if os.path.exists(INDEX_FILE):
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
existing = json.load(f)
print(f" 上次更新: {existing.get('fetch_date')}")
print(f" 故事数量: {len(existing.get('stories', []))}")
response = input("\n是否重新获取? (y/N): ")
if response.lower() != 'y':
print("👋 保持现有索引")
return
# 获取并保存
stories = fetch_story_list()
index = build_index(stories)
save_index(index)
print("\n✅ Phase 1 完成")
print("下一步: 运行 download_stories.py 开始下载")
if __name__ == '__main__':
main()
+165
View File
@@ -0,0 +1,165 @@
#!/bin/bash
# Library of Short Stories 下载器启动脚本
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
LOG_DIR="$SCRIPT_DIR/logs"
mkdir -p "$LOG_DIR"
show_help() {
echo "📚 Library of Short Stories 下载器"
echo ""
echo "用法:"
echo " ./start.sh fetch # Phase 1: 获取故事索引"
echo " ./start.sh download # Phase 2: 开始下载(前台)"
echo " ./start.sh download-bg # Phase 2: 后台下载"
echo " ./start.sh status # 查看下载状态"
echo " ./start.sh stop # 停止后台下载"
echo " ./start.sh reset # 重置索引(重新获取链接)"
echo ""
echo "目录:"
echo " 索引: config/stories_index.json"
echo " 故事: /home/ubuntu/media/libraryofshortstories/"
echo " 日志: logs/"
}
case "${1:-help}" in
fetch|index)
echo "🔍 Phase 1: 获取故事索引..."
python3 "$SCRIPT_DIR/scripts/fetch_index.py"
;;
download)
echo "📥 Phase 2: 开始下载(前台模式)..."
echo "按 Ctrl+C 可以暂停,下次会继续从断点下载"
echo ""
python3 "$SCRIPT_DIR/scripts/download_stories.py"
;;
download-bg|bg)
echo "📥 Phase 2: 开始后台下载..."
# 检查是否已在运行
if pgrep -f "download_stories.py" > /dev/null; then
echo "⚠️ 下载进程已在运行"
echo "查看日志: tail -f $LOG_DIR/download.log"
exit 1
fi
# 后台运行
nohup python3 "$SCRIPT_DIR/scripts/download_stories.py" \
> "$LOG_DIR/download.log" 2>&1 &
PID=$!
echo "✅ 后台进程已启动 (PID: $PID)"
echo ""
echo "查看状态:"
echo " 实时日志: tail -f $LOG_DIR/download.log"
echo " 查看进程: ps aux | grep download_stories"
echo " 停止下载: ./start.sh stop"
;;
status)
echo "📊 下载状态"
echo ""
# 检查进程
if pgrep -f "download_stories.py" > /dev/null; then
echo "✅ 后台下载正在运行"
pgrep -f "download_stories.py" | xargs ps -o pid,etime,%cpu,%mem,command -p
else
echo "⏹️ 后台下载未运行"
fi
echo ""
# 显示索引统计
if [ -f "$SCRIPT_DIR/config/stories_index.json" ]; then
python3 -c "
import json
import os
from datetime import datetime
with open('$SCRIPT_DIR/config/stories_index.json') as f:
data = json.load(f)
stories = data.get('stories', [])
pending = len([s for s in stories if s['status'] == 'pending'])
downloaded = len([s for s in stories if s['status'] == 'downloaded'])
failed = len([s for s in stories if s['status'] == 'failed'])
# 实际目录数量
media_dir = '/home/ubuntu/media/libraryofshortstories'
if os.path.exists(media_dir):
actual_count = len([d for d in os.listdir(media_dir) if os.path.isdir(os.path.join(media_dir, d))])
else:
actual_count = 0
# 计算进度和预计时间
total = len(stories)
progress = (downloaded / total) * 100 if total > 0 else 0
eta_minutes = (pending * 7) // 60
eta_hours = eta_minutes // 60
eta_remain = eta_minutes % 60
print(f'📚 下载统计:')
print(f' 总计: {total}')
print(f' 已完成: {downloaded} ({progress:.1f}%)')
print(f' 待下载: {pending}')
print(f' 失败: {failed}')
print(f' 实际目录: {actual_count}')
print(f'')
if pending > 0:
if eta_hours > 0:
print(f' 预计剩余: ~{eta_hours}小时{eta_remain}分钟')
else:
print(f' 预计剩余: ~{eta_remain}分钟')
print(f' 上次索引更新: {data.get(\"fetch_date\", \"unknown\")[:19]}')
"
else
echo "❌ 索引文件不存在,请先运行: ./start.sh fetch"
fi
echo ""
echo "📁 存储位置: /home/ubuntu/media/libraryofshortstories/"
;;
stop)
echo "⏹️ 停止后台下载..."
if pgrep -f "download_stories.py" > /dev/null; then
pkill -f "download_stories.py"
echo "✅ 已停止"
else
echo "⚠️ 没有运行的下载进程"
fi
;;
reset)
echo "⚠️ 这将重置索引,重新获取链接"
read -p "确定继续? (y/N): " confirm
if [ "$confirm" = "y" ] || [ "$confirm" = "Y" ]; then
rm -f "$SCRIPT_DIR/config/stories_index.json"
echo "✅ 索引已重置"
echo "现在可以重新运行: ./start.sh fetch"
else
echo "👋 已取消"
fi
;;
log|logs)
if [ -f "$LOG_DIR/download.log" ]; then
tail -50 "$LOG_DIR/download.log"
else
echo "暂无日志文件"
fi
;;
help|--help|-h)
show_help
;;
*)
echo "❌ 未知命令: $1"
show_help
exit 1
;;
esac