a
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
.venv-*/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# Node dependencies
|
||||
node_modules/
|
||||
|
||||
# 运行时数据
|
||||
config/stories_index.json
|
||||
|
||||
# 日志
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# 环境变量
|
||||
.env
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# 下载的故事内容(存储在 /home/ubuntu/media/)
|
||||
# 这个目录在 git 外
|
||||
@@ -0,0 +1,101 @@
|
||||
# Library of Short Stories 下载器
|
||||
|
||||
从 https://www.libraryofshortstories.com 下载英语短篇故事。
|
||||
|
||||
## 功能特性
|
||||
|
||||
- ✅ **API 直接获取**: 无需浏览器,直接调用网站 API
|
||||
- ✅ **断点续传**: 中断后自动从上次位置继续
|
||||
- ✅ **后台运行**: 支持后台下载,不影响其他操作
|
||||
- ✅ **7秒间隔**: 礼貌下载,避免被封
|
||||
- ✅ **元数据保存**: 保存作者、年份、类型等信息
|
||||
- ✅ **纯文本输出**: 去除 HTML 标签,干净阅读
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 获取故事索引(Phase 1)
|
||||
|
||||
```bash
|
||||
./start.sh fetch
|
||||
```
|
||||
|
||||
这会从 API 获取最多 1000 个故事的链接,保存到 `config/stories_index.json`
|
||||
|
||||
### 2. 下载故事(Phase 2)
|
||||
|
||||
**前台模式**(适合少量下载):
|
||||
```bash
|
||||
./start.sh download
|
||||
```
|
||||
|
||||
**后台模式**(推荐,适合大量下载):
|
||||
```bash
|
||||
./start.sh download-bg
|
||||
```
|
||||
|
||||
### 3. 查看状态
|
||||
|
||||
```bash
|
||||
./start.sh status
|
||||
```
|
||||
|
||||
### 4. 查看日志
|
||||
|
||||
```bash
|
||||
./start.sh log # 查看最近日志
|
||||
tail -f logs/download.log # 实时跟踪
|
||||
```
|
||||
|
||||
### 5. 停止下载
|
||||
|
||||
```bash
|
||||
./start.sh stop
|
||||
```
|
||||
|
||||
## 存储结构
|
||||
|
||||
```
|
||||
/home/ubuntu/media/libraryofshortstories/
|
||||
├── the-adventure-of-the-red-circle/
|
||||
│ ├── content.txt # 故事正文
|
||||
│ └── metadata.md # 元数据
|
||||
├── the-1000000-bank-note/
|
||||
│ ├── content.txt
|
||||
│ └── metadata.md
|
||||
└── ...
|
||||
```
|
||||
|
||||
## 索引文件格式
|
||||
|
||||
`config/stories_index.json`:
|
||||
```json
|
||||
{
|
||||
"fetch_date": "2026-03-27T10:00:00",
|
||||
"total_available": 1500,
|
||||
"max_limit": 1000,
|
||||
"stories": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "The £1,000,000 Bank-Note",
|
||||
"author": "Mark Twain",
|
||||
"code": "the-1000000-bank-note",
|
||||
"url": "https://www.libraryofshortstories.com/onlinereader/the-1000000-bank-note",
|
||||
"status": "downloaded",
|
||||
"download_date": "2026-03-27T10:05:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## 技术细节
|
||||
|
||||
- **下载间隔**: 7 秒
|
||||
- **超时设置**: 30 秒
|
||||
- **重试机制**: 失败标记为 failed,下次运行会重试
|
||||
- **并发数**: 单线程,避免被封
|
||||
|
||||
## 注意事项
|
||||
|
||||
1. 下载约 1000 个故事需要约 **2 小时** (1000 × 7秒)
|
||||
2. 建议首次使用后台模式 `./start.sh download-bg`
|
||||
3. 可以安全中断 (Ctrl+C 或 `./start.sh stop`),进度会自动保存
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 2: 下载故事内容
|
||||
使用网站 API 直接获取纯文本
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import time
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
# 路径配置
|
||||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||||
INDEX_FILE = os.path.join(CONFIG_DIR, 'stories_index.json')
|
||||
STORAGE_DIR = '/home/ubuntu/media/libraryofshortstories'
|
||||
DELAY = 7 # 秒
|
||||
|
||||
|
||||
def load_index():
|
||||
"""加载索引"""
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引"""
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def fetch_story_content(code):
|
||||
"""获取故事内容(纯文本)"""
|
||||
url = f"https://www.libraryofshortstories.com/storiesgzip/{code}.txt"
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept-Encoding': 'gzip',
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def fetch_story_metadata(code):
|
||||
"""获取故事元数据"""
|
||||
url = f"https://www.libraryofshortstories.com/storymetadata/{code}"
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept-Encoding': 'gzip',
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def save_story(story_info, content, metadata):
|
||||
"""保存故事到目录"""
|
||||
# 创建目录
|
||||
story_dir = os.path.join(STORAGE_DIR, story_info['code'])
|
||||
os.makedirs(story_dir, exist_ok=True)
|
||||
|
||||
# 保存内容
|
||||
content_file = os.path.join(story_dir, 'content.txt')
|
||||
with open(content_file, 'w', encoding='utf-8') as f:
|
||||
f.write(f"# {story_info['title']}\n\n")
|
||||
f.write(f"Author: {story_info['author']}\n")
|
||||
if story_info.get('publish_year'):
|
||||
f.write(f"Year: {story_info['publish_year']}\n")
|
||||
if story_info.get('genres'):
|
||||
f.write(f"Genres: {', '.join(story_info['genres'])}\n")
|
||||
f.write(f"\n{'='*60}\n\n")
|
||||
f.write(content)
|
||||
|
||||
# 保存元数据
|
||||
metadata_file = os.path.join(story_dir, 'metadata.md')
|
||||
with open(metadata_file, 'w', encoding='utf-8') as f:
|
||||
f.write(f"# {story_info['title']} - Metadata\n\n")
|
||||
f.write(f"- **Title**: {story_info['title']}\n")
|
||||
f.write(f"- **Author**: {story_info['author']}\n")
|
||||
f.write(f"- **Year**: {story_info.get('publish_year', 'Unknown')}\n")
|
||||
f.write(f"- **Words**: {story_info.get('words', 'Unknown')}\n")
|
||||
f.write(f"- **Genres**: {', '.join(story_info.get('genres', []))}\n")
|
||||
f.write(f"- **Source URL**: {story_info['url']}\n")
|
||||
f.write(f"- **Download Date**: {datetime.now().isoformat()}\n")
|
||||
|
||||
if metadata:
|
||||
f.write(f"\n## Additional Metadata\n\n")
|
||||
for key, value in metadata.items():
|
||||
if isinstance(value, list):
|
||||
f.write(f"- **{key}**: {', '.join(str(v) for v in value)}\n")
|
||||
else:
|
||||
f.write(f"- **{key}**: {value}\n")
|
||||
|
||||
if story_info.get('description'):
|
||||
f.write(f"\n## Description\n\n{story_info['description']}\n")
|
||||
|
||||
return story_dir
|
||||
|
||||
|
||||
def download_one_story(story_info):
|
||||
"""下载单个故事"""
|
||||
try:
|
||||
print(f" 📥 下载: {story_info['title'][:50]}...")
|
||||
|
||||
# 获取内容
|
||||
content = fetch_story_content(story_info['code'])
|
||||
|
||||
# 获取额外元数据(可选)
|
||||
metadata = fetch_story_metadata(story_info['code'])
|
||||
|
||||
# 保存
|
||||
story_dir = save_story(story_info, content, metadata)
|
||||
|
||||
print(f" ✅ 已保存: {story_dir}")
|
||||
print(f" 字数: {len(content)} 字符")
|
||||
return True, None
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 失败: {e}")
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def main():
|
||||
print("="*60)
|
||||
print("📚 Library of Short Stories - 下载器")
|
||||
print("="*60)
|
||||
|
||||
# 加载索引
|
||||
if not os.path.exists(INDEX_FILE):
|
||||
print("❌ 索引文件不存在,请先运行: ./start.sh fetch")
|
||||
return
|
||||
|
||||
index = load_index()
|
||||
stories = index['stories']
|
||||
|
||||
# 统计
|
||||
pending = [s for s in stories if s['status'] == 'pending']
|
||||
downloaded = [s for s in stories if s['status'] == 'downloaded']
|
||||
failed = [s for s in stories if s['status'] == 'failed']
|
||||
|
||||
print(f"\n📊 当前状态:")
|
||||
print(f" 总计: {len(stories)}")
|
||||
print(f" 待下载: {len(pending)}")
|
||||
print(f" 已完成: {len(downloaded)}")
|
||||
print(f" 失败: {len(failed)}")
|
||||
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
|
||||
print(f" 预计时间: {len(pending) * DELAY // 60} 分钟\n")
|
||||
|
||||
if not pending:
|
||||
print("✅ 所有故事已下载完成!")
|
||||
return
|
||||
|
||||
# 开始下载
|
||||
print("开始下载... (按 Ctrl+C 暂停)\n")
|
||||
|
||||
try:
|
||||
for i, story in enumerate(pending, 1):
|
||||
print(f"[{i}/{len(pending)}] ID: {story['id']}")
|
||||
|
||||
success, error = download_one_story(story)
|
||||
|
||||
# 更新状态
|
||||
story['status'] = 'downloaded' if success else 'failed'
|
||||
story['download_date'] = datetime.now().isoformat() if success else None
|
||||
story['error'] = error
|
||||
|
||||
# 每下载一个就保存索引(断点续传)
|
||||
save_index(index)
|
||||
|
||||
# 延迟
|
||||
if i < len(pending):
|
||||
time.sleep(DELAY)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ 下载完成!")
|
||||
print("="*60)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ 用户中断")
|
||||
print(f"进度已保存,下次会继续从断点下载")
|
||||
save_index(index)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 1: 获取故事索引
|
||||
从 API 获取所有故事链接并保存
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
CONFIG_DIR = os.path.join(os.path.dirname(__file__), '..', 'config')
|
||||
INDEX_FILE = os.path.join(CONFIG_DIR, 'stories_index.json')
|
||||
|
||||
|
||||
def fetch_story_list():
|
||||
"""从 API 获取故事列表"""
|
||||
url = "https://www.libraryofshortstories.com/storylist-reduced"
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept': 'application/json',
|
||||
'Accept-Encoding': 'gzip',
|
||||
}
|
||||
|
||||
print("🔍 正在获取故事列表...")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
stories = response.json()
|
||||
print(f"✅ 获取到 {len(stories)} 个故事")
|
||||
return stories
|
||||
|
||||
|
||||
def build_index(stories):
|
||||
"""构建索引(最多1000个)"""
|
||||
index = {
|
||||
'fetch_date': datetime.now().isoformat(),
|
||||
'source_url': 'https://www.libraryofshortstories.com/storylist-reduced',
|
||||
'total_available': len(stories),
|
||||
'max_limit': 1000,
|
||||
'stories': []
|
||||
}
|
||||
|
||||
for i, story in enumerate(stories[:1000], 1):
|
||||
story_entry = {
|
||||
'id': i,
|
||||
'title': story.get('title', 'Unknown'),
|
||||
'author': story.get('author', 'Unknown'),
|
||||
'code': story.get('code'),
|
||||
'url': f"https://www.libraryofshortstories.com/onlinereader/{story.get('code')}",
|
||||
'words': story.get('words', 0),
|
||||
'publish_year': story.get('publish_year'),
|
||||
'genres': story.get('genres', []),
|
||||
'description': story.get('description', ''),
|
||||
'status': 'pending', # pending, downloaded, failed
|
||||
'download_date': None,
|
||||
'error': None
|
||||
}
|
||||
index['stories'].append(story_entry)
|
||||
|
||||
return index
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引到文件"""
|
||||
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
|
||||
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"💾 索引已保存: {INDEX_FILE}")
|
||||
print(f"📊 总计: {len(index['stories'])} 个故事")
|
||||
|
||||
|
||||
def main():
|
||||
# 检查是否已有索引
|
||||
if os.path.exists(INDEX_FILE):
|
||||
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
existing = json.load(f)
|
||||
print(f" 上次更新: {existing.get('fetch_date')}")
|
||||
print(f" 故事数量: {len(existing.get('stories', []))}")
|
||||
|
||||
response = input("\n是否重新获取? (y/N): ")
|
||||
if response.lower() != 'y':
|
||||
print("👋 保持现有索引")
|
||||
return
|
||||
|
||||
# 获取并保存
|
||||
stories = fetch_story_list()
|
||||
index = build_index(stories)
|
||||
save_index(index)
|
||||
|
||||
print("\n✅ Phase 1 完成")
|
||||
print("下一步: 运行 download_stories.py 开始下载")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+165
@@ -0,0 +1,165 @@
|
||||
#!/bin/bash
|
||||
# Library of Short Stories 下载器启动脚本
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
LOG_DIR="$SCRIPT_DIR/logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
show_help() {
|
||||
echo "📚 Library of Short Stories 下载器"
|
||||
echo ""
|
||||
echo "用法:"
|
||||
echo " ./start.sh fetch # Phase 1: 获取故事索引"
|
||||
echo " ./start.sh download # Phase 2: 开始下载(前台)"
|
||||
echo " ./start.sh download-bg # Phase 2: 后台下载"
|
||||
echo " ./start.sh status # 查看下载状态"
|
||||
echo " ./start.sh stop # 停止后台下载"
|
||||
echo " ./start.sh reset # 重置索引(重新获取链接)"
|
||||
echo ""
|
||||
echo "目录:"
|
||||
echo " 索引: config/stories_index.json"
|
||||
echo " 故事: /home/ubuntu/media/libraryofshortstories/"
|
||||
echo " 日志: logs/"
|
||||
}
|
||||
|
||||
case "${1:-help}" in
|
||||
fetch|index)
|
||||
echo "🔍 Phase 1: 获取故事索引..."
|
||||
python3 "$SCRIPT_DIR/scripts/fetch_index.py"
|
||||
;;
|
||||
|
||||
download)
|
||||
echo "📥 Phase 2: 开始下载(前台模式)..."
|
||||
echo "按 Ctrl+C 可以暂停,下次会继续从断点下载"
|
||||
echo ""
|
||||
python3 "$SCRIPT_DIR/scripts/download_stories.py"
|
||||
;;
|
||||
|
||||
download-bg|bg)
|
||||
echo "📥 Phase 2: 开始后台下载..."
|
||||
|
||||
# 检查是否已在运行
|
||||
if pgrep -f "download_stories.py" > /dev/null; then
|
||||
echo "⚠️ 下载进程已在运行"
|
||||
echo "查看日志: tail -f $LOG_DIR/download.log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 后台运行
|
||||
nohup python3 "$SCRIPT_DIR/scripts/download_stories.py" \
|
||||
> "$LOG_DIR/download.log" 2>&1 &
|
||||
|
||||
PID=$!
|
||||
echo "✅ 后台进程已启动 (PID: $PID)"
|
||||
echo ""
|
||||
echo "查看状态:"
|
||||
echo " 实时日志: tail -f $LOG_DIR/download.log"
|
||||
echo " 查看进程: ps aux | grep download_stories"
|
||||
echo " 停止下载: ./start.sh stop"
|
||||
;;
|
||||
|
||||
status)
|
||||
echo "📊 下载状态"
|
||||
echo ""
|
||||
|
||||
# 检查进程
|
||||
if pgrep -f "download_stories.py" > /dev/null; then
|
||||
echo "✅ 后台下载正在运行"
|
||||
pgrep -f "download_stories.py" | xargs ps -o pid,etime,%cpu,%mem,command -p
|
||||
else
|
||||
echo "⏹️ 后台下载未运行"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# 显示索引统计
|
||||
if [ -f "$SCRIPT_DIR/config/stories_index.json" ]; then
|
||||
python3 -c "
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
with open('$SCRIPT_DIR/config/stories_index.json') as f:
|
||||
data = json.load(f)
|
||||
stories = data.get('stories', [])
|
||||
pending = len([s for s in stories if s['status'] == 'pending'])
|
||||
downloaded = len([s for s in stories if s['status'] == 'downloaded'])
|
||||
failed = len([s for s in stories if s['status'] == 'failed'])
|
||||
|
||||
# 实际目录数量
|
||||
media_dir = '/home/ubuntu/media/libraryofshortstories'
|
||||
if os.path.exists(media_dir):
|
||||
actual_count = len([d for d in os.listdir(media_dir) if os.path.isdir(os.path.join(media_dir, d))])
|
||||
else:
|
||||
actual_count = 0
|
||||
|
||||
# 计算进度和预计时间
|
||||
total = len(stories)
|
||||
progress = (downloaded / total) * 100 if total > 0 else 0
|
||||
eta_minutes = (pending * 7) // 60
|
||||
eta_hours = eta_minutes // 60
|
||||
eta_remain = eta_minutes % 60
|
||||
|
||||
print(f'📚 下载统计:')
|
||||
print(f' 总计: {total}')
|
||||
print(f' 已完成: {downloaded} ({progress:.1f}%)')
|
||||
print(f' 待下载: {pending}')
|
||||
print(f' 失败: {failed}')
|
||||
print(f' 实际目录: {actual_count}')
|
||||
print(f'')
|
||||
if pending > 0:
|
||||
if eta_hours > 0:
|
||||
print(f' 预计剩余: ~{eta_hours}小时{eta_remain}分钟')
|
||||
else:
|
||||
print(f' 预计剩余: ~{eta_remain}分钟')
|
||||
print(f' 上次索引更新: {data.get(\"fetch_date\", \"unknown\")[:19]}')
|
||||
"
|
||||
else
|
||||
echo "❌ 索引文件不存在,请先运行: ./start.sh fetch"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "📁 存储位置: /home/ubuntu/media/libraryofshortstories/"
|
||||
;;
|
||||
|
||||
stop)
|
||||
echo "⏹️ 停止后台下载..."
|
||||
|
||||
if pgrep -f "download_stories.py" > /dev/null; then
|
||||
pkill -f "download_stories.py"
|
||||
echo "✅ 已停止"
|
||||
else
|
||||
echo "⚠️ 没有运行的下载进程"
|
||||
fi
|
||||
;;
|
||||
|
||||
reset)
|
||||
echo "⚠️ 这将重置索引,重新获取链接"
|
||||
read -p "确定继续? (y/N): " confirm
|
||||
if [ "$confirm" = "y" ] || [ "$confirm" = "Y" ]; then
|
||||
rm -f "$SCRIPT_DIR/config/stories_index.json"
|
||||
echo "✅ 索引已重置"
|
||||
echo "现在可以重新运行: ./start.sh fetch"
|
||||
else
|
||||
echo "👋 已取消"
|
||||
fi
|
||||
;;
|
||||
|
||||
log|logs)
|
||||
if [ -f "$LOG_DIR/download.log" ]; then
|
||||
tail -50 "$LOG_DIR/download.log"
|
||||
else
|
||||
echo "暂无日志文件"
|
||||
fi
|
||||
;;
|
||||
|
||||
help|--help|-h)
|
||||
show_help
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "❌ 未知命令: $1"
|
||||
show_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user