Ubuntu
277 lines
8.6 KiB
Python
Executable File
277 lines
8.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Phase 2: 下载 Loyal Books 书籍内容
|
|
支持断点续传，避免重复下载大文件
|
|
"""
|
|
|
|
import json
|
|
import requests
|
|
import os
|
|
import re
|
|
from datetime import datetime
|
|
from time import sleep
|
|
from pathlib import Path
|
|
|
|
# 注意：使用小写 children
|
|
STORAGE_DIR = Path("/home/ubuntu/media/loyalbooks/children")
|
|
CONFIG_DIR = Path(__file__).parent.parent / "config"
|
|
INDEX_FILE = CONFIG_DIR / "loyalbooks_children_index.json"
|
|
DELAY = 5 # 下载间隔
|
|
|
|
# 最小文件大小（字节），小于此值认为下载不完整
|
|
MIN_TXT_SIZE = 100 # TXT 至少 100 字节
|
|
MIN_ZIP_SIZE = 1024 * 1024 # ZIP 至少 1MB
|
|
|
|
|
|
def load_index():
|
|
"""加载索引"""
|
|
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
|
|
|
|
def save_index(index):
|
|
"""保存索引"""
|
|
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
|
json.dump(index, f, indent=2, ensure_ascii=False)
|
|
|
|
|
|
def sanitize_filename(name):
|
|
"""清理文件名，用于目录名"""
|
|
# 移除不安全字符，限制长度
|
|
name = re.sub(r'[<>:"/\\|?*]', '_', name)
|
|
return name[:100] # 限制长度
|
|
|
|
|
|
def file_exists_and_valid(path, min_size):
|
|
"""检查文件是否存在且大小有效"""
|
|
if not path.exists():
|
|
return False
|
|
size = path.stat().st_size
|
|
return size >= min_size
|
|
|
|
|
|
def download_file_resume(url, dest_path, min_size, timeout=300):
|
|
"""
|
|
下载文件，支持断点续传
|
|
如果文件已存在且大小有效，则跳过
|
|
"""
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
|
}
|
|
|
|
# 检查是否已存在有效文件
|
|
if file_exists_and_valid(dest_path, min_size):
|
|
existing_size = dest_path.stat().st_size
|
|
print(f" ⏭️ 已存在 ({existing_size // 1024}KB)，跳过")
|
|
return existing_size
|
|
|
|
# 检查是否有部分下载的文件（断点续传）
|
|
temp_path = dest_path.with_suffix('.tmp')
|
|
resume_byte_pos = 0
|
|
|
|
if temp_path.exists():
|
|
resume_byte_pos = temp_path.stat().st_size
|
|
print(f" 🔄 断点续传 from {resume_byte_pos // 1024}KB...", end=' ')
|
|
headers['Range'] = f'bytes={resume_byte_pos}-'
|
|
else:
|
|
print(f" 下载中...", end=' ')
|
|
|
|
try:
|
|
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
|
|
|
|
# 如果服务器不支持断点续传，从头下载
|
|
if response.status_code == 416: # Range not satisfiable
|
|
print("⚠️ 不支持续传，重新下载...", end=' ')
|
|
headers.pop('Range', None)
|
|
resume_byte_pos = 0
|
|
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
|
|
|
|
response.raise_for_status()
|
|
|
|
# 确定写入模式
|
|
mode = 'ab' if resume_byte_pos > 0 and response.status_code == 206 else 'wb'
|
|
|
|
with open(temp_path, mode) as f:
|
|
for chunk in response.iter_content(chunk_size=8192):
|
|
if chunk:
|
|
f.write(chunk)
|
|
|
|
# 下载完成，重命名为正式文件
|
|
temp_path.rename(dest_path)
|
|
|
|
final_size = dest_path.stat().st_size
|
|
if final_size < min_size:
|
|
raise Exception(f"文件大小不足: {final_size} bytes")
|
|
|
|
return final_size
|
|
|
|
except Exception as e:
|
|
# 下载失败，保留 tmp 文件以便下次续传
|
|
print(f"❌ {e}")
|
|
raise
|
|
|
|
|
|
def download_book(book):
|
|
"""下载单本书籍，支持断点续传"""
|
|
book_name = book['name']
|
|
book_slug = sanitize_filename(book_name)
|
|
book_dir = STORAGE_DIR / book_slug
|
|
|
|
print(f"\n📖 {book_name[:50]}...")
|
|
|
|
# 创建书籍目录
|
|
book_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
results = {'txt': False, 'mp3': False, 'txt_size': 0, 'mp3_size': 0}
|
|
|
|
# 下载 TXT
|
|
if book.get('txt_url'):
|
|
txt_path = book_dir / f"{book_slug}.txt"
|
|
try:
|
|
print(f" TXT:", end=' ')
|
|
size = download_file_resume(book['txt_url'], txt_path, MIN_TXT_SIZE)
|
|
print(f"✅ {size // 1024}KB")
|
|
results['txt'] = True
|
|
results['txt_size'] = size
|
|
sleep(DELAY)
|
|
except Exception as e:
|
|
print(f"❌ {e}")
|
|
else:
|
|
print(f" TXT: 无链接")
|
|
|
|
# 下载 MP3 ZIP（大文件，特别小心）
|
|
if book.get('mp3_zip_url'):
|
|
zip_path = book_dir / "mp3_64kb.zip"
|
|
try:
|
|
print(f" MP3 ZIP:", end=' ')
|
|
size = download_file_resume(book['mp3_zip_url'], zip_path, MIN_ZIP_SIZE, timeout=600)
|
|
print(f"✅ {size // 1024 // 1024}MB")
|
|
results['mp3'] = True
|
|
results['mp3_size'] = size
|
|
sleep(DELAY)
|
|
except Exception as e:
|
|
print(f"❌ {e}")
|
|
else:
|
|
print(f" MP3 ZIP: 无链接")
|
|
|
|
return results
|
|
|
|
|
|
def scan_existing_downloads():
|
|
"""扫描已下载的文件，更新索引状态"""
|
|
index = load_index()
|
|
|
|
if not STORAGE_DIR.exists():
|
|
return index
|
|
|
|
print("🔍 扫描已下载的文件...")
|
|
updated = 0
|
|
for book in index['books']:
|
|
if book['status'] == 'downloaded':
|
|
continue
|
|
|
|
book_slug = sanitize_filename(book['name'])
|
|
book_dir = STORAGE_DIR / book_slug
|
|
|
|
if not book_dir.exists():
|
|
continue
|
|
|
|
# 检查文件
|
|
txt_path = book_dir / f"{book_slug}.txt"
|
|
zip_path = book_dir / "mp3_64kb.zip"
|
|
|
|
has_txt = file_exists_and_valid(txt_path, MIN_TXT_SIZE)
|
|
has_mp3 = file_exists_and_valid(zip_path, MIN_ZIP_SIZE)
|
|
|
|
if has_txt or has_mp3:
|
|
book['status'] = 'downloaded'
|
|
book['download_date'] = datetime.now().isoformat()
|
|
book['downloaded_files'] = {
|
|
'txt': has_txt,
|
|
'mp3': has_mp3,
|
|
'txt_size': txt_path.stat().st_size if has_txt else 0,
|
|
'mp3_size': zip_path.stat().st_size if has_mp3 else 0
|
|
}
|
|
updated += 1
|
|
|
|
save_index(index)
|
|
if updated > 0:
|
|
print(f" 发现 {updated} 本已下载的书籍")
|
|
return index
|
|
|
|
|
|
def main():
|
|
print("="*60)
|
|
print("📚 Loyal Books - Children 下载器")
|
|
print("="*60)
|
|
print(f"💾 存储路径: {STORAGE_DIR}")
|
|
print(f"🔄 支持断点续传，不重复下载")
|
|
print()
|
|
|
|
# 加载索引
|
|
if not INDEX_FILE.exists():
|
|
print("❌ 索引文件不存在，请先运行 fetch_books.py")
|
|
return
|
|
|
|
# 扫描已下载的文件
|
|
index = scan_existing_downloads()
|
|
books = index['books']
|
|
|
|
# 统计
|
|
pending = [b for b in books if b['status'] == 'pending']
|
|
downloaded = [b for b in books if b['status'] == 'downloaded']
|
|
failed = [b for b in books if b['status'] == 'failed']
|
|
|
|
# 统计存储空间
|
|
total_txt = sum(b.get('downloaded_files', {}).get('txt_size', 0) for b in downloaded)
|
|
total_mp3 = sum(b.get('downloaded_files', {}).get('mp3_size', 0) for b in downloaded)
|
|
|
|
print(f"📊 当前状态:")
|
|
print(f" 总计书籍: {len(books)}")
|
|
print(f" 待下载: {len(pending)}")
|
|
print(f" 已完成: {len(downloaded)}")
|
|
print(f" 失败: {len(failed)}")
|
|
print(f" 已占用空间: TXT {total_txt//1024//1024}MB, MP3 {total_mp3//1024//1024}MB")
|
|
|
|
if not pending:
|
|
print("\n✅ 所有书籍已下载完成！")
|
|
return
|
|
|
|
# 预估剩余空间
|
|
avg_mp3 = total_mp3 / len(downloaded) if downloaded else 50 * 1024 * 1024 # 默认50MB
|
|
estimated_need = len(pending) * avg_mp3
|
|
print(f" 预计还需空间: ~{estimated_need//1024//1024}MB")
|
|
|
|
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
|
|
print(f" 开始下载... (按 Ctrl+C 暂停，支持断点续传)\n")
|
|
|
|
try:
|
|
for i, book in enumerate(pending, 1):
|
|
print(f"[{i}/{len(pending)}] 书籍: {book['name'][:40]}...")
|
|
|
|
results = download_book(book)
|
|
|
|
# 更新状态（至少成功一个就算完成）
|
|
if results['txt'] or results['mp3']:
|
|
book['status'] = 'downloaded'
|
|
book['download_date'] = datetime.now().isoformat()
|
|
book['downloaded_files'] = results
|
|
else:
|
|
book['status'] = 'failed'
|
|
|
|
# 保存索引
|
|
save_index(index)
|
|
|
|
print("\n" + "="*60)
|
|
print("✅ 下载完成！")
|
|
print("="*60)
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n\n⚠️ 用户中断")
|
|
print(f"进度已保存，下次会自动从断点续传")
|
|
save_index(index)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|