This commit is contained in:
Ubuntu
2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
+276
View File
@@ -0,0 +1,276 @@
#!/usr/bin/env python3
"""
Phase 2: 下载 Loyal Books 书籍内容
支持断点续传,避免重复下载大文件
"""
import json
import requests
import os
import re
from datetime import datetime
from time import sleep
from pathlib import Path
# 注意:使用小写 children
STORAGE_DIR = Path("/home/ubuntu/media/loyalbooks/children")
CONFIG_DIR = Path(__file__).parent.parent / "config"
INDEX_FILE = CONFIG_DIR / "loyalbooks_children_index.json"
DELAY = 5 # 下载间隔
# 最小文件大小(字节),小于此值认为下载不完整
MIN_TXT_SIZE = 100 # TXT 至少 100 字节
MIN_ZIP_SIZE = 1024 * 1024 # ZIP 至少 1MB
def load_index():
"""加载索引"""
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
return json.load(f)
def save_index(index):
"""保存索引"""
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
json.dump(index, f, indent=2, ensure_ascii=False)
def sanitize_filename(name):
"""清理文件名,用于目录名"""
# 移除不安全字符,限制长度
name = re.sub(r'[<>:"/\\|?*]', '_', name)
return name[:100] # 限制长度
def file_exists_and_valid(path, min_size):
"""检查文件是否存在且大小有效"""
if not path.exists():
return False
size = path.stat().st_size
return size >= min_size
def download_file_resume(url, dest_path, min_size, timeout=300):
"""
下载文件,支持断点续传
如果文件已存在且大小有效,则跳过
"""
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
}
# 检查是否已存在有效文件
if file_exists_and_valid(dest_path, min_size):
existing_size = dest_path.stat().st_size
print(f" ⏭️ 已存在 ({existing_size // 1024}KB),跳过")
return existing_size
# 检查是否有部分下载的文件(断点续传)
temp_path = dest_path.with_suffix('.tmp')
resume_byte_pos = 0
if temp_path.exists():
resume_byte_pos = temp_path.stat().st_size
print(f" 🔄 断点续传 from {resume_byte_pos // 1024}KB...", end=' ')
headers['Range'] = f'bytes={resume_byte_pos}-'
else:
print(f" 下载中...", end=' ')
try:
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
# 如果服务器不支持断点续传,从头下载
if response.status_code == 416: # Range not satisfiable
print("⚠️ 不支持续传,重新下载...", end=' ')
headers.pop('Range', None)
resume_byte_pos = 0
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
response.raise_for_status()
# 确定写入模式
mode = 'ab' if resume_byte_pos > 0 and response.status_code == 206 else 'wb'
with open(temp_path, mode) as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
# 下载完成,重命名为正式文件
temp_path.rename(dest_path)
final_size = dest_path.stat().st_size
if final_size < min_size:
raise Exception(f"文件大小不足: {final_size} bytes")
return final_size
except Exception as e:
# 下载失败,保留 tmp 文件以便下次续传
print(f"❌ {e}")
raise
def download_book(book):
"""下载单本书籍,支持断点续传"""
book_name = book['name']
book_slug = sanitize_filename(book_name)
book_dir = STORAGE_DIR / book_slug
print(f"\n📖 {book_name[:50]}...")
# 创建书籍目录
book_dir.mkdir(parents=True, exist_ok=True)
results = {'txt': False, 'mp3': False, 'txt_size': 0, 'mp3_size': 0}
# 下载 TXT
if book.get('txt_url'):
txt_path = book_dir / f"{book_slug}.txt"
try:
print(f" TXT:", end=' ')
size = download_file_resume(book['txt_url'], txt_path, MIN_TXT_SIZE)
print(f"✅ {size // 1024}KB")
results['txt'] = True
results['txt_size'] = size
sleep(DELAY)
except Exception as e:
print(f"❌ {e}")
else:
print(f" TXT: 无链接")
# 下载 MP3 ZIP(大文件,特别小心)
if book.get('mp3_zip_url'):
zip_path = book_dir / "mp3_64kb.zip"
try:
print(f" MP3 ZIP:", end=' ')
size = download_file_resume(book['mp3_zip_url'], zip_path, MIN_ZIP_SIZE, timeout=600)
print(f"✅ {size // 1024 // 1024}MB")
results['mp3'] = True
results['mp3_size'] = size
sleep(DELAY)
except Exception as e:
print(f"❌ {e}")
else:
print(f" MP3 ZIP: 无链接")
return results
def scan_existing_downloads():
"""扫描已下载的文件,更新索引状态"""
index = load_index()
if not STORAGE_DIR.exists():
return index
print("🔍 扫描已下载的文件...")
updated = 0
for book in index['books']:
if book['status'] == 'downloaded':
continue
book_slug = sanitize_filename(book['name'])
book_dir = STORAGE_DIR / book_slug
if not book_dir.exists():
continue
# 检查文件
txt_path = book_dir / f"{book_slug}.txt"
zip_path = book_dir / "mp3_64kb.zip"
has_txt = file_exists_and_valid(txt_path, MIN_TXT_SIZE)
has_mp3 = file_exists_and_valid(zip_path, MIN_ZIP_SIZE)
if has_txt or has_mp3:
book['status'] = 'downloaded'
book['download_date'] = datetime.now().isoformat()
book['downloaded_files'] = {
'txt': has_txt,
'mp3': has_mp3,
'txt_size': txt_path.stat().st_size if has_txt else 0,
'mp3_size': zip_path.stat().st_size if has_mp3 else 0
}
updated += 1
save_index(index)
if updated > 0:
print(f" 发现 {updated} 本已下载的书籍")
return index
def main():
print("="*60)
print("📚 Loyal Books - Children 下载器")
print("="*60)
print(f"💾 存储路径: {STORAGE_DIR}")
print(f"🔄 支持断点续传,不重复下载")
print()
# 加载索引
if not INDEX_FILE.exists():
print("❌ 索引文件不存在,请先运行 fetch_books.py")
return
# 扫描已下载的文件
index = scan_existing_downloads()
books = index['books']
# 统计
pending = [b for b in books if b['status'] == 'pending']
downloaded = [b for b in books if b['status'] == 'downloaded']
failed = [b for b in books if b['status'] == 'failed']
# 统计存储空间
total_txt = sum(b.get('downloaded_files', {}).get('txt_size', 0) for b in downloaded)
total_mp3 = sum(b.get('downloaded_files', {}).get('mp3_size', 0) for b in downloaded)
print(f"📊 当前状态:")
print(f" 总计书籍: {len(books)}")
print(f" 待下载: {len(pending)}")
print(f" 已完成: {len(downloaded)}")
print(f" 失败: {len(failed)}")
print(f" 已占用空间: TXT {total_txt//1024//1024}MB, MP3 {total_mp3//1024//1024}MB")
if not pending:
print("\n✅ 所有书籍已下载完成!")
return
# 预估剩余空间
avg_mp3 = total_mp3 / len(downloaded) if downloaded else 50 * 1024 * 1024 # 默认50MB
estimated_need = len(pending) * avg_mp3
print(f" 预计还需空间: ~{estimated_need//1024//1024}MB")
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
print(f" 开始下载... (按 Ctrl+C 暂停,支持断点续传)\n")
try:
for i, book in enumerate(pending, 1):
print(f"[{i}/{len(pending)}] 书籍: {book['name'][:40]}...")
results = download_book(book)
# 更新状态(至少成功一个就算完成)
if results['txt'] or results['mp3']:
book['status'] = 'downloaded'
book['download_date'] = datetime.now().isoformat()
book['downloaded_files'] = results
else:
book['status'] = 'failed'
# 保存索引
save_index(index)
print("\n" + "="*60)
print("✅ 下载完成!")
print("="*60)
except KeyboardInterrupt:
print("\n\n⚠️ 用户中断")
print(f"进度已保存,下次会自动从断点续传")
save_index(index)
if __name__ == '__main__':
main()
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""
Phase 1: 获取 Loyal Books 书籍列表和下载链接
Children 分类,6页,约585本书
"""
import json
import requests
import re
import os
from datetime import datetime
from time import sleep
from pathlib import Path
from urllib.parse import urljoin
BASE_URL = "https://www.loyalbooks.com"
CATEGORY = "Children"
PAGES = 6
DELAY = 3 # 列表页间隔
CONFIG_DIR = Path(__file__).parent.parent / "config"
INDEX_FILE = CONFIG_DIR / "loyalbooks_children_index.json"
def fetch_page(page_num):
"""抓取单页书籍列表"""
if page_num == 1:
url = f"{BASE_URL}/genre/{CATEGORY}?results=100"
else:
url = f"{BASE_URL}/genre/{CATEGORY}?results=100&page={page_num}"
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
}
print(f" 抓取第 {page_num} 页...")
response = requests.get(url, headers=headers, timeout=30)
response.raise_for_status()
return response.text
def parse_books_from_page(html):
"""从页面解析书籍链接"""
books = []
# 匹配书籍链接: href="/book/book-name-by-author"
pattern = r'href="(/book/[^"]+)"'
matches = re.findall(pattern, html)
# 去重
seen = set()
for match in matches:
book_slug = match.strip()
if book_slug not in seen:
seen.add(book_slug)
# 从 slug 提取书名
book_name = book_slug.replace('/book/', '')
books.append({
'slug': book_slug,
'name': book_name,
'detail_url': f"{BASE_URL}{book_slug}",
'txt_url': None, # 稍后填充
'mp3_zip_url': None, # 稍后填充
'status': 'pending',
'download_date': None
})
return books
def fetch_book_detail(book):
"""抓取书籍详情页,获取下载链接"""
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
}
try:
print(f" 获取详情: {book['name'][:40]}...", end=' ')
response = requests.get(book['detail_url'], headers=headers, timeout=30)
response.raise_for_status()
html = response.text
# 提取 TXT 链接
txt_pattern = r'href="(/download/text/[^"]+\.txt)"'
txt_match = re.search(txt_pattern, html)
if txt_match:
book['txt_url'] = urljoin(BASE_URL, txt_match.group(1))
# 提取 MP3 ZIP 链接 (archive.org)
mp3_pattern = r'href="(https?://www\.archive\.org/download/[^"]*_64kb_mp3\.zip)"'
mp3_match = re.search(mp3_pattern, html)
if mp3_match:
book['mp3_zip_url'] = mp3_match.group(1)
print(f"✅ TXT:{('Y' if book['txt_url'] else 'N')} MP3:{('Y' if book['mp3_zip_url'] else 'N')}")
return True
except Exception as e:
print(f"❌ {e}")
return False
def build_index():
"""构建书籍索引"""
index = {
'fetch_date': datetime.now().isoformat(),
'source_url': BASE_URL,
'category': CATEGORY,
'pages': PAGES,
'books': []
}
all_books = []
# Step 1: 获取所有书籍列表
print("\n📚 Step 1: 获取书籍列表...")
for page in range(1, PAGES + 1):
print(f"\n[{page}/{PAGES}] 处理第 {page} 页")
try:
html = fetch_page(page)
books = parse_books_from_page(html)
print(f" ✅ 找到 {len(books)} 本书")
all_books.extend(books)
except Exception as e:
print(f" ❌ 错误: {e}")
if page < PAGES:
sleep(DELAY)
print(f"\n📊 列表抓取完成,共 {len(all_books)} 本书")
# Step 2: 获取每本书的详情和下载链接
print("\n🔍 Step 2: 获取下载链接...")
print(f"预计时间: {len(all_books) * 5 // 60} 分钟\n")
for i, book in enumerate(all_books, 1):
print(f"[{i}/{len(all_books)}]", end=' ')
fetch_book_detail(book)
if i < len(all_books):
sleep(5) # 详情页间隔
index['books'] = all_books
# 统计
with_txt = sum(1 for b in all_books if b['txt_url'])
with_mp3 = sum(1 for b in all_books if b['mp3_zip_url'])
print(f"\n📈 统计:")
print(f" 有 TXT: {with_txt}/{len(all_books)}")
print(f" 有 MP3: {with_mp3}/{len(all_books)}")
return index
def save_index(index):
"""保存索引"""
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
json.dump(index, f, indent=2, ensure_ascii=False)
print(f"\n💾 索引已保存: {INDEX_FILE}")
print(f"📊 总计: {len(index['books'])} 本书")
def main():
# 检查是否已有索引
if INDEX_FILE.exists():
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
existing = json.load(f)
print(f" 上次更新: {existing.get('fetch_date')}")
print(f" 书籍数量: {len(existing.get('books', []))}")
response = input("\n是否重新获取? (y/N): ")
if response.lower() != 'y':
print("👋 保持现有索引")
return
# 获取并保存
print("\n🔍 开始抓取 Loyal Books - Children 分类...")
print(f"⏱️ 预计时间: {PAGES * 3 + 585 * 5 // 60} 分钟\n")
index = build_index()
save_index(index)
print("\n✅ Phase 1 完成")
print("下一步: 运行 download_books.py 开始下载书籍内容")
if __name__ == '__main__':
main()