a
This commit is contained in:
Executable
+276
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 2: 下载 Loyal Books 书籍内容
|
||||
支持断点续传,避免重复下载大文件
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
from pathlib import Path
|
||||
|
||||
# 注意:使用小写 children
|
||||
STORAGE_DIR = Path("/home/ubuntu/media/loyalbooks/children")
|
||||
CONFIG_DIR = Path(__file__).parent.parent / "config"
|
||||
INDEX_FILE = CONFIG_DIR / "loyalbooks_children_index.json"
|
||||
DELAY = 5 # 下载间隔
|
||||
|
||||
# 最小文件大小(字节),小于此值认为下载不完整
|
||||
MIN_TXT_SIZE = 100 # TXT 至少 100 字节
|
||||
MIN_ZIP_SIZE = 1024 * 1024 # ZIP 至少 1MB
|
||||
|
||||
|
||||
def load_index():
|
||||
"""加载索引"""
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引"""
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def sanitize_filename(name):
|
||||
"""清理文件名,用于目录名"""
|
||||
# 移除不安全字符,限制长度
|
||||
name = re.sub(r'[<>:"/\\|?*]', '_', name)
|
||||
return name[:100] # 限制长度
|
||||
|
||||
|
||||
def file_exists_and_valid(path, min_size):
|
||||
"""检查文件是否存在且大小有效"""
|
||||
if not path.exists():
|
||||
return False
|
||||
size = path.stat().st_size
|
||||
return size >= min_size
|
||||
|
||||
|
||||
def download_file_resume(url, dest_path, min_size, timeout=300):
|
||||
"""
|
||||
下载文件,支持断点续传
|
||||
如果文件已存在且大小有效,则跳过
|
||||
"""
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
# 检查是否已存在有效文件
|
||||
if file_exists_and_valid(dest_path, min_size):
|
||||
existing_size = dest_path.stat().st_size
|
||||
print(f" ⏭️ 已存在 ({existing_size // 1024}KB),跳过")
|
||||
return existing_size
|
||||
|
||||
# 检查是否有部分下载的文件(断点续传)
|
||||
temp_path = dest_path.with_suffix('.tmp')
|
||||
resume_byte_pos = 0
|
||||
|
||||
if temp_path.exists():
|
||||
resume_byte_pos = temp_path.stat().st_size
|
||||
print(f" 🔄 断点续传 from {resume_byte_pos // 1024}KB...", end=' ')
|
||||
headers['Range'] = f'bytes={resume_byte_pos}-'
|
||||
else:
|
||||
print(f" 下载中...", end=' ')
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
|
||||
|
||||
# 如果服务器不支持断点续传,从头下载
|
||||
if response.status_code == 416: # Range not satisfiable
|
||||
print("⚠️ 不支持续传,重新下载...", end=' ')
|
||||
headers.pop('Range', None)
|
||||
resume_byte_pos = 0
|
||||
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
# 确定写入模式
|
||||
mode = 'ab' if resume_byte_pos > 0 and response.status_code == 206 else 'wb'
|
||||
|
||||
with open(temp_path, mode) as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
# 下载完成,重命名为正式文件
|
||||
temp_path.rename(dest_path)
|
||||
|
||||
final_size = dest_path.stat().st_size
|
||||
if final_size < min_size:
|
||||
raise Exception(f"文件大小不足: {final_size} bytes")
|
||||
|
||||
return final_size
|
||||
|
||||
except Exception as e:
|
||||
# 下载失败,保留 tmp 文件以便下次续传
|
||||
print(f"❌ {e}")
|
||||
raise
|
||||
|
||||
|
||||
def download_book(book):
|
||||
"""下载单本书籍,支持断点续传"""
|
||||
book_name = book['name']
|
||||
book_slug = sanitize_filename(book_name)
|
||||
book_dir = STORAGE_DIR / book_slug
|
||||
|
||||
print(f"\n📖 {book_name[:50]}...")
|
||||
|
||||
# 创建书籍目录
|
||||
book_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
results = {'txt': False, 'mp3': False, 'txt_size': 0, 'mp3_size': 0}
|
||||
|
||||
# 下载 TXT
|
||||
if book.get('txt_url'):
|
||||
txt_path = book_dir / f"{book_slug}.txt"
|
||||
try:
|
||||
print(f" TXT:", end=' ')
|
||||
size = download_file_resume(book['txt_url'], txt_path, MIN_TXT_SIZE)
|
||||
print(f"✅ {size // 1024}KB")
|
||||
results['txt'] = True
|
||||
results['txt_size'] = size
|
||||
sleep(DELAY)
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
else:
|
||||
print(f" TXT: 无链接")
|
||||
|
||||
# 下载 MP3 ZIP(大文件,特别小心)
|
||||
if book.get('mp3_zip_url'):
|
||||
zip_path = book_dir / "mp3_64kb.zip"
|
||||
try:
|
||||
print(f" MP3 ZIP:", end=' ')
|
||||
size = download_file_resume(book['mp3_zip_url'], zip_path, MIN_ZIP_SIZE, timeout=600)
|
||||
print(f"✅ {size // 1024 // 1024}MB")
|
||||
results['mp3'] = True
|
||||
results['mp3_size'] = size
|
||||
sleep(DELAY)
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
else:
|
||||
print(f" MP3 ZIP: 无链接")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def scan_existing_downloads():
|
||||
"""扫描已下载的文件,更新索引状态"""
|
||||
index = load_index()
|
||||
|
||||
if not STORAGE_DIR.exists():
|
||||
return index
|
||||
|
||||
print("🔍 扫描已下载的文件...")
|
||||
updated = 0
|
||||
for book in index['books']:
|
||||
if book['status'] == 'downloaded':
|
||||
continue
|
||||
|
||||
book_slug = sanitize_filename(book['name'])
|
||||
book_dir = STORAGE_DIR / book_slug
|
||||
|
||||
if not book_dir.exists():
|
||||
continue
|
||||
|
||||
# 检查文件
|
||||
txt_path = book_dir / f"{book_slug}.txt"
|
||||
zip_path = book_dir / "mp3_64kb.zip"
|
||||
|
||||
has_txt = file_exists_and_valid(txt_path, MIN_TXT_SIZE)
|
||||
has_mp3 = file_exists_and_valid(zip_path, MIN_ZIP_SIZE)
|
||||
|
||||
if has_txt or has_mp3:
|
||||
book['status'] = 'downloaded'
|
||||
book['download_date'] = datetime.now().isoformat()
|
||||
book['downloaded_files'] = {
|
||||
'txt': has_txt,
|
||||
'mp3': has_mp3,
|
||||
'txt_size': txt_path.stat().st_size if has_txt else 0,
|
||||
'mp3_size': zip_path.stat().st_size if has_mp3 else 0
|
||||
}
|
||||
updated += 1
|
||||
|
||||
save_index(index)
|
||||
if updated > 0:
|
||||
print(f" 发现 {updated} 本已下载的书籍")
|
||||
return index
|
||||
|
||||
|
||||
def main():
|
||||
print("="*60)
|
||||
print("📚 Loyal Books - Children 下载器")
|
||||
print("="*60)
|
||||
print(f"💾 存储路径: {STORAGE_DIR}")
|
||||
print(f"🔄 支持断点续传,不重复下载")
|
||||
print()
|
||||
|
||||
# 加载索引
|
||||
if not INDEX_FILE.exists():
|
||||
print("❌ 索引文件不存在,请先运行 fetch_books.py")
|
||||
return
|
||||
|
||||
# 扫描已下载的文件
|
||||
index = scan_existing_downloads()
|
||||
books = index['books']
|
||||
|
||||
# 统计
|
||||
pending = [b for b in books if b['status'] == 'pending']
|
||||
downloaded = [b for b in books if b['status'] == 'downloaded']
|
||||
failed = [b for b in books if b['status'] == 'failed']
|
||||
|
||||
# 统计存储空间
|
||||
total_txt = sum(b.get('downloaded_files', {}).get('txt_size', 0) for b in downloaded)
|
||||
total_mp3 = sum(b.get('downloaded_files', {}).get('mp3_size', 0) for b in downloaded)
|
||||
|
||||
print(f"📊 当前状态:")
|
||||
print(f" 总计书籍: {len(books)}")
|
||||
print(f" 待下载: {len(pending)}")
|
||||
print(f" 已完成: {len(downloaded)}")
|
||||
print(f" 失败: {len(failed)}")
|
||||
print(f" 已占用空间: TXT {total_txt//1024//1024}MB, MP3 {total_mp3//1024//1024}MB")
|
||||
|
||||
if not pending:
|
||||
print("\n✅ 所有书籍已下载完成!")
|
||||
return
|
||||
|
||||
# 预估剩余空间
|
||||
avg_mp3 = total_mp3 / len(downloaded) if downloaded else 50 * 1024 * 1024 # 默认50MB
|
||||
estimated_need = len(pending) * avg_mp3
|
||||
print(f" 预计还需空间: ~{estimated_need//1024//1024}MB")
|
||||
|
||||
print(f"\n⏱️ 下载间隔: {DELAY} 秒")
|
||||
print(f" 开始下载... (按 Ctrl+C 暂停,支持断点续传)\n")
|
||||
|
||||
try:
|
||||
for i, book in enumerate(pending, 1):
|
||||
print(f"[{i}/{len(pending)}] 书籍: {book['name'][:40]}...")
|
||||
|
||||
results = download_book(book)
|
||||
|
||||
# 更新状态(至少成功一个就算完成)
|
||||
if results['txt'] or results['mp3']:
|
||||
book['status'] = 'downloaded'
|
||||
book['download_date'] = datetime.now().isoformat()
|
||||
book['downloaded_files'] = results
|
||||
else:
|
||||
book['status'] = 'failed'
|
||||
|
||||
# 保存索引
|
||||
save_index(index)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ 下载完成!")
|
||||
print("="*60)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ 用户中断")
|
||||
print(f"进度已保存,下次会自动从断点续传")
|
||||
save_index(index)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+194
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 1: 获取 Loyal Books 书籍列表和下载链接
|
||||
Children 分类,6页,约585本书
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import re
|
||||
import os
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
BASE_URL = "https://www.loyalbooks.com"
|
||||
CATEGORY = "Children"
|
||||
PAGES = 6
|
||||
DELAY = 3 # 列表页间隔
|
||||
|
||||
CONFIG_DIR = Path(__file__).parent.parent / "config"
|
||||
INDEX_FILE = CONFIG_DIR / "loyalbooks_children_index.json"
|
||||
|
||||
|
||||
def fetch_page(page_num):
|
||||
"""抓取单页书籍列表"""
|
||||
if page_num == 1:
|
||||
url = f"{BASE_URL}/genre/{CATEGORY}?results=100"
|
||||
else:
|
||||
url = f"{BASE_URL}/genre/{CATEGORY}?results=100&page={page_num}"
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
print(f" 抓取第 {page_num} 页...")
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
def parse_books_from_page(html):
|
||||
"""从页面解析书籍链接"""
|
||||
books = []
|
||||
|
||||
# 匹配书籍链接: href="/book/book-name-by-author"
|
||||
pattern = r'href="(/book/[^"]+)"'
|
||||
matches = re.findall(pattern, html)
|
||||
|
||||
# 去重
|
||||
seen = set()
|
||||
for match in matches:
|
||||
book_slug = match.strip()
|
||||
if book_slug not in seen:
|
||||
seen.add(book_slug)
|
||||
# 从 slug 提取书名
|
||||
book_name = book_slug.replace('/book/', '')
|
||||
books.append({
|
||||
'slug': book_slug,
|
||||
'name': book_name,
|
||||
'detail_url': f"{BASE_URL}{book_slug}",
|
||||
'txt_url': None, # 稍后填充
|
||||
'mp3_zip_url': None, # 稍后填充
|
||||
'status': 'pending',
|
||||
'download_date': None
|
||||
})
|
||||
|
||||
return books
|
||||
|
||||
|
||||
def fetch_book_detail(book):
|
||||
"""抓取书籍详情页,获取下载链接"""
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
try:
|
||||
print(f" 获取详情: {book['name'][:40]}...", end=' ')
|
||||
response = requests.get(book['detail_url'], headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
html = response.text
|
||||
|
||||
# 提取 TXT 链接
|
||||
txt_pattern = r'href="(/download/text/[^"]+\.txt)"'
|
||||
txt_match = re.search(txt_pattern, html)
|
||||
if txt_match:
|
||||
book['txt_url'] = urljoin(BASE_URL, txt_match.group(1))
|
||||
|
||||
# 提取 MP3 ZIP 链接 (archive.org)
|
||||
mp3_pattern = r'href="(https?://www\.archive\.org/download/[^"]*_64kb_mp3\.zip)"'
|
||||
mp3_match = re.search(mp3_pattern, html)
|
||||
if mp3_match:
|
||||
book['mp3_zip_url'] = mp3_match.group(1)
|
||||
|
||||
print(f"✅ TXT:{('Y' if book['txt_url'] else 'N')} MP3:{('Y' if book['mp3_zip_url'] else 'N')}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
return False
|
||||
|
||||
|
||||
def build_index():
|
||||
"""构建书籍索引"""
|
||||
index = {
|
||||
'fetch_date': datetime.now().isoformat(),
|
||||
'source_url': BASE_URL,
|
||||
'category': CATEGORY,
|
||||
'pages': PAGES,
|
||||
'books': []
|
||||
}
|
||||
|
||||
all_books = []
|
||||
|
||||
# Step 1: 获取所有书籍列表
|
||||
print("\n📚 Step 1: 获取书籍列表...")
|
||||
for page in range(1, PAGES + 1):
|
||||
print(f"\n[{page}/{PAGES}] 处理第 {page} 页")
|
||||
|
||||
try:
|
||||
html = fetch_page(page)
|
||||
books = parse_books_from_page(html)
|
||||
print(f" ✅ 找到 {len(books)} 本书")
|
||||
all_books.extend(books)
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 错误: {e}")
|
||||
|
||||
if page < PAGES:
|
||||
sleep(DELAY)
|
||||
|
||||
print(f"\n📊 列表抓取完成,共 {len(all_books)} 本书")
|
||||
|
||||
# Step 2: 获取每本书的详情和下载链接
|
||||
print("\n🔍 Step 2: 获取下载链接...")
|
||||
print(f"预计时间: {len(all_books) * 5 // 60} 分钟\n")
|
||||
|
||||
for i, book in enumerate(all_books, 1):
|
||||
print(f"[{i}/{len(all_books)}]", end=' ')
|
||||
fetch_book_detail(book)
|
||||
|
||||
if i < len(all_books):
|
||||
sleep(5) # 详情页间隔
|
||||
|
||||
index['books'] = all_books
|
||||
|
||||
# 统计
|
||||
with_txt = sum(1 for b in all_books if b['txt_url'])
|
||||
with_mp3 = sum(1 for b in all_books if b['mp3_zip_url'])
|
||||
print(f"\n📈 统计:")
|
||||
print(f" 有 TXT: {with_txt}/{len(all_books)}")
|
||||
print(f" 有 MP3: {with_mp3}/{len(all_books)}")
|
||||
|
||||
return index
|
||||
|
||||
|
||||
def save_index(index):
|
||||
"""保存索引"""
|
||||
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n💾 索引已保存: {INDEX_FILE}")
|
||||
print(f"📊 总计: {len(index['books'])} 本书")
|
||||
|
||||
|
||||
def main():
|
||||
# 检查是否已有索引
|
||||
if INDEX_FILE.exists():
|
||||
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
existing = json.load(f)
|
||||
print(f" 上次更新: {existing.get('fetch_date')}")
|
||||
print(f" 书籍数量: {len(existing.get('books', []))}")
|
||||
|
||||
response = input("\n是否重新获取? (y/N): ")
|
||||
if response.lower() != 'y':
|
||||
print("👋 保持现有索引")
|
||||
return
|
||||
|
||||
# 获取并保存
|
||||
print("\n🔍 开始抓取 Loyal Books - Children 分类...")
|
||||
print(f"⏱️ 预计时间: {PAGES * 3 + 585 * 5 // 60} 分钟\n")
|
||||
|
||||
index = build_index()
|
||||
save_index(index)
|
||||
|
||||
print("\n✅ Phase 1 完成")
|
||||
print("下一步: 运行 download_books.py 开始下载书籍内容")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user