Ubuntu
195 lines
5.7 KiB
Python
Executable File
195 lines
5.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Phase 1: 获取 Loyal Books 书籍列表和下载链接
|
|
Children 分类，6页，约585本书
|
|
"""
|
|
|
|
import json
|
|
import requests
|
|
import re
|
|
import os
|
|
from datetime import datetime
|
|
from time import sleep
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin
|
|
|
|
BASE_URL = "https://www.loyalbooks.com"
|
|
CATEGORY = "Children"
|
|
PAGES = 6
|
|
DELAY = 3 # 列表页间隔
|
|
|
|
CONFIG_DIR = Path(__file__).parent.parent / "config"
|
|
INDEX_FILE = CONFIG_DIR / "loyalbooks_children_index.json"
|
|
|
|
|
|
def fetch_page(page_num):
|
|
"""抓取单页书籍列表"""
|
|
if page_num == 1:
|
|
url = f"{BASE_URL}/genre/{CATEGORY}?results=100"
|
|
else:
|
|
url = f"{BASE_URL}/genre/{CATEGORY}?results=100&page={page_num}"
|
|
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
|
}
|
|
|
|
print(f" 抓取第 {page_num} 页...")
|
|
response = requests.get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.text
|
|
|
|
|
|
def parse_books_from_page(html):
|
|
"""从页面解析书籍链接"""
|
|
books = []
|
|
|
|
# 匹配书籍链接: href="/book/book-name-by-author"
|
|
pattern = r'href="(/book/[^"]+)"'
|
|
matches = re.findall(pattern, html)
|
|
|
|
# 去重
|
|
seen = set()
|
|
for match in matches:
|
|
book_slug = match.strip()
|
|
if book_slug not in seen:
|
|
seen.add(book_slug)
|
|
# 从 slug 提取书名
|
|
book_name = book_slug.replace('/book/', '')
|
|
books.append({
|
|
'slug': book_slug,
|
|
'name': book_name,
|
|
'detail_url': f"{BASE_URL}{book_slug}",
|
|
'txt_url': None, # 稍后填充
|
|
'mp3_zip_url': None, # 稍后填充
|
|
'status': 'pending',
|
|
'download_date': None
|
|
})
|
|
|
|
return books
|
|
|
|
|
|
def fetch_book_detail(book):
|
|
"""抓取书籍详情页，获取下载链接"""
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
|
|
}
|
|
|
|
try:
|
|
print(f" 获取详情: {book['name'][:40]}...", end=' ')
|
|
response = requests.get(book['detail_url'], headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
html = response.text
|
|
|
|
# 提取 TXT 链接
|
|
txt_pattern = r'href="(/download/text/[^"]+\.txt)"'
|
|
txt_match = re.search(txt_pattern, html)
|
|
if txt_match:
|
|
book['txt_url'] = urljoin(BASE_URL, txt_match.group(1))
|
|
|
|
# 提取 MP3 ZIP 链接 (archive.org)
|
|
mp3_pattern = r'href="(https?://www\.archive\.org/download/[^"]*_64kb_mp3\.zip)"'
|
|
mp3_match = re.search(mp3_pattern, html)
|
|
if mp3_match:
|
|
book['mp3_zip_url'] = mp3_match.group(1)
|
|
|
|
print(f"✅ TXT:{('Y' if book['txt_url'] else 'N')} MP3:{('Y' if book['mp3_zip_url'] else 'N')}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
print(f"❌ {e}")
|
|
return False
|
|
|
|
|
|
def build_index():
|
|
"""构建书籍索引"""
|
|
index = {
|
|
'fetch_date': datetime.now().isoformat(),
|
|
'source_url': BASE_URL,
|
|
'category': CATEGORY,
|
|
'pages': PAGES,
|
|
'books': []
|
|
}
|
|
|
|
all_books = []
|
|
|
|
# Step 1: 获取所有书籍列表
|
|
print("\n📚 Step 1: 获取书籍列表...")
|
|
for page in range(1, PAGES + 1):
|
|
print(f"\n[{page}/{PAGES}] 处理第 {page} 页")
|
|
|
|
try:
|
|
html = fetch_page(page)
|
|
books = parse_books_from_page(html)
|
|
print(f" ✅ 找到 {len(books)} 本书")
|
|
all_books.extend(books)
|
|
|
|
except Exception as e:
|
|
print(f" ❌ 错误: {e}")
|
|
|
|
if page < PAGES:
|
|
sleep(DELAY)
|
|
|
|
print(f"\n📊 列表抓取完成，共 {len(all_books)} 本书")
|
|
|
|
# Step 2: 获取每本书的详情和下载链接
|
|
print("\n🔍 Step 2: 获取下载链接...")
|
|
print(f"预计时间: {len(all_books) * 5 // 60} 分钟\n")
|
|
|
|
for i, book in enumerate(all_books, 1):
|
|
print(f"[{i}/{len(all_books)}]", end=' ')
|
|
fetch_book_detail(book)
|
|
|
|
if i < len(all_books):
|
|
sleep(5) # 详情页间隔
|
|
|
|
index['books'] = all_books
|
|
|
|
# 统计
|
|
with_txt = sum(1 for b in all_books if b['txt_url'])
|
|
with_mp3 = sum(1 for b in all_books if b['mp3_zip_url'])
|
|
print(f"\n📈 统计:")
|
|
print(f" 有 TXT: {with_txt}/{len(all_books)}")
|
|
print(f" 有 MP3: {with_mp3}/{len(all_books)}")
|
|
|
|
return index
|
|
|
|
|
|
def save_index(index):
|
|
"""保存索引"""
|
|
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
|
json.dump(index, f, indent=2, ensure_ascii=False)
|
|
|
|
print(f"\n💾 索引已保存: {INDEX_FILE}")
|
|
print(f"📊 总计: {len(index['books'])} 本书")
|
|
|
|
|
|
def main():
|
|
# 检查是否已有索引
|
|
if INDEX_FILE.exists():
|
|
print(f"⚠️ 索引文件已存在: {INDEX_FILE}")
|
|
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
|
existing = json.load(f)
|
|
print(f" 上次更新: {existing.get('fetch_date')}")
|
|
print(f" 书籍数量: {len(existing.get('books', []))}")
|
|
|
|
response = input("\n是否重新获取? (y/N): ")
|
|
if response.lower() != 'y':
|
|
print("👋 保持现有索引")
|
|
return
|
|
|
|
# 获取并保存
|
|
print("\n🔍 开始抓取 Loyal Books - Children 分类...")
|
|
print(f"⏱️ 预计时间: {PAGES * 3 + 585 * 5 // 60} 分钟\n")
|
|
|
|
index = build_index()
|
|
save_index(index)
|
|
|
|
print("\n✅ Phase 1 完成")
|
|
print("下一步: 运行 download_books.py 开始下载书籍内容")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|