Ubuntu
72 lines
2.4 KiB
Python
72 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""测试单篇文章采集"""
|
|
import os
|
|
import sys
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
from batch_crawl import (
|
|
get_article_detail, download_image, html_to_markdown,
|
|
clean_markdown, insert_image_to_db, save_article, DB_PATH
|
|
)
|
|
from playwright.sync_api import sync_playwright
|
|
import sqlite3
|
|
|
|
TEST_URL = "https://niaodaifu.cn/health-information/detail?menu_id=15&article_id=992"
|
|
CATEGORY_ID = 2 # 科普文章
|
|
|
|
print("=" * 60)
|
|
print("测试采集单篇文章")
|
|
print("=" * 60)
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page()
|
|
|
|
print(f"\n1. 获取详情页: {TEST_URL}")
|
|
detail = get_article_detail(page, TEST_URL)
|
|
print(f" 标题: {detail['title']}")
|
|
print(f" 来源: {detail['source']}")
|
|
print(f" 时间: {detail['time']}")
|
|
print(f" 正文HTML长度: {len(detail['content_html'])}")
|
|
|
|
print("\n2. 转换 Markdown...")
|
|
from bs4 import BeautifulSoup
|
|
soup = BeautifulSoup(detail['content_html'], 'html.parser')
|
|
content_div = soup.find('div', class_='article-content')
|
|
downloaded_images = {}
|
|
content_md = html_to_markdown(content_div, TEST_URL, downloaded_images)
|
|
content_md = clean_markdown(content_md)
|
|
print(f" Markdown长度: {len(content_md)}")
|
|
print(f" 下载图片数: {len(downloaded_images)}")
|
|
for src, path in downloaded_images.items():
|
|
print(f" {src} -> {path}")
|
|
|
|
print("\n3. Markdown 前500字符预览:")
|
|
print("-" * 40)
|
|
print(content_md[:500])
|
|
print("-" * 40)
|
|
|
|
print("\n4. 保存到数据库...")
|
|
article = {
|
|
'article_id': 992,
|
|
'title': detail['title'] or '2024吴试纸春节放假安排',
|
|
'href': TEST_URL,
|
|
'cover_url': 'https://image.niaodaifu.cn/news/2024-01/d756f6171be42e0000a7f1b7375193da.jpg',
|
|
'time': detail['time'],
|
|
}
|
|
cover_path, cover_info = download_image(article['cover_url'])
|
|
print(f" 封面图: {cover_path}")
|
|
if cover_info:
|
|
conn = sqlite3.connect(DB_PATH)
|
|
img_id = insert_image_to_db(conn, cover_info)
|
|
conn.commit()
|
|
conn.close()
|
|
print(f" 封面图入库 ID: {img_id}")
|
|
|
|
post_id = save_article(article, detail, CATEGORY_ID, cover_path, content_md)
|
|
print(f" 文章入库 ID: {post_id}")
|
|
|
|
browser.close()
|
|
|
|
print("\n测试完成！")
|