#!/usr/bin/env python3 """测试单篇文章采集""" import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from batch_crawl import ( get_article_detail, download_image, html_to_markdown, clean_markdown, insert_image_to_db, save_article, DB_PATH ) from playwright.sync_api import sync_playwright import sqlite3 TEST_URL = "https://niaodaifu.cn/health-information/detail?menu_id=15&article_id=992" CATEGORY_ID = 2 # 科普文章 print("=" * 60) print("测试采集单篇文章") print("=" * 60) with sync_playwright() as p: browser = p.chromium.launch(headless=True) page = browser.new_page() print(f"\n1. 获取详情页: {TEST_URL}") detail = get_article_detail(page, TEST_URL) print(f" 标题: {detail['title']}") print(f" 来源: {detail['source']}") print(f" 时间: {detail['time']}") print(f" 正文HTML长度: {len(detail['content_html'])}") print("\n2. 转换 Markdown...") from bs4 import BeautifulSoup soup = BeautifulSoup(detail['content_html'], 'html.parser') content_div = soup.find('div', class_='article-content') downloaded_images = {} content_md = html_to_markdown(content_div, TEST_URL, downloaded_images) content_md = clean_markdown(content_md) print(f" Markdown长度: {len(content_md)}") print(f" 下载图片数: {len(downloaded_images)}") for src, path in downloaded_images.items(): print(f" {src} -> {path}") print("\n3. Markdown 前500字符预览:") print("-" * 40) print(content_md[:500]) print("-" * 40) print("\n4. 保存到数据库...") article = { 'article_id': 992, 'title': detail['title'] or '2024吴试纸春节放假安排', 'href': TEST_URL, 'cover_url': 'https://image.niaodaifu.cn/news/2024-01/d756f6171be42e0000a7f1b7375193da.jpg', 'time': detail['time'], } cover_path, cover_info = download_image(article['cover_url']) print(f" 封面图: {cover_path}") if cover_info: conn = sqlite3.connect(DB_PATH) img_id = insert_image_to_db(conn, cover_info) conn.commit() conn.close() print(f" 封面图入库 ID: {img_id}") post_id = save_article(article, detail, CATEGORY_ID, cover_path, content_md) print(f" 文章入库 ID: {post_id}") browser.close() print("\n测试完成!")