Ubuntu
544 lines
18 KiB
Python
544 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
批量采集 niaodaifu.cn 健康资讯文章
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import hashlib
|
|
import requests
|
|
import sqlite3
|
|
from datetime import datetime
|
|
from urllib.parse import urljoin, urlparse
|
|
from playwright.sync_api import sync_playwright
|
|
from bs4 import BeautifulSoup, NavigableString
|
|
from PIL import Image
|
|
import io
|
|
|
|
# ============ 配置 ============
|
|
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
UPLOAD_DIR = os.path.join(BASE_DIR, "..", "storage", "admin", "images")
|
|
DB_PATH = os.path.join(BASE_DIR, "cms.db")
|
|
|
|
MENU_TO_CATEGORY = {
|
|
15: (2, "科普文章"),
|
|
7: (3, "肾友"),
|
|
8: (4, "糖友"),
|
|
6: (5, "健康"),
|
|
}
|
|
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
}
|
|
|
|
|
|
# ============ 工具函数 ============
|
|
def slugify(title):
|
|
"""生成 URL 友好的 slug"""
|
|
import unicodedata
|
|
slug = unicodedata.normalize('NFKC', title)
|
|
slug = re.sub(r'[^\w\s-]', '', slug).strip().lower()
|
|
slug = re.sub(r'[-\s]+', '-', slug)
|
|
if not slug:
|
|
slug = 'article'
|
|
return slug
|
|
|
|
|
|
def ensure_unique_slug(conn, slug):
|
|
"""确保 slug 唯一"""
|
|
cursor = conn.cursor()
|
|
original = slug
|
|
suffix = 1
|
|
while True:
|
|
cursor.execute('SELECT 1 FROM news_posts WHERE slug = ?', (slug,))
|
|
if not cursor.fetchone():
|
|
break
|
|
slug = f"{original}-{suffix}"
|
|
suffix += 1
|
|
return slug
|
|
|
|
|
|
def download_image(url, module='news'):
|
|
"""下载图片到本地存储目录，返回文件路径和数据库记录信息"""
|
|
if not url or url.startswith('data:'):
|
|
return None, None
|
|
|
|
# 过滤 135editor 占位图
|
|
if '135editor.com' in url and url.endswith('.gif'):
|
|
return None, None
|
|
|
|
try:
|
|
resp = requests.get(url, headers=HEADERS, timeout=30)
|
|
if resp.status_code != 200:
|
|
return None, None
|
|
|
|
content = resp.content
|
|
if len(content) < 100:
|
|
return None, None
|
|
|
|
# 判断扩展名
|
|
ext = os.path.splitext(urlparse(url).path)[1].lower()
|
|
if ext not in ('.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp'):
|
|
# 尝试通过内容判断
|
|
if content[:2] == b'\xff\xd8':
|
|
ext = '.jpg'
|
|
elif content[:8] == b'\x89PNG\r\n\x1a\n':
|
|
ext = '.png'
|
|
elif content[:4] == b'GIF8':
|
|
ext = '.gif'
|
|
elif content[:4] == b'<svg':
|
|
ext = '.svg'
|
|
else:
|
|
ext = '.jpg'
|
|
|
|
# 生成文件名
|
|
filename = hashlib.md5(url.encode()).hexdigest() + ext
|
|
subdir = datetime.now().strftime('%Y%m')
|
|
dest_dir = os.path.join(UPLOAD_DIR, subdir)
|
|
os.makedirs(dest_dir, exist_ok=True)
|
|
|
|
filepath = os.path.join(dest_dir, filename)
|
|
with open(filepath, 'wb') as f:
|
|
f.write(content)
|
|
|
|
# 生成缩略图
|
|
thumb_path = None
|
|
if ext in ('.jpg', '.jpeg', '.png', '.webp', '.bmp'):
|
|
try:
|
|
img = Image.open(io.BytesIO(content))
|
|
img.thumbnail((300, 300))
|
|
thumb_filename = f"thumb_{filename}"
|
|
thumb_filepath = os.path.join(dest_dir, thumb_filename)
|
|
if ext == '.png':
|
|
img.save(thumb_filepath, 'PNG')
|
|
else:
|
|
img.save(thumb_filepath, 'JPEG', quality=85)
|
|
thumb_path = f'/storage/admin/images/{subdir}/{thumb_filename}'
|
|
except Exception:
|
|
pass
|
|
|
|
rel_path = f'/storage/admin/images/{subdir}/{filename}'
|
|
file_size = len(content)
|
|
|
|
# 获取图片尺寸
|
|
width, height = None, None
|
|
try:
|
|
img = Image.open(io.BytesIO(content))
|
|
width, height = img.size
|
|
except Exception:
|
|
pass
|
|
|
|
return rel_path, {
|
|
'filename': filename,
|
|
'original_name': os.path.basename(urlparse(url).path) or filename,
|
|
'file_path': rel_path,
|
|
'thumb_path': thumb_path,
|
|
'file_size': file_size,
|
|
'width': width,
|
|
'height': height,
|
|
'module': module,
|
|
}
|
|
except Exception as e:
|
|
print(f" [图片下载失败] {url}: {e}")
|
|
return None, None
|
|
|
|
|
|
def insert_image_to_db(conn, img_info):
|
|
"""插入图片记录到 images 表，返回 image_id"""
|
|
if not img_info:
|
|
return None
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
INSERT INTO images (filename, original_name, file_path, thumb_path, description, file_size, width, height, module, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
''', (
|
|
img_info['filename'],
|
|
img_info['original_name'],
|
|
img_info['file_path'],
|
|
img_info.get('thumb_path'),
|
|
None,
|
|
img_info['file_size'],
|
|
img_info.get('width'),
|
|
img_info.get('height'),
|
|
img_info['module'],
|
|
datetime.now().isoformat(),
|
|
))
|
|
return cursor.lastrowid
|
|
|
|
|
|
def html_to_markdown(element, base_url, downloaded_images):
|
|
"""递归将 HTML 转换为 Markdown，同时下载图片"""
|
|
if element is None:
|
|
return ''
|
|
|
|
parts = []
|
|
for child in element.children:
|
|
if isinstance(child, NavigableString):
|
|
text = str(child)
|
|
# 保留换行，但压缩多余空格
|
|
parts.append(text)
|
|
continue
|
|
|
|
tag = child.name
|
|
if tag is None:
|
|
continue
|
|
|
|
# 跳过 135editor 装饰性元素和脚本
|
|
if tag in ('script', 'style', 'iframe', 'video', 'audio'):
|
|
continue
|
|
|
|
# 跳过纯装饰性 section/div（没有文本内容且没有图片）
|
|
inner_text = child.get_text(strip=True)
|
|
has_img = child.find('img') is not None
|
|
if tag in ('section', 'div') and not inner_text and not has_img:
|
|
continue
|
|
|
|
if tag == 'p':
|
|
text = html_to_markdown(child, base_url, downloaded_images).strip()
|
|
if text:
|
|
parts.append(text + '\n\n')
|
|
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
|
|
level = int(tag[1])
|
|
text = html_to_markdown(child, base_url, downloaded_images).strip()
|
|
if text:
|
|
parts.append('#' * level + ' ' + text + '\n\n')
|
|
elif tag == 'img':
|
|
src = child.get('src') or child.get('_src') or child.get('data-src', '')
|
|
if src and not src.endswith('.gif'):
|
|
full_src = urljoin(base_url, src)
|
|
# 下载图片
|
|
if full_src not in downloaded_images:
|
|
rel_path, img_info = download_image(full_src)
|
|
if rel_path:
|
|
downloaded_images[full_src] = rel_path
|
|
# 同时记录到数据库
|
|
conn = sqlite3.connect(DB_PATH)
|
|
insert_image_to_db(conn, img_info)
|
|
conn.commit()
|
|
conn.close()
|
|
if full_src in downloaded_images:
|
|
alt = child.get('alt', '图片')
|
|
parts.append(f'\n\n')
|
|
elif tag in ('strong', 'b'):
|
|
text = html_to_markdown(child, base_url, downloaded_images).strip()
|
|
if text:
|
|
parts.append(f'**{text}**')
|
|
elif tag in ('em', 'i'):
|
|
text = html_to_markdown(child, base_url, downloaded_images).strip()
|
|
if text:
|
|
parts.append(f'*{text}*')
|
|
elif tag == 'br':
|
|
parts.append('\n')
|
|
elif tag == 'a':
|
|
href = child.get('href', '')
|
|
text = html_to_markdown(child, base_url, downloaded_images).strip()
|
|
if text and href:
|
|
full_href = urljoin(base_url, href)
|
|
parts.append(f'[{text}]({full_href})')
|
|
elif text:
|
|
parts.append(text)
|
|
elif tag in ('ul', 'ol'):
|
|
is_ordered = (tag == 'ol')
|
|
items = []
|
|
for li in child.find_all('li', recursive=False):
|
|
li_text = html_to_markdown(li, base_url, downloaded_images).strip()
|
|
if li_text:
|
|
prefix = f"{len(items)+1}. " if is_ordered else "- "
|
|
items.append(prefix + li_text)
|
|
if items:
|
|
parts.append('\n' + '\n'.join(items) + '\n\n')
|
|
elif tag == 'blockquote':
|
|
text = html_to_markdown(child, base_url, downloaded_images).strip()
|
|
if text:
|
|
lines = text.split('\n')
|
|
quoted = '\n'.join('> ' + line for line in lines)
|
|
parts.append(quoted + '\n\n')
|
|
elif tag in ('table', 'thead', 'tbody', 'tr', 'td', 'th'):
|
|
# 简单处理表格：提取文本
|
|
text = html_to_markdown(child, base_url, downloaded_images).strip()
|
|
if text:
|
|
parts.append(text + ' ')
|
|
elif tag == 'pre':
|
|
code = child.get_text()
|
|
parts.append(f'\n```\n{code}\n```\n\n')
|
|
elif tag == 'code':
|
|
code = child.get_text()
|
|
parts.append(f'`{code}`')
|
|
else:
|
|
# 其他标签递归处理
|
|
text = html_to_markdown(child, base_url, downloaded_images)
|
|
if text:
|
|
parts.append(text)
|
|
|
|
return ''.join(parts)
|
|
|
|
|
|
def clean_markdown(md):
|
|
"""清理 Markdown 文本"""
|
|
# 合并多个空行
|
|
md = re.sub(r'\n{3,}', '\n\n', md)
|
|
# 去除行首行尾空白
|
|
md = md.strip()
|
|
return md
|
|
|
|
|
|
def parse_articles_from_page(page):
|
|
"""从当前页面解析文章列表"""
|
|
articles = []
|
|
html = page.content()
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
medias = soup.find_all('div', class_='media')
|
|
for media in medias:
|
|
heading_a = media.find('a', class_='media-heading')
|
|
if not heading_a:
|
|
continue
|
|
title = heading_a.get_text(strip=True)
|
|
href = heading_a.get('href', '')
|
|
if not href:
|
|
continue
|
|
m = re.search(r'article_id=(\d+)', href)
|
|
if not m:
|
|
continue
|
|
article_id = int(m.group(1))
|
|
cover_url = None
|
|
left_a = media.find('a', class_='media-left')
|
|
if left_a:
|
|
img = left_a.find('img')
|
|
if img:
|
|
cover_url = img.get('src')
|
|
time_str = ''
|
|
time_el = media.find('span', class_='time')
|
|
if time_el:
|
|
time_str = time_el.get_text(strip=True)
|
|
articles.append({
|
|
'article_id': article_id,
|
|
'title': title,
|
|
'href': urljoin('https://niaodaifu.cn', href),
|
|
'cover_url': cover_url,
|
|
'time': time_str,
|
|
})
|
|
return articles, soup
|
|
|
|
|
|
# ============ 采集函数 ============
|
|
def get_article_list(page, menu_id):
|
|
"""获取指定栏目的所有文章列表（通过点击分页按钮翻页）"""
|
|
base_url = f"https://niaodaifu.cn/health-information?menu_id={menu_id}"
|
|
all_articles = []
|
|
seen_ids = set()
|
|
|
|
page.goto(base_url, wait_until='networkidle', timeout=30000)
|
|
page.wait_for_timeout(3000)
|
|
|
|
# 获取总条数
|
|
html = page.content()
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
total = 0
|
|
total_el = soup.find('span', class_='el-pagination__total')
|
|
if total_el:
|
|
m = re.search(r'(\d+)', total_el.get_text())
|
|
if m:
|
|
total = int(m.group(1))
|
|
|
|
# 获取页码按钮总数（最后一个数字页码）
|
|
pager_btns = page.query_selector_all('.el-pager li.number')
|
|
total_pages = 1
|
|
if pager_btns:
|
|
# 最后一个页码按钮的文本就是总页数（可能被省略，需要点击更多后才知道）
|
|
# 先查 el-pagination__total
|
|
if total > 0:
|
|
page_size_text = ''
|
|
size_select = soup.find('input', {'class': 'el-input__inner'})
|
|
page_size = 10 # 默认每页10条
|
|
total_pages = max(1, (total + page_size - 1) // page_size)
|
|
|
|
print(f" 栏目 {menu_id} 共 {total} 篇文章，预计 {total_pages} 页")
|
|
|
|
current_page = 1
|
|
while True:
|
|
articles, soup = parse_articles_from_page(page)
|
|
new_count = 0
|
|
for art in articles:
|
|
if art['article_id'] not in seen_ids:
|
|
seen_ids.add(art['article_id'])
|
|
all_articles.append(art)
|
|
new_count += 1
|
|
print(f" 第 {current_page} 页采集到 {len(articles)} 篇（新增 {new_count} 篇）")
|
|
|
|
# 查找下一页按钮
|
|
next_btn = page.query_selector('button.btn-next')
|
|
if not next_btn:
|
|
break
|
|
# 检查是否禁用
|
|
is_disabled = next_btn.get_attribute('disabled')
|
|
if is_disabled is not None:
|
|
break
|
|
|
|
next_btn.click()
|
|
page.wait_for_timeout(2000)
|
|
current_page += 1
|
|
|
|
return all_articles
|
|
|
|
|
|
def get_article_detail(page, url):
|
|
"""获取文章详情"""
|
|
page.goto(url, wait_until='networkidle', timeout=30000)
|
|
page.wait_for_timeout(3000)
|
|
|
|
html = page.content()
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
# 标题：在 article > h3 中，格式 "首页 》栏目 》标题"
|
|
title = ''
|
|
article_el = soup.find('article')
|
|
if article_el:
|
|
h3 = article_el.find('h3')
|
|
if h3:
|
|
text = h3.get_text(strip=True)
|
|
parts = [p.strip() for p in text.split('》')]
|
|
# 过滤 "首页" 和栏目名，取最后的标题
|
|
if len(parts) >= 2:
|
|
title = parts[-1]
|
|
else:
|
|
title = text
|
|
|
|
# 来源和时间
|
|
source = ''
|
|
time_str = ''
|
|
if article_el:
|
|
info_p = article_el.find('p', class_='more-info')
|
|
if info_p:
|
|
src_el = info_p.find('span', class_='source')
|
|
if src_el:
|
|
source = src_el.get_text(strip=True).replace('来源：', '')
|
|
time_el = info_p.find('span', class_='time')
|
|
if time_el:
|
|
time_str = time_el.get_text(strip=True)
|
|
|
|
# 正文 HTML
|
|
content_div = soup.find('div', class_='article-content')
|
|
content_html = ''
|
|
if content_div:
|
|
content_html = str(content_div)
|
|
|
|
return {
|
|
'title': title,
|
|
'source': source,
|
|
'time': time_str,
|
|
'content_html': content_html,
|
|
}
|
|
|
|
|
|
def save_article(article, detail, category_id, cover_path, content_md):
|
|
"""保存文章到数据库"""
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
|
|
title = detail['title'] or article['title']
|
|
slug = ensure_unique_slug(conn, slugify(title))
|
|
|
|
# 解析发布时间
|
|
created_at = datetime.now().isoformat()
|
|
if detail.get('time') or article.get('time'):
|
|
t = detail.get('time') or article.get('time')
|
|
try:
|
|
created_at = datetime.strptime(t, '%Y-%m-%d %H:%M').isoformat()
|
|
except ValueError:
|
|
try:
|
|
created_at = datetime.strptime(t, '%Y-%m-%d').isoformat()
|
|
except ValueError:
|
|
pass
|
|
|
|
summary = title[:200]
|
|
if detail.get('source'):
|
|
summary = f"来源：{detail['source']} | {summary}"
|
|
|
|
cursor.execute('''
|
|
INSERT INTO news_posts (title, slug, content, content_type, summary, cover_image, category_id, is_published, is_draft, view_count, created_at, updated_at)
|
|
VALUES (?, ?, ?, 'markdown', ?, ?, ?, 1, 0, 0, ?, ?)
|
|
''', (
|
|
title,
|
|
slug,
|
|
content_md,
|
|
summary,
|
|
cover_path,
|
|
category_id,
|
|
created_at,
|
|
created_at,
|
|
))
|
|
|
|
post_id = cursor.lastrowid
|
|
conn.commit()
|
|
conn.close()
|
|
return post_id
|
|
|
|
|
|
# ============ 主程序 ============
|
|
def main():
|
|
print("=" * 60)
|
|
print("开始批量采集 niaodaifu.cn 文章")
|
|
print("=" * 60)
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page(viewport={'width': 1920, 'height': 1080})
|
|
|
|
for menu_id, (category_id, cat_name) in MENU_TO_CATEGORY.items():
|
|
print(f"\n>>> 采集栏目 [{cat_name}] menu_id={menu_id} -> category_id={category_id}")
|
|
|
|
articles = get_article_list(page, menu_id)
|
|
print(f" 共找到 {len(articles)} 篇文章")
|
|
|
|
for idx, article in enumerate(articles, 1):
|
|
print(f" [{idx}/{len(articles)}] {article['title']}")
|
|
|
|
try:
|
|
detail = get_article_detail(page, article['href'])
|
|
except Exception as e:
|
|
print(f" [详情获取失败] {e}")
|
|
continue
|
|
|
|
# 下载封面图
|
|
cover_path = None
|
|
if article.get('cover_url'):
|
|
cover_path, cover_info = download_image(article['cover_url'])
|
|
if cover_info:
|
|
conn = sqlite3.connect(DB_PATH)
|
|
insert_image_to_db(conn, cover_info)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
# 转换正文为 Markdown
|
|
content_md = ''
|
|
if detail.get('content_html'):
|
|
soup = BeautifulSoup(detail['content_html'], 'html.parser')
|
|
content_div = soup.find('div', class_='article-content')
|
|
if content_div:
|
|
downloaded_images = {}
|
|
content_md = html_to_markdown(content_div, article['href'], downloaded_images)
|
|
content_md = clean_markdown(content_md)
|
|
|
|
if not content_md.strip():
|
|
print(f" [警告] 正文为空，跳过")
|
|
continue
|
|
|
|
# 保存
|
|
try:
|
|
post_id = save_article(article, detail, category_id, cover_path, content_md)
|
|
print(f" [已入库] post_id={post_id}")
|
|
except Exception as e:
|
|
print(f" [入库失败] {e}")
|
|
|
|
browser.close()
|
|
|
|
print("\n" + "=" * 60)
|
|
print("采集完成")
|
|
print("=" * 60)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|