admin/nanocode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

a

Browse Source
This commit is contained in:
Ubuntu
2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
new-media-platform/website-new/backend/batch_crawl.py
+543
View File
@@ -0,0 +1,543 @@
#!/usr/bin/env python3
"""
批量采集 niaodaifu.cn 健康资讯文章
"""
import os
import re
import hashlib
import requests
import sqlite3
from datetime import datetime
from urllib.parse import urljoin, urlparse
from playwright.sync_api import sync_playwright
from bs4 import BeautifulSoup, NavigableString
from PIL import Image
import io
# ============ 配置 ============
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
UPLOAD_DIR = os.path.join(BASE_DIR, "..", "storage", "admin", "images")
DB_PATH = os.path.join(BASE_DIR, "cms.db")
MENU_TO_CATEGORY = {
15: (2, "科普文章"),
7: (3, "肾友"),
8: (4, "糖友"),
6: (5, "健康"),
}
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
}
# ============ 工具函数 ============
def slugify(title):
"""生成 URL 友好的 slug"""
import unicodedata
slug = unicodedata.normalize('NFKC', title)
slug = re.sub(r'[^\w\s-]', '', slug).strip().lower()
slug = re.sub(r'[-\s]+', '-', slug)
if not slug:
slug = 'article'
return slug
def ensure_unique_slug(conn, slug):
"""确保 slug 唯一"""
cursor = conn.cursor()
original = slug
suffix = 1
while True:
cursor.execute('SELECT 1 FROM news_posts WHERE slug = ?', (slug,))
if not cursor.fetchone():
break
slug = f"{original}-{suffix}"
suffix += 1
return slug
def download_image(url, module='news'):
"""下载图片到本地存储目录，返回文件路径和数据库记录信息"""
if not url or url.startswith('data:'):
return None, None
# 过滤 135editor 占位图
if '135editor.com' in url and url.endswith('.gif'):
return None, None
try:
resp = requests.get(url, headers=HEADERS, timeout=30)
if resp.status_code != 200:
return None, None
content = resp.content
if len(content) < 100:
return None, None
# 判断扩展名
ext = os.path.splitext(urlparse(url).path)[1].lower()
if ext not in ('.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp'):
# 尝试通过内容判断
if content[:2] == b'\xff\xd8':
ext = '.jpg'
elif content[:8] == b'\x89PNG\r\n\x1a\n':
ext = '.png'
elif content[:4] == b'GIF8':
ext = '.gif'
elif content[:4] == b'<svg':
ext = '.svg'
else:
ext = '.jpg'
# 生成文件名
filename = hashlib.md5(url.encode()).hexdigest() + ext
subdir = datetime.now().strftime('%Y%m')
dest_dir = os.path.join(UPLOAD_DIR, subdir)
os.makedirs(dest_dir, exist_ok=True)
filepath = os.path.join(dest_dir, filename)
with open(filepath, 'wb') as f:
f.write(content)
# 生成缩略图
thumb_path = None
if ext in ('.jpg', '.jpeg', '.png', '.webp', '.bmp'):
try:
img = Image.open(io.BytesIO(content))
img.thumbnail((300, 300))
thumb_filename = f"thumb_{filename}"
thumb_filepath = os.path.join(dest_dir, thumb_filename)
if ext == '.png':
img.save(thumb_filepath, 'PNG')
else:
img.save(thumb_filepath, 'JPEG', quality=85)
thumb_path = f'/storage/admin/images/{subdir}/{thumb_filename}'
except Exception:
pass
rel_path = f'/storage/admin/images/{subdir}/{filename}'
file_size = len(content)
# 获取图片尺寸
width, height = None, None
try:
img = Image.open(io.BytesIO(content))
width, height = img.size
except Exception:
pass
return rel_path, {
'filename': filename,
'original_name': os.path.basename(urlparse(url).path) or filename,
'file_path': rel_path,
'thumb_path': thumb_path,
'file_size': file_size,
'width': width,
'height': height,
'module': module,
}
except Exception as e:
print(f" [图片下载失败] {url}: {e}")
return None, None
def insert_image_to_db(conn, img_info):
"""插入图片记录到 images 表，返回 image_id"""
if not img_info:
return None
cursor = conn.cursor()
cursor.execute('''
INSERT INTO images (filename, original_name, file_path, thumb_path, description, file_size, width, height, module, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
img_info['filename'],
img_info['original_name'],
img_info['file_path'],
img_info.get('thumb_path'),
None,
img_info['file_size'],
img_info.get('width'),
img_info.get('height'),
img_info['module'],
datetime.now().isoformat(),
))
return cursor.lastrowid
def html_to_markdown(element, base_url, downloaded_images):
"""递归将 HTML 转换为 Markdown，同时下载图片"""
if element is None:
return ''
parts = []
for child in element.children:
if isinstance(child, NavigableString):
text = str(child)
# 保留换行，但压缩多余空格
parts.append(text)
continue
tag = child.name
if tag is None:
continue
# 跳过 135editor 装饰性元素和脚本
if tag in ('script', 'style', 'iframe', 'video', 'audio'):
continue
# 跳过纯装饰性 section/div（没有文本内容且没有图片）
inner_text = child.get_text(strip=True)
has_img = child.find('img') is not None
if tag in ('section', 'div') and not inner_text and not has_img:
continue
if tag == 'p':
text = html_to_markdown(child, base_url, downloaded_images).strip()
if text:
parts.append(text + '\n\n')
elif tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
level = int(tag[1])
text = html_to_markdown(child, base_url, downloaded_images).strip()
if text:
parts.append('#' * level + ' ' + text + '\n\n')
elif tag == 'img':
src = child.get('src') or child.get('_src') or child.get('data-src', '')
if src and not src.endswith('.gif'):
full_src = urljoin(base_url, src)
# 下载图片
if full_src not in downloaded_images:
rel_path, img_info = download_image(full_src)
if rel_path:
downloaded_images[full_src] = rel_path
# 同时记录到数据库
conn = sqlite3.connect(DB_PATH)
insert_image_to_db(conn, img_info)
conn.commit()
conn.close()
if full_src in downloaded_images:
alt = child.get('alt', '图片')
parts.append(f'\n![{alt}]({downloaded_images[full_src]})\n')
elif tag in ('strong', 'b'):
text = html_to_markdown(child, base_url, downloaded_images).strip()
if text:
parts.append(f'**{text}**')
elif tag in ('em', 'i'):
text = html_to_markdown(child, base_url, downloaded_images).strip()
if text:
parts.append(f'*{text}*')
elif tag == 'br':
parts.append('\n')
elif tag == 'a':
href = child.get('href', '')
text = html_to_markdown(child, base_url, downloaded_images).strip()
if text and href:
full_href = urljoin(base_url, href)
parts.append(f'[{text}]({full_href})')
elif text:
parts.append(text)
elif tag in ('ul', 'ol'):
is_ordered = (tag == 'ol')
items = []
for li in child.find_all('li', recursive=False):
li_text = html_to_markdown(li, base_url, downloaded_images).strip()
if li_text:
prefix = f"{len(items)+1}. " if is_ordered else "- "
items.append(prefix + li_text)
if items:
parts.append('\n' + '\n'.join(items) + '\n\n')
elif tag == 'blockquote':
text = html_to_markdown(child, base_url, downloaded_images).strip()
if text:
lines = text.split('\n')
quoted = '\n'.join('> ' + line for line in lines)
parts.append(quoted + '\n\n')
elif tag in ('table', 'thead', 'tbody', 'tr', 'td', 'th'):
# 简单处理表格：提取文本
text = html_to_markdown(child, base_url, downloaded_images).strip()
if text:
parts.append(text + ' ')
elif tag == 'pre':
code = child.get_text()
parts.append(f'\n```\n{code}\n```\n\n')
elif tag == 'code':
code = child.get_text()
parts.append(f'`{code}`')
else:
# 其他标签递归处理
text = html_to_markdown(child, base_url, downloaded_images)
if text:
parts.append(text)
return ''.join(parts)
def clean_markdown(md):
"""清理 Markdown 文本"""
# 合并多个空行
md = re.sub(r'\n{3,}', '\n\n', md)
# 去除行首行尾空白
md = md.strip()
return md
def parse_articles_from_page(page):
"""从当前页面解析文章列表"""
articles = []
html = page.content()
soup = BeautifulSoup(html, 'html.parser')
medias = soup.find_all('div', class_='media')
for media in medias:
heading_a = media.find('a', class_='media-heading')
if not heading_a:
continue
title = heading_a.get_text(strip=True)
href = heading_a.get('href', '')
if not href:
continue
m = re.search(r'article_id=(\d+)', href)
if not m:
continue
article_id = int(m.group(1))
cover_url = None
left_a = media.find('a', class_='media-left')
if left_a:
img = left_a.find('img')
if img:
cover_url = img.get('src')
time_str = ''
time_el = media.find('span', class_='time')
if time_el:
time_str = time_el.get_text(strip=True)
articles.append({
'article_id': article_id,
'title': title,
'href': urljoin('https://niaodaifu.cn', href),
'cover_url': cover_url,
'time': time_str,
})
return articles, soup
# ============ 采集函数 ============
def get_article_list(page, menu_id):
"""获取指定栏目的所有文章列表（通过点击分页按钮翻页）"""
base_url = f"https://niaodaifu.cn/health-information?menu_id={menu_id}"
all_articles = []
seen_ids = set()
page.goto(base_url, wait_until='networkidle', timeout=30000)
page.wait_for_timeout(3000)
# 获取总条数
html = page.content()
soup = BeautifulSoup(html, 'html.parser')
total = 0
total_el = soup.find('span', class_='el-pagination__total')
if total_el:
m = re.search(r'(\d+)', total_el.get_text())
if m:
total = int(m.group(1))
# 获取页码按钮总数（最后一个数字页码）
pager_btns = page.query_selector_all('.el-pager li.number')
total_pages = 1
if pager_btns:
# 最后一个页码按钮的文本就是总页数（可能被省略，需要点击更多后才知道）
# 先查 el-pagination__total
if total > 0:
page_size_text = ''
size_select = soup.find('input', {'class': 'el-input__inner'})
page_size = 10 # 默认每页10条
total_pages = max(1, (total + page_size - 1) // page_size)
print(f" 栏目 {menu_id} 共 {total} 篇文章，预计 {total_pages} 页")
current_page = 1
while True:
articles, soup = parse_articles_from_page(page)
new_count = 0
for art in articles:
if art['article_id'] not in seen_ids:
seen_ids.add(art['article_id'])
all_articles.append(art)
new_count += 1
print(f" 第 {current_page} 页采集到 {len(articles)} 篇（新增 {new_count} 篇）")
# 查找下一页按钮
next_btn = page.query_selector('button.btn-next')
if not next_btn:
break
# 检查是否禁用
is_disabled = next_btn.get_attribute('disabled')
if is_disabled is not None:
break
next_btn.click()
page.wait_for_timeout(2000)
current_page += 1
return all_articles
def get_article_detail(page, url):
"""获取文章详情"""
page.goto(url, wait_until='networkidle', timeout=30000)
page.wait_for_timeout(3000)
html = page.content()
soup = BeautifulSoup(html, 'html.parser')
# 标题：在 article > h3 中，格式 "首页 》栏目 》标题"
title = ''
article_el = soup.find('article')
if article_el:
h3 = article_el.find('h3')
if h3:
text = h3.get_text(strip=True)
parts = [p.strip() for p in text.split('》')]
# 过滤 "首页" 和栏目名，取最后的标题
if len(parts) >= 2:
title = parts[-1]
else:
title = text
# 来源和时间
source = ''
time_str = ''
if article_el:
info_p = article_el.find('p', class_='more-info')
if info_p:
src_el = info_p.find('span', class_='source')
if src_el:
source = src_el.get_text(strip=True).replace('来源：', '')
time_el = info_p.find('span', class_='time')
if time_el:
time_str = time_el.get_text(strip=True)
# 正文 HTML
content_div = soup.find('div', class_='article-content')
content_html = ''
if content_div:
content_html = str(content_div)
return {
'title': title,
'source': source,
'time': time_str,
'content_html': content_html,
}
def save_article(article, detail, category_id, cover_path, content_md):
"""保存文章到数据库"""
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
title = detail['title'] or article['title']
slug = ensure_unique_slug(conn, slugify(title))
# 解析发布时间
created_at = datetime.now().isoformat()
if detail.get('time') or article.get('time'):
t = detail.get('time') or article.get('time')
try:
created_at = datetime.strptime(t, '%Y-%m-%d %H:%M').isoformat()
except ValueError:
try:
created_at = datetime.strptime(t, '%Y-%m-%d').isoformat()
except ValueError:
pass
summary = title[:200]
if detail.get('source'):
summary = f"来源：{detail['source']} | {summary}"
cursor.execute('''
INSERT INTO news_posts (title, slug, content, content_type, summary, cover_image, category_id, is_published, is_draft, view_count, created_at, updated_at)
VALUES (?, ?, ?, 'markdown', ?, ?, ?, 1, 0, 0, ?, ?)
''', (
title,
slug,
content_md,
summary,
cover_path,
category_id,
created_at,
created_at,
))
post_id = cursor.lastrowid
conn.commit()
conn.close()
return post_id
# ============ 主程序 ============
def main():
print("=" * 60)
print("开始批量采集 niaodaifu.cn 文章")
print("=" * 60)
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page(viewport={'width': 1920, 'height': 1080})
for menu_id, (category_id, cat_name) in MENU_TO_CATEGORY.items():
print(f"\n>>> 采集栏目 [{cat_name}] menu_id={menu_id} -> category_id={category_id}")
articles = get_article_list(page, menu_id)
print(f" 共找到 {len(articles)} 篇文章")
for idx, article in enumerate(articles, 1):
print(f" [{idx}/{len(articles)}] {article['title']}")
try:
detail = get_article_detail(page, article['href'])
except Exception as e:
print(f" [详情获取失败] {e}")
continue
# 下载封面图
cover_path = None
if article.get('cover_url'):
cover_path, cover_info = download_image(article['cover_url'])
if cover_info:
conn = sqlite3.connect(DB_PATH)
insert_image_to_db(conn, cover_info)
conn.commit()
conn.close()
# 转换正文为 Markdown
content_md = ''
if detail.get('content_html'):
soup = BeautifulSoup(detail['content_html'], 'html.parser')
content_div = soup.find('div', class_='article-content')
if content_div:
downloaded_images = {}
content_md = html_to_markdown(content_div, article['href'], downloaded_images)
content_md = clean_markdown(content_md)
if not content_md.strip():
print(f" [警告] 正文为空，跳过")
continue
# 保存
try:
post_id = save_article(article, detail, category_id, cover_path, content_md)
print(f" [已入库] post_id={post_id}")
except Exception as e:
print(f" [入库失败] {e}")
browser.close()
print("\n" + "=" * 60)
print("采集完成")
print("=" * 60)
if __name__ == '__main__':
main()