a
This commit is contained in:
Ubuntu
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
增强版IT资讯抓取系统模板
|
||||
请根据实际环境配置路径
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import random
|
||||
import sqlite3
|
||||
import datetime
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlparse
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import mimetypes
|
||||
|
||||
# ==================== 配置部分 ====================
|
||||
# 请根据实际环境修改以下配置
|
||||
CONFIG = {
|
||||
'database_path': '{{DATABASE_PATH}}', # 数据库文件路径
|
||||
'base_dir': '{{BASE_DIR}}', # 基础目录
|
||||
'min_word_count': 500, # 最小字数要求
|
||||
'max_days_old': 7, # 最大天数（最近N天）
|
||||
'websites': [
|
||||
{'name': 'InfoQ', 'url': 'https://www.infoq.com', 'category': 'tech'},
|
||||
{'name': 'Wired', 'url': 'https://www.wired.com', 'category': 'tech'},
|
||||
{'name': 'TechCrunch', 'url': 'https://techcrunch.com', 'category': 'tech'},
|
||||
# 可以添加更多网站
|
||||
],
|
||||
'user_agents': [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36',
|
||||
],
|
||||
'request_delay': 2, # 请求延迟（秒）
|
||||
'timeout': 30, # 请求超时（秒）
|
||||
}
|
||||
|
||||
# ==================== 数据库操作 ====================
|
||||
def init_database():
|
||||
"""初始化数据库"""
|
||||
conn = sqlite3.connect(CONFIG['database_path'])
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 创建文章表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS articles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
url TEXT UNIQUE NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
category TEXT,
|
||||
publish_date TEXT,
|
||||
word_count INTEGER,
|
||||
processed INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建索引
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON articles(url)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_processed ON articles(processed)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_date ON articles(publish_date)')
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"✅ 数据库初始化完成: {CONFIG['database_path']}")
|
||||
|
||||
def save_article(title, url, source, category, publish_date, word_count):
|
||||
"""保存文章到数据库"""
|
||||
try:
|
||||
conn = sqlite3.connect(CONFIG['database_path'])
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute('''
|
||||
INSERT OR IGNORE INTO articles
|
||||
(title, url, source, category, publish_date, word_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
''', (title, url, source, category, publish_date, word_count))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ 保存文章失败: {e}")
|
||||
return False
|
||||
|
||||
# ==================== 网页抓取 ====================
|
||||
def fetch_with_retry(url, max_retries=3):
|
||||
"""带重试的网页抓取"""
|
||||
headers = {
|
||||
'User-Agent': random.choice(CONFIG['user_agents']),
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Connection': 'keep-alive',
|
||||
}
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
time.sleep(CONFIG['request_delay'])
|
||||
response = requests.get(url, headers=headers, timeout=CONFIG['timeout'])
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
print(f"❌ 抓取失败 ({attempt+1}/{max_retries}): {url} - {e}")
|
||||
return None
|
||||
print(f"⚠️ 重试抓取 ({attempt+1}/{max_retries}): {url}")
|
||||
time.sleep(2 ** attempt) # 指数退避
|
||||
|
||||
def extract_articles_from_html(html, base_url, source_name):
|
||||
"""从HTML中提取文章（需要根据具体网站实现）"""
|
||||
# 这是一个模板函数，需要根据具体网站结构实现
|
||||
articles = []
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# 示例：查找文章链接
|
||||
for link in soup.find_all('a', href=True):
|
||||
href = link['href']
|
||||
title = link.get_text(strip=True)
|
||||
|
||||
# 过滤非文章链接
|
||||
if not title or len(title) < 10:
|
||||
continue
|
||||
|
||||
# 构建完整URL
|
||||
full_url = urljoin(base_url, href)
|
||||
|
||||
# 添加到文章列表
|
||||
articles.append({
|
||||
'title': title,
|
||||
'url': full_url,
|
||||
'source': source_name,
|
||||
'category': 'tech', # 默认分类
|
||||
})
|
||||
|
||||
return articles
|
||||
|
||||
# ==================== 主函数 ====================
|
||||
def main():
|
||||
"""主函数"""
|
||||
print("🚀 开始IT资讯抓取")
|
||||
print(f"📁 工作目录: {CONFIG['base_dir']}")
|
||||
print(f"🗄️ 数据库: {CONFIG['database_path']}")
|
||||
|
||||
# 初始化数据库
|
||||
init_database()
|
||||
|
||||
# 创建当天目录
|
||||
today = datetime.datetime.now().strftime('%Y-%m-%d')
|
||||
day_dir = os.path.join(CONFIG['base_dir'], today)
|
||||
os.makedirs(day_dir, exist_ok=True)
|
||||
|
||||
total_articles = 0
|
||||
|
||||
# 遍历网站
|
||||
for site in CONFIG['websites']:
|
||||
print(f"\n🌐 抓取网站: {site['name']} ({site['url']})")
|
||||
|
||||
response = fetch_with_retry(site['url'])
|
||||
if not response:
|
||||
continue
|
||||
|
||||
articles = extract_articles_from_html(response.text, site['url'], site['name'])
|
||||
|
||||
print(f"📰 找到 {len(articles)} 篇文章")
|
||||
|
||||
# 保存文章
|
||||
for article in articles:
|
||||
# 这里可以添加更多处理逻辑
|
||||
saved = save_article(
|
||||
article['title'],
|
||||
article['url'],
|
||||
article['source'],
|
||||
article.get('category', 'tech'),
|
||||
datetime.datetime.now().strftime('%Y-%m-%d'),
|
||||
0 # 字数需要后续处理
|
||||
)
|
||||
|
||||
if saved:
|
||||
total_articles += 1
|
||||
|
||||
print(f"\n✅ 抓取完成，共保存 {total_articles} 篇文章")
|
||||
return total_articles
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user