a
This commit is contained in:
+448
@@ -0,0 +1,448 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
增强版IT资讯抓取脚本
|
||||
基于新仓库代码架构,集成配置管理
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import sqlite3
|
||||
import datetime
|
||||
import time
|
||||
import random
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class EnhancedCrawler:
|
||||
"""增强版抓取器"""
|
||||
|
||||
def __init__(self, config_path=None):
|
||||
"""初始化抓取器"""
|
||||
if config_path is None:
|
||||
config_path = "/home/ubuntu/nanocode/it_automation_enhanced/config/config.json"
|
||||
|
||||
self.config_path = config_path
|
||||
self.load_config()
|
||||
self.setup_directories()
|
||||
|
||||
logger.info(f"增强版抓取器初始化完成")
|
||||
logger.info(f"工作目录: {self.config['paths']['work_dir']}")
|
||||
logger.info(f"数据库: {self.config['paths']['database_path']}")
|
||||
|
||||
def load_config(self):
|
||||
"""加载配置"""
|
||||
try:
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
self.config = json.load(f)
|
||||
logger.info(f"配置加载成功: {self.config_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"配置加载失败: {e}")
|
||||
# 使用默认配置
|
||||
self.config = self.get_default_config()
|
||||
|
||||
def get_default_config(self):
|
||||
"""获取默认配置"""
|
||||
return {
|
||||
"paths": {
|
||||
"work_dir": "/home/ubuntu/nanocode/it_automation_enhanced",
|
||||
"database_path": "/home/ubuntu/nanocode/it_automation_enhanced/data/articles.db",
|
||||
"log_dir": "/home/ubuntu/nanocode/it_automation_enhanced/logs"
|
||||
},
|
||||
"crawler": {
|
||||
"min_word_count": 500,
|
||||
"max_days_old": 7,
|
||||
"request_delay": 3,
|
||||
"timeout": 30,
|
||||
"max_retries": 3
|
||||
},
|
||||
"websites": [
|
||||
{
|
||||
"name": "InfoQ",
|
||||
"url": "https://www.infoq.com",
|
||||
"category": "tech",
|
||||
"enabled": True
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
def setup_directories(self):
|
||||
"""设置目录"""
|
||||
paths = self.config['paths']
|
||||
|
||||
for key, path in paths.items():
|
||||
if key.endswith('_dir') or key.endswith('_path'):
|
||||
dir_path = os.path.dirname(path) if '.' in os.path.basename(path) else path
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
logger.debug(f"目录已创建/确认: {dir_path}")
|
||||
|
||||
def init_database(self):
|
||||
"""初始化数据库"""
|
||||
db_path = self.config['paths']['database_path']
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 创建文章表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS articles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
url TEXT UNIQUE NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
category TEXT,
|
||||
publish_date TEXT,
|
||||
word_count INTEGER DEFAULT 0,
|
||||
processed INTEGER DEFAULT 0,
|
||||
translation_status INTEGER DEFAULT 0,
|
||||
humanizer_status INTEGER DEFAULT 0,
|
||||
xhs_status INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建索引
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON articles(url)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_processed ON articles(processed)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_source ON articles(source)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_date ON articles(publish_date)')
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(f"数据库初始化完成: {db_path}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"数据库初始化失败: {e}")
|
||||
return False
|
||||
|
||||
def fetch_with_retry(self, url, max_retries=None):
|
||||
"""带重试的网页抓取"""
|
||||
if max_retries is None:
|
||||
max_retries = self.config['crawler']['max_retries']
|
||||
|
||||
crawler_config = self.config['crawler']
|
||||
headers = {
|
||||
'User-Agent': random.choice(crawler_config['user_agents']),
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Connection': 'keep-alive',
|
||||
'Referer': 'https://www.google.com/'
|
||||
}
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
time.sleep(crawler_config['request_delay'])
|
||||
logger.debug(f"抓取尝试 ({attempt+1}/{max_retries}): {url[:80]}...")
|
||||
|
||||
response = requests.get(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=crawler_config['timeout'],
|
||||
allow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.debug(f"抓取成功: {url[:80]}... (状态码: {response.status_code})")
|
||||
return response
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
if attempt == max_retries - 1:
|
||||
logger.warning(f"抓取失败 ({attempt+1}/{max_retries}): {url[:80]}... - {e}")
|
||||
return None
|
||||
|
||||
wait_time = 2 ** attempt # 指数退避
|
||||
logger.debug(f"重试等待 {wait_time}秒...")
|
||||
time.sleep(wait_time)
|
||||
|
||||
return None
|
||||
|
||||
def extract_infoq_articles(self, html, base_url):
|
||||
"""从InfoQ提取文章(示例)"""
|
||||
articles = []
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# 尝试多种选择器
|
||||
article_selectors = [
|
||||
'.card__title a', # 卡片标题
|
||||
'.article-card__title a', # 文章卡片
|
||||
'h2 a', 'h3 a', # 标题链接
|
||||
'a[href*="/news/"]', # 新闻链接
|
||||
'a[href*="/articles/"]', # 文章链接
|
||||
'.featured__content a' # 特色内容
|
||||
]
|
||||
|
||||
seen_urls = set()
|
||||
|
||||
for selector in article_selectors:
|
||||
elements = soup.select(selector)
|
||||
if not elements:
|
||||
continue
|
||||
|
||||
for element in elements:
|
||||
if len(articles) >= 10: # 限制数量
|
||||
break
|
||||
|
||||
href = element.get('href', '')
|
||||
title = element.get_text(strip=True)
|
||||
|
||||
if not href or not title or len(title) < 10:
|
||||
continue
|
||||
|
||||
# 构建完整URL
|
||||
if href.startswith('http'):
|
||||
full_url = href
|
||||
else:
|
||||
full_url = urljoin(base_url, href)
|
||||
|
||||
# 去重
|
||||
if full_url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(full_url)
|
||||
|
||||
# 提取可能的描述
|
||||
description = ""
|
||||
parent = element.parent
|
||||
if parent:
|
||||
desc_elem = parent.find('p')
|
||||
if desc_elem:
|
||||
description = desc_elem.get_text(strip=True)[:200]
|
||||
|
||||
articles.append({
|
||||
'title': title,
|
||||
'url': full_url,
|
||||
'description': description,
|
||||
'source': 'InfoQ',
|
||||
'category': 'tech'
|
||||
})
|
||||
|
||||
return articles
|
||||
|
||||
def extract_wired_articles(self, html, base_url):
|
||||
"""从Wired提取文章(示例)"""
|
||||
articles = []
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# Wired的选择器
|
||||
selectors = [
|
||||
'h2 a', 'h3 a',
|
||||
'.SummaryItemHedLink-civMjp',
|
||||
'.card__title a',
|
||||
'a[data-testid="SummaryItemHedLink"]'
|
||||
]
|
||||
|
||||
seen_urls = set()
|
||||
|
||||
for selector in selectors:
|
||||
elements = soup.select(selector)
|
||||
if not elements:
|
||||
continue
|
||||
|
||||
for element in elements:
|
||||
if len(articles) >= 10:
|
||||
break
|
||||
|
||||
href = element.get('href', '')
|
||||
title = element.get_text(strip=True)
|
||||
|
||||
if not href or not title or len(title) < 10:
|
||||
continue
|
||||
|
||||
full_url = urljoin(base_url, href)
|
||||
|
||||
if full_url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(full_url)
|
||||
|
||||
articles.append({
|
||||
'title': title,
|
||||
'url': full_url,
|
||||
'source': 'Wired',
|
||||
'category': 'tech'
|
||||
})
|
||||
|
||||
return articles
|
||||
|
||||
def save_article(self, article):
|
||||
"""保存文章到数据库
|
||||
|
||||
Returns:
|
||||
tuple: (success: bool, is_new: bool)
|
||||
success: 操作是否成功
|
||||
is_new: 是否是新文章(True=新插入,False=已存在)
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.config['paths']['database_path'])
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 检查content字段是否存在,如果不存在则添加
|
||||
cursor.execute("PRAGMA table_info(articles)")
|
||||
columns = [col[1] for col in cursor.fetchall()]
|
||||
if 'content' not in columns:
|
||||
cursor.execute('ALTER TABLE articles ADD COLUMN content TEXT')
|
||||
conn.commit()
|
||||
logger.info("数据库表已更新,添加content字段")
|
||||
|
||||
# 首先检查URL是否已存在
|
||||
cursor.execute('SELECT id FROM articles WHERE url = ?', (article['url'],))
|
||||
existing = cursor.fetchone()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"文章已存在,跳过: {article['title'][:50]}...")
|
||||
conn.close()
|
||||
return True, False # 成功,但不是新文章
|
||||
|
||||
# 插入新文章
|
||||
cursor.execute('''
|
||||
INSERT INTO articles
|
||||
(title, url, source, category, publish_date, word_count, content)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
article['title'],
|
||||
article['url'],
|
||||
article.get('source', 'unknown'),
|
||||
article.get('category', 'tech'),
|
||||
datetime.datetime.now().strftime('%Y-%m-%d'),
|
||||
article.get('word_count', 0),
|
||||
article.get('content', '')
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(f"新文章保存成功: {article['title'][:50]}...")
|
||||
return True, True # 成功,是新文章
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"文章保存失败: {e}")
|
||||
return False, False
|
||||
|
||||
def crawl_website(self, website):
|
||||
"""抓取单个网站"""
|
||||
if not website.get('enabled', True):
|
||||
logger.info(f"网站已禁用: {website['name']}")
|
||||
return 0
|
||||
|
||||
logger.info(f"开始抓取网站: {website['name']} ({website['url']})")
|
||||
|
||||
response = self.fetch_with_retry(website['url'])
|
||||
if not response:
|
||||
logger.warning(f"网站抓取失败: {website['name']}")
|
||||
return 0
|
||||
|
||||
# 根据网站类型调用不同的提取器
|
||||
if website['name'] == 'InfoQ':
|
||||
articles = self.extract_infoq_articles(response.text, website['url'])
|
||||
elif website['name'] == 'Wired':
|
||||
articles = self.extract_wired_articles(response.text, website['url'])
|
||||
else:
|
||||
# 通用提取器
|
||||
articles = self.extract_infoq_articles(response.text, website['url'])
|
||||
|
||||
logger.info(f"找到 {len(articles)} 篇文章")
|
||||
|
||||
# 保存文章
|
||||
saved_count = 0
|
||||
new_count = 0
|
||||
for article in articles:
|
||||
success, is_new = self.save_article(article)
|
||||
if success:
|
||||
saved_count += 1
|
||||
if is_new:
|
||||
new_count += 1
|
||||
|
||||
logger.info(f"处理 {saved_count} 篇文章,其中 {new_count} 篇为新文章")
|
||||
return new_count
|
||||
|
||||
def run(self):
|
||||
"""运行抓取器"""
|
||||
logger.info("=" * 60)
|
||||
logger.info("🚀 开始增强版资讯抓取")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# 初始化数据库
|
||||
if not self.init_database():
|
||||
logger.error("数据库初始化失败,退出")
|
||||
return 0
|
||||
|
||||
total_saved = 0
|
||||
|
||||
# 抓取所有启用的网站
|
||||
for website in self.config['websites']:
|
||||
saved = self.crawl_website(website)
|
||||
total_saved += saved
|
||||
|
||||
# 统计结果
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"📊 抓取完成,共保存 {total_saved} 篇文章")
|
||||
|
||||
# 显示最新文章
|
||||
if total_saved > 0:
|
||||
self.show_recent_articles()
|
||||
|
||||
return total_saved
|
||||
|
||||
def show_recent_articles(self):
|
||||
"""显示最近的文章"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.config['paths']['database_path'])
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute('''
|
||||
SELECT title, source, created_at
|
||||
FROM articles
|
||||
WHERE processed = 0
|
||||
ORDER BY id DESC
|
||||
LIMIT 5
|
||||
''')
|
||||
|
||||
articles = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
if articles:
|
||||
logger.info("📋 最新未处理文章:")
|
||||
for title, source, created_at in articles:
|
||||
logger.info(f" • [{source}] {title[:50]}... ({created_at})")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"显示文章失败: {e}")
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
crawler = EnhancedCrawler()
|
||||
|
||||
try:
|
||||
saved_count = crawler.run()
|
||||
|
||||
if saved_count > 0:
|
||||
print(f"\n🎉 抓取成功,保存 {saved_count} 篇文章")
|
||||
return 0
|
||||
else:
|
||||
print("\n⚠️ 抓取完成,但未保存新文章")
|
||||
return 1
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n⏹️ 用户中断")
|
||||
return 130
|
||||
except Exception as e:
|
||||
print(f"\n❌ 抓取过程中出现错误: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
Reference in New Issue
Block a user