Ubuntu
448 lines
15 KiB
Python
Executable File
448 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
增强版IT资讯抓取脚本
|
|
基于新仓库代码架构，集成配置管理
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import sqlite3
|
|
import datetime
|
|
import time
|
|
import random
|
|
import logging
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
# 配置日志
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class EnhancedCrawler:
|
|
"""增强版抓取器"""
|
|
|
|
def __init__(self, config_path=None):
|
|
"""初始化抓取器"""
|
|
if config_path is None:
|
|
config_path = "/home/ubuntu/nanocode/it_automation_enhanced/config/config.json"
|
|
|
|
self.config_path = config_path
|
|
self.load_config()
|
|
self.setup_directories()
|
|
|
|
logger.info(f"增强版抓取器初始化完成")
|
|
logger.info(f"工作目录: {self.config['paths']['work_dir']}")
|
|
logger.info(f"数据库: {self.config['paths']['database_path']}")
|
|
|
|
def load_config(self):
|
|
"""加载配置"""
|
|
try:
|
|
with open(self.config_path, 'r', encoding='utf-8') as f:
|
|
self.config = json.load(f)
|
|
logger.info(f"配置加载成功: {self.config_path}")
|
|
except Exception as e:
|
|
logger.error(f"配置加载失败: {e}")
|
|
# 使用默认配置
|
|
self.config = self.get_default_config()
|
|
|
|
def get_default_config(self):
|
|
"""获取默认配置"""
|
|
return {
|
|
"paths": {
|
|
"work_dir": "/home/ubuntu/nanocode/it_automation_enhanced",
|
|
"database_path": "/home/ubuntu/nanocode/it_automation_enhanced/data/articles.db",
|
|
"log_dir": "/home/ubuntu/nanocode/it_automation_enhanced/logs"
|
|
},
|
|
"crawler": {
|
|
"min_word_count": 500,
|
|
"max_days_old": 7,
|
|
"request_delay": 3,
|
|
"timeout": 30,
|
|
"max_retries": 3
|
|
},
|
|
"websites": [
|
|
{
|
|
"name": "InfoQ",
|
|
"url": "https://www.infoq.com",
|
|
"category": "tech",
|
|
"enabled": True
|
|
}
|
|
]
|
|
}
|
|
|
|
def setup_directories(self):
|
|
"""设置目录"""
|
|
paths = self.config['paths']
|
|
|
|
for key, path in paths.items():
|
|
if key.endswith('_dir') or key.endswith('_path'):
|
|
dir_path = os.path.dirname(path) if '.' in os.path.basename(path) else path
|
|
os.makedirs(dir_path, exist_ok=True)
|
|
logger.debug(f"目录已创建/确认: {dir_path}")
|
|
|
|
def init_database(self):
|
|
"""初始化数据库"""
|
|
db_path = self.config['paths']['database_path']
|
|
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
|
|
# 创建文章表
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS articles (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
title TEXT NOT NULL,
|
|
url TEXT UNIQUE NOT NULL,
|
|
source TEXT NOT NULL,
|
|
category TEXT,
|
|
publish_date TEXT,
|
|
word_count INTEGER DEFAULT 0,
|
|
processed INTEGER DEFAULT 0,
|
|
translation_status INTEGER DEFAULT 0,
|
|
humanizer_status INTEGER DEFAULT 0,
|
|
xhs_status INTEGER DEFAULT 0,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
''')
|
|
|
|
# 创建索引
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON articles(url)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_processed ON articles(processed)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_source ON articles(source)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_date ON articles(publish_date)')
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.info(f"数据库初始化完成: {db_path}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"数据库初始化失败: {e}")
|
|
return False
|
|
|
|
def fetch_with_retry(self, url, max_retries=None):
|
|
"""带重试的网页抓取"""
|
|
if max_retries is None:
|
|
max_retries = self.config['crawler']['max_retries']
|
|
|
|
crawler_config = self.config['crawler']
|
|
headers = {
|
|
'User-Agent': random.choice(crawler_config['user_agents']),
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
'Connection': 'keep-alive',
|
|
'Referer': 'https://www.google.com/'
|
|
}
|
|
|
|
for attempt in range(max_retries):
|
|
try:
|
|
time.sleep(crawler_config['request_delay'])
|
|
logger.debug(f"抓取尝试 ({attempt+1}/{max_retries}): {url[:80]}...")
|
|
|
|
response = requests.get(
|
|
url,
|
|
headers=headers,
|
|
timeout=crawler_config['timeout'],
|
|
allow_redirects=True
|
|
)
|
|
response.raise_for_status()
|
|
|
|
logger.debug(f"抓取成功: {url[:80]}... (状态码: {response.status_code})")
|
|
return response
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
if attempt == max_retries - 1:
|
|
logger.warning(f"抓取失败 ({attempt+1}/{max_retries}): {url[:80]}... - {e}")
|
|
return None
|
|
|
|
wait_time = 2 ** attempt # 指数退避
|
|
logger.debug(f"重试等待 {wait_time}秒...")
|
|
time.sleep(wait_time)
|
|
|
|
return None
|
|
|
|
def extract_infoq_articles(self, html, base_url):
|
|
"""从InfoQ提取文章（示例）"""
|
|
articles = []
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
# 尝试多种选择器
|
|
article_selectors = [
|
|
'.card__title a', # 卡片标题
|
|
'.article-card__title a', # 文章卡片
|
|
'h2 a', 'h3 a', # 标题链接
|
|
'a[href*="/news/"]', # 新闻链接
|
|
'a[href*="/articles/"]', # 文章链接
|
|
'.featured__content a' # 特色内容
|
|
]
|
|
|
|
seen_urls = set()
|
|
|
|
for selector in article_selectors:
|
|
elements = soup.select(selector)
|
|
if not elements:
|
|
continue
|
|
|
|
for element in elements:
|
|
if len(articles) >= 10: # 限制数量
|
|
break
|
|
|
|
href = element.get('href', '')
|
|
title = element.get_text(strip=True)
|
|
|
|
if not href or not title or len(title) < 10:
|
|
continue
|
|
|
|
# 构建完整URL
|
|
if href.startswith('http'):
|
|
full_url = href
|
|
else:
|
|
full_url = urljoin(base_url, href)
|
|
|
|
# 去重
|
|
if full_url in seen_urls:
|
|
continue
|
|
seen_urls.add(full_url)
|
|
|
|
# 提取可能的描述
|
|
description = ""
|
|
parent = element.parent
|
|
if parent:
|
|
desc_elem = parent.find('p')
|
|
if desc_elem:
|
|
description = desc_elem.get_text(strip=True)[:200]
|
|
|
|
articles.append({
|
|
'title': title,
|
|
'url': full_url,
|
|
'description': description,
|
|
'source': 'InfoQ',
|
|
'category': 'tech'
|
|
})
|
|
|
|
return articles
|
|
|
|
def extract_wired_articles(self, html, base_url):
|
|
"""从Wired提取文章（示例）"""
|
|
articles = []
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
# Wired的选择器
|
|
selectors = [
|
|
'h2 a', 'h3 a',
|
|
'.SummaryItemHedLink-civMjp',
|
|
'.card__title a',
|
|
'a[data-testid="SummaryItemHedLink"]'
|
|
]
|
|
|
|
seen_urls = set()
|
|
|
|
for selector in selectors:
|
|
elements = soup.select(selector)
|
|
if not elements:
|
|
continue
|
|
|
|
for element in elements:
|
|
if len(articles) >= 10:
|
|
break
|
|
|
|
href = element.get('href', '')
|
|
title = element.get_text(strip=True)
|
|
|
|
if not href or not title or len(title) < 10:
|
|
continue
|
|
|
|
full_url = urljoin(base_url, href)
|
|
|
|
if full_url in seen_urls:
|
|
continue
|
|
seen_urls.add(full_url)
|
|
|
|
articles.append({
|
|
'title': title,
|
|
'url': full_url,
|
|
'source': 'Wired',
|
|
'category': 'tech'
|
|
})
|
|
|
|
return articles
|
|
|
|
def save_article(self, article):
|
|
"""保存文章到数据库
|
|
|
|
Returns:
|
|
tuple: (success: bool, is_new: bool)
|
|
success: 操作是否成功
|
|
is_new: 是否是新文章（True=新插入，False=已存在）
|
|
"""
|
|
try:
|
|
conn = sqlite3.connect(self.config['paths']['database_path'])
|
|
cursor = conn.cursor()
|
|
|
|
# 检查content字段是否存在，如果不存在则添加
|
|
cursor.execute("PRAGMA table_info(articles)")
|
|
columns = [col[1] for col in cursor.fetchall()]
|
|
if 'content' not in columns:
|
|
cursor.execute('ALTER TABLE articles ADD COLUMN content TEXT')
|
|
conn.commit()
|
|
logger.info("数据库表已更新，添加content字段")
|
|
|
|
# 首先检查URL是否已存在
|
|
cursor.execute('SELECT id FROM articles WHERE url = ?', (article['url'],))
|
|
existing = cursor.fetchone()
|
|
|
|
if existing:
|
|
logger.debug(f"文章已存在，跳过: {article['title'][:50]}...")
|
|
conn.close()
|
|
return True, False # 成功，但不是新文章
|
|
|
|
# 插入新文章
|
|
cursor.execute('''
|
|
INSERT INTO articles
|
|
(title, url, source, category, publish_date, word_count, content)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
''', (
|
|
article['title'],
|
|
article['url'],
|
|
article.get('source', 'unknown'),
|
|
article.get('category', 'tech'),
|
|
datetime.datetime.now().strftime('%Y-%m-%d'),
|
|
article.get('word_count', 0),
|
|
article.get('content', '')
|
|
))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.info(f"新文章保存成功: {article['title'][:50]}...")
|
|
return True, True # 成功，是新文章
|
|
|
|
except Exception as e:
|
|
logger.error(f"文章保存失败: {e}")
|
|
return False, False
|
|
|
|
def crawl_website(self, website):
|
|
"""抓取单个网站"""
|
|
if not website.get('enabled', True):
|
|
logger.info(f"网站已禁用: {website['name']}")
|
|
return 0
|
|
|
|
logger.info(f"开始抓取网站: {website['name']} ({website['url']})")
|
|
|
|
response = self.fetch_with_retry(website['url'])
|
|
if not response:
|
|
logger.warning(f"网站抓取失败: {website['name']}")
|
|
return 0
|
|
|
|
# 根据网站类型调用不同的提取器
|
|
if website['name'] == 'InfoQ':
|
|
articles = self.extract_infoq_articles(response.text, website['url'])
|
|
elif website['name'] == 'Wired':
|
|
articles = self.extract_wired_articles(response.text, website['url'])
|
|
else:
|
|
# 通用提取器
|
|
articles = self.extract_infoq_articles(response.text, website['url'])
|
|
|
|
logger.info(f"找到 {len(articles)} 篇文章")
|
|
|
|
# 保存文章
|
|
saved_count = 0
|
|
new_count = 0
|
|
for article in articles:
|
|
success, is_new = self.save_article(article)
|
|
if success:
|
|
saved_count += 1
|
|
if is_new:
|
|
new_count += 1
|
|
|
|
logger.info(f"处理 {saved_count} 篇文章，其中 {new_count} 篇为新文章")
|
|
return new_count
|
|
|
|
def run(self):
|
|
"""运行抓取器"""
|
|
logger.info("=" * 60)
|
|
logger.info("🚀 开始增强版资讯抓取")
|
|
logger.info("=" * 60)
|
|
|
|
# 初始化数据库
|
|
if not self.init_database():
|
|
logger.error("数据库初始化失败，退出")
|
|
return 0
|
|
|
|
total_saved = 0
|
|
|
|
# 抓取所有启用的网站
|
|
for website in self.config['websites']:
|
|
saved = self.crawl_website(website)
|
|
total_saved += saved
|
|
|
|
# 统计结果
|
|
logger.info("=" * 60)
|
|
logger.info(f"📊 抓取完成，共保存 {total_saved} 篇文章")
|
|
|
|
# 显示最新文章
|
|
if total_saved > 0:
|
|
self.show_recent_articles()
|
|
|
|
return total_saved
|
|
|
|
def show_recent_articles(self):
|
|
"""显示最近的文章"""
|
|
try:
|
|
conn = sqlite3.connect(self.config['paths']['database_path'])
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute('''
|
|
SELECT title, source, created_at
|
|
FROM articles
|
|
WHERE processed = 0
|
|
ORDER BY id DESC
|
|
LIMIT 5
|
|
''')
|
|
|
|
articles = cursor.fetchall()
|
|
conn.close()
|
|
|
|
if articles:
|
|
logger.info("📋 最新未处理文章:")
|
|
for title, source, created_at in articles:
|
|
logger.info(f" • [{source}] {title[:50]}... ({created_at})")
|
|
|
|
except Exception as e:
|
|
logger.error(f"显示文章失败: {e}")
|
|
|
|
def main():
|
|
"""主函数"""
|
|
crawler = EnhancedCrawler()
|
|
|
|
try:
|
|
saved_count = crawler.run()
|
|
|
|
if saved_count > 0:
|
|
print(f"\n🎉 抓取成功，保存 {saved_count} 篇文章")
|
|
return 0
|
|
else:
|
|
print("\n⚠️ 抓取完成，但未保存新文章")
|
|
return 1
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n⏹️ 用户中断")
|
|
return 130
|
|
except Exception as e:
|
|
print(f"\n❌ 抓取过程中出现错误: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
return 1
|
|
|
|
if __name__ == "__main__":
|
|
exit(main()) |