Ubuntu
443 lines
16 KiB
Python
443 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
文章处理器 - 从数据库读取文章，抓取详情，AI处理，保存结果
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin
|
|
from datetime import datetime
|
|
|
|
# 设置日志
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# 导入AI处理器
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from deepseek_processor import AIProcessor
|
|
|
|
# 导入去AI化处理器
|
|
try:
|
|
sys.path.insert(0, '/home/ubuntu/.nanobot/workspace/skills/anti-ai-detector')
|
|
from it_integration import ITContentHumanizer
|
|
HUMANIZER_AVAILABLE = True
|
|
except ImportError:
|
|
HUMANIZER_AVAILABLE = False
|
|
logger.warning("去AI化模块未安装，将跳过人性化处理")
|
|
|
|
|
|
class ArticleProcessor:
|
|
"""文章处理器"""
|
|
|
|
def __init__(self, config_path=None):
|
|
self.config_path = config_path or "/home/ubuntu/nanocode/it_automation_enhanced/config/config.json"
|
|
self.load_config()
|
|
self.ai_processor = AIProcessor(self.config_path)
|
|
|
|
# 初始化去AI化处理器
|
|
if HUMANIZER_AVAILABLE:
|
|
self.humanizer = ITContentHumanizer()
|
|
logger.info("去AI化模块已加载")
|
|
else:
|
|
self.humanizer = None
|
|
|
|
def load_config(self):
|
|
"""加载配置"""
|
|
try:
|
|
with open(self.config_path, 'r', encoding='utf-8') as f:
|
|
self.config = json.load(f)
|
|
logger.info("配置加载成功")
|
|
except Exception as e:
|
|
logger.error(f"配置加载失败: {e}")
|
|
self.config = {}
|
|
|
|
def get_db_connection(self):
|
|
"""获取数据库连接"""
|
|
db_path = self.config.get('paths', {}).get('database_path', 'data/articles.db')
|
|
return sqlite3.connect(db_path)
|
|
|
|
def fetch_article_content(self, url: str, source: str = '') -> str:
|
|
"""抓取文章详情页内容"""
|
|
logger.info(f"抓取文章详情: {url}")
|
|
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
}
|
|
|
|
try:
|
|
response = requests.get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
|
|
# 根据来源使用不同的提取策略
|
|
if 'wired.com' in url:
|
|
content = self._extract_wired_content(soup)
|
|
elif 'infoq.com' in url or 'infoq.cn' in url:
|
|
content = self._extract_infoq_content(soup)
|
|
elif 'techcrunch.com' in url:
|
|
content = self._extract_techcrunch_content(soup)
|
|
elif 'juejin.cn' in url:
|
|
content = self._extract_juejin_content(soup)
|
|
elif 'oschina.net' in url or 'my.oschina.net' in url:
|
|
content = self._extract_oschina_content(soup)
|
|
elif 'zhihu.com' in url:
|
|
content = self._extract_zhihu_content(soup)
|
|
else:
|
|
# 通用提取
|
|
content = self._extract_generic_content(soup)
|
|
|
|
logger.info(f"抓取成功，内容长度: {len(content)} 字符")
|
|
return content
|
|
|
|
except Exception as e:
|
|
logger.error(f"抓取失败: {e}")
|
|
return ""
|
|
|
|
def _extract_wired_content(self, soup: BeautifulSoup) -> str:
|
|
"""提取Wired文章内容"""
|
|
selectors = [
|
|
'article p',
|
|
'.article-body p',
|
|
'[data-testid="ArticleBodyContent"] p',
|
|
'.body__inner-container p'
|
|
]
|
|
|
|
for selector in selectors:
|
|
paragraphs = soup.select(selector)
|
|
if paragraphs:
|
|
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
|
if len(content) > 200:
|
|
return content
|
|
|
|
return ""
|
|
|
|
def _extract_infoq_content(self, soup: BeautifulSoup) -> str:
|
|
"""提取InfoQ文章内容"""
|
|
selectors = [
|
|
'.article__content p',
|
|
'.article-content p',
|
|
'.content__body p',
|
|
'article p'
|
|
]
|
|
|
|
for selector in selectors:
|
|
paragraphs = soup.select(selector)
|
|
if paragraphs:
|
|
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
|
if len(content) > 200:
|
|
return content
|
|
|
|
return ""
|
|
|
|
def _extract_techcrunch_content(self, soup: BeautifulSoup) -> str:
|
|
"""提取TechCrunch文章内容"""
|
|
selectors = [
|
|
'.article-content p',
|
|
'.article__body p',
|
|
'.entry-content p',
|
|
'article p'
|
|
]
|
|
|
|
for selector in selectors:
|
|
paragraphs = soup.select(selector)
|
|
if paragraphs:
|
|
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
|
if len(content) > 200:
|
|
return content
|
|
|
|
return ""
|
|
|
|
def _extract_juejin_content(self, soup: BeautifulSoup) -> str:
|
|
"""提取掘金文章内容"""
|
|
selectors = [
|
|
'.article-content p',
|
|
'.markdown-body p',
|
|
'.article-content',
|
|
'.markdown-body'
|
|
]
|
|
|
|
for selector in selectors:
|
|
if selector.endswith(' p'):
|
|
paragraphs = soup.select(selector)
|
|
if paragraphs:
|
|
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
|
if len(content) > 200:
|
|
return content
|
|
else:
|
|
content_elem = soup.select_one(selector)
|
|
if content_elem:
|
|
text = content_elem.get_text(separator='\n', strip=True)
|
|
if len(text) > 200:
|
|
return text
|
|
|
|
return ""
|
|
|
|
def _extract_oschina_content(self, soup: BeautifulSoup) -> str:
|
|
"""提取OSChina博客内容"""
|
|
selectors = [
|
|
'.article-detail .content p',
|
|
'.blog-content p',
|
|
'.article-content p',
|
|
'.content p'
|
|
]
|
|
|
|
for selector in selectors:
|
|
paragraphs = soup.select(selector)
|
|
if paragraphs:
|
|
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
|
if len(content) > 200:
|
|
return content
|
|
|
|
return ""
|
|
|
|
def _extract_zhihu_content(self, soup: BeautifulSoup) -> str:
|
|
"""提取知乎回答内容"""
|
|
selectors = [
|
|
'.RichContent-inner p',
|
|
'.RichContent-inner',
|
|
'.AnswerCard .ContentItem-RichContent p',
|
|
'.ContentItem-RichContent'
|
|
]
|
|
|
|
for selector in selectors:
|
|
if selector.endswith(' p'):
|
|
paragraphs = soup.select(selector)
|
|
if paragraphs:
|
|
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
|
if len(content) > 200:
|
|
return content
|
|
else:
|
|
content_elem = soup.select_one(selector)
|
|
if content_elem:
|
|
text = content_elem.get_text(separator='\n', strip=True)
|
|
if len(text) > 200:
|
|
return text
|
|
|
|
return ""
|
|
|
|
def _extract_generic_content(self, soup: BeautifulSoup) -> str:
|
|
"""通用内容提取"""
|
|
# 移除脚本和样式
|
|
for script in soup(['script', 'style', 'nav', 'header', 'footer']):
|
|
script.decompose()
|
|
|
|
# 尝试常见内容选择器
|
|
selectors = [
|
|
'article',
|
|
'.article-content',
|
|
'.content',
|
|
'.post-content',
|
|
'main',
|
|
'.main-content'
|
|
]
|
|
|
|
for selector in selectors:
|
|
content_elem = soup.select_one(selector)
|
|
if content_elem:
|
|
text = content_elem.get_text(separator='\n', strip=True)
|
|
if len(text) > 200:
|
|
return text
|
|
|
|
# 回退到body
|
|
body = soup.find('body')
|
|
if body:
|
|
return body.get_text(separator='\n', strip=True)
|
|
|
|
return ""
|
|
|
|
def process_single_article(self, article_id: int) -> bool:
|
|
"""处理单篇文章"""
|
|
logger.info(f"处理文章 ID: {article_id}")
|
|
|
|
conn = self.get_db_connection()
|
|
cursor = conn.cursor()
|
|
|
|
try:
|
|
# 获取文章信息
|
|
cursor.execute(
|
|
"SELECT id, title, url, source, content FROM articles WHERE id = ?",
|
|
(article_id,)
|
|
)
|
|
row = cursor.fetchone()
|
|
|
|
if not row:
|
|
logger.error(f"文章不存在: {article_id}")
|
|
return False
|
|
|
|
article_id, title, url, source, existing_content = row
|
|
|
|
# 如果数据库中没有内容，抓取详情页
|
|
content = existing_content if existing_content else self.fetch_article_content(url, source)
|
|
|
|
if not content or len(content) < 500:
|
|
logger.warning(f"文章内容太短或为空，跳过: {title}")
|
|
# 标记为已处理（但失败）
|
|
cursor.execute(
|
|
"UPDATE articles SET processed = 1, updated_at = ? WHERE id = ?",
|
|
(datetime.now().isoformat(), article_id)
|
|
)
|
|
conn.commit()
|
|
return False
|
|
|
|
# 更新数据库中的内容
|
|
if not existing_content:
|
|
cursor.execute(
|
|
"UPDATE articles SET content = ? WHERE id = ?",
|
|
(content, article_id)
|
|
)
|
|
conn.commit()
|
|
|
|
# AI处理：判断是否需要翻译（国内源已经是中文）
|
|
is_chinese_source = source in ['掘金', 'OSChina', '知乎热榜'] or 'juejin' in url or 'oschina' in url or 'zhihu' in url
|
|
|
|
if is_chinese_source:
|
|
# 中文源：改写润色 + 总结
|
|
logger.info("中文文章，进行改写润色...")
|
|
rewritten = self.ai_processor.rewrite_article(content)
|
|
|
|
if not rewritten:
|
|
logger.warning("改写失败，使用原文")
|
|
rewritten = content
|
|
|
|
# 去AI化处理（改写后）
|
|
if self.humanizer:
|
|
logger.info("开始去AI化处理（改写）...")
|
|
rewritten = self.humanizer.process_translation(rewritten)
|
|
|
|
logger.info("开始AI总结...")
|
|
summary = self.ai_processor.summarize_article(content)
|
|
|
|
# 去AI化处理（总结后）
|
|
if self.humanizer:
|
|
logger.info("开始去AI化处理（总结）...")
|
|
summary = self.humanizer.process_summary(summary, platform='xiaohongshu')
|
|
|
|
translated = rewritten
|
|
else:
|
|
# 英文源：翻译 + 总结（原有逻辑）
|
|
logger.info("开始AI翻译...")
|
|
translated = self.ai_processor.translate_article(content)
|
|
|
|
if not translated:
|
|
logger.error("翻译失败")
|
|
return False
|
|
|
|
# 去AI化处理（翻译后）
|
|
if self.humanizer:
|
|
logger.info("开始去AI化处理（翻译）...")
|
|
translated = self.humanizer.process_translation(translated)
|
|
|
|
logger.info("开始AI总结...")
|
|
summary = self.ai_processor.summarize_article(content)
|
|
|
|
# 去AI化处理（总结后）
|
|
if self.humanizer:
|
|
logger.info("开始去AI化处理（总结）...")
|
|
summary = self.humanizer.process_summary(summary, platform='xiaohongshu')
|
|
|
|
# 保存处理结果到文件（供邮件和小红书使用）
|
|
output_dir = self.config.get('paths', {}).get('output_dir', 'output')
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
result = {
|
|
'id': article_id,
|
|
'original_title': title,
|
|
'url': url,
|
|
'source': source,
|
|
'translated_content': translated,
|
|
'summary': summary,
|
|
'processed_at': datetime.now().isoformat()
|
|
}
|
|
|
|
output_file = os.path.join(output_dir, f"article_{article_id}_processed.json")
|
|
with open(output_file, 'w', encoding='utf-8') as f:
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
|
|
logger.info(f"处理结果保存到: {output_file}")
|
|
|
|
# 更新数据库状态
|
|
cursor.execute(
|
|
"""UPDATE articles
|
|
SET processed = 1,
|
|
content = ?,
|
|
updated_at = ?
|
|
WHERE id = ?""",
|
|
(content, datetime.now().isoformat(), article_id)
|
|
)
|
|
conn.commit()
|
|
|
|
logger.info(f"文章处理完成: {title}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"处理文章失败: {e}")
|
|
return False
|
|
finally:
|
|
conn.close()
|
|
|
|
def process_unprocessed_articles(self, limit: int = 5) -> list:
|
|
"""处理所有未处理的文章"""
|
|
logger.info("获取未处理文章列表...")
|
|
|
|
conn = self.get_db_connection()
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute(
|
|
"""SELECT id, title, url, source FROM articles
|
|
WHERE processed = 0
|
|
ORDER BY created_at DESC
|
|
LIMIT ?""",
|
|
(limit,)
|
|
)
|
|
|
|
articles = cursor.fetchall()
|
|
conn.close()
|
|
|
|
logger.info(f"找到 {len(articles)} 篇未处理文章")
|
|
|
|
processed_ids = []
|
|
for row in articles:
|
|
article_id = row[0]
|
|
if self.process_single_article(article_id):
|
|
processed_ids.append(article_id)
|
|
|
|
return processed_ids
|
|
|
|
|
|
def main():
|
|
"""主函数"""
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description='处理文章')
|
|
parser.add_argument('--article-id', type=int, help='处理指定ID的文章')
|
|
parser.add_argument('--limit', type=int, default=5, help='最多处理多少篇文章')
|
|
parser.add_argument('--all', action='store_true', help='处理所有未处理文章')
|
|
|
|
args = parser.parse_args()
|
|
|
|
processor = ArticleProcessor()
|
|
|
|
if args.article_id:
|
|
success = processor.process_single_article(args.article_id)
|
|
sys.exit(0 if success else 1)
|
|
else:
|
|
limit = args.limit if not args.all else 100
|
|
processed = processor.process_unprocessed_articles(limit)
|
|
logger.info(f"成功处理 {len(processed)} 篇文章")
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|