a
This commit is contained in:
Ubuntu
@@ -0,0 +1,442 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
文章处理器 - 从数据库读取文章，抓取详情，AI处理，保存结果
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.parse import urljoin
|
||||
from datetime import datetime
|
||||
|
||||
# 设置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 导入AI处理器
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from deepseek_processor import AIProcessor
|
||||
|
||||
# 导入去AI化处理器
|
||||
try:
|
||||
sys.path.insert(0, '/home/ubuntu/.nanobot/workspace/skills/anti-ai-detector')
|
||||
from it_integration import ITContentHumanizer
|
||||
HUMANIZER_AVAILABLE = True
|
||||
except ImportError:
|
||||
HUMANIZER_AVAILABLE = False
|
||||
logger.warning("去AI化模块未安装，将跳过人性化处理")
|
||||
|
||||
|
||||
class ArticleProcessor:
|
||||
"""文章处理器"""
|
||||
|
||||
def __init__(self, config_path=None):
|
||||
self.config_path = config_path or "/home/ubuntu/nanocode/it_automation_enhanced/config/config.json"
|
||||
self.load_config()
|
||||
self.ai_processor = AIProcessor(self.config_path)
|
||||
|
||||
# 初始化去AI化处理器
|
||||
if HUMANIZER_AVAILABLE:
|
||||
self.humanizer = ITContentHumanizer()
|
||||
logger.info("去AI化模块已加载")
|
||||
else:
|
||||
self.humanizer = None
|
||||
|
||||
def load_config(self):
|
||||
"""加载配置"""
|
||||
try:
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
self.config = json.load(f)
|
||||
logger.info("配置加载成功")
|
||||
except Exception as e:
|
||||
logger.error(f"配置加载失败: {e}")
|
||||
self.config = {}
|
||||
|
||||
def get_db_connection(self):
|
||||
"""获取数据库连接"""
|
||||
db_path = self.config.get('paths', {}).get('database_path', 'data/articles.db')
|
||||
return sqlite3.connect(db_path)
|
||||
|
||||
def fetch_article_content(self, url: str, source: str = '') -> str:
|
||||
"""抓取文章详情页内容"""
|
||||
logger.info(f"抓取文章详情: {url}")
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
|
||||
# 根据来源使用不同的提取策略
|
||||
if 'wired.com' in url:
|
||||
content = self._extract_wired_content(soup)
|
||||
elif 'infoq.com' in url or 'infoq.cn' in url:
|
||||
content = self._extract_infoq_content(soup)
|
||||
elif 'techcrunch.com' in url:
|
||||
content = self._extract_techcrunch_content(soup)
|
||||
elif 'juejin.cn' in url:
|
||||
content = self._extract_juejin_content(soup)
|
||||
elif 'oschina.net' in url or 'my.oschina.net' in url:
|
||||
content = self._extract_oschina_content(soup)
|
||||
elif 'zhihu.com' in url:
|
||||
content = self._extract_zhihu_content(soup)
|
||||
else:
|
||||
# 通用提取
|
||||
content = self._extract_generic_content(soup)
|
||||
|
||||
logger.info(f"抓取成功，内容长度: {len(content)} 字符")
|
||||
return content
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"抓取失败: {e}")
|
||||
return ""
|
||||
|
||||
def _extract_wired_content(self, soup: BeautifulSoup) -> str:
|
||||
"""提取Wired文章内容"""
|
||||
selectors = [
|
||||
'article p',
|
||||
'.article-body p',
|
||||
'[data-testid="ArticleBodyContent"] p',
|
||||
'.body__inner-container p'
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
paragraphs = soup.select(selector)
|
||||
if paragraphs:
|
||||
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
||||
if len(content) > 200:
|
||||
return content
|
||||
|
||||
return ""
|
||||
|
||||
def _extract_infoq_content(self, soup: BeautifulSoup) -> str:
|
||||
"""提取InfoQ文章内容"""
|
||||
selectors = [
|
||||
'.article__content p',
|
||||
'.article-content p',
|
||||
'.content__body p',
|
||||
'article p'
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
paragraphs = soup.select(selector)
|
||||
if paragraphs:
|
||||
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
||||
if len(content) > 200:
|
||||
return content
|
||||
|
||||
return ""
|
||||
|
||||
def _extract_techcrunch_content(self, soup: BeautifulSoup) -> str:
|
||||
"""提取TechCrunch文章内容"""
|
||||
selectors = [
|
||||
'.article-content p',
|
||||
'.article__body p',
|
||||
'.entry-content p',
|
||||
'article p'
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
paragraphs = soup.select(selector)
|
||||
if paragraphs:
|
||||
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
||||
if len(content) > 200:
|
||||
return content
|
||||
|
||||
return ""
|
||||
|
||||
def _extract_juejin_content(self, soup: BeautifulSoup) -> str:
|
||||
"""提取掘金文章内容"""
|
||||
selectors = [
|
||||
'.article-content p',
|
||||
'.markdown-body p',
|
||||
'.article-content',
|
||||
'.markdown-body'
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
if selector.endswith(' p'):
|
||||
paragraphs = soup.select(selector)
|
||||
if paragraphs:
|
||||
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
||||
if len(content) > 200:
|
||||
return content
|
||||
else:
|
||||
content_elem = soup.select_one(selector)
|
||||
if content_elem:
|
||||
text = content_elem.get_text(separator='\n', strip=True)
|
||||
if len(text) > 200:
|
||||
return text
|
||||
|
||||
return ""
|
||||
|
||||
def _extract_oschina_content(self, soup: BeautifulSoup) -> str:
|
||||
"""提取OSChina博客内容"""
|
||||
selectors = [
|
||||
'.article-detail .content p',
|
||||
'.blog-content p',
|
||||
'.article-content p',
|
||||
'.content p'
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
paragraphs = soup.select(selector)
|
||||
if paragraphs:
|
||||
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
||||
if len(content) > 200:
|
||||
return content
|
||||
|
||||
return ""
|
||||
|
||||
def _extract_zhihu_content(self, soup: BeautifulSoup) -> str:
|
||||
"""提取知乎回答内容"""
|
||||
selectors = [
|
||||
'.RichContent-inner p',
|
||||
'.RichContent-inner',
|
||||
'.AnswerCard .ContentItem-RichContent p',
|
||||
'.ContentItem-RichContent'
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
if selector.endswith(' p'):
|
||||
paragraphs = soup.select(selector)
|
||||
if paragraphs:
|
||||
content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
|
||||
if len(content) > 200:
|
||||
return content
|
||||
else:
|
||||
content_elem = soup.select_one(selector)
|
||||
if content_elem:
|
||||
text = content_elem.get_text(separator='\n', strip=True)
|
||||
if len(text) > 200:
|
||||
return text
|
||||
|
||||
return ""
|
||||
|
||||
def _extract_generic_content(self, soup: BeautifulSoup) -> str:
|
||||
"""通用内容提取"""
|
||||
# 移除脚本和样式
|
||||
for script in soup(['script', 'style', 'nav', 'header', 'footer']):
|
||||
script.decompose()
|
||||
|
||||
# 尝试常见内容选择器
|
||||
selectors = [
|
||||
'article',
|
||||
'.article-content',
|
||||
'.content',
|
||||
'.post-content',
|
||||
'main',
|
||||
'.main-content'
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
content_elem = soup.select_one(selector)
|
||||
if content_elem:
|
||||
text = content_elem.get_text(separator='\n', strip=True)
|
||||
if len(text) > 200:
|
||||
return text
|
||||
|
||||
# 回退到body
|
||||
body = soup.find('body')
|
||||
if body:
|
||||
return body.get_text(separator='\n', strip=True)
|
||||
|
||||
return ""
|
||||
|
||||
def process_single_article(self, article_id: int) -> bool:
|
||||
"""处理单篇文章"""
|
||||
logger.info(f"处理文章 ID: {article_id}")
|
||||
|
||||
conn = self.get_db_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
try:
|
||||
# 获取文章信息
|
||||
cursor.execute(
|
||||
"SELECT id, title, url, source, content FROM articles WHERE id = ?",
|
||||
(article_id,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
if not row:
|
||||
logger.error(f"文章不存在: {article_id}")
|
||||
return False
|
||||
|
||||
article_id, title, url, source, existing_content = row
|
||||
|
||||
# 如果数据库中没有内容，抓取详情页
|
||||
content = existing_content if existing_content else self.fetch_article_content(url, source)
|
||||
|
||||
if not content or len(content) < 500:
|
||||
logger.warning(f"文章内容太短或为空，跳过: {title}")
|
||||
# 标记为已处理（但失败）
|
||||
cursor.execute(
|
||||
"UPDATE articles SET processed = 1, updated_at = ? WHERE id = ?",
|
||||
(datetime.now().isoformat(), article_id)
|
||||
)
|
||||
conn.commit()
|
||||
return False
|
||||
|
||||
# 更新数据库中的内容
|
||||
if not existing_content:
|
||||
cursor.execute(
|
||||
"UPDATE articles SET content = ? WHERE id = ?",
|
||||
(content, article_id)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# AI处理：判断是否需要翻译（国内源已经是中文）
|
||||
is_chinese_source = source in ['掘金', 'OSChina', '知乎热榜'] or 'juejin' in url or 'oschina' in url or 'zhihu' in url
|
||||
|
||||
if is_chinese_source:
|
||||
# 中文源：改写润色 + 总结
|
||||
logger.info("中文文章，进行改写润色...")
|
||||
rewritten = self.ai_processor.rewrite_article(content)
|
||||
|
||||
if not rewritten:
|
||||
logger.warning("改写失败，使用原文")
|
||||
rewritten = content
|
||||
|
||||
# 去AI化处理（改写后）
|
||||
if self.humanizer:
|
||||
logger.info("开始去AI化处理（改写）...")
|
||||
rewritten = self.humanizer.process_translation(rewritten)
|
||||
|
||||
logger.info("开始AI总结...")
|
||||
summary = self.ai_processor.summarize_article(content)
|
||||
|
||||
# 去AI化处理（总结后）
|
||||
if self.humanizer:
|
||||
logger.info("开始去AI化处理（总结）...")
|
||||
summary = self.humanizer.process_summary(summary, platform='xiaohongshu')
|
||||
|
||||
translated = rewritten
|
||||
else:
|
||||
# 英文源：翻译 + 总结（原有逻辑）
|
||||
logger.info("开始AI翻译...")
|
||||
translated = self.ai_processor.translate_article(content)
|
||||
|
||||
if not translated:
|
||||
logger.error("翻译失败")
|
||||
return False
|
||||
|
||||
# 去AI化处理（翻译后）
|
||||
if self.humanizer:
|
||||
logger.info("开始去AI化处理（翻译）...")
|
||||
translated = self.humanizer.process_translation(translated)
|
||||
|
||||
logger.info("开始AI总结...")
|
||||
summary = self.ai_processor.summarize_article(content)
|
||||
|
||||
# 去AI化处理（总结后）
|
||||
if self.humanizer:
|
||||
logger.info("开始去AI化处理（总结）...")
|
||||
summary = self.humanizer.process_summary(summary, platform='xiaohongshu')
|
||||
|
||||
# 保存处理结果到文件（供邮件和小红书使用）
|
||||
output_dir = self.config.get('paths', {}).get('output_dir', 'output')
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
result = {
|
||||
'id': article_id,
|
||||
'original_title': title,
|
||||
'url': url,
|
||||
'source': source,
|
||||
'translated_content': translated,
|
||||
'summary': summary,
|
||||
'processed_at': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
output_file = os.path.join(output_dir, f"article_{article_id}_processed.json")
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f"处理结果保存到: {output_file}")
|
||||
|
||||
# 更新数据库状态
|
||||
cursor.execute(
|
||||
"""UPDATE articles
|
||||
SET processed = 1,
|
||||
content = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?""",
|
||||
(content, datetime.now().isoformat(), article_id)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
logger.info(f"文章处理完成: {title}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理文章失败: {e}")
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def process_unprocessed_articles(self, limit: int = 5) -> list:
|
||||
"""处理所有未处理的文章"""
|
||||
logger.info("获取未处理文章列表...")
|
||||
|
||||
conn = self.get_db_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute(
|
||||
"""SELECT id, title, url, source FROM articles
|
||||
WHERE processed = 0
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ?""",
|
||||
(limit,)
|
||||
)
|
||||
|
||||
articles = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
logger.info(f"找到 {len(articles)} 篇未处理文章")
|
||||
|
||||
processed_ids = []
|
||||
for row in articles:
|
||||
article_id = row[0]
|
||||
if self.process_single_article(article_id):
|
||||
processed_ids.append(article_id)
|
||||
|
||||
return processed_ids
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='处理文章')
|
||||
parser.add_argument('--article-id', type=int, help='处理指定ID的文章')
|
||||
parser.add_argument('--limit', type=int, default=5, help='最多处理多少篇文章')
|
||||
parser.add_argument('--all', action='store_true', help='处理所有未处理文章')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
processor = ArticleProcessor()
|
||||
|
||||
if args.article_id:
|
||||
success = processor.process_single_article(args.article_id)
|
||||
sys.exit(0 if success else 1)
|
||||
else:
|
||||
limit = args.limit if not args.all else 100
|
||||
processed = processor.process_unprocessed_articles(limit)
|
||||
logger.info(f"成功处理 {len(processed)} 篇文章")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user