#!/usr/bin/env python3
"""
文章处理器 - 从数据库读取文章，抓取详情，AI处理，保存结果
"""

import os
import sys
import json
import logging
import sqlite3
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from datetime import datetime

# 设置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# 导入AI处理器
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from deepseek_processor import AIProcessor

# 导入去AI化处理器
try:
    sys.path.insert(0, '/home/ubuntu/.nanobot/workspace/skills/anti-ai-detector')
    from it_integration import ITContentHumanizer
    HUMANIZER_AVAILABLE = True
except ImportError:
    HUMANIZER_AVAILABLE = False
    logger.warning("去AI化模块未安装，将跳过人性化处理")


class ArticleProcessor:
    """文章处理器"""
    
    def __init__(self, config_path=None):
        self.config_path = config_path or "/home/ubuntu/nanocode/it_automation_enhanced/config/config.json"
        self.load_config()
        self.ai_processor = AIProcessor(self.config_path)
        
        # 初始化去AI化处理器
        if HUMANIZER_AVAILABLE:
            self.humanizer = ITContentHumanizer()
            logger.info("去AI化模块已加载")
        else:
            self.humanizer = None
        
    def load_config(self):
        """加载配置"""
        try:
            with open(self.config_path, 'r', encoding='utf-8') as f:
                self.config = json.load(f)
            logger.info("配置加载成功")
        except Exception as e:
            logger.error(f"配置加载失败: {e}")
            self.config = {}
    
    def get_db_connection(self):
        """获取数据库连接"""
        db_path = self.config.get('paths', {}).get('database_path', 'data/articles.db')
        return sqlite3.connect(db_path)
    
    def fetch_article_content(self, url: str, source: str = '') -> str:
        """抓取文章详情页内容"""
        logger.info(f"抓取文章详情: {url}")
        
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
        }
        
        try:
            response = requests.get(url, headers=headers, timeout=30)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.text, 'html.parser')
            
            # 根据来源使用不同的提取策略
            if 'wired.com' in url:
                content = self._extract_wired_content(soup)
            elif 'infoq.com' in url or 'infoq.cn' in url:
                content = self._extract_infoq_content(soup)
            elif 'techcrunch.com' in url:
                content = self._extract_techcrunch_content(soup)
            elif 'juejin.cn' in url:
                content = self._extract_juejin_content(soup)
            elif 'oschina.net' in url or 'my.oschina.net' in url:
                content = self._extract_oschina_content(soup)
            elif 'zhihu.com' in url:
                content = self._extract_zhihu_content(soup)
            else:
                # 通用提取
                content = self._extract_generic_content(soup)
            
            logger.info(f"抓取成功，内容长度: {len(content)} 字符")
            return content
            
        except Exception as e:
            logger.error(f"抓取失败: {e}")
            return ""
    
    def _extract_wired_content(self, soup: BeautifulSoup) -> str:
        """提取Wired文章内容"""
        selectors = [
            'article p',
            '.article-body p',
            '[data-testid="ArticleBodyContent"] p',
            '.body__inner-container p'
        ]
        
        for selector in selectors:
            paragraphs = soup.select(selector)
            if paragraphs:
                content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
                if len(content) > 200:
                    return content
        
        return ""
    
    def _extract_infoq_content(self, soup: BeautifulSoup) -> str:
        """提取InfoQ文章内容"""
        selectors = [
            '.article__content p',
            '.article-content p',
            '.content__body p',
            'article p'
        ]
        
        for selector in selectors:
            paragraphs = soup.select(selector)
            if paragraphs:
                content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
                if len(content) > 200:
                    return content
        
        return ""
    
    def _extract_techcrunch_content(self, soup: BeautifulSoup) -> str:
        """提取TechCrunch文章内容"""
        selectors = [
            '.article-content p',
            '.article__body p',
            '.entry-content p',
            'article p'
        ]
        
        for selector in selectors:
            paragraphs = soup.select(selector)
            if paragraphs:
                content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
                if len(content) > 200:
                    return content
        
        return ""
    
    def _extract_juejin_content(self, soup: BeautifulSoup) -> str:
        """提取掘金文章内容"""
        selectors = [
            '.article-content p',
            '.markdown-body p',
            '.article-content',
            '.markdown-body'
        ]
        
        for selector in selectors:
            if selector.endswith(' p'):
                paragraphs = soup.select(selector)
                if paragraphs:
                    content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
                    if len(content) > 200:
                        return content
            else:
                content_elem = soup.select_one(selector)
                if content_elem:
                    text = content_elem.get_text(separator='\n', strip=True)
                    if len(text) > 200:
                        return text
        
        return ""
    
    def _extract_oschina_content(self, soup: BeautifulSoup) -> str:
        """提取OSChina博客内容"""
        selectors = [
            '.article-detail .content p',
            '.blog-content p',
            '.article-content p',
            '.content p'
        ]
        
        for selector in selectors:
            paragraphs = soup.select(selector)
            if paragraphs:
                content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
                if len(content) > 200:
                    return content
        
        return ""
    
    def _extract_zhihu_content(self, soup: BeautifulSoup) -> str:
        """提取知乎回答内容"""
        selectors = [
            '.RichContent-inner p',
            '.RichContent-inner',
            '.AnswerCard .ContentItem-RichContent p',
            '.ContentItem-RichContent'
        ]
        
        for selector in selectors:
            if selector.endswith(' p'):
                paragraphs = soup.select(selector)
                if paragraphs:
                    content = '\n\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
                    if len(content) > 200:
                        return content
            else:
                content_elem = soup.select_one(selector)
                if content_elem:
                    text = content_elem.get_text(separator='\n', strip=True)
                    if len(text) > 200:
                        return text
        
        return ""
    
    def _extract_generic_content(self, soup: BeautifulSoup) -> str:
        """通用内容提取"""
        # 移除脚本和样式
        for script in soup(['script', 'style', 'nav', 'header', 'footer']):
            script.decompose()
        
        # 尝试常见内容选择器
        selectors = [
            'article',
            '.article-content',
            '.content',
            '.post-content',
            'main',
            '.main-content'
        ]
        
        for selector in selectors:
            content_elem = soup.select_one(selector)
            if content_elem:
                text = content_elem.get_text(separator='\n', strip=True)
                if len(text) > 200:
                    return text
        
        # 回退到body
        body = soup.find('body')
        if body:
            return body.get_text(separator='\n', strip=True)
        
        return ""
    
    def process_single_article(self, article_id: int) -> bool:
        """处理单篇文章"""
        logger.info(f"处理文章 ID: {article_id}")
        
        conn = self.get_db_connection()
        cursor = conn.cursor()
        
        try:
            # 获取文章信息
            cursor.execute(
                "SELECT id, title, url, source, content FROM articles WHERE id = ?",
                (article_id,)
            )
            row = cursor.fetchone()
            
            if not row:
                logger.error(f"文章不存在: {article_id}")
                return False
            
            article_id, title, url, source, existing_content = row
            
            # 如果数据库中没有内容，抓取详情页
            content = existing_content if existing_content else self.fetch_article_content(url, source)
            
            if not content or len(content) < 500:
                logger.warning(f"文章内容太短或为空，跳过: {title}")
                # 标记为已处理（但失败）
                cursor.execute(
                    "UPDATE articles SET processed = 1, updated_at = ? WHERE id = ?",
                    (datetime.now().isoformat(), article_id)
                )
                conn.commit()
                return False
            
            # 更新数据库中的内容
            if not existing_content:
                cursor.execute(
                    "UPDATE articles SET content = ? WHERE id = ?",
                    (content, article_id)
                )
                conn.commit()
            
            # AI处理：判断是否需要翻译（国内源已经是中文）
            is_chinese_source = source in ['掘金', 'OSChina', '知乎热榜'] or 'juejin' in url or 'oschina' in url or 'zhihu' in url
            
            if is_chinese_source:
                # 中文源：改写润色 + 总结
                logger.info("中文文章，进行改写润色...")
                rewritten = self.ai_processor.rewrite_article(content)
                
                if not rewritten:
                    logger.warning("改写失败，使用原文")
                    rewritten = content
                
                # 去AI化处理（改写后）
                if self.humanizer:
                    logger.info("开始去AI化处理（改写）...")
                    rewritten = self.humanizer.process_translation(rewritten)
                
                logger.info("开始AI总结...")
                summary = self.ai_processor.summarize_article(content)
                
                # 去AI化处理（总结后）
                if self.humanizer:
                    logger.info("开始去AI化处理（总结）...")
                    summary = self.humanizer.process_summary(summary, platform='xiaohongshu')
                
                translated = rewritten
            else:
                # 英文源：翻译 + 总结（原有逻辑）
                logger.info("开始AI翻译...")
                translated = self.ai_processor.translate_article(content)
                
                if not translated:
                    logger.error("翻译失败")
                    return False
                
                # 去AI化处理（翻译后）
                if self.humanizer:
                    logger.info("开始去AI化处理（翻译）...")
                    translated = self.humanizer.process_translation(translated)
                
                logger.info("开始AI总结...")
                summary = self.ai_processor.summarize_article(content)
                
                # 去AI化处理（总结后）
                if self.humanizer:
                    logger.info("开始去AI化处理（总结）...")
                    summary = self.humanizer.process_summary(summary, platform='xiaohongshu')
            
            # 保存处理结果到文件（供邮件和小红书使用）
            output_dir = self.config.get('paths', {}).get('output_dir', 'output')
            os.makedirs(output_dir, exist_ok=True)
            
            result = {
                'id': article_id,
                'original_title': title,
                'url': url,
                'source': source,
                'translated_content': translated,
                'summary': summary,
                'processed_at': datetime.now().isoformat()
            }
            
            output_file = os.path.join(output_dir, f"article_{article_id}_processed.json")
            with open(output_file, 'w', encoding='utf-8') as f:
                json.dump(result, f, ensure_ascii=False, indent=2)
            
            logger.info(f"处理结果保存到: {output_file}")
            
            # 更新数据库状态
            cursor.execute(
                """UPDATE articles 
                   SET processed = 1, 
                       content = ?,
                       updated_at = ? 
                   WHERE id = ?""",
                (content, datetime.now().isoformat(), article_id)
            )
            conn.commit()
            
            logger.info(f"文章处理完成: {title}")
            return True
            
        except Exception as e:
            logger.error(f"处理文章失败: {e}")
            return False
        finally:
            conn.close()
    
    def process_unprocessed_articles(self, limit: int = 5) -> list:
        """处理所有未处理的文章"""
        logger.info("获取未处理文章列表...")
        
        conn = self.get_db_connection()
        cursor = conn.cursor()
        
        cursor.execute(
            """SELECT id, title, url, source FROM articles 
               WHERE processed = 0 
               ORDER BY created_at DESC 
               LIMIT ?""",
            (limit,)
        )
        
        articles = cursor.fetchall()
        conn.close()
        
        logger.info(f"找到 {len(articles)} 篇未处理文章")
        
        processed_ids = []
        for row in articles:
            article_id = row[0]
            if self.process_single_article(article_id):
                processed_ids.append(article_id)
        
        return processed_ids


def main():
    """主函数"""
    import argparse
    
    parser = argparse.ArgumentParser(description='处理文章')
    parser.add_argument('--article-id', type=int, help='处理指定ID的文章')
    parser.add_argument('--limit', type=int, default=5, help='最多处理多少篇文章')
    parser.add_argument('--all', action='store_true', help='处理所有未处理文章')
    
    args = parser.parse_args()
    
    processor = ArticleProcessor()
    
    if args.article_id:
        success = processor.process_single_article(args.article_id)
        sys.exit(0 if success else 1)
    else:
        limit = args.limit if not args.all else 100
        processed = processor.process_unprocessed_articles(limit)
        logger.info(f"成功处理 {len(processed)} 篇文章")
        sys.exit(0)


if __name__ == '__main__':
    main()