#!/usr/bin/env python3
"""
增强版IT资讯抓取脚本
基于新仓库代码架构，集成配置管理
"""

import os
import sys
import json
import sqlite3
import datetime
import time
import random
import logging
from pathlib import Path
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup

# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

class EnhancedCrawler:
    """增强版抓取器"""
    
    def __init__(self, config_path=None):
        """初始化抓取器"""
        if config_path is None:
            config_path = "/home/ubuntu/nanocode/it_automation_enhanced/config/config.json"
        
        self.config_path = config_path
        self.load_config()
        self.setup_directories()
        
        logger.info(f"增强版抓取器初始化完成")
        logger.info(f"工作目录: {self.config['paths']['work_dir']}")
        logger.info(f"数据库: {self.config['paths']['database_path']}")
    
    def load_config(self):
        """加载配置"""
        try:
            with open(self.config_path, 'r', encoding='utf-8') as f:
                self.config = json.load(f)
            logger.info(f"配置加载成功: {self.config_path}")
        except Exception as e:
            logger.error(f"配置加载失败: {e}")
            # 使用默认配置
            self.config = self.get_default_config()
    
    def get_default_config(self):
        """获取默认配置"""
        return {
            "paths": {
                "work_dir": "/home/ubuntu/nanocode/it_automation_enhanced",
                "database_path": "/home/ubuntu/nanocode/it_automation_enhanced/data/articles.db",
                "log_dir": "/home/ubuntu/nanocode/it_automation_enhanced/logs"
            },
            "crawler": {
                "min_word_count": 500,
                "max_days_old": 7,
                "request_delay": 3,
                "timeout": 30,
                "max_retries": 3
            },
            "websites": [
                {
                    "name": "InfoQ",
                    "url": "https://www.infoq.com",
                    "category": "tech",
                    "enabled": True
                }
            ]
        }
    
    def setup_directories(self):
        """设置目录"""
        paths = self.config['paths']
        
        for key, path in paths.items():
            if key.endswith('_dir') or key.endswith('_path'):
                dir_path = os.path.dirname(path) if '.' in os.path.basename(path) else path
                os.makedirs(dir_path, exist_ok=True)
                logger.debug(f"目录已创建/确认: {dir_path}")
    
    def init_database(self):
        """初始化数据库"""
        db_path = self.config['paths']['database_path']
        
        try:
            conn = sqlite3.connect(db_path)
            cursor = conn.cursor()
            
            # 创建文章表
            cursor.execute('''
            CREATE TABLE IF NOT EXISTS articles (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                title TEXT NOT NULL,
                url TEXT UNIQUE NOT NULL,
                source TEXT NOT NULL,
                category TEXT,
                publish_date TEXT,
                word_count INTEGER DEFAULT 0,
                processed INTEGER DEFAULT 0,
                translation_status INTEGER DEFAULT 0,
                humanizer_status INTEGER DEFAULT 0,
                xhs_status INTEGER DEFAULT 0,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
            ''')
            
            # 创建索引
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON articles(url)')
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_processed ON articles(processed)')
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_source ON articles(source)')
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_date ON articles(publish_date)')
            
            conn.commit()
            conn.close()
            
            logger.info(f"数据库初始化完成: {db_path}")
            return True
            
        except Exception as e:
            logger.error(f"数据库初始化失败: {e}")
            return False
    
    def fetch_with_retry(self, url, max_retries=None):
        """带重试的网页抓取"""
        if max_retries is None:
            max_retries = self.config['crawler']['max_retries']
        
        crawler_config = self.config['crawler']
        headers = {
            'User-Agent': random.choice(crawler_config['user_agents']),
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate',
            'Connection': 'keep-alive',
            'Referer': 'https://www.google.com/'
        }
        
        for attempt in range(max_retries):
            try:
                time.sleep(crawler_config['request_delay'])
                logger.debug(f"抓取尝试 ({attempt+1}/{max_retries}): {url[:80]}...")
                
                response = requests.get(
                    url, 
                    headers=headers, 
                    timeout=crawler_config['timeout'],
                    allow_redirects=True
                )
                response.raise_for_status()
                
                logger.debug(f"抓取成功: {url[:80]}... (状态码: {response.status_code})")
                return response
                
            except requests.exceptions.RequestException as e:
                if attempt == max_retries - 1:
                    logger.warning(f"抓取失败 ({attempt+1}/{max_retries}): {url[:80]}... - {e}")
                    return None
                
                wait_time = 2 ** attempt  # 指数退避
                logger.debug(f"重试等待 {wait_time}秒...")
                time.sleep(wait_time)
        
        return None
    
    def extract_infoq_articles(self, html, base_url):
        """从InfoQ提取文章（示例）"""
        articles = []
        soup = BeautifulSoup(html, 'html.parser')
        
        # 尝试多种选择器
        article_selectors = [
            '.card__title a',           # 卡片标题
            '.article-card__title a',   # 文章卡片
            'h2 a', 'h3 a',             # 标题链接
            'a[href*="/news/"]',        # 新闻链接
            'a[href*="/articles/"]',    # 文章链接
            '.featured__content a'      # 特色内容
        ]
        
        seen_urls = set()
        
        for selector in article_selectors:
            elements = soup.select(selector)
            if not elements:
                continue
                
            for element in elements:
                if len(articles) >= 10:  # 限制数量
                    break
                
                href = element.get('href', '')
                title = element.get_text(strip=True)
                
                if not href or not title or len(title) < 10:
                    continue
                
                # 构建完整URL
                if href.startswith('http'):
                    full_url = href
                else:
                    full_url = urljoin(base_url, href)
                
                # 去重
                if full_url in seen_urls:
                    continue
                seen_urls.add(full_url)
                
                # 提取可能的描述
                description = ""
                parent = element.parent
                if parent:
                    desc_elem = parent.find('p')
                    if desc_elem:
                        description = desc_elem.get_text(strip=True)[:200]
                
                articles.append({
                    'title': title,
                    'url': full_url,
                    'description': description,
                    'source': 'InfoQ',
                    'category': 'tech'
                })
        
        return articles
    
    def extract_wired_articles(self, html, base_url):
        """从Wired提取文章（示例）"""
        articles = []
        soup = BeautifulSoup(html, 'html.parser')
        
        # Wired的选择器
        selectors = [
            'h2 a', 'h3 a',
            '.SummaryItemHedLink-civMjp',
            '.card__title a',
            'a[data-testid="SummaryItemHedLink"]'
        ]
        
        seen_urls = set()
        
        for selector in selectors:
            elements = soup.select(selector)
            if not elements:
                continue
                
            for element in elements:
                if len(articles) >= 10:
                    break
                
                href = element.get('href', '')
                title = element.get_text(strip=True)
                
                if not href or not title or len(title) < 10:
                    continue
                
                full_url = urljoin(base_url, href)
                
                if full_url in seen_urls:
                    continue
                seen_urls.add(full_url)
                
                articles.append({
                    'title': title,
                    'url': full_url,
                    'source': 'Wired',
                    'category': 'tech'
                })
        
        return articles
    
    def save_article(self, article):
        """保存文章到数据库
        
        Returns:
            tuple: (success: bool, is_new: bool) 
                success: 操作是否成功
                is_new: 是否是新文章（True=新插入，False=已存在）
        """
        try:
            conn = sqlite3.connect(self.config['paths']['database_path'])
            cursor = conn.cursor()
            
            # 检查content字段是否存在，如果不存在则添加
            cursor.execute("PRAGMA table_info(articles)")
            columns = [col[1] for col in cursor.fetchall()]
            if 'content' not in columns:
                cursor.execute('ALTER TABLE articles ADD COLUMN content TEXT')
                conn.commit()
                logger.info("数据库表已更新，添加content字段")
            
            # 首先检查URL是否已存在
            cursor.execute('SELECT id FROM articles WHERE url = ?', (article['url'],))
            existing = cursor.fetchone()
            
            if existing:
                logger.debug(f"文章已存在，跳过: {article['title'][:50]}...")
                conn.close()
                return True, False  # 成功，但不是新文章
            
            # 插入新文章
            cursor.execute('''
            INSERT INTO articles 
            (title, url, source, category, publish_date, word_count, content)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            ''', (
                article['title'],
                article['url'],
                article.get('source', 'unknown'),
                article.get('category', 'tech'),
                datetime.datetime.now().strftime('%Y-%m-%d'),
                article.get('word_count', 0),
                article.get('content', '')
            ))
            
            conn.commit()
            conn.close()
            
            logger.info(f"新文章保存成功: {article['title'][:50]}...")
            return True, True  # 成功，是新文章
            
        except Exception as e:
            logger.error(f"文章保存失败: {e}")
            return False, False
    
    def crawl_website(self, website):
        """抓取单个网站"""
        if not website.get('enabled', True):
            logger.info(f"网站已禁用: {website['name']}")
            return 0
        
        logger.info(f"开始抓取网站: {website['name']} ({website['url']})")
        
        response = self.fetch_with_retry(website['url'])
        if not response:
            logger.warning(f"网站抓取失败: {website['name']}")
            return 0
        
        # 根据网站类型调用不同的提取器
        if website['name'] == 'InfoQ':
            articles = self.extract_infoq_articles(response.text, website['url'])
        elif website['name'] == 'Wired':
            articles = self.extract_wired_articles(response.text, website['url'])
        else:
            # 通用提取器
            articles = self.extract_infoq_articles(response.text, website['url'])
        
        logger.info(f"找到 {len(articles)} 篇文章")
        
        # 保存文章
        saved_count = 0
        new_count = 0
        for article in articles:
            success, is_new = self.save_article(article)
            if success:
                saved_count += 1
                if is_new:
                    new_count += 1
        
        logger.info(f"处理 {saved_count} 篇文章，其中 {new_count} 篇为新文章")
        return new_count
    
    def run(self):
        """运行抓取器"""
        logger.info("=" * 60)
        logger.info("🚀 开始增强版资讯抓取")
        logger.info("=" * 60)
        
        # 初始化数据库
        if not self.init_database():
            logger.error("数据库初始化失败，退出")
            return 0
        
        total_saved = 0
        
        # 抓取所有启用的网站
        for website in self.config['websites']:
            saved = self.crawl_website(website)
            total_saved += saved
        
        # 统计结果
        logger.info("=" * 60)
        logger.info(f"📊 抓取完成，共保存 {total_saved} 篇文章")
        
        # 显示最新文章
        if total_saved > 0:
            self.show_recent_articles()
        
        return total_saved
    
    def show_recent_articles(self):
        """显示最近的文章"""
        try:
            conn = sqlite3.connect(self.config['paths']['database_path'])
            cursor = conn.cursor()
            
            cursor.execute('''
            SELECT title, source, created_at 
            FROM articles 
            WHERE processed = 0 
            ORDER BY id DESC 
            LIMIT 5
            ''')
            
            articles = cursor.fetchall()
            conn.close()
            
            if articles:
                logger.info("📋 最新未处理文章:")
                for title, source, created_at in articles:
                    logger.info(f"  • [{source}] {title[:50]}... ({created_at})")
            
        except Exception as e:
            logger.error(f"显示文章失败: {e}")

def main():
    """主函数"""
    crawler = EnhancedCrawler()
    
    try:
        saved_count = crawler.run()
        
        if saved_count > 0:
            print(f"\n🎉 抓取成功，保存 {saved_count} 篇文章")
            return 0
        else:
            print("\n⚠️ 抓取完成，但未保存新文章")
            return 1
            
    except KeyboardInterrupt:
        print("\n⏹️ 用户中断")
        return 130
    except Exception as e:
        print(f"\n❌ 抓取过程中出现错误: {e}")
        import traceback
        traceback.print_exc()
        return 1

if __name__ == "__main__":
    exit(main())