#!/usr/bin/env python3
"""
增强版IT资讯抓取系统模板
请根据实际环境配置路径
"""

import sys
import os
import re
import json
import random
import sqlite3
import datetime
import time
from pathlib import Path
from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
import mimetypes

# ==================== 配置部分 ====================
# 请根据实际环境修改以下配置
CONFIG = {
    'database_path': '{{DATABASE_PATH}}',  # 数据库文件路径
    'base_dir': '{{BASE_DIR}}',            # 基础目录
    'min_word_count': 500,                 # 最小字数要求
    'max_days_old': 7,                     # 最大天数（最近N天）
    'websites': [
        {'name': 'InfoQ', 'url': 'https://www.infoq.com', 'category': 'tech'},
        {'name': 'Wired', 'url': 'https://www.wired.com', 'category': 'tech'},
        {'name': 'TechCrunch', 'url': 'https://techcrunch.com', 'category': 'tech'},
        # 可以添加更多网站
    ],
    'user_agents': [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36',
    ],
    'request_delay': 2,  # 请求延迟（秒）
    'timeout': 30,       # 请求超时（秒）
}

# ==================== 数据库操作 ====================
def init_database():
    """初始化数据库"""
    conn = sqlite3.connect(CONFIG['database_path'])
    cursor = conn.cursor()
    
    # 创建文章表
    cursor.execute('''
    CREATE TABLE IF NOT EXISTS articles (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT NOT NULL,
        url TEXT UNIQUE NOT NULL,
        source TEXT NOT NULL,
        category TEXT,
        publish_date TEXT,
        word_count INTEGER,
        processed INTEGER DEFAULT 0,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
    )
    ''')
    
    # 创建索引
    cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON articles(url)')
    cursor.execute('CREATE INDEX IF NOT EXISTS idx_processed ON articles(processed)')
    cursor.execute('CREATE INDEX IF NOT EXISTS idx_date ON articles(publish_date)')
    
    conn.commit()
    conn.close()
    print(f"✅ 数据库初始化完成: {CONFIG['database_path']}")

def save_article(title, url, source, category, publish_date, word_count):
    """保存文章到数据库"""
    try:
        conn = sqlite3.connect(CONFIG['database_path'])
        cursor = conn.cursor()
        
        cursor.execute('''
        INSERT OR IGNORE INTO articles 
        (title, url, source, category, publish_date, word_count)
        VALUES (?, ?, ?, ?, ?, ?)
        ''', (title, url, source, category, publish_date, word_count))
        
        conn.commit()
        conn.close()
        return True
    except Exception as e:
        print(f"❌ 保存文章失败: {e}")
        return False

# ==================== 网页抓取 ====================
def fetch_with_retry(url, max_retries=3):
    """带重试的网页抓取"""
    headers = {
        'User-Agent': random.choice(CONFIG['user_agents']),
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Accept-Encoding': 'gzip, deflate',
        'Connection': 'keep-alive',
    }
    
    for attempt in range(max_retries):
        try:
            time.sleep(CONFIG['request_delay'])
            response = requests.get(url, headers=headers, timeout=CONFIG['timeout'])
            response.raise_for_status()
            return response
        except Exception as e:
            if attempt == max_retries - 1:
                print(f"❌ 抓取失败 ({attempt+1}/{max_retries}): {url} - {e}")
                return None
            print(f"⚠️ 重试抓取 ({attempt+1}/{max_retries}): {url}")
            time.sleep(2 ** attempt)  # 指数退避

def extract_articles_from_html(html, base_url, source_name):
    """从HTML中提取文章（需要根据具体网站实现）"""
    # 这是一个模板函数，需要根据具体网站结构实现
    articles = []
    soup = BeautifulSoup(html, 'html.parser')
    
    # 示例：查找文章链接
    for link in soup.find_all('a', href=True):
        href = link['href']
        title = link.get_text(strip=True)
        
        # 过滤非文章链接
        if not title or len(title) < 10:
            continue
            
        # 构建完整URL
        full_url = urljoin(base_url, href)
        
        # 添加到文章列表
        articles.append({
            'title': title,
            'url': full_url,
            'source': source_name,
            'category': 'tech',  # 默认分类
        })
    
    return articles

# ==================== 主函数 ====================
def main():
    """主函数"""
    print("🚀 开始IT资讯抓取")
    print(f"📁 工作目录: {CONFIG['base_dir']}")
    print(f"🗄️ 数据库: {CONFIG['database_path']}")
    
    # 初始化数据库
    init_database()
    
    # 创建当天目录
    today = datetime.datetime.now().strftime('%Y-%m-%d')
    day_dir = os.path.join(CONFIG['base_dir'], today)
    os.makedirs(day_dir, exist_ok=True)
    
    total_articles = 0
    
    # 遍历网站
    for site in CONFIG['websites']:
        print(f"\n🌐 抓取网站: {site['name']} ({site['url']})")
        
        response = fetch_with_retry(site['url'])
        if not response:
            continue
            
        articles = extract_articles_from_html(response.text, site['url'], site['name'])
        
        print(f"📰 找到 {len(articles)} 篇文章")
        
        # 保存文章
        for article in articles:
            # 这里可以添加更多处理逻辑
            saved = save_article(
                article['title'],
                article['url'],
                article['source'],
                article.get('category', 'tech'),
                datetime.datetime.now().strftime('%Y-%m-%d'),
                0  # 字数需要后续处理
            )
            
            if saved:
                total_articles += 1
    
    print(f"\n✅ 抓取完成，共保存 {total_articles} 篇文章")
    return total_articles

if __name__ == '__main__':
    main()