nanocode/playwright-hotspots/scripts/search-tech-puppeteer.js

#!/usr/bin/env node
/**
 * 使用 Puppeteer 搜索技术博客
 */

const puppeteer = require('puppeteer-core');
const { getLaunchOptions } = require('../config/browser');
const fs = require('fs');
const path = require('path');

function saveResult(platform, keyword, items) {
  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
  const data = {
    platform,
    keyword,
    timestamp: new Date().toISOString(),
    count: items.length,
    items,
  };
  
  const filepath = path.join(__dirname, '../data', `${platform}_search_${timestamp}.json`);
  if (!fs.existsSync(path.dirname(filepath))) {
    fs.mkdirSync(path.dirname(filepath), { recursive: true });
  }
  
  fs.writeFileSync(filepath, JSON.stringify(data, null, 2), 'utf-8');
  return filepath;
}

function printResults(items, platform) {
  console.log('\n' + '='.repeat(70));
  items.forEach((item, i) => {
    console.log(`${i + 1}. ${item.title.substring(0, 60)}${item.title.length > 60 ? '...' : ''}`);
    console.log(`   ${item.url}`);
  });
  console.log('='.repeat(70));
}

async function searchJuejin(browser, keyword) {
  console.log(`\n📱 搜索掘金: "${keyword}"`);
  
  const page = await browser.newPage();
  
  try {
    await page.setUserAgent(
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
    );
    
    const url = `https://juejin.cn/search?query=${encodeURIComponent(keyword)}&type=0`;
    
    await page.goto(url, {
      waitUntil: 'networkidle2',
      timeout: 60000,
    });
    
    // 等待搜索结果加载
    await new Promise(r => setTimeout(r, 3000));
    
    // 提取搜索结果
    const items = await page.evaluate(() => {
      const results = [];
      
      // 掘金搜索结果选择器
      const selectors = [
        '.search-result-list .item a.title',
        '.result-list .item .title a',
        '[class*="search"] [class*="item"] a[href*="/post/"]',
        '.article-item a',
      ];
      
      for (const selector of selectors) {
        const elements = document.querySelectorAll(selector);
        if (elements.length > 0) {
          elements.forEach((el) => {
            const title = el.textContent?.trim();
            const href = el.getAttribute('href');
            
            if (title && href && title.length > 5 && !results.find(r => r.title === title)) {
              const articleId = href.match(/\/post\/(\d+)/)?.[1];
              if (articleId) {
                results.push({
                  title: title.substring(0, 100),
                  url: `https://juejin.cn/post/${articleId}`,
                });
              }
            }
          });
          break;
        }
      }
      
      return results.slice(0, 10);
    });
    
    if (items.length > 0) {
      printResults(items, '掘金');
      const filepath = saveResult('juejin', keyword, items);
      console.log(`✅ 找到 ${items.length} 条结果`);
      console.log(`💾 已保存: ${filepath}`);
    } else {
      console.log('⚠️ 未找到结果，页面可能需要登录');
    }
    
    return items;
    
  } catch (e) {
    console.error('❌ 掘金搜索失败:', e.message);
    return [];
  } finally {
    await page.close();
  }
}

async function searchCnblogs(browser, keyword) {
  console.log(`\n💻 搜索博客园: "${keyword}"`);
  
  const page = await browser.newPage();
  
  try {
    await page.setUserAgent(
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
    );
    
    const url = `https://zzk.cnblogs.com/s?w=${encodeURIComponent(keyword)}`;
    
    await page.goto(url, {
      waitUntil: 'networkidle2',
      timeout: 60000,
    });
    
    await new Promise(r => setTimeout(r, 3000));
    
    // 提取搜索结果
    const items = await page.evaluate(() => {
      const results = [];
      
      const selectors = [
        '.searchItem .searchItemTitle a',
        '.search-item .title a',
        '#search_result .searchItem a',
      ];
      
      for (const selector of selectors) {
        const elements = document.querySelectorAll(selector);
        if (elements.length > 0) {
          elements.forEach((el) => {
            const title = el.textContent?.trim();
            const href = el.getAttribute('href');
            
            if (title && href && title.length > 5 && !results.find(r => r.title === title)) {
              results.push({
                title: title.substring(0, 100),
                url: href.startsWith('http') ? href : `https:${href}`,
              });
            }
          });
          break;
        }
      }
      
      return results.slice(0, 10);
    });
    
    if (items.length > 0) {
      printResults(items, '博客园');
      const filepath = saveResult('cnblogs', keyword, items);
      console.log(`✅ 找到 ${items.length} 条结果`);
      console.log(`💾 已保存: ${filepath}`);
    } else {
      console.log('⚠️ 未找到结果');
    }
    
    return items;
    
  } catch (e) {
    console.error('❌ 博客园搜索失败:', e.message);
    return [];
  } finally {
    await page.close();
  }
}

async function main() {
  const keyword = process.argv[2] || 'android 开发';
  const platform = process.argv[3] || 'all';
  
  console.log('='.repeat(70));
  console.log(`🔍 技术博客搜索: "${keyword}"`);
  console.log('='.repeat(70));
  
  const browser = await puppeteer.launch(getLaunchOptions());
  
  try {
    if (platform === 'all' || platform === 'juejin') {
      await searchJuejin(browser, keyword);
    }
    
    if (platform === 'all' || platform === 'cnblogs') {
      await searchCnblogs(browser, keyword);
    }
    
  } finally {
    await browser.close();
  }
  
  console.log('\n' + '='.repeat(70));
  console.log('搜索完成');
  console.log('='.repeat(70));
}

main().catch(console.error);