Ubuntu
99 lines
2.5 KiB
JavaScript
99 lines
2.5 KiB
JavaScript
/**
|
|
* 微博热搜抓取
|
|
* https://s.weibo.com/top/summary
|
|
*/
|
|
|
|
const puppeteer = require('puppeteer-core');
|
|
const { getLaunchOptions } = require('../../config/browser');
|
|
const { saveData, formatHotNumber, randomDelay, printTable } = require('../utils/helpers');
|
|
|
|
const PLATFORM = 'weibo';
|
|
const URL = 'https://s.weibo.com/top/summary';
|
|
|
|
/**
|
|
* 抓取微博热搜
|
|
*/
|
|
async function fetchWeiboHot() {
|
|
const browser = await puppeteer.launch(getLaunchOptions());
|
|
|
|
try {
|
|
console.log('🔥 开始抓取微博热搜...');
|
|
|
|
const page = await browser.newPage();
|
|
|
|
// 设置 User-Agent
|
|
await page.setUserAgent(
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
);
|
|
|
|
// 打开页面
|
|
await page.goto(URL, {
|
|
waitUntil: 'networkidle2',
|
|
timeout: 60000
|
|
});
|
|
|
|
await randomDelay(1000, 2000);
|
|
|
|
// 提取热搜数据
|
|
const hotspots = await page.evaluate(() => {
|
|
const items = [];
|
|
const rows = document.querySelectorAll('#pl_top_realtimehot tbody tr');
|
|
|
|
rows.forEach((row, index) => {
|
|
// 跳过表头
|
|
if (index === 0) return;
|
|
|
|
const rankEl = row.querySelector('.td-01.ranktop');
|
|
const titleEl = row.querySelector('.td-02 a');
|
|
const hotEl = row.querySelector('.td-02 span');
|
|
|
|
if (titleEl) {
|
|
const rank = rankEl ? rankEl.textContent.trim() : String(index);
|
|
const title = titleEl.textContent.trim();
|
|
const hot = hotEl ? hotEl.textContent.trim() : '';
|
|
const url = titleEl.href;
|
|
|
|
items.push({
|
|
rank: parseInt(rank) || index,
|
|
title,
|
|
hot,
|
|
url,
|
|
});
|
|
}
|
|
});
|
|
|
|
return items;
|
|
});
|
|
|
|
const result = {
|
|
platform: PLATFORM,
|
|
timestamp: new Date().toISOString(),
|
|
url: URL,
|
|
count: hotspots.length,
|
|
items: hotspots.slice(0, 20), // 只取前20条
|
|
};
|
|
|
|
// 打印结果
|
|
printTable(result.items);
|
|
|
|
// 保存数据
|
|
saveData(PLATFORM, result);
|
|
|
|
console.log(`✅ 成功抓取 ${hotspots.length} 条微博热搜`);
|
|
return result;
|
|
|
|
} catch (error) {
|
|
console.error('❌ 抓取失败:', error.message);
|
|
throw error;
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
module.exports = { fetchWeiboHot };
|
|
|
|
// 直接运行
|
|
if (require.main === module) {
|
|
fetchWeiboHot().catch(console.error);
|
|
}
|