a
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* 微博热搜抓取
|
||||
* https://s.weibo.com/top/summary
|
||||
*/
|
||||
|
||||
const puppeteer = require('puppeteer-core');
|
||||
const { getLaunchOptions } = require('../../config/browser');
|
||||
const { saveData, formatHotNumber, randomDelay, printTable } = require('../utils/helpers');
|
||||
|
||||
const PLATFORM = 'weibo';
|
||||
const URL = 'https://s.weibo.com/top/summary';
|
||||
|
||||
/**
|
||||
* 抓取微博热搜
|
||||
*/
|
||||
async function fetchWeiboHot() {
|
||||
const browser = await puppeteer.launch(getLaunchOptions());
|
||||
|
||||
try {
|
||||
console.log('🔥 开始抓取微博热搜...');
|
||||
|
||||
const page = await browser.newPage();
|
||||
|
||||
// 设置 User-Agent
|
||||
await page.setUserAgent(
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
);
|
||||
|
||||
// 打开页面
|
||||
await page.goto(URL, {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
await randomDelay(1000, 2000);
|
||||
|
||||
// 提取热搜数据
|
||||
const hotspots = await page.evaluate(() => {
|
||||
const items = [];
|
||||
const rows = document.querySelectorAll('#pl_top_realtimehot tbody tr');
|
||||
|
||||
rows.forEach((row, index) => {
|
||||
// 跳过表头
|
||||
if (index === 0) return;
|
||||
|
||||
const rankEl = row.querySelector('.td-01.ranktop');
|
||||
const titleEl = row.querySelector('.td-02 a');
|
||||
const hotEl = row.querySelector('.td-02 span');
|
||||
|
||||
if (titleEl) {
|
||||
const rank = rankEl ? rankEl.textContent.trim() : String(index);
|
||||
const title = titleEl.textContent.trim();
|
||||
const hot = hotEl ? hotEl.textContent.trim() : '';
|
||||
const url = titleEl.href;
|
||||
|
||||
items.push({
|
||||
rank: parseInt(rank) || index,
|
||||
title,
|
||||
hot,
|
||||
url,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return items;
|
||||
});
|
||||
|
||||
const result = {
|
||||
platform: PLATFORM,
|
||||
timestamp: new Date().toISOString(),
|
||||
url: URL,
|
||||
count: hotspots.length,
|
||||
items: hotspots.slice(0, 20), // 只取前20条
|
||||
};
|
||||
|
||||
// 打印结果
|
||||
printTable(result.items);
|
||||
|
||||
// 保存数据
|
||||
saveData(PLATFORM, result);
|
||||
|
||||
console.log(`✅ 成功抓取 ${hotspots.length} 条微博热搜`);
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ 抓取失败:', error.message);
|
||||
throw error;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { fetchWeiboHot };
|
||||
|
||||
// 直接运行
|
||||
if (require.main === module) {
|
||||
fetchWeiboHot().catch(console.error);
|
||||
}
|
||||
Reference in New Issue
Block a user