admin/nanocode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

a

Browse Source
This commit is contained in:
Ubuntu
2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playwright-hotspots/src/platforms/bilibili.js
+90
View File
@@ -0,0 +1,90 @@
/**
* B站热搜抓取
* https://s.search.bilibili.com/main/hotword
*/
const https = require('https');
const { saveData, formatHotNumber, printTable } = require('../utils/helpers');
const PLATFORM = 'bilibili';
/**
* HTTP GET 请求
*/
function httpGet(url) {
return new Promise((resolve, reject) => {
https.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://search.bilibili.com/',
}
}, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
try {
resolve(JSON.parse(data));
} catch (e) {
reject(new Error('JSON parse error: ' + e.message));
}
});
}).on('error', reject);
});
}
/**
* 抓取B站热搜
*/
async function fetchBilibiliTrending() {
console.log('📺 开始抓取B站热搜...');
try {
const apiUrl = 'https://s.search.bilibili.com/main/hotword';
const response = await httpGet(apiUrl);
if (response && response.code === 0 && response.list) {
const items = response.list.map((item, index) => ({
rank: index + 1,
title: item.keyword,
showName: item.show_name,
hot: item.icon ? '热' : '',
heatScore: item.heat_score || 0,
url: `https://search.bilibili.com/all?keyword=${encodeURIComponent(item.keyword)}`,
}));
const result = {
platform: PLATFORM,
timestamp: new Date().toISOString(),
url: 'https://www.bilibili.com',
count: items.length,
items: items.slice(0, 20),
};
// 打印结果
printTable(result.items.map(i => ({
rank: i.rank,
title: i.title,
hot: i.hot + (i.heatScore ? ` ${formatHotNumber(i.heatScore)}` : '')
})));
// 保存数据
saveData(PLATFORM, result);
console.log(`✅ 成功抓取 ${items.length} 条B站热搜`);
return result;
} else {
throw new Error('API 返回格式异常: ' + (response.message || 'unknown'));
}
} catch (error) {
console.error('❌ 抓取失败:', error.message);
throw error;
}
}
module.exports = { fetchBilibiliTrending };
// 直接运行
if (require.main === module) {
fetchBilibiliTrending().catch(console.error);
}
playwright-hotspots/src/platforms/techblogs/cnblogs.js
+194
View File
@@ -0,0 +1,194 @@
/**
* 博客园热门/搜索抓取
* https://www.cnblogs.com/
*/
const puppeteer = require('puppeteer-core');
const { getLaunchOptions } = require('../../../config/browser');
const { saveData, randomDelay, printTable } = require('../../utils/helpers');
const PLATFORM = 'cnblogs';
/**
* 抓取博客园热门文章
*/
async function fetchCnblogsHot() {
console.log('💻 开始抓取博客园热门文章...');
const browser = await puppeteer.launch(getLaunchOptions());
try {
const page = await browser.newPage();
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
);
// 访问博客园首页
await page.goto('https://www.cnblogs.com/', {
waitUntil: 'networkidle2',
timeout: 60000,
});
await randomDelay(2000, 3000);
// 提取文章数据
const items = await page.evaluate(() => {
const results = [];
// 博客园文章卡片选择器
const selectors = [
'.post-item .post-item-title a',
'.article-item .title a',
'#post_list .post_item .titlelnk',
'.entrylist .entrylistItem .entrylistTitle a',
];
for (const selector of selectors) {
const elements = document.querySelectorAll(selector);
if (elements.length > 0) {
elements.forEach((el, index) => {
const title = el.textContent?.trim();
const href = el.getAttribute('href');
if (title && href && title.length > 5) {
results.push({
rank: results.length + 1,
title: title.substring(0, 100),
url: href.startsWith('http') ? href : `https:${href}`,
});
}
});
break;
}
}
return results;
});
const result = {
platform: PLATFORM,
timestamp: new Date().toISOString(),
url: 'https://www.cnblogs.com/',
count: items.length,
items: items.slice(0, 20),
};
if (items.length > 0) {
printTable(result.items);
saveData(PLATFORM, result);
console.log(`✅ 成功抓取 ${items.length} 条博客园文章`);
} else {
console.log('⚠️ 未解析到文章');
}
return result;
} catch (error) {
console.error('❌ 抓取失败:', error.message);
throw error;
} finally {
await browser.close();
}
}
/**
* 搜索博客园文章
* @param {string} keyword - 搜索关键词
*/
async function searchCnblogs(keyword) {
console.log(`🔍 搜索博客园: "${keyword}"...`);
const browser = await puppeteer.launch(getLaunchOptions());
try {
const page = await browser.newPage();
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
);
// 访问搜索页面
const searchUrl = `https://zzk.cnblogs.com/s?w=${encodeURIComponent(keyword)}`;
await page.goto(searchUrl, {
waitUntil: 'networkidle2',
timeout: 60000,
});
await randomDelay(2000, 3000);
// 提取搜索结果
const items = await page.evaluate(() => {
const results = [];
// 搜索结果选择器
const selectors = [
'.searchItem .searchItemTitle a',
'.search-item .title a',
'.article-list .article-item a',
];
for (const selector of selectors) {
const elements = document.querySelectorAll(selector);
if (elements.length > 0) {
elements.forEach((el) => {
const title = el.textContent?.trim();
const href = el.getAttribute('href');
if (title && href && title.length > 5) {
// 去重
if (!results.find(r => r.title === title)) {
results.push({
rank: results.length + 1,
title: title.substring(0, 100),
url: href.startsWith('http') ? href : `https:${href}`,
});
}
}
});
break;
}
}
return results;
});
const result = {
platform: PLATFORM,
keyword,
timestamp: new Date().toISOString(),
url: searchUrl,
count: items.length,
items: items.slice(0, 20),
};
if (items.length > 0) {
printTable(result.items);
saveData(`${PLATFORM}_search`, result);
console.log(`✅ 成功搜索到 ${items.length} 条博客园文章`);
} else {
console.log('⚠️ 未找到相关文章');
}
return result;
} catch (error) {
console.error('❌ 搜索失败:', error.message);
throw error;
} finally {
await browser.close();
}
}
module.exports = { fetchCnblogsHot, searchCnblogs };
// 直接运行
if (require.main === module) {
const keyword = process.argv[2];
if (keyword) {
searchCnblogs(keyword).catch(console.error);
} else {
fetchCnblogsHot().catch(console.error);
}
}
playwright-hotspots/src/platforms/techblogs/juejin.js
+198
View File
@@ -0,0 +1,198 @@
/**
* 掘金热门文章抓取（使用 Puppeteer）
* https://juejin.cn/hot/articles
* 更新: 2026-03-28 修复选择器
*/
const puppeteer = require('puppeteer-core');
const { getLaunchOptions } = require('../../../config/browser');
const { saveData, randomDelay, printTable } = require('../../utils/helpers');
const PLATFORM = 'juejin';
/**
* 抓取掘金热门文章
*/
async function fetchJuejinHot(category = 'all') {
console.log(`📱 开始抓取掘金热门文章...`);
const browser = await puppeteer.launch(getLaunchOptions());
try {
const page = await browser.newPage();
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
);
// 打开掘金热榜
await page.goto('https://juejin.cn/hot/articles', {
waitUntil: 'networkidle2',
timeout: 60000,
});
await randomDelay(3000, 5000);
// 提取文章数据（2026-03-28 更新选择器）
const items = await page.evaluate(() => {
const results = [];
const seen = new Set();
// 查找所有包含 /post/ 的链接
const allLinks = document.querySelectorAll('a[href*="/post/"]');
allLinks.forEach((el) => {
const href = el.getAttribute('href');
const text = el.textContent?.trim();
// 提取文章ID
const match = href.match(/\/post\/(\d+)/);
if (!match) return;
const articleId = match[1];
// 跳过重复
if (seen.has(articleId)) return;
seen.add(articleId);
// 清理标题（去除作者名、数字等）
let title = text
?.replace(/\d+\s*$/, '') // 去除末尾数字
?.replace(/\s{2,}/g, ' ') // 合并多余空格
?.trim();
// 过滤无效标题
if (!title || title.length < 10 || title.length > 100) return;
if (title.match(/^\d+$/) || title.includes('http')) return;
results.push({
rank: results.length + 1,
title,
url: `https://juejin.cn/post/${articleId}`,
});
});
return results;
});
const result = {
platform: PLATFORM,
category,
timestamp: new Date().toISOString(),
url: 'https://juejin.cn/hot/articles',
count: items.length,
items: items.slice(0, 20),
};
if (items.length > 0) {
printTable(result.items);
saveData(PLATFORM, result);
console.log(`✅ 成功抓取 ${items.length} 条掘金文章`);
} else {
console.log('⚠️ 未解析到文章，页面结构可能已变化');
}
return result;
} catch (error) {
console.error('❌ 抓取失败:', error.message);
throw error;
} finally {
await browser.close();
}
}
/**
* 搜索掘金文章
* @param {string} keyword - 搜索关键词
*/
async function searchJuejin(keyword) {
console.log(`🔍 搜索掘金: "${keyword}"...`);
const browser = await puppeteer.launch(getLaunchOptions());
try {
const page = await browser.newPage();
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
);
const searchUrl = `https://juejin.cn/search?query=${encodeURIComponent(keyword)}&type=0`;
await page.goto(searchUrl, {
waitUntil: 'networkidle2',
timeout: 60000,
});
await randomDelay(3000, 5000);
const items = await page.evaluate(() => {
const results = [];
const seen = new Set();
const allLinks = document.querySelectorAll('a[href*="/post/"]');
allLinks.forEach((el) => {
const text = el.textContent?.trim();
const href = el.getAttribute('href');
const match = href.match(/\/post\/(\d+)/);
if (!match) return;
const articleId = match[1];
if (seen.has(articleId)) return;
seen.add(articleId);
let title = text
?.replace(/\d+\s*$/, '')
?.replace(/\s{2,}/g, ' ')
?.trim();
if (!title || title.length < 10 || title.length > 100) return;
if (title.match(/^\d+$/) || title.includes('http')) return;
results.push({
rank: results.length + 1,
title,
url: `https://juejin.cn/post/${articleId}`,
});
});
return results;
});
const result = {
platform: PLATFORM,
keyword,
timestamp: new Date().toISOString(),
url: searchUrl,
count: items.length,
items: items.slice(0, 20),
};
printTable(result.items);
saveData(`${PLATFORM}_search`, result);
console.log(`✅ 成功搜索到 ${items.length} 条掘金文章`);
return result;
} catch (error) {
console.error('❌ 搜索失败:', error.message);
throw error;
} finally {
await browser.close();
}
}
module.exports = { fetchJuejinHot, searchJuejin };
// 直接运行
if (require.main === module) {
const keyword = process.argv[2];
if (keyword) {
searchJuejin(keyword).catch(console.error);
} else {
fetchJuejinHot().catch(console.error);
}
}
playwright-hotspots/src/platforms/weibo.js
+98
View File
@@ -0,0 +1,98 @@
/**
* 微博热搜抓取
* https://s.weibo.com/top/summary
*/
const puppeteer = require('puppeteer-core');
const { getLaunchOptions } = require('../../config/browser');
const { saveData, formatHotNumber, randomDelay, printTable } = require('../utils/helpers');
const PLATFORM = 'weibo';
const URL = 'https://s.weibo.com/top/summary';
/**
* 抓取微博热搜
*/
async function fetchWeiboHot() {
const browser = await puppeteer.launch(getLaunchOptions());
try {
console.log('🔥 开始抓取微博热搜...');
const page = await browser.newPage();
// 设置 User-Agent
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
);
// 打开页面
await page.goto(URL, {
waitUntil: 'networkidle2',
timeout: 60000
});
await randomDelay(1000, 2000);
// 提取热搜数据
const hotspots = await page.evaluate(() => {
const items = [];
const rows = document.querySelectorAll('#pl_top_realtimehot tbody tr');
rows.forEach((row, index) => {
// 跳过表头
if (index === 0) return;
const rankEl = row.querySelector('.td-01.ranktop');
const titleEl = row.querySelector('.td-02 a');
const hotEl = row.querySelector('.td-02 span');
if (titleEl) {
const rank = rankEl ? rankEl.textContent.trim() : String(index);
const title = titleEl.textContent.trim();
const hot = hotEl ? hotEl.textContent.trim() : '';
const url = titleEl.href;
items.push({
rank: parseInt(rank) || index,
title,
hot,
url,
});
}
});
return items;
});
const result = {
platform: PLATFORM,
timestamp: new Date().toISOString(),
url: URL,
count: hotspots.length,
items: hotspots.slice(0, 20), // 只取前20条
};
// 打印结果
printTable(result.items);
// 保存数据
saveData(PLATFORM, result);
console.log(`✅ 成功抓取 ${hotspots.length} 条微博热搜`);
return result;
} catch (error) {
console.error('❌ 抓取失败:', error.message);
throw error;
} finally {
await browser.close();
}
}
module.exports = { fetchWeiboHot };
// 直接运行
if (require.main === module) {
fetchWeiboHot().catch(console.error);
}
playwright-hotspots/src/platforms/xiaohongshu.js
+96
View File
@@ -0,0 +1,96 @@
/**
* 小红书搜索/热点抓取
* 注意：小红书需要登录才能获取完整数据
*/
const puppeteer = require('puppeteer-core');
const { getLaunchOptions } = require('../../config/browser');
const { saveData, randomDelay, printTable } = require('../utils/helpers');
const PLATFORM = 'xiaohongshu';
/**
* 抓取小红书搜索建议/热点
* @param {string} keyword - 搜索关键词
*/
async function fetchXiaohongshuSearch(keyword = 'AI') {
const browser = await puppeteer.launch(getLaunchOptions());
try {
console.log(`🔴 开始抓取小红书搜索: "${keyword}"...`);
const page = await browser.newPage();
// 设置 User-Agent
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
);
// 小红书搜索页
const searchUrl = `https://www.xiaohongshu.com/search_result?keyword=${encodeURIComponent(keyword)}`;
await page.goto(searchUrl, {
waitUntil: 'networkidle2',
timeout: 60000
});
await randomDelay(2000, 4000);
// 尝试提取笔记数据
const notes = await page.evaluate(() => {
const items = [];
// 小红书的笔记卡片选择器（可能随时间变化）
const cards = document.querySelectorAll('.note-item, [class*="note"], .feeds-page > div > div');
cards.forEach((card, index) => {
const titleEl = card.querySelector('.title, .desc, [class*="title"]');
const linkEl = card.querySelector('a[href*="/explore/"]');
if (titleEl) {
items.push({
rank: index + 1,
title: titleEl.textContent.trim().substring(0, 100),
url: linkEl ? linkEl.href : '',
});
}
});
return items;
});
const result = {
platform: PLATFORM,
keyword,
timestamp: new Date().toISOString(),
url: searchUrl,
count: notes.length,
items: notes.slice(0, 10),
note: '小红书需要登录才能获取完整数据',
};
if (notes.length > 0) {
printTable(result.items);
} else {
console.log('⚠️ 未获取到数据，可能需要登录');
}
saveData(PLATFORM, result);
console.log(`✅ 完成`);
return result;
} catch (error) {
console.error('❌ 抓取失败:', error.message);
throw error;
} finally {
await browser.close();
}
}
module.exports = { fetchXiaohongshuSearch };
// 直接运行
if (require.main === module) {
const keyword = process.argv[2] || 'AI';
fetchXiaohongshuSearch(keyword).catch(console.error);
}
playwright-hotspots/src/platforms/zhihu.js
+163
View File
@@ -0,0 +1,163 @@
/**
* 知乎热榜抓取（支持 Cookie）
* https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total
*/
const https = require('https');
const fs = require('fs');
const path = require('path');
const { saveData, formatHotNumber, printTable } = require('../utils/helpers');
const PLATFORM = 'zhihu';
const COOKIE_FILE = '/home/ubuntu/zhihu_cookies.txt';
/**
* 解析 Cookie 文件
* 支持格式1: JSON (从浏览器插件导出)
* 支持格式2: 纯文本 name=value; name2=value2
*/
function loadCookies() {
try {
if (!fs.existsSync(COOKIE_FILE)) {
console.warn(`⚠️ Cookie 文件不存在: ${COOKIE_FILE}`);
return null;
}
const content = fs.readFileSync(COOKIE_FILE, 'utf-8').trim();
// 尝试解析为 JSON
if (content.startsWith('[') || content.startsWith('{')) {
try {
const cookies = JSON.parse(content);
// 处理数组格式 [{name, value, domain, ...}, ...]
if (Array.isArray(cookies)) {
const cookieStr = cookies
.filter(c => c.name && c.value)
.map(c => `${c.name}=${c.value}`)
.join('; ');
console.log(`📋 已加载 JSON Cookie: ${cookies.length} 条`);
return cookieStr;
}
// 处理对象格式 {name: value, name2: value2}
else if (typeof cookies === 'object') {
const cookieStr = Object.entries(cookies)
.map(([k, v]) => `${k}=${v}`)
.join('; ');
console.log(`📋 已加载 JSON Object Cookie`);
return cookieStr;
}
} catch (e) {
// JSON 解析失败，当作纯文本处理
}
}
// 纯文本格式
console.log(`📋 已加载文本 Cookie: ${content.length} 字符`);
return content;
} catch (e) {
console.warn('⚠️ 读取 Cookie 文件失败:', e.message);
return null;
}
}
/**
* HTTP GET 请求（带 Cookie）
*/
function httpGet(url, cookie = null) {
return new Promise((resolve, reject) => {
const headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://www.zhihu.com/',
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
};
if (cookie) {
headers['Cookie'] = cookie;
}
https.get(url, { headers }, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
try {
resolve(JSON.parse(data));
} catch (e) {
reject(new Error('JSON parse error: ' + e.message + '\nData: ' + data.substring(0, 200)));
}
});
}).on('error', reject);
});
}
/**
* 抓取知乎热榜
*/
async function fetchZhihuHot() {
console.log('📚 开始抓取知乎热榜...');
// 加载 Cookie
const cookie = loadCookies();
if (!cookie) {
console.warn('⚠️ 未找到有效 Cookie，知乎抓取可能失败');
}
try {
const apiUrl = 'https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50';
const response = await httpGet(apiUrl, cookie);
// 检查是否需要登录
if (response.error) {
if (response.error.code === 101 || response.error.name === 'AuthenticationError') {
throw new Error('Cookie 已过期或无效，请更新 zhihu_cookies.txt');
}
throw new Error(`API 错误: ${response.error.message || JSON.stringify(response.error)}`);
}
if (response && response.data) {
const items = response.data.map((item, index) => {
const target = item.target || {};
const detailText = target.detail_text || ''; // "1234 万热度"
return {
rank: index + 1,
title: target.title,
excerpt: (target.excerpt || '').substring(0, 100),
hot: detailText,
url: `https://www.zhihu.com/question/${target.id}`,
};
});
const result = {
platform: PLATFORM,
timestamp: new Date().toISOString(),
url: 'https://www.zhihu.com/hot',
count: items.length,
items: items.slice(0, 20),
};
// 打印结果
printTable(result.items);
// 保存数据
saveData(PLATFORM, result);
console.log(`✅ 成功抓取 ${items.length} 条知乎热榜`);
return result;
} else {
throw new Error('API 返回格式异常: ' + JSON.stringify(response).substring(0, 200));
}
} catch (error) {
console.error('❌ 抓取失败:', error.message);
throw error;
}
}
module.exports = { fetchZhihuHot };
// 直接运行
if (require.main === module) {
fetchZhihuHot().catch(console.error);
}