Ubuntu
254 lines
9.9 KiB
Python
254 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
技术博客文章抓取器
|
||
从指定技术博客源抓取最近3天内的短文章
|
||
"""
|
||
import requests
|
||
import json
|
||
import random
|
||
import re
|
||
from datetime import datetime, timedelta
|
||
from typing import List, Dict, Optional
|
||
from urllib.parse import urljoin, urlparse
|
||
import feedparser
|
||
|
||
# 可用技术博客源（2026-03-30 检测）
|
||
TECH_SOURCES = [
|
||
{"name": "Coding Horror", "url": "https://blog.codinghorror.com/", "type": "rss"},
|
||
{"name": "Joel on Software", "url": "https://www.joelonsoftware.com/", "type": "rss"},
|
||
{"name": "TechCrunch", "url": "https://techcrunch.com/", "type": "rss"},
|
||
{"name": "Stratechery", "url": "https://stratechery.com/", "type": "rss"},
|
||
{"name": "Wired", "url": "https://wired.com/", "type": "rss"},
|
||
{"name": "Dan Luu", "url": "https://danluu.com/", "type": "rss"},
|
||
{"name": "Simon Willison", "url": "https://simonwillison.net/", "type": "rss"},
|
||
{"name": "Martin Fowler", "url": "https://martinfowler.com/", "type": "rss"},
|
||
{"name": "WorryDream", "url": "https://worrydream.com/", "type": "html"},
|
||
{"name": "Microsoft DevBlogs", "url": "https://devblogs.microsoft.com/", "type": "rss"},
|
||
{"name": "Flavio Copes", "url": "https://flaviocopes.com/", "type": "rss"},
|
||
{"name": "Gwern", "url": "https://gwern.net/", "type": "rss"},
|
||
{"name": "GitHub Blog", "url": "https://github.blog/", "type": "rss"},
|
||
{"name": "InfoQ中文", "url": "https://www.infoq.cn/", "type": "api"},
|
||
{"name": "NVIDIA Blog", "url": "https://developer.nvidia.com/blog/", "type": "rss"},
|
||
]
|
||
|
||
|
||
class TechBlogFetcher:
|
||
"""技术博客文章抓取器"""
|
||
|
||
def __init__(self):
|
||
self.headers = {
|
||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||
}
|
||
self.max_days = 7 # 放宽到7天内
|
||
self.max_chars = 5000 # 放宽字数限制到5000
|
||
self.min_chars = 300 # 最小字数
|
||
|
||
def get_random_sources(self, count: int = 3) -> List[Dict]:
|
||
"""随机选择n个数据源"""
|
||
return random.sample(TECH_SOURCES, k=min(count, len(TECH_SOURCES)))
|
||
|
||
def fetch_from_rss(self, source: Dict) -> List[Dict]:
|
||
"""从RSS源抓取文章"""
|
||
articles = []
|
||
try:
|
||
# 已知可用的RSS feed（2026-03-30验证）
|
||
known_feeds = {
|
||
"Coding Horror": "https://blog.codinghorror.com/rss/",
|
||
"Joel on Software": "https://www.joelonsoftware.com/feed/",
|
||
"TechCrunch": "https://techcrunch.com/feed/",
|
||
"Wired": "https://www.wired.com/feed/",
|
||
"Dan Luu": "https://danluu.com/atom.xml",
|
||
"Simon Willison": "https://simonwillison.net/atom/everything/",
|
||
"Martin Fowler": "https://martinfowler.com/feed.atom",
|
||
"Gwern": "https://gwern.net/atom.xml",
|
||
"Flavio Copes": "https://flaviocopes.com/rss.xml",
|
||
"GitHub Blog": "https://github.blog/feed/",
|
||
"Microsoft DevBlogs": "https://devblogs.microsoft.com/feed",
|
||
"NVIDIA Blog": "https://developer.nvidia.com/blog/feed/",
|
||
}
|
||
|
||
# 获取RSS URL
|
||
rss_url = known_feeds.get(source["name"])
|
||
if not rss_url:
|
||
# 尝试通用路径
|
||
rss_url = urljoin(source["url"], "/feed")
|
||
|
||
print(f" 尝试RSS: {rss_url}")
|
||
feed = feedparser.parse(rss_url)
|
||
|
||
if not feed.entries:
|
||
print(f" ⚠️ RSS无条目")
|
||
return []
|
||
|
||
print(f" ✅ 获取到 {len(feed.entries)} 篇RSS条目")
|
||
|
||
cutoff_date = datetime.now() - timedelta(days=self.max_days)
|
||
|
||
for entry in feed.entries[:10]: # 最近10篇
|
||
# 解析日期
|
||
pub_date = None
|
||
for date_field in ['published_parsed', 'updated_parsed', 'created_parsed']:
|
||
if hasattr(entry, date_field) and getattr(entry, date_field):
|
||
pub_date = datetime(*getattr(entry, date_field)[:6])
|
||
break
|
||
|
||
if not pub_date or pub_date < cutoff_date:
|
||
continue
|
||
|
||
article = {
|
||
"title": entry.get("title", ""),
|
||
"url": entry.get("link", ""),
|
||
"source": source["name"],
|
||
"published": pub_date.isoformat(),
|
||
"summary": entry.get("summary", entry.get("description", ""))[:500],
|
||
}
|
||
articles.append(article)
|
||
|
||
except Exception as e:
|
||
print(f"RSS抓取失败 {source['name']}: {e}")
|
||
|
||
return articles
|
||
|
||
def fetch_from_infoq_cn(self) -> List[Dict]:
|
||
"""从InfoQ中文抓取"""
|
||
articles = []
|
||
try:
|
||
url = "https://www.infoq.cn/public/v1/my/recommends"
|
||
headers = {
|
||
**self.headers,
|
||
"Accept": "application/json",
|
||
"Referer": "https://www.infoq.cn/",
|
||
}
|
||
resp = requests.post(url, headers=headers, json={"size": 20}, timeout=15)
|
||
data = resp.json()
|
||
|
||
if data.get("code") == 0:
|
||
cutoff_date = datetime.now() - timedelta(days=self.max_days)
|
||
|
||
for item in data.get("data", []):
|
||
pub_time = item.get("publish_time", "")
|
||
if pub_time:
|
||
try:
|
||
pub_date = datetime.fromtimestamp(pub_time / 1000)
|
||
if pub_date < cutoff_date:
|
||
continue
|
||
except:
|
||
pass
|
||
|
||
articles.append({
|
||
"title": item.get("article_title", ""),
|
||
"url": f"https://www.infoq.cn/article/{item.get('uuid', '')}",
|
||
"source": "InfoQ中文",
|
||
"published": pub_time,
|
||
"summary": item.get("article_summary", "")[:500],
|
||
})
|
||
except Exception as e:
|
||
print(f"InfoQ抓取失败: {e}")
|
||
|
||
return articles
|
||
|
||
def fetch_full_content(self, url: str) -> str:
|
||
"""抓取文章完整内容"""
|
||
try:
|
||
resp = requests.get(url, headers=self.headers, timeout=15)
|
||
|
||
# 使用readability或简单提取
|
||
from bs4 import BeautifulSoup
|
||
soup = BeautifulSoup(resp.text, 'html.parser')
|
||
|
||
# 移除脚本和样式
|
||
for tag in soup(["script", "style", "nav", "header", "footer"]):
|
||
tag.decompose()
|
||
|
||
# 提取正文
|
||
article = soup.find('article') or soup.find('main') or soup.find('div', class_=re.compile('content|post|entry'))
|
||
if article:
|
||
text = article.get_text(separator='\n', strip=True)
|
||
else:
|
||
text = soup.get_text(separator='\n', strip=True)
|
||
|
||
# 清理
|
||
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
||
text = '\n'.join(lines)
|
||
|
||
return text
|
||
except Exception as e:
|
||
print(f"抓取全文失败 {url}: {e}")
|
||
return ""
|
||
|
||
def filter_short_articles(self, articles: List[Dict]) -> List[Dict]:
|
||
"""筛选短文章（< 5000字符）并获取完整内容"""
|
||
result = []
|
||
|
||
for article in articles:
|
||
# 优先使用RSS摘要
|
||
content = article.get("summary", "")
|
||
|
||
# 如果摘要太短，尝试抓取全文
|
||
if len(content) < self.min_chars:
|
||
content = self.fetch_full_content(article["url"])
|
||
|
||
if content and len(content) < self.max_chars and len(content) > self.min_chars:
|
||
article["content"] = content
|
||
result.append(article)
|
||
if len(result) >= 3: # 最多3篇
|
||
break
|
||
|
||
return result
|
||
|
||
def fetch_daily_articles(self) -> Optional[Dict]:
|
||
"""
|
||
每日抓取主流程
|
||
返回一篇适合发布的文章
|
||
"""
|
||
print("🔍 开始抓取技术博客...")
|
||
|
||
# 随机选择3个源
|
||
sources = self.get_random_sources(3)
|
||
all_articles = []
|
||
|
||
for source in sources:
|
||
print(f" 📡 检查 {source['name']}...")
|
||
|
||
if source["name"] == "InfoQ中文":
|
||
articles = self.fetch_from_infoq_cn()
|
||
elif source["type"] == "rss":
|
||
articles = self.fetch_from_rss(source)
|
||
else:
|
||
continue
|
||
|
||
all_articles.extend(articles)
|
||
print(f" 找到 {len(articles)} 篇候选")
|
||
|
||
if not all_articles:
|
||
print("❌ 没有找到候选文章")
|
||
return None
|
||
|
||
print(f"\n📄 共 {len(all_articles)} 篇候选，筛选短文章...")
|
||
|
||
# 筛选短文章
|
||
short_articles = self.filter_short_articles(all_articles)
|
||
|
||
if not short_articles:
|
||
print("❌ 没有符合字数要求的文章")
|
||
return None
|
||
|
||
# 随机选择一篇
|
||
selected = random.choice(short_articles)
|
||
print(f"\n✅ 选中文章: {selected['title'][:50]}...")
|
||
print(f" 来源: {selected['source']}")
|
||
print(f" 字数: {len(selected['content'])}")
|
||
|
||
return selected
|
||
|
||
|
||
if __name__ == "__main__":
|
||
fetcher = TechBlogFetcher()
|
||
article = fetcher.fetch_daily_articles()
|
||
|
||
if article:
|
||
print("\n" + "="*50)
|
||
print(f"标题: {article['title']}")
|
||
print(f"URL: {article['url']}")
|
||
print(f"内容预览:\n{article['content'][:300]}...")
|