Ubuntu
580 lines
24 KiB
Python
Executable File
580 lines
24 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
小红书数据分析模块
|
||
分析发布效果，优化内容策略
|
||
"""
|
||
|
||
import os
|
||
import json
|
||
import sqlite3
|
||
from datetime import datetime, timedelta
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from collections import Counter
|
||
|
||
class XiaohongshuDataAnalyzer:
|
||
"""小红书数据分析器"""
|
||
|
||
def __init__(self):
|
||
self.publish_db = "/home/ubuntu/.nanobot/workspace/xiaohongshu_publish_records"
|
||
self.monitor_db = "/home/ubuntu/.nanobot/workspace/xiaohongshu_monitor.db"
|
||
self.ab_test_db = "/home/ubuntu/.nanobot/workspace/xiaohongshu_ab_test.db"
|
||
self.output_dir = "/home/ubuntu/.nanobot/workspace/xiaohongshu_analysis"
|
||
|
||
os.makedirs(self.output_dir, exist_ok=True)
|
||
|
||
def analyze_publish_patterns(self, days=30):
|
||
"""分析发布模式"""
|
||
print(f"📊 分析最近{days}天发布模式...")
|
||
|
||
# 收集发布记录
|
||
publish_files = []
|
||
for filename in os.listdir(self.publish_db):
|
||
if filename.endswith('.json'):
|
||
filepath = os.path.join(self.publish_db, filename)
|
||
mtime = os.path.getmtime(filepath)
|
||
file_age_days = (time.time() - mtime) / (24 * 3600)
|
||
|
||
if file_age_days <= days:
|
||
publish_files.append((filepath, mtime))
|
||
|
||
if not publish_files:
|
||
print(f"ℹ️ 最近{days}天无发布记录")
|
||
return None
|
||
|
||
# 分析数据
|
||
publish_dates = []
|
||
sources = []
|
||
titles = []
|
||
|
||
for filepath, mtime in publish_files:
|
||
try:
|
||
with open(filepath, 'r', encoding='utf-8') as f:
|
||
record = json.load(f)
|
||
|
||
publish_time = record.get('publish_time')
|
||
if publish_time:
|
||
publish_dates.append(datetime.fromisoformat(publish_time))
|
||
|
||
article = record.get('article', {})
|
||
sources.append(article.get('source', '未知'))
|
||
titles.append(article.get('title', ''))
|
||
|
||
except Exception as e:
|
||
print(f"❌ 读取文件失败 {filepath}: {e}")
|
||
|
||
# 生成分析报告
|
||
analysis = {
|
||
'total_publishes': len(publish_files),
|
||
'date_range': {
|
||
'start': min(publish_dates).strftime('%Y-%m-%d') if publish_dates else '无',
|
||
'end': max(publish_dates).strftime('%Y-%m-%d') if publish_dates else '无'
|
||
},
|
||
'source_distribution': dict(Counter(sources)),
|
||
'avg_publishes_per_day': len(publish_files) / days if days > 0 else 0,
|
||
'title_length_stats': {
|
||
'avg': np.mean([len(t) for t in titles]) if titles else 0,
|
||
'min': min([len(t) for t in titles]) if titles else 0,
|
||
'max': max([len(t) for t in titles]) if titles else 0
|
||
}
|
||
}
|
||
|
||
# 生成可视化
|
||
self._create_publish_visualization(publish_dates, sources, days)
|
||
|
||
return analysis
|
||
|
||
def _create_publish_visualization(self, publish_dates, sources, days):
|
||
"""创建发布可视化"""
|
||
try:
|
||
# 按日期统计
|
||
date_counts = {}
|
||
for date in publish_dates:
|
||
date_str = date.strftime('%Y-%m-%d')
|
||
date_counts[date_str] = date_counts.get(date_str, 0) + 1
|
||
|
||
# 按来源统计
|
||
source_counts = Counter(sources)
|
||
|
||
# 创建图表
|
||
fig, axes = plt.subplots(2, 2, figsize=(15, 10))
|
||
|
||
# 1. 发布时间分布
|
||
if date_counts:
|
||
dates = list(date_counts.keys())
|
||
counts = list(date_counts.values())
|
||
|
||
axes[0, 0].bar(dates, counts)
|
||
axes[0, 0].set_title('每日发布数量')
|
||
axes[0, 0].set_xlabel('日期')
|
||
axes[0, 0].set_ylabel('发布数量')
|
||
axes[0, 0].tick_params(axis='x', rotation=45)
|
||
|
||
# 2. 来源分布
|
||
if source_counts:
|
||
sources_list = list(source_counts.keys())
|
||
source_values = list(source_counts.values())
|
||
|
||
axes[0, 1].pie(source_values, labels=sources_list, autopct='%1.1f%%')
|
||
axes[0, 1].set_title('文章来源分布')
|
||
|
||
# 3. 发布时间点分布
|
||
if publish_dates:
|
||
hours = [d.hour for d in publish_dates]
|
||
hour_counts = Counter(hours)
|
||
|
||
hour_keys = list(hour_counts.keys())
|
||
hour_values = [hour_counts.get(h, 0) for h in range(24)]
|
||
|
||
axes[1, 0].bar(range(24), hour_values)
|
||
axes[1, 0].set_title('发布时间点分布')
|
||
axes[1, 0].set_xlabel('小时')
|
||
axes[1, 0].set_ylabel('发布数量')
|
||
axes[1, 0].set_xticks(range(0, 24, 2))
|
||
|
||
# 4. 发布趋势
|
||
if date_counts:
|
||
# 按日期排序
|
||
sorted_dates = sorted(date_counts.items())
|
||
trend_dates = [d[0] for d in sorted_dates]
|
||
trend_counts = [d[1] for d in sorted_dates]
|
||
|
||
axes[1, 1].plot(trend_dates, trend_counts, marker='o')
|
||
axes[1, 1].set_title('发布趋势')
|
||
axes[1, 1].set_xlabel('日期')
|
||
axes[1, 1].set_ylabel('发布数量')
|
||
axes[1, 1].tick_params(axis='x', rotation=45)
|
||
|
||
plt.tight_layout()
|
||
|
||
# 保存图表
|
||
chart_file = os.path.join(self.output_dir, f'publish_analysis_{days}d.png')
|
||
plt.savefig(chart_file, dpi=150, bbox_inches='tight')
|
||
plt.close()
|
||
|
||
print(f"📈 可视化图表已保存: {chart_file}")
|
||
|
||
except Exception as e:
|
||
print(f"❌ 创建可视化失败: {e}")
|
||
|
||
def analyze_cookie_health(self, days=30):
|
||
"""分析Cookie健康状况"""
|
||
print(f"🔍 分析最近{days}天Cookie健康状况...")
|
||
|
||
try:
|
||
conn = sqlite3.connect(self.monitor_db)
|
||
cursor = conn.cursor()
|
||
|
||
# 查询最近数据
|
||
cursor.execute('''
|
||
SELECT
|
||
DATE(check_time) as check_date,
|
||
COUNT(*) as total_checks,
|
||
SUM(CASE WHEN cookie_valid THEN 1 ELSE 0 END) as valid_checks
|
||
FROM cookie_monitor
|
||
WHERE check_time >= datetime('now', ?)
|
||
GROUP BY DATE(check_time)
|
||
ORDER BY check_date DESC
|
||
''', (f'-{days} days',))
|
||
|
||
daily_stats = cursor.fetchall()
|
||
|
||
# 计算总体统计
|
||
cursor.execute('''
|
||
SELECT
|
||
COUNT(*) as total_checks,
|
||
SUM(CASE WHEN cookie_valid THEN 1 ELSE 0 END) as valid_checks
|
||
FROM cookie_monitor
|
||
WHERE check_time >= datetime('now', ?)
|
||
''', (f'-{days} days',))
|
||
|
||
overall_stats = cursor.fetchone()
|
||
|
||
conn.close()
|
||
|
||
if not overall_stats or overall_stats[0] == 0:
|
||
print(f"ℹ️ 最近{days}天无Cookie检查记录")
|
||
return None
|
||
|
||
total_checks, valid_checks = overall_stats
|
||
success_rate = (valid_checks / total_checks) * 100 if total_checks > 0 else 0
|
||
|
||
analysis = {
|
||
'total_checks': total_checks,
|
||
'valid_checks': valid_checks,
|
||
'success_rate': success_rate,
|
||
'daily_stats': []
|
||
}
|
||
|
||
for check_date, daily_total, daily_valid in daily_stats:
|
||
daily_rate = (daily_valid / daily_total) * 100 if daily_total > 0 else 0
|
||
analysis['daily_stats'].append({
|
||
'date': check_date,
|
||
'total': daily_total,
|
||
'valid': daily_valid,
|
||
'rate': daily_rate
|
||
})
|
||
|
||
# 生成健康报告
|
||
health_status = "良好"
|
||
if success_rate < 80:
|
||
health_status = "一般"
|
||
if success_rate < 50:
|
||
health_status = "较差"
|
||
if success_rate < 20:
|
||
health_status = "危险"
|
||
|
||
analysis['health_status'] = health_status
|
||
analysis['recommendation'] = self._get_cookie_recommendation(success_rate)
|
||
|
||
return analysis
|
||
|
||
except Exception as e:
|
||
print(f"❌ 分析Cookie健康失败: {e}")
|
||
return None
|
||
|
||
def _get_cookie_recommendation(self, success_rate):
|
||
"""获取Cookie建议"""
|
||
if success_rate >= 90:
|
||
return "Cookie状态优秀，无需操作"
|
||
elif success_rate >= 80:
|
||
return "Cookie状态良好，建议定期检查"
|
||
elif success_rate >= 60:
|
||
return "Cookie状态一般，建议近期检查更新"
|
||
elif success_rate >= 40:
|
||
return "Cookie状态较差，建议尽快更新"
|
||
else:
|
||
return "Cookie状态危险，需要立即更新"
|
||
|
||
def analyze_ab_test_results(self, days=30):
|
||
"""分析A/B测试结果"""
|
||
print(f"🎯 分析最近{days}天A/B测试结果...")
|
||
|
||
try:
|
||
conn = sqlite3.connect(self.ab_test_db)
|
||
cursor = conn.cursor()
|
||
|
||
# 查询测试数据
|
||
cursor.execute('''
|
||
SELECT
|
||
style_used,
|
||
COUNT(*) as test_count,
|
||
AVG(engagement_score) as avg_score,
|
||
MAX(engagement_score) as max_score,
|
||
SUM(likes) as total_likes,
|
||
SUM(collects) as total_collects,
|
||
SUM(comments) as total_comments,
|
||
SUM(shares) as total_shares
|
||
FROM ab_tests
|
||
WHERE test_date >= datetime('now', ?)
|
||
AND engagement_score > 0
|
||
GROUP BY style_used
|
||
ORDER BY avg_score DESC
|
||
''', (f'-{days} days',))
|
||
|
||
style_results = cursor.fetchall()
|
||
|
||
# 查询最佳表现
|
||
cursor.execute('''
|
||
SELECT
|
||
article_title,
|
||
style_used,
|
||
engagement_score,
|
||
likes,
|
||
collects,
|
||
comments,
|
||
shares
|
||
FROM ab_tests
|
||
WHERE test_date >= datetime('now', ?)
|
||
AND engagement_score > 0
|
||
ORDER BY engagement_score DESC
|
||
LIMIT 5
|
||
''', (f'-{days} days',))
|
||
|
||
top_performers = cursor.fetchall()
|
||
|
||
conn.close()
|
||
|
||
if not style_results:
|
||
print(f"ℹ️ 最近{days}天无A/B测试数据")
|
||
return None
|
||
|
||
analysis = {
|
||
'total_tests': sum(r[1] for r in style_results),
|
||
'style_performance': [],
|
||
'top_performers': [],
|
||
'recommendations': []
|
||
}
|
||
|
||
# 分析风格表现
|
||
for style, count, avg_score, max_score, likes, collects, comments, shares in style_results:
|
||
analysis['style_performance'].append({
|
||
'style': style,
|
||
'test_count': count,
|
||
'avg_score': avg_score,
|
||
'max_score': max_score,
|
||
'total_likes': likes or 0,
|
||
'total_collects': collects or 0,
|
||
'total_comments': comments or 0,
|
||
'total_shares': shares or 0
|
||
})
|
||
|
||
# 分析最佳表现
|
||
for title, style, score, likes, collects, comments, shares in top_performers:
|
||
analysis['top_performers'].append({
|
||
'title': title[:50] + '...' if len(title) > 50 else title,
|
||
'style': style,
|
||
'score': score,
|
||
'likes': likes or 0,
|
||
'collects': collects or 0,
|
||
'comments': comments or 0,
|
||
'shares': shares or 0
|
||
})
|
||
|
||
# 生成推荐
|
||
best_style = max(analysis['style_performance'], key=lambda x: x['avg_score'])
|
||
worst_style = min(analysis['style_performance'], key=lambda x: x['avg_score'])
|
||
|
||
analysis['recommendations'].append({
|
||
'type': '最佳风格',
|
||
'style': best_style['style'],
|
||
'reason': f"平均互动分数最高: {best_style['avg_score']:.2f}",
|
||
'confidence': min(best_style['test_count'] / 5, 1.0)
|
||
})
|
||
|
||
analysis['recommendations'].append({
|
||
'type': '避免风格',
|
||
'style': worst_style['style'],
|
||
'reason': f"平均互动分数最低: {worst_style['avg_score']:.2f}",
|
||
'suggestion': '减少使用或优化该风格'
|
||
})
|
||
|
||
# 互动类型分析
|
||
total_likes = sum(s['total_likes'] for s in analysis['style_performance'])
|
||
total_collects = sum(s['total_collects'] for s in analysis['style_performance'])
|
||
total_comments = sum(s['total_comments'] for s in analysis['style_performance'])
|
||
total_shares = sum(s['total_shares'] for s in analysis['style_performance'])
|
||
|
||
total_engagement = total_likes + total_collects + total_comments + total_shares
|
||
|
||
if total_engagement > 0:
|
||
engagement_distribution = {
|
||
'likes': total_likes / total_engagement,
|
||
'collects': total_collects / total_engagement,
|
||
'comments': total_comments / total_engagement,
|
||
'shares': total_shares / total_engagement
|
||
}
|
||
|
||
# 分析哪种互动最多
|
||
max_engagement = max(engagement_distribution.items(), key=lambda x: x[1])
|
||
analysis['engagement_insight'] = {
|
||
'distribution': engagement_distribution,
|
||
'most_common': max_engagement[0],
|
||
'percentage': max_engagement[1] * 100
|
||
}
|
||
|
||
return analysis
|
||
|
||
except Exception as e:
|
||
print(f"❌ 分析A/B测试失败: {e}")
|
||
return None
|
||
|
||
def generate_comprehensive_report(self, days=30):
|
||
"""生成综合分析报告"""
|
||
print(f"📋 生成{days}天综合分析报告...")
|
||
|
||
report = f"📊 小红书自动化系统综合分析报告\n"
|
||
report += "=" * 60 + "\n\n"
|
||
report += f"📅 报告周期: 最近{days}天\n"
|
||
report += f"⏰ 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n"
|
||
|
||
# 1. 发布模式分析
|
||
publish_analysis = self.analyze_publish_patterns(days)
|
||
if publish_analysis:
|
||
report += "1. 📱 发布模式分析\n"
|
||
report += f" 总发布数量: {publish_analysis['total_publishes']} 篇\n"
|
||
report += f" 时间范围: {publish_analysis['date_range']['start']} 至 {publish_analysis['date_range']['end']}\n"
|
||
report += f" 日均发布: {publish_analysis['avg_publishes_per_day']:.2f} 篇\n\n"
|
||
|
||
report += " 文章来源分布:\n"
|
||
for source, count in publish_analysis['source_distribution'].items():
|
||
percentage = (count / publish_analysis['total_publishes']) * 100
|
||
report += f" {source}: {count} 篇 ({percentage:.1f}%)\n"
|
||
|
||
report += f"\n 标题长度统计:\n"
|
||
report += f" 平均: {publish_analysis['title_length_stats']['avg']:.1f} 字符\n"
|
||
report += f" 最短: {publish_analysis['title_length_stats']['min']} 字符\n"
|
||
report += f" 最长: {publish_analysis['title_length_stats']['max']} 字符\n\n"
|
||
else:
|
||
report += "1. 📱 发布模式分析: 无数据\n\n"
|
||
|
||
# 2. Cookie健康分析
|
||
cookie_analysis = self.analyze_cookie_health(days)
|
||
if cookie_analysis:
|
||
report += "2. 🔐 Cookie健康分析\n"
|
||
report += f" 总检查次数: {cookie_analysis['total_checks']} 次\n"
|
||
report += f" 有效检查: {cookie_analysis['valid_checks']} 次\n"
|
||
report += f" 成功率: {cookie_analysis['success_rate']:.1f}%\n"
|
||
report += f" 健康状态: {cookie_analysis['health_status']}\n"
|
||
report += f" 建议: {cookie_analysis['recommendation']}\n\n"
|
||
|
||
# 显示最近7天数据
|
||
recent_days = min(7, len(cookie_analysis['daily_stats']))
|
||
if recent_days > 0:
|
||
report += f" 最近{recent_days}天成功率:\n"
|
||
for i in range(recent_days):
|
||
daily = cookie_analysis['daily_stats'][i]
|
||
report += f" {daily['date']}: {daily['rate']:.1f}% ({daily['valid']}/{daily['total']})\n"
|
||
report += "\n"
|
||
else:
|
||
report += "2. 🔐 Cookie健康分析: 无数据\n\n"
|
||
|
||
# 3. A/B测试分析
|
||
ab_test_analysis = self.analyze_ab_test_results(days)
|
||
if ab_test_analysis:
|
||
report += "3. 🎯 A/B测试分析\n"
|
||
report += f" 总测试数量: {ab_test_analysis['total_tests']} 次\n\n"
|
||
|
||
report += " 风格表现排名:\n"
|
||
for i, style in enumerate(ab_test_analysis['style_performance'], 1):
|
||
report += f" {i}. {style['style']}: {style['avg_score']:.2f} 分 (测试{style['test_count']}次)\n"
|
||
report += "\n"
|
||
|
||
report += " 最佳表现文章:\n"
|
||
for performer in ab_test_analysis['top_performers'][:3]:
|
||
report += f" 📝 {performer['title']}\n"
|
||
report += f" 风格: {performer['style']}, 分数: {performer['score']:.2f}\n"
|
||
report += f" 互动: 👍{performer['likes']} ⭐{performer['collects']} 💬{performer['comments']} 📤{performer['shares']}\n"
|
||
report += "\n"
|
||
|
||
report += " 优化建议:\n"
|
||
for rec in ab_test_analysis['recommendations']:
|
||
report += f" {rec['type']}: {rec['style']}\n"
|
||
report += f" 理由: {rec['reason']}\n"
|
||
if 'confidence' in rec:
|
||
report += f" 置信度: {rec['confidence']:.1%}\n"
|
||
if 'suggestion' in rec:
|
||
report += f" 建议: {rec['suggestion']}\n"
|
||
report += "\n"
|
||
|
||
if 'engagement_insight' in ab_test_analysis:
|
||
insight = ab_test_analysis['engagement_insight']
|
||
report += " 互动洞察:\n"
|
||
report += f" 最多互动类型: {insight['most_common']} ({insight['percentage']:.1f}%)\n"
|
||
report += f" 点赞占比: {insight['distribution']['likes']*100:.1f}%\n"
|
||
report += f" 收藏占比: {insight['distribution']['collects']*100:.1f}%\n"
|
||
report += f" 评论占比: {insight['distribution']['comments']*100:.1f}%\n"
|
||
report += f" 分享占比: {insight['distribution']['shares']*100:.1f}%\n\n"
|
||
else:
|
||
report += "3. 🎯 A/B测试分析: 无数据\n\n"
|
||
|
||
# 4. 系统建议
|
||
report += "4. 💡 系统优化建议\n"
|
||
|
||
suggestions = []
|
||
|
||
# 基于发布模式建议
|
||
if publish_analysis:
|
||
if publish_analysis['avg_publishes_per_day'] < 0.5:
|
||
suggestions.append("增加发布频率，建议每天至少发布1篇")
|
||
|
||
# 检查来源多样性
|
||
source_count = len(publish_analysis['source_distribution'])
|
||
if source_count < 3:
|
||
suggestions.append(f"增加文章来源多样性，当前只有{source_count}个来源")
|
||
|
||
# 基于Cookie健康建议
|
||
if cookie_analysis:
|
||
if cookie_analysis['success_rate'] < 80:
|
||
suggestions.append("Cookie成功率较低，建议检查更新")
|
||
|
||
# 基于A/B测试建议
|
||
if ab_test_analysis:
|
||
if ab_test_analysis['total_tests'] < 10:
|
||
suggestions.append("A/B测试数据不足，建议继续收集数据")
|
||
|
||
if suggestions:
|
||
for i, suggestion in enumerate(suggestions, 1):
|
||
report += f" {i}. {suggestion}\n"
|
||
else:
|
||
report += " 系统运行良好，暂无优化建议\n"
|
||
|
||
report += "\n" + "=" * 60 + "\n"
|
||
report += "📁 相关文件:\n"
|
||
report += f" 发布记录: {self.publish_db}/\n"
|
||
report += f" 监控数据库: {self.monitor_db}\n"
|
||
report += f" A/B测试数据库: {self.ab_test_db}\n"
|
||
report += f" 分析输出: {self.output_dir}/\n"
|
||
report += "\n⚡ 快速命令:\n"
|
||
report += " 查看报告: python3 data_analyzer.py report --days 30\n"
|
||
report += " 分析发布: python3 data_analyzer.py publish --days 30\n"
|
||
report += " 分析Cookie: python3 data_analyzer.py cookie --days 30\n"
|
||
report += " 分析A/B测试: python3 data_analyzer.py abtest --days 30\n"
|
||
|
||
# 保存报告到文件
|
||
report_file = os.path.join(self.output_dir, f'comprehensive_report_{days}d_{datetime.now().strftime("%Y%m%d_%H%M%S")}.txt')
|
||
with open(report_file, 'w', encoding='utf-8') as f:
|
||
f.write(report)
|
||
|
||
print(f"✅ 综合分析报告已保存: {report_file}")
|
||
|
||
return report
|
||
|
||
def main():
|
||
"""主函数"""
|
||
import argparse
|
||
import time
|
||
|
||
parser = argparse.ArgumentParser(description='小红书数据分析器')
|
||
subparsers = parser.add_subparsers(dest='command', help='命令')
|
||
|
||
# report命令
|
||
report_parser = subparsers.add_parser('report', help='生成综合分析报告')
|
||
report_parser.add_argument('--days', type=int, default=30, help='分析天数')
|
||
|
||
# publish命令
|
||
publish_parser = subparsers.add_parser('publish', help='分析发布模式')
|
||
publish_parser.add_argument('--days', type=int, default=30, help='分析天数')
|
||
|
||
# cookie命令
|
||
cookie_parser = subparsers.add_parser('cookie', help='分析Cookie健康')
|
||
cookie_parser.add_argument('--days', type=int, default=30, help='分析天数')
|
||
|
||
# abtest命令
|
||
abtest_parser = subparsers.add_parser('abtest', help='分析A/B测试')
|
||
abtest_parser.add_argument('--days', type=int, default=30, help='分析天数')
|
||
|
||
args = parser.parse_args()
|
||
|
||
analyzer = XiaohongshuDataAnalyzer()
|
||
|
||
if args.command == 'report':
|
||
report = analyzer.generate_comprehensive_report(args.days)
|
||
print(report)
|
||
|
||
elif args.command == 'publish':
|
||
analysis = analyzer.analyze_publish_patterns(args.days)
|
||
if analysis:
|
||
print(json.dumps(analysis, indent=2, ensure_ascii=False))
|
||
else:
|
||
print(f"最近{args.days}天无发布数据")
|
||
|
||
elif args.command == 'cookie':
|
||
analysis = analyzer.analyze_cookie_health(args.days)
|
||
if analysis:
|
||
print(json.dumps(analysis, indent=2, ensure_ascii=False))
|
||
else:
|
||
print(f"最近{args.days}天无Cookie数据")
|
||
|
||
elif args.command == 'abtest':
|
||
analysis = analyzer.analyze_ab_test_results(args.days)
|
||
if analysis:
|
||
print(json.dumps(analysis, indent=2, ensure_ascii=False))
|
||
else:
|
||
print(f"最近{args.days}天无A/B测试数据")
|
||
|
||
else:
|
||
parser.print_help()
|
||
|
||
if __name__ == "__main__":
|
||
main() |