a
This commit is contained in:
@@ -0,0 +1,383 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
小红书内容转换器 - 修复版
|
||||
专门处理IT资讯自动化系统生成的Markdown文件
|
||||
"""
|
||||
|
||||
import re
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class Article:
|
||||
"""文章数据结构"""
|
||||
title: str
|
||||
content: str
|
||||
source: str = ""
|
||||
url: str = ""
|
||||
keywords: List[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.keywords is None:
|
||||
self.keywords = []
|
||||
|
||||
@dataclass
|
||||
class XiaohongshuContent:
|
||||
"""小红书内容结构"""
|
||||
title: str
|
||||
content: str
|
||||
cover_suggestions: List[str] = None
|
||||
tags: List[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.cover_suggestions is None:
|
||||
self.cover_suggestions = []
|
||||
if self.tags is None:
|
||||
self.tags = []
|
||||
|
||||
class ContentConverterFixed:
|
||||
"""修复版内容转换器"""
|
||||
|
||||
def __init__(self, config: Optional[Dict] = None):
|
||||
self.config = config or {}
|
||||
logger.info("修复版内容转换器初始化完成")
|
||||
|
||||
def parse_markdown(self, markdown_content: str) -> Article:
|
||||
"""
|
||||
解析Markdown文件,提取真正的文章内容
|
||||
|
||||
Args:
|
||||
markdown_content: Markdown文件内容
|
||||
|
||||
Returns:
|
||||
文章对象
|
||||
"""
|
||||
lines = markdown_content.split('\n')
|
||||
|
||||
# 提取标题(第一行)
|
||||
title = ""
|
||||
if lines and lines[0].startswith('# '):
|
||||
title = lines[0][2:].strip()
|
||||
|
||||
# 查找文章内容部分
|
||||
content_start = -1
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip() == "## 📰 文章内容(DeepSeek翻译)":
|
||||
content_start = i + 1
|
||||
break
|
||||
|
||||
# 提取真正的文章内容 - 修复:只提取DeepSeek翻译的正文
|
||||
content_lines = []
|
||||
if content_start > 0:
|
||||
# 跳过空行和可能的标题标记行
|
||||
i = content_start
|
||||
while i < len(lines):
|
||||
line = lines[i].strip()
|
||||
# 跳过空行和标题标记行
|
||||
if not line or line.startswith('**标题:') or line.startswith('**核心内容翻译:'):
|
||||
i += 1
|
||||
continue
|
||||
# 找到真正的内容开始
|
||||
break
|
||||
|
||||
# 收集真正的文章内容(直到下一个##标题或---分隔符)
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
# 如果遇到下一个标题或分隔符,停止收集
|
||||
if line.startswith('## ') or line.strip() == '---':
|
||||
break
|
||||
content_lines.append(line)
|
||||
i += 1
|
||||
|
||||
# 如果还是没有找到,尝试查找中文内容
|
||||
if not content_lines:
|
||||
for i, line in enumerate(lines):
|
||||
# 查找典型的中文内容开始
|
||||
if '本文报道了' in line or '文章指出' in line or '主要观点' in line:
|
||||
# 从这个段落开始收集
|
||||
j = i
|
||||
while j < len(lines) and not lines[j].startswith('## ') and lines[j].strip() != '---':
|
||||
if lines[j].strip():
|
||||
content_lines.append(lines[j])
|
||||
j += 1
|
||||
break
|
||||
|
||||
# 如果还是没有,使用整个内容
|
||||
if not content_lines:
|
||||
content_lines = lines
|
||||
|
||||
content = '\n'.join(content_lines).strip()
|
||||
|
||||
# 提取来源
|
||||
source = ""
|
||||
for line in lines:
|
||||
if "**来源**:" in line or "来源:" in line:
|
||||
match = re.search(r'[::]\s*(.+)', line)
|
||||
if match:
|
||||
source = match.group(1).strip()
|
||||
break
|
||||
|
||||
# 提取原文链接
|
||||
url = ""
|
||||
for line in lines:
|
||||
if "原文链接:" in line or "**原文链接**:" in line:
|
||||
match = re.search(r'https?://[^\s]+', line)
|
||||
if match:
|
||||
url = match.group(0).strip()
|
||||
break
|
||||
|
||||
return Article(
|
||||
title=title,
|
||||
content=content,
|
||||
source=source,
|
||||
url=url
|
||||
)
|
||||
|
||||
def extract_main_content(self, content: str) -> str:
|
||||
"""
|
||||
从文章内容中提取核心部分
|
||||
|
||||
Args:
|
||||
content: 原始文章内容
|
||||
|
||||
Returns:
|
||||
提取后的核心内容
|
||||
"""
|
||||
# 移除标题行
|
||||
lines = content.split('\n')
|
||||
filtered_lines = []
|
||||
|
||||
for line in lines:
|
||||
# 跳过空行和标题行
|
||||
if not line.strip():
|
||||
continue
|
||||
if line.startswith('**标题:'):
|
||||
continue
|
||||
if line.startswith('**核心内容翻译:'):
|
||||
continue
|
||||
if line.startswith('---'):
|
||||
continue
|
||||
|
||||
filtered_lines.append(line)
|
||||
|
||||
return '\n\n'.join(filtered_lines)
|
||||
|
||||
def to_xiaohongshu_basic(self, article: Article) -> str:
|
||||
"""
|
||||
基础转换:将文章转换为小红书格式
|
||||
|
||||
Args:
|
||||
article: 文章对象
|
||||
|
||||
Returns:
|
||||
小红书格式的内容
|
||||
"""
|
||||
# 提取核心内容
|
||||
main_content = self.extract_main_content(article.content)
|
||||
|
||||
# 处理标题
|
||||
title = self._process_title(article.title)
|
||||
|
||||
# 构建小红书内容
|
||||
xhs_content = ""
|
||||
|
||||
# 开头
|
||||
xhs_content += f"姐妹们!今天分享一个超实用的技术资讯\n\n"
|
||||
xhs_content += f"往下看 ⬇️\n\n"
|
||||
|
||||
# 标题
|
||||
xhs_content += f"【{title}】\n\n"
|
||||
|
||||
# 来源
|
||||
if article.source:
|
||||
xhs_content += f"来源: {article.source}\n\n"
|
||||
xhs_content += "—————\n\n"
|
||||
|
||||
# 核心内容(限制长度)
|
||||
if main_content:
|
||||
# 限制内容长度(小红书适合短内容)
|
||||
if len(main_content) > 800:
|
||||
# 取前800字符,并在完整句子处截断
|
||||
truncated = main_content[:800]
|
||||
last_period = max(
|
||||
truncated.rfind('。'),
|
||||
truncated.rfind('!'),
|
||||
truncated.rfind('?'),
|
||||
truncated.rfind('.')
|
||||
)
|
||||
if last_period > 600: # 确保有足够内容
|
||||
main_content = truncated[:last_period + 1]
|
||||
else:
|
||||
main_content = truncated + "..."
|
||||
|
||||
xhs_content += f"{main_content}\n\n"
|
||||
|
||||
# 结尾
|
||||
xhs_content += "—————\n\n"
|
||||
xhs_content += "有用的话记得收藏 📌\n\n"
|
||||
|
||||
# 标签
|
||||
tags = self._generate_tags(article)
|
||||
xhs_content += f"{tags}\n\n"
|
||||
|
||||
# 互动提示
|
||||
xhs_content += "💬 你对这个话题有什么看法?欢迎在评论区讨论!\n"
|
||||
xhs_content += "⭐ 如果觉得有用,记得点赞收藏哦~\n\n"
|
||||
|
||||
# 原文链接
|
||||
if article.url:
|
||||
xhs_content += f"🔗 原文链接: {article.url}"
|
||||
|
||||
return xhs_content
|
||||
|
||||
def _process_title(self, title: str) -> str:
|
||||
"""处理标题"""
|
||||
# 缩短过长的标题
|
||||
if len(title) > 40:
|
||||
title = title[:37] + "..."
|
||||
return title
|
||||
|
||||
def _generate_tags(self, article: Article) -> str:
|
||||
"""生成标签"""
|
||||
base_tags = ["#来源", "#干货分享", "#实用技巧", "#技术资讯", "#科技新闻", "#技术"]
|
||||
|
||||
# 根据来源添加特定标签
|
||||
if "MIT" in article.source or "Technology Review" in article.source:
|
||||
base_tags.append("#MIT")
|
||||
base_tags.append("#AI技术")
|
||||
|
||||
if "The Verge" in article.source:
|
||||
base_tags.append("#TheVerge")
|
||||
base_tags.append("#科技媒体")
|
||||
|
||||
if "InfoQ" in article.source:
|
||||
base_tags.append("#InfoQ")
|
||||
base_tags.append("#开发者")
|
||||
|
||||
return " ".join(base_tags)
|
||||
|
||||
def convert_markdown_to_xiaohongshu(self, markdown_content: str) -> str:
|
||||
"""
|
||||
主转换函数:将Markdown文件转换为小红书格式
|
||||
|
||||
Args:
|
||||
markdown_content: Markdown文件内容
|
||||
|
||||
Returns:
|
||||
小红书格式的内容
|
||||
"""
|
||||
try:
|
||||
# 1. 解析Markdown文件
|
||||
article = self.parse_markdown(markdown_content)
|
||||
|
||||
logger.info(f"解析文章: {article.title}")
|
||||
logger.info(f"来源: {article.source}")
|
||||
logger.info(f"内容长度: {len(article.content)} 字符")
|
||||
|
||||
# 2. 转换为小红书格式
|
||||
xhs_content = self.to_xiaohongshu_basic(article)
|
||||
|
||||
logger.info(f"转换完成,标题: {article.title[:30]}...")
|
||||
return xhs_content
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"转换失败: {e}")
|
||||
|
||||
# 回退:简单转换
|
||||
lines = markdown_content.split('\n')
|
||||
title = lines[0][2:].strip() if lines and lines[0].startswith('# ') else "技术资讯"
|
||||
|
||||
return f"""姐妹们!今天分享一个技术资讯
|
||||
|
||||
往下看 ⬇️
|
||||
|
||||
【{title[:30]}...】
|
||||
|
||||
来源: IT资讯
|
||||
|
||||
—————
|
||||
|
||||
💡 内容处理中遇到问题,请查看原文链接获取完整内容。
|
||||
|
||||
—————
|
||||
|
||||
有用的话记得收藏 📌
|
||||
|
||||
#技术资讯 #科技新闻 #干货分享
|
||||
|
||||
💬 你对这个话题有什么看法?欢迎在评论区讨论!
|
||||
⭐ 如果觉得有用,记得点赞收藏哦~"""
|
||||
|
||||
|
||||
def test_conversion():
|
||||
"""测试转换功能"""
|
||||
converter = ContentConverterFixed()
|
||||
|
||||
# 测试内容
|
||||
test_md = """# Where OpenAI's technology could show up in Iran
|
||||
|
||||
## 📋 文章信息
|
||||
|
||||
- **来源**: MIT Technology Review
|
||||
- **原文标题**: Where OpenAI's technology could show up in Iran
|
||||
- **原文链接**: https://www.technologyreview.com/2026/03/16/1134315/where-openais-technology-could-show-up-in-iran/
|
||||
- **发布日期**: None
|
||||
- **原文字数**: 1,227
|
||||
- **文章类型**: 技术类
|
||||
- **处理方式**: DeepSeek翻译
|
||||
- **处理时间**: 2026-03-17 16:01:00
|
||||
- **抓取时间**: 2026-03-17T16:00:27.836284
|
||||
|
||||
---
|
||||
|
||||
## 📰 文章内容(DeepSeek翻译)
|
||||
|
||||
**标题:OpenAI技术可能在伊朗战场出现的三个关键领域**
|
||||
|
||||
**核心内容翻译:**
|
||||
|
||||
从战争边缘到战斗核心,值得关注的三个领域。
|
||||
|
||||
OpenAI与五角大楼达成允许其在涉密环境中使用AI技术的争议性协议已逾两周。该协议的具体授权范围仍存紧迫疑问:萨姆·奥尔特曼虽宣称军方不得用其技术开发生化武器,但协议仅要求军方遵循自身()相关准则。OpenAI另一主要声明——协议将禁止其技术用于国内监控——同样显得可疑。
|
||||
|
||||
OpenAI的动机尚不明确。这并非科技巨头首次违背"永不涉足"的誓言承接军事合同,但其立场转变速度之快值得注意。或许出于资金压力:OpenAI正投入巨资进行AI训练并寻求更多收入来源(包括广告)。亦或奥尔特曼真心信奉其常提及的意识形态框架:自由民主国家(及其军队)必须掌握最强AI以与中国竞争。
|
||||
|
||||
更关键的问题在于后续发展。随着美国升级对伊朗打击(AI在其中发挥空前作用),OpenAI已决定深入复杂的战斗核心。那么其技术具体可能出现在哪些作战环节?哪些应用会获得客户(及员工)容忍?
|
||||
|
||||
**目标定位与打击**
|
||||
尽管协议已生效,OpenAI技术何时能适配涉密环境仍不确定,因其需与军方现有工具整合(埃隆·马斯克的xAI近期也与五角大楼达成协议,其AI模型Grok预计将经历相同流程)。但整合压力迫在眉睫:在Anthropic拒绝允许其AI用于"任何合法用途"后,特朗普总统下令军方停用该技术,五角大楼更将Anthropic列为供应链风险。
|
||||
|
||||
**作战平台部署**
|
||||
五角大楼去年秋季推出的生成式AI平台GenAI.mil,允许军事单位接入商业AI模型并执行与商业领域类似的任务。谷歌Gemini是首批入驻模型。今年1月五角大楼宣布将纳入xAI的Grok(尽管该模型曾传播反犹内容并生成非自愿深度伪造视频)。OpenAI于2月跟进,宣布其模型将用于起草政策文件、合同及协助任务行政支持。
|
||||
|
||||
虽然使用该平台处理非密任务的人员对伊朗敏感决策影响有限,但OpenAI入驻具有更深层意义:这呼应了赫格塞斯在五角大楼全力推行的AI全覆盖战略(即便许多早期用户并不完全明确使用场景)。其传递的信号是:AI正在重塑美国作战的每个环节——从目标判定到文书工作,而OpenAI正日益赢得其中份额。
|
||||
---
|
||||
处理信息: DeepSeek API | 耗时: 14.24秒
|
||||
|
||||
---
|
||||
原文链接: https://www.technologyreview.com/2026/03/16/1134315/where-openais-technology-could-show-up-in-iran/
|
||||
处理时间: 2026-03-17 16:01:00
|
||||
|
||||
|
||||
## 🖼️ 文章图片
|
||||
|
||||
1. 
|
||||
*描述: Sam Altman collage*
|
||||
"""
|
||||
|
||||
result = converter.convert_markdown_to_xiaohongshu(test_md)
|
||||
|
||||
print("=" * 60)
|
||||
print("修复版转换器测试结果")
|
||||
print("=" * 60)
|
||||
print(result)
|
||||
print("=" * 60)
|
||||
print(f"内容长度: {len(result)} 字符")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
test_conversion()
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
增强版IT资讯抓取系统模板
|
||||
请根据实际环境配置路径
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import random
|
||||
import sqlite3
|
||||
import datetime
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlparse
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import mimetypes
|
||||
|
||||
# ==================== 配置部分 ====================
|
||||
# 请根据实际环境修改以下配置
|
||||
CONFIG = {
|
||||
'database_path': '{{DATABASE_PATH}}', # 数据库文件路径
|
||||
'base_dir': '{{BASE_DIR}}', # 基础目录
|
||||
'min_word_count': 500, # 最小字数要求
|
||||
'max_days_old': 7, # 最大天数(最近N天)
|
||||
'websites': [
|
||||
{'name': 'InfoQ', 'url': 'https://www.infoq.com', 'category': 'tech'},
|
||||
{'name': 'Wired', 'url': 'https://www.wired.com', 'category': 'tech'},
|
||||
{'name': 'TechCrunch', 'url': 'https://techcrunch.com', 'category': 'tech'},
|
||||
# 可以添加更多网站
|
||||
],
|
||||
'user_agents': [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36',
|
||||
],
|
||||
'request_delay': 2, # 请求延迟(秒)
|
||||
'timeout': 30, # 请求超时(秒)
|
||||
}
|
||||
|
||||
# ==================== 数据库操作 ====================
|
||||
def init_database():
|
||||
"""初始化数据库"""
|
||||
conn = sqlite3.connect(CONFIG['database_path'])
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 创建文章表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS articles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
url TEXT UNIQUE NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
category TEXT,
|
||||
publish_date TEXT,
|
||||
word_count INTEGER,
|
||||
processed INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建索引
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON articles(url)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_processed ON articles(processed)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_date ON articles(publish_date)')
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"✅ 数据库初始化完成: {CONFIG['database_path']}")
|
||||
|
||||
def save_article(title, url, source, category, publish_date, word_count):
|
||||
"""保存文章到数据库"""
|
||||
try:
|
||||
conn = sqlite3.connect(CONFIG['database_path'])
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute('''
|
||||
INSERT OR IGNORE INTO articles
|
||||
(title, url, source, category, publish_date, word_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
''', (title, url, source, category, publish_date, word_count))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ 保存文章失败: {e}")
|
||||
return False
|
||||
|
||||
# ==================== 网页抓取 ====================
|
||||
def fetch_with_retry(url, max_retries=3):
|
||||
"""带重试的网页抓取"""
|
||||
headers = {
|
||||
'User-Agent': random.choice(CONFIG['user_agents']),
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Connection': 'keep-alive',
|
||||
}
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
time.sleep(CONFIG['request_delay'])
|
||||
response = requests.get(url, headers=headers, timeout=CONFIG['timeout'])
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
print(f"❌ 抓取失败 ({attempt+1}/{max_retries}): {url} - {e}")
|
||||
return None
|
||||
print(f"⚠️ 重试抓取 ({attempt+1}/{max_retries}): {url}")
|
||||
time.sleep(2 ** attempt) # 指数退避
|
||||
|
||||
def extract_articles_from_html(html, base_url, source_name):
|
||||
"""从HTML中提取文章(需要根据具体网站实现)"""
|
||||
# 这是一个模板函数,需要根据具体网站结构实现
|
||||
articles = []
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# 示例:查找文章链接
|
||||
for link in soup.find_all('a', href=True):
|
||||
href = link['href']
|
||||
title = link.get_text(strip=True)
|
||||
|
||||
# 过滤非文章链接
|
||||
if not title or len(title) < 10:
|
||||
continue
|
||||
|
||||
# 构建完整URL
|
||||
full_url = urljoin(base_url, href)
|
||||
|
||||
# 添加到文章列表
|
||||
articles.append({
|
||||
'title': title,
|
||||
'url': full_url,
|
||||
'source': source_name,
|
||||
'category': 'tech', # 默认分类
|
||||
})
|
||||
|
||||
return articles
|
||||
|
||||
# ==================== 主函数 ====================
|
||||
def main():
|
||||
"""主函数"""
|
||||
print("🚀 开始IT资讯抓取")
|
||||
print(f"📁 工作目录: {CONFIG['base_dir']}")
|
||||
print(f"🗄️ 数据库: {CONFIG['database_path']}")
|
||||
|
||||
# 初始化数据库
|
||||
init_database()
|
||||
|
||||
# 创建当天目录
|
||||
today = datetime.datetime.now().strftime('%Y-%m-%d')
|
||||
day_dir = os.path.join(CONFIG['base_dir'], today)
|
||||
os.makedirs(day_dir, exist_ok=True)
|
||||
|
||||
total_articles = 0
|
||||
|
||||
# 遍历网站
|
||||
for site in CONFIG['websites']:
|
||||
print(f"\n🌐 抓取网站: {site['name']} ({site['url']})")
|
||||
|
||||
response = fetch_with_retry(site['url'])
|
||||
if not response:
|
||||
continue
|
||||
|
||||
articles = extract_articles_from_html(response.text, site['url'], site['name'])
|
||||
|
||||
print(f"📰 找到 {len(articles)} 篇文章")
|
||||
|
||||
# 保存文章
|
||||
for article in articles:
|
||||
# 这里可以添加更多处理逻辑
|
||||
saved = save_article(
|
||||
article['title'],
|
||||
article['url'],
|
||||
article['source'],
|
||||
article.get('category', 'tech'),
|
||||
datetime.datetime.now().strftime('%Y-%m-%d'),
|
||||
0 # 字数需要后续处理
|
||||
)
|
||||
|
||||
if saved:
|
||||
total_articles += 1
|
||||
|
||||
print(f"\n✅ 抓取完成,共保存 {total_articles} 篇文章")
|
||||
return total_articles
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+388
@@ -0,0 +1,388 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
中文去AI化工具
|
||||
基于 oh-my-writing-skill 的 humanizer-cn 技能
|
||||
消除AI写作痕迹,让文章更自然
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
class HumanizerCN:
|
||||
"""中文去AI化处理器"""
|
||||
|
||||
def __init__(self):
|
||||
# AI写作特征模式
|
||||
self.ai_patterns = {
|
||||
# 过度强调意义和历史地位
|
||||
'overemphasis': [
|
||||
r'彰显[了着]', r'见证[了着]', r'标志着', r'具有里程碑意义',
|
||||
r'揭示了', r'体现了', r'承载着', r'奠定了基础',
|
||||
r'树立了标杆', r'开创了先河', r'具有划时代意义', r'承前启后'
|
||||
],
|
||||
|
||||
# 过度强调知名度和媒体报道
|
||||
'overhype': [
|
||||
r'广受关注', r'引发热议', r'备受瞩目', r'获得广泛报道',
|
||||
r'在业界引起强烈反响', r'各大媒体纷纷报道'
|
||||
],
|
||||
|
||||
# "-ing 式"假深度分析
|
||||
'fake_depth': [
|
||||
r'致力于', r'旨在', r'聚焦于', r'着眼于',
|
||||
r'立足于', r'依托于', r'围绕着', r'基于', r'借助于'
|
||||
],
|
||||
|
||||
# 广告式推销语言
|
||||
'advertising': [
|
||||
r'匠心独运', r'精心打造', r'倾力呈现', r'独具匠心',
|
||||
r'别具一格', r'独树一帜', r'引领潮流', r'开启新篇章',
|
||||
r'重新定义'
|
||||
],
|
||||
|
||||
# 模糊归因和万金油说法
|
||||
'vague_attribution': [
|
||||
r'业内人士指出', r'专家表示', r'有观点认为',
|
||||
r'据了解', r'据悉', r'相关人士透露', r'多方消息显示'
|
||||
],
|
||||
|
||||
# 套路化的"挑战与展望"段落
|
||||
'cliche_ending': [
|
||||
r'尽管如此', r'虽然面临挑战', r'但我们有理由相信',
|
||||
r'未来可期', r'前景广阔', r'任重道远', r'砥砺前行'
|
||||
],
|
||||
|
||||
# 高频AI词汇
|
||||
'ai_vocabulary': [
|
||||
r'此外', r'值得一提的是', r'值得注意的是',
|
||||
r'不言而喻', r'毋庸置疑', r'显而易见',
|
||||
r'与此同时', r'综上所述', r'总而言之',
|
||||
r'一言以蔽之', r'鉴于此', r'基于此',
|
||||
r'有鉴于此', r'就此而言', r'从某种意义上说'
|
||||
],
|
||||
|
||||
# 回避简单动词
|
||||
'avoid_simple_verbs': [
|
||||
r'作为[^,。]*而存在', r'扮演着[^,。]*的角色',
|
||||
r'承担着[^,。]*的使命', r'发挥着[^,。]*的作用',
|
||||
r'被视为', r'被认为是'
|
||||
],
|
||||
|
||||
# 否定式并列句
|
||||
'negative_parallel': [
|
||||
r'不仅[^,。]*更是[^,。]*',
|
||||
r'不只是[^,。]*而是[^,。]*'
|
||||
],
|
||||
|
||||
# 三段式堆砌
|
||||
'triple_structure': [
|
||||
r'创新、品质与服务的完美结合',
|
||||
r'高效、稳定、易用三大特点'
|
||||
]
|
||||
}
|
||||
|
||||
# 替换映射
|
||||
self.replacement_map = {
|
||||
# 语气转换
|
||||
'本文将介绍': '今天来聊聊',
|
||||
'建议用户': '建议大家',
|
||||
'该产品性能优秀': '这个真的很好用',
|
||||
'综上所述': '总之就是',
|
||||
'值得注意的是': '划重点',
|
||||
'需要指出的是': '这里要注意',
|
||||
'根据研究显示': '我发现',
|
||||
'一般来说': '通常来说',
|
||||
'具体来说': '具体来说',
|
||||
'换句话说': '简单说就是',
|
||||
'例如': '比如',
|
||||
'此外': '另外',
|
||||
'然而': '不过',
|
||||
'因此': '所以',
|
||||
'最终': '最后',
|
||||
|
||||
# 简化表达
|
||||
'为了能够实现这一目标': '为了',
|
||||
'由于这一事实的存在': '因为',
|
||||
'在当前这个时间点上': '现在',
|
||||
'在如果需要帮助的情况下': '如果需要帮助',
|
||||
'具有能够处理的能力': '能处理',
|
||||
'需要特别指出的是': '',
|
||||
|
||||
# 具体化
|
||||
'该项目一经发布便引发业界广泛关注': '发布后反响不错',
|
||||
'业内专家普遍认为': '有专家认为',
|
||||
'具有广阔的应用前景': '应用前景不错',
|
||||
'对行业产生深远影响': '对行业影响挺大'
|
||||
}
|
||||
|
||||
def detect_ai_patterns(self, text: str) -> Dict:
|
||||
"""检测AI写作模式"""
|
||||
detection = {
|
||||
'total_patterns': 0,
|
||||
'patterns_by_type': {},
|
||||
'suggestions': []
|
||||
}
|
||||
|
||||
for pattern_type, patterns in self.ai_patterns.items():
|
||||
count = 0
|
||||
found_patterns = []
|
||||
|
||||
for pattern in patterns:
|
||||
matches = re.findall(pattern, text)
|
||||
if matches:
|
||||
count += len(matches)
|
||||
found_patterns.extend(matches)
|
||||
|
||||
if count > 0:
|
||||
detection['patterns_by_type'][pattern_type] = {
|
||||
'count': count,
|
||||
'examples': list(set(found_patterns))[:3] # 最多显示3个例子
|
||||
}
|
||||
detection['total_patterns'] += count
|
||||
|
||||
# 生成建议
|
||||
if detection['total_patterns'] > 10:
|
||||
detection['suggestions'].append('AI痕迹较重,建议全面重写')
|
||||
elif detection['total_patterns'] > 5:
|
||||
detection['suggestions'].append('有较多AI痕迹,建议重点修改')
|
||||
elif detection['total_patterns'] > 0:
|
||||
detection['suggestions'].append('有少量AI痕迹,建议微调')
|
||||
else:
|
||||
detection['suggestions'].append('AI痕迹较少,质量不错')
|
||||
|
||||
return detection
|
||||
|
||||
def humanize_text(self, text: str, aggressive: bool = False) -> str:
|
||||
"""去AI化处理"""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
# 第一步:替换模式化表达
|
||||
humanized = text
|
||||
for pattern, replacement in self.replacement_map.items():
|
||||
humanized = humanized.replace(pattern, replacement)
|
||||
|
||||
# 第二步:处理段落结构
|
||||
paragraphs = humanized.split('\n\n')
|
||||
processed_paragraphs = []
|
||||
|
||||
for para in paragraphs:
|
||||
if not para.strip():
|
||||
processed_paragraphs.append(para)
|
||||
continue
|
||||
|
||||
# 缩短长段落
|
||||
if len(para) > 200:
|
||||
sentences = re.split(r'[。!?]', para)
|
||||
if len(sentences) > 3:
|
||||
para = '。'.join(sentences[:3]) + '。'
|
||||
|
||||
# 添加一点"人味"
|
||||
if aggressive and len(para) > 50:
|
||||
# 随机添加一些口语化表达
|
||||
import random
|
||||
human_touches = [
|
||||
'说实话',
|
||||
'我个人觉得',
|
||||
'其实吧',
|
||||
'怎么说呢',
|
||||
'你猜怎么着'
|
||||
]
|
||||
|
||||
if random.random() < 0.3: # 30%概率添加
|
||||
touch = random.choice(human_touches)
|
||||
para = f"{touch},{para}"
|
||||
|
||||
processed_paragraphs.append(para)
|
||||
|
||||
humanized = '\n\n'.join(processed_paragraphs)
|
||||
|
||||
# 第三步:修复过度修饰
|
||||
# 移除过多的修饰词
|
||||
modifiers = ['非常', '极其', '十分', '特别', '相当', '颇为']
|
||||
for modifier in modifiers:
|
||||
pattern = rf'{modifier}[的得地]?[^,。]*?的'
|
||||
humanized = re.sub(pattern, '', humanized)
|
||||
|
||||
# 第四步:简化列表结构
|
||||
lines = humanized.split('\n')
|
||||
simplified_lines = []
|
||||
|
||||
for line in lines:
|
||||
# 简化加粗列表项
|
||||
if re.search(r'\*\*.*?\*\*:', line):
|
||||
line = re.sub(r'\*\*(.*?)\*\*:', r'\1:', line)
|
||||
|
||||
simplified_lines.append(line)
|
||||
|
||||
humanized = '\n'.join(simplified_lines)
|
||||
|
||||
return humanized
|
||||
|
||||
def add_human_touch(self, text: str) -> str:
|
||||
"""添加人味"""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
# 添加一些个人化的表达
|
||||
touches = [
|
||||
'我试过',
|
||||
'我的经验是',
|
||||
'说实话',
|
||||
'个人觉得',
|
||||
'其实吧',
|
||||
'你猜怎么着',
|
||||
'没想到的是'
|
||||
]
|
||||
|
||||
import random
|
||||
|
||||
# 在开头添加
|
||||
if random.random() < 0.4:
|
||||
touch = random.choice(touches)
|
||||
text = f"{touch},{text}"
|
||||
|
||||
# 在中间随机添加
|
||||
sentences = re.split(r'[。!?]', text)
|
||||
if len(sentences) > 2:
|
||||
insert_pos = random.randint(1, len(sentences)-1)
|
||||
touch = random.choice(touches)
|
||||
sentences.insert(insert_pos, f"{touch},")
|
||||
text = '。'.join(sentences)
|
||||
|
||||
return text
|
||||
|
||||
def analyze_and_humanize(self, text: str) -> Dict:
|
||||
"""分析并去AI化"""
|
||||
print("🔍 分析文本中的AI痕迹...")
|
||||
|
||||
# 检测AI模式
|
||||
detection = self.detect_ai_patterns(text)
|
||||
|
||||
print(f"📊 检测结果:")
|
||||
print(f" 总AI模式数: {detection['total_patterns']}")
|
||||
|
||||
for pattern_type, info in detection['patterns_by_type'].items():
|
||||
print(f" {pattern_type}: {info['count']} 处")
|
||||
if info['examples']:
|
||||
print(f" 示例: {', '.join(info['examples'][:2])}")
|
||||
|
||||
# 去AI化处理
|
||||
print("\n🛠️ 进行去AI化处理...")
|
||||
humanized = self.humanize_text(text, aggressive=detection['total_patterns'] > 5)
|
||||
|
||||
# 添加人味(如果需要)
|
||||
if detection['total_patterns'] > 3:
|
||||
print("💬 添加人味...")
|
||||
humanized = self.add_human_touch(humanized)
|
||||
|
||||
# 计算改进程度
|
||||
original_length = len(text)
|
||||
humanized_length = len(humanized)
|
||||
length_change = ((humanized_length - original_length) / original_length) * 100
|
||||
|
||||
result = {
|
||||
'original_text': text,
|
||||
'humanized_text': humanized,
|
||||
'detection': detection,
|
||||
'stats': {
|
||||
'original_length': original_length,
|
||||
'humanized_length': humanized_length,
|
||||
'length_change_percent': round(length_change, 2),
|
||||
'ai_patterns_removed': detection['total_patterns']
|
||||
},
|
||||
'suggestions': detection['suggestions']
|
||||
}
|
||||
|
||||
print(f"✅ 处理完成")
|
||||
print(f" 原文长度: {original_length} 字符")
|
||||
print(f" 处理后长度: {humanized_length} 字符")
|
||||
print(f" 长度变化: {length_change:.1f}%")
|
||||
print(f" 建议: {', '.join(detection['suggestions'])}")
|
||||
|
||||
return result
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='中文去AI化工具')
|
||||
subparsers = parser.add_subparsers(dest='command', help='命令')
|
||||
|
||||
# analyze命令
|
||||
analyze_parser = subparsers.add_parser('analyze', help='分析文本')
|
||||
analyze_parser.add_argument('--text', help='要分析的文本')
|
||||
analyze_parser.add_argument('--file', help='文本文件路径')
|
||||
|
||||
# humanize命令
|
||||
humanize_parser = subparsers.add_parser('humanize', help='去AI化处理')
|
||||
humanize_parser.add_argument('--text', help='要处理的文本')
|
||||
humanize_parser.add_argument('--file', help='文本文件路径')
|
||||
humanize_parser.add_argument('--output', help='输出文件路径')
|
||||
humanize_parser.add_argument('--aggressive', action='store_true', help='激进模式')
|
||||
|
||||
# test命令
|
||||
subparsers.add_parser('test', help='测试功能')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
humanizer = HumanizerCN()
|
||||
|
||||
if args.command == 'analyze':
|
||||
text = args.text
|
||||
if args.file:
|
||||
with open(args.file, 'r', encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
|
||||
if not text:
|
||||
print("❌ 请提供文本")
|
||||
return
|
||||
|
||||
result = humanizer.analyze_and_humanize(text)
|
||||
print("\n📊 分析结果:")
|
||||
print(json.dumps(result['detection'], indent=2, ensure_ascii=False))
|
||||
|
||||
elif args.command == 'humanize':
|
||||
text = args.text
|
||||
if args.file:
|
||||
with open(args.file, 'r', encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
|
||||
if not text:
|
||||
print("❌ 请提供文本")
|
||||
return
|
||||
|
||||
result = humanizer.analyze_and_humanize(text)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(result['humanized_text'])
|
||||
print(f"✅ 已保存到: {args.output}")
|
||||
else:
|
||||
print("\n📝 处理后的文本:")
|
||||
print("=" * 60)
|
||||
print(result['humanized_text'])
|
||||
print("=" * 60)
|
||||
|
||||
elif args.command == 'test':
|
||||
test_text = """该软件的发布标志着人工智能领域的一个重要里程碑,彰显了公司在技术创新方面的不懈追求。此外,值得一提的是,该产品具有高性能、低功耗、易扩展三大优势。业内专家普遍认为,这将对整个行业产生深远影响。尽管面临诸多挑战,但我们有理由相信,未来可期。"""
|
||||
|
||||
print("🧪 测试去AI化功能")
|
||||
print("=" * 60)
|
||||
print("测试文本:")
|
||||
print(test_text)
|
||||
print("=" * 60)
|
||||
|
||||
result = humanizer.analyze_and_humanize(test_text)
|
||||
|
||||
print("\n📝 处理后的文本:")
|
||||
print("=" * 60)
|
||||
print(result['humanized_text'])
|
||||
print("=" * 60)
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/bin/bash
|
||||
# IT资讯自动化系统 - 主调度脚本模板
|
||||
# 请根据实际环境配置路径
|
||||
|
||||
echo "============================================================"
|
||||
echo "🚀 IT资讯自动化系统启动"
|
||||
echo "============================================================"
|
||||
echo "执行时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
# ==================== 配置区域 ====================
|
||||
# 请根据实际环境修改以下路径
|
||||
WORK_DIR="{{WORK_DIR}}" # 工作目录
|
||||
XHS_SKILL_DIR="{{XHS_SKILL_DIR}}" # 小红书技能目录
|
||||
LOG_DIR="{{LOG_DIR}}" # 日志目录
|
||||
|
||||
cd "$WORK_DIR" || { echo "无法进入工作目录: $WORK_DIR"; exit 1; }
|
||||
|
||||
# 创建日志目录
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# 日志文件
|
||||
LOG_FILE="$LOG_DIR/main_scheduler_$(date '+%H%M%S').log"
|
||||
|
||||
log() {
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
local message="[$timestamp] $1"
|
||||
echo "$message"
|
||||
echo "$message" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
log "开始执行IT资讯自动化流程"
|
||||
|
||||
# ==================== 步骤0: 系统检查 ====================
|
||||
log "步骤0: 系统检查"
|
||||
|
||||
# 检查小红书技能目录
|
||||
if [ ! -d "$XHS_SKILL_DIR" ]; then
|
||||
log "❌ 小红书技能目录不存在: $XHS_SKILL_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查小红书发布器
|
||||
XHS_PUBLISHER="$XHS_SKILL_DIR/xiaohongshu_publisher.py"
|
||||
if [ ! -f "$XHS_PUBLISHER" ]; then
|
||||
log "❌ 小红书发布器不存在: $XHS_PUBLISHER"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==================== 步骤1: 网页抓取 ====================
|
||||
log "步骤1: 开始网页抓取"
|
||||
python3 enhanced_crawler.py >> "$LOG_FILE" 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
log "✅ 网页抓取完成"
|
||||
else
|
||||
log "❌ 网页抓取失败"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==================== 步骤2: 文章处理 ====================
|
||||
log "步骤2: 开始文章处理"
|
||||
python3 complete_processor.py >> "$LOG_FILE" 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
log "✅ 文章处理完成"
|
||||
else
|
||||
log "❌ 文章处理失败"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==================== 步骤3: 小红书发布 ====================
|
||||
log "步骤3: 开始小红书发布"
|
||||
python3 "$XHS_PUBLISHER" >> "$LOG_FILE" 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
log "✅ 小红书发布完成"
|
||||
else
|
||||
log "❌ 小红书发布失败"
|
||||
fi
|
||||
|
||||
# ==================== 完成 ====================
|
||||
log "✅ IT资讯自动化流程执行完成"
|
||||
echo "============================================================"
|
||||
echo "📊 执行日志: $LOG_FILE"
|
||||
echo "============================================================"
|
||||
Reference in New Issue
Block a user