This commit is contained in:
Ubuntu
2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
+446
View File
@@ -0,0 +1,446 @@
#!/usr/bin/env python3
"""
Anti-AI Detector - 去AI化工具
支持中文、英文和小红书风格优化
"""
import re
import os
import sys
import argparse
from pathlib import Path
from typing import Dict, List, Tuple
class AntiAIDetector:
"""去AI化检测器"""
def __init__(self):
self.init_cn_patterns()
self.init_en_patterns()
self.init_xhs_patterns()
# ==================== 中文AI痕迹模式 ====================
def init_cn_patterns(self):
"""初始化中文AI检测模式"""
self.cn_patterns = {
# 标志性短语
"值得注意的是": ["需要关注的是", "关键点在于", ""],
"我们可以看到": ["从数据看", "实际情况是", ""],
"换句话说": ["也就是说", "简单来说", ""],
"不难发现": ["可以看出", "", ""],
"显而易见": ["很明显", "显然", ""],
"众所周知": ["大家都知道", "", ""],
"综上所述": ["总结来说", "", ""],
"归根结底": ["根本原因是", "说到底", ""],
"由此可见": ["这说明", "", ""],
# 结构模式
"首先": ["第一", "开始", ""],
"其次": ["第二", "然后", ""],
"最后": ["最终", "", ""],
"总而言之": ["总的来说", "", ""],
# 过度修饰
"非常": ["很", "挺", ""],
"极其": ["很", "特别", ""],
"毫无疑问": ["确实", "", ""],
"一定程度上": ["", "", ""],
# 模板化表达
"随着...的发展": ["", "", ""],
"在当今社会": ["", "", ""],
"引起了广泛关注": ["", "", ""],
"具有重要的意义": ["", "", ""],
}
self.cn_structures = [
# 排比结构
(r'不仅[^,]+,而且[^,]+,还[^。]+', "避免三重排比"),
(r'一方面[^,]+,另一方面[^,]+', "减少固定结构"),
# 过渡词堆砌
(r'首先.*?其次.*?最后.*?(综上所述|总而言之)', "去除模板化结构"),
# 学术腔
(r'从[^,]+角度(来看|分析|出发)', "口语化表达"),
(r'基于[^,]+(的)?(背景|基础|之上)', "简化表达"),
]
# ==================== 英文AI痕迹模式(Stop Slop)====================
def init_en_patterns(self):
"""初始化英文AI检测模式(基于Stop Slop)"""
self.en_patterns = {
# 标志性短语
"Here's the thing": ["The point is", "Fact is", ""],
"Here's the deal": ["The reality is", "", ""],
"Let that sink in": ["", "", ""],
"The uncomfortable truth is": ["Truth is", "", ""],
"The reality is": ["", "", ""],
"This matters because": ["Because", "", ""],
"What matters is": ["", "", ""],
"Full stop": ["", "", ""],
"Period": ["", "", ""],
"Let me be clear": ["", "", ""],
"I'm going to be honest": ["", "", ""],
"Make no mistake": ["", "", ""],
# 过度使用的动词
"navigate": ["handle", "deal with", "work through"],
"unpack": ["explain", "break down", ""],
"leverage": ["use", "take advantage of", ""],
"delve": ["explore", "look at", ""],
"dive into": ["explore", "examine", ""],
"peel back the layers": ["examine", "analyze", ""],
# 固定结构
"Not because": ["Not from", "", ""],
"But because": ["But from", "", ""],
"That's it": ["", "", ""],
"That's the": ["", "", ""],
"And that's okay": ["", "", ""],
# 过度修饰
"In today's world": ["Today", "Now", ""],
"In the digital age": ["", "", ""],
"Rapidly evolving": ["changing", "", ""],
"Fast-paced": ["busy", "", ""],
# 模糊表达
"It's important to note": ["Note", "", ""],
"It's worth mentioning": ["", "", ""],
"As mentioned earlier": ["", "", ""],
}
self.en_structures = [
# "Not because X. Because Y."
(r'Not because[^.]+\. Because[^.]+\.', "合并为一句"),
# "[Noun]. That's it. That's the [thing]."
(r'(\w+)\. That\'s it\. That\'s the[^.]+\.', "简化表达"),
# 三重形容词
(r'\w+, \w+, and \w+ (landscape|environment|space)', "减少形容词"),
# 过度使用hedging
(r'arguably|potentially|likely|generally|typically', "直接表达"),
]
# ==================== 小红书风格优化 ====================
def init_xhs_patterns(self):
"""初始化小红书风格检测模式"""
self.xhs_patterns = {
# 去除营销感
"强烈推荐": ["推荐", "我觉得", ""],
"必买": ["值得买", "可以考虑", ""],
"神器": ["好用", "不错", ""],
"绝绝子": ["很好", "不错", ""],
"YYDS": ["很棒", "", ""],
"宝藏": ["好", "不错", ""],
"闭眼入": ["推荐", "", ""],
"天花板": ["很好", "顶级", ""],
# 去模板化
"今天给大家分享": ["分享", "说说", ""],
"姐妹们": ["大家", "", ""],
"干货": ["内容", "", ""],
"码住": ["收藏", "记住", ""],
"抄作业": ["参考", "", ""],
# 过度emoji提示
"emoji堆砌": ["适量使用emoji", "", ""],
"标签堆砌": ["减少标签", "", ""],
}
self.xhs_structures = [
# 过于规整的排版
(r'^[0-9]+[.、\s]', "避免过于规整的序号"),
(r'✅|❌|⚠️|⭐|🌟{3,}', "emoji过多"),
(r'#[^#\s]+#[^#\s]+#[^#\s]+', "标签过多"),
]
# ==================== 检测方法 ====================
def detect_cn(self, text: str) -> List[Tuple[str, str, List[str]]]:
"""检测中文AI痕迹"""
findings = []
for pattern, replacements in self.cn_patterns.items():
if pattern in text or re.search(pattern.replace('...', '.*?'), text):
findings.append(("phrase", pattern, replacements))
for pattern, suggestion in self.cn_structures:
matches = re.findall(pattern, text, re.DOTALL)
if matches:
findings.append(("structure", pattern, [suggestion]))
return findings
def detect_en(self, text: str) -> List[Tuple[str, str, List[str]]]:
"""检测英文AI痕迹"""
findings = []
text_lower = text.lower()
for pattern, replacements in self.en_patterns.items():
if pattern.lower() in text_lower:
findings.append(("phrase", pattern, replacements))
for pattern, suggestion in self.en_structures:
matches = re.findall(pattern, text, re.IGNORECASE)
if matches:
findings.append(("structure", pattern, [suggestion]))
return findings
def detect_xhs(self, text: str) -> List[Tuple[str, str, List[str]]]:
"""检测小红书风格问题"""
findings = []
for pattern, replacements in self.xhs_patterns.items():
if pattern in text:
findings.append(("xhs_phrase", pattern, replacements))
for pattern, suggestion in self.xhs_structures:
matches = re.findall(pattern, text, re.MULTILINE)
if matches:
findings.append(("xhs_structure", pattern, [suggestion]))
# 检测emoji密度
emoji_count = len(re.findall(r'[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]', text))
if emoji_count > 5:
findings.append(("emoji_density", f"{emoji_count}个emoji", ["建议减少到3-5个"]))
return findings
# ==================== 处理方法 ====================
def humanize_cn(self, text: str, aggressive: bool = False) -> str:
"""中文去AI化"""
result = text
# 替换标志性短语
replacements = {
"值得注意的是": "",
"我们可以看到": "",
"不难发现": "",
"显而易见": "很明显",
"众所周知": "大家都知道",
"综上所述": "总结来说",
"归根结底": "说到底",
"由此可见": "这说明",
"换句话说": "也就是说",
}
for old, new in replacements.items():
if new:
result = result.replace(old, new)
else:
# 删除或根据上下文调整
result = result.replace(old + ",", "")
result = result.replace(old + ",", "")
# 简化过渡词
result = re.sub(r'首先[,,]?', "第一,", result)
result = re.sub(r'其次[,,]?', "第二,", result)
result = re.sub(r'最后[,,]?', "最后,", result)
# 去除过度修饰
result = re.sub(r'非常([的地])', r'很\1', result)
result = re.sub(r'极其([的地])', r'特别\1', result)
# 去除模板化开头
result = re.sub(r'随着[^,]+的发展[,,]', "", result)
result = re.sub(r'在当今社会[,,]', "", result)
if aggressive:
# 更激进的优化
result = re.sub(r'[,,]+', ",", result) # 合并多个逗号
result = re.sub(r'。+', "。", result) # 合并多个句号
return result.strip()
def humanize_en(self, text: str, aggressive: bool = False) -> str:
"""英文去AI化(基于Stop Slop)"""
result = text
# 替换标志性短语
replacements = {
r"Here's the thing[:,.]*\s*": "",
r"Here's the deal[:,.]*\s*": "",
r"Let that sink in[:,.]*\s*": "",
r"The uncomfortable truth is[:,.]*\s*": "",
r"Let me be clear[:,.]*\s*": "",
r"I'm going to be honest[:,.]*\s*": "",
r"Make no mistake[:,.]*\s*": "",
r"Full stop[.]*": ".",
r"Period[.]*": ".",
# 过度使用的动词
r"navigate\s+(the\s+)?": "handle ",
r"unpack\s*": "explain ",
r"leverage\s*": "use ",
r"delve\s+(into\s+)?": "explore ",
r"dive into\s*": "explore ",
}
for pattern, replacement in replacements.items():
result = re.sub(pattern, replacement, result, flags=re.IGNORECASE)
# 简化结构
result = re.sub(r'Not because([^,.]+)[,.]\s*Because([^,.]+)[,.]',
r'Not from\1, but from\2.', result, flags=re.IGNORECASE)
result = re.sub(r"(\w+)\.\s*That's it\.\s*That's the[^.]+\.",
r'\1 is what matters.', result, flags=re.IGNORECASE)
# 去除过度修饰
result = re.sub(r'In today\'s world[,\s]*', '', result, flags=re.IGNORECASE)
result = re.sub(r'In the digital age[,\s]*', '', result, flags=re.IGNORECASE)
result = re.sub(r'rapidly evolving', 'changing', result, flags=re.IGNORECASE)
result = re.sub(r'fast-paced', 'busy', result, flags=re.IGNORECASE)
# 简化hedging
result = re.sub(r'It\'s important to note that\s*', '', result, flags=re.IGNORECASE)
result = re.sub(r'It\'s worth mentioning that\s*', '', result, flags=re.IGNORECASE)
if aggressive:
# 去除更多学术腔
result = re.sub(r'arguably\s*[,]*\s*', '', result, flags=re.IGNORECASE)
result = re.sub(r'potentially\s*[,]*\s*', '', result, flags=re.IGNORECASE)
return result.strip()
def humanize_xhs(self, text: str) -> str:
"""小红书风格优化"""
result = text
# 替换营销词汇
replacements = {
"强烈推荐": "推荐",
"必买": "值得买",
"闭眼入": "推荐",
"神器": "好用",
"绝绝子": "很好",
"YYDS": "很棒",
"宝藏": "不错",
"天花板": "顶级",
"干货": "内容",
"码住": "收藏",
"抄作业": "参考",
}
for old, new in replacements.items():
result = result.replace(old, new)
# 简化称呼
result = result.replace("姐妹们", "大家")
result = result.replace("集美们", "大家")
result = result.replace("宝子们", "大家")
# 优化emoji分布(保留但不过度)
# 如果emoji过多,提示用户手动调整
# 减少标签堆砌
result = re.sub(r'#{3,}', '#', result)
# 去除过于规整的序号
result = re.sub(r'^\d+[.、]\s*', '', result, flags=re.MULTILINE)
return result.strip()
def auto_detect_and_humanize(self, text: str) -> Tuple[str, str]:
"""自动检测语言并去AI化"""
# 简单语言检测
chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', text))
english_chars = len(re.findall(r'[a-zA-Z]', text))
if chinese_chars > english_chars:
# 中文为主
return self.humanize_cn(text), "zh"
else:
# 英文为主
return self.humanize_en(text), "en"
def analyze(self, text: str) -> Dict:
"""全面分析文本的AI痕迹"""
cn_findings = self.detect_cn(text)
en_findings = self.detect_en(text)
xhs_findings = self.detect_xhs(text)
return {
"cn_findings": cn_findings,
"en_findings": en_findings,
"xhs_findings": xhs_findings,
"cn_score": len(cn_findings),
"en_score": len(en_findings),
"xhs_score": len(xhs_findings),
"total_risk": len(cn_findings) + len(en_findings) + len(xhs_findings),
}
def main():
"""命令行入口"""
parser = argparse.ArgumentParser(description='去AI化工具')
parser.add_argument('text', nargs='?', help='输入文本')
parser.add_argument('-f', '--file', help='输入文件')
parser.add_argument('-o', '--output', help='输出文件')
parser.add_argument('-t', '--type', choices=['zh', 'en', 'xhs', 'auto'],
default='auto', help='处理类型')
parser.add_argument('-a', '--aggressive', action='store_true',
help='激进模式')
parser.add_argument('--analyze', action='store_true',
help='仅分析不处理')
args = parser.parse_args()
detector = AntiAIDetector()
# 获取输入文本
if args.file:
with open(args.file, 'r', encoding='utf-8') as f:
text = f.read()
elif args.text:
text = args.text
else:
print("请提供文本或使用-f指定文件")
return
# 分析模式
if args.analyze:
result = detector.analyze(text)
print("=" * 50)
print("AI痕迹分析报告")
print("=" * 50)
print(f"中文AI痕迹: {result['cn_score']} 处")
print(f"英文AI痕迹: {result['en_score']} 处")
print(f"小红书风格问题: {result['xhs_score']} 处")
print(f"总体风险: {result['total_risk']} 处")
print("\n详细发现:")
for finding in result['cn_findings'][:5]:
print(f" [CN] {finding[1]}")
for finding in result['en_findings'][:5]:
print(f" [EN] {finding[1]}")
return
# 处理模式
if args.type == 'auto':
result, lang = detector.auto_detect_and_humanize(text)
print(f"检测到语言: {lang}")
elif args.type == 'zh':
result = detector.humanize_cn(text, args.aggressive)
elif args.type == 'en':
result = detector.humanize_en(text, args.aggressive)
elif args.type == 'xhs':
result = detector.humanize_xhs(text)
# 输出结果
if args.output:
with open(args.output, 'w', encoding='utf-8') as f:
f.write(result)
print(f"结果已保存到: {args.output}")
else:
print("\n" + "=" * 50)
print("处理结果:")
print("=" * 50)
print(result)
if __name__ == '__main__':
main()