This commit is contained in:
Ubuntu
2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
+168
View File
@@ -0,0 +1,168 @@
"""
知识库管理模块 - 使用 BAAI/bge-small-zh-v1.5 中文嵌入模型(本地离线版)
"""
import os
from pathlib import Path
from typing import List
from langchain_chroma import Chroma
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import TextLoader
from langchain_core.documents import Document
from config import (
KNOWLEDGE_DIR, CHROMA_DB_DIR,
CHUNK_SIZE, CHUNK_OVERLAP,
RETRIEVAL_TOP_K
)
class KnowledgeBase:
"""知识库管理类 - 基于本地 BGE 中文嵌入模型"""
def __init__(self):
self.db_dir = str(CHROMA_DB_DIR)
self.collection_name = "customer_service_kb"
# 从本地加载 BGE 模型
print("🔧 正在加载本地 BGE 中文嵌入模型...")
# 直接使用 sentence_transformers 加载本地模型
from sentence_transformers import SentenceTransformer
# 本地模型路径
local_model_path = '/home/ubuntu/.cache/huggingface/models--BAAI--bge-small-zh-v1.5/snapshots/'
# 找到具体的 snapshot 目录
if os.path.exists(local_model_path):
snapshots = os.listdir(local_model_path)
if snapshots:
model_path = os.path.join(local_model_path, snapshots[0])
else:
model_path = 'BAAI/bge-small-zh-v1.5' # fallback
else:
model_path = 'BAAI/bge-small-zh-v1.5'
print(f" 模型路径: {model_path}")
# 加载模型
self.model = SentenceTransformer(model_path, device='cpu')
# 创建自定义 Embeddings 类
from langchain_core.embeddings import Embeddings
class LocalBGEEmbeddings(Embeddings):
def __init__(self, model):
self.model = model
def embed_documents(self, texts: List[str]) -> List[List[float]]:
"""嵌入文档"""
embeddings = self.model.encode(texts, normalize_embeddings=True)
return embeddings.tolist()
def embed_query(self, text: str) -> List[float]:
"""嵌入查询"""
embedding = self.model.encode(text, normalize_embeddings=True)
return embedding.tolist()
self.embeddings = LocalBGEEmbeddings(self.model)
print("✅ BGE 模型加载完成")
# 初始化或加载向量数据库
self.vectorstore = self._load_or_create_db()
def _load_or_create_db(self) -> Chroma:
"""加载或创建向量数据库"""
if os.path.exists(self.db_dir) and os.listdir(self.db_dir):
print(f"📚 加载已有知识库: {self.db_dir}")
return Chroma(
collection_name=self.collection_name,
embedding_function=self.embeddings,
persist_directory=self.db_dir
)
else:
print(f"🆕 创建新知识库: {self.db_dir}")
os.makedirs(self.db_dir, exist_ok=True)
return Chroma(
collection_name=self.collection_name,
embedding_function=self.embeddings,
persist_directory=self.db_dir
)
def load_documents(self, directory: str = None) -> List[Document]:
"""从目录加载所有文档"""
directory = directory or str(KNOWLEDGE_DIR)
documents = []
if not os.path.exists(directory):
print(f"⚠️ 目录不存在: {directory}")
return documents
for file_path in Path(directory).glob("*"):
if file_path.suffix.lower() == ".txt":
loader = TextLoader(str(file_path), encoding="utf-8")
documents.extend(loader.load())
print(f"✅ 加载 TXT: {file_path.name}")
elif file_path.suffix.lower() == ".md":
loader = TextLoader(str(file_path), encoding="utf-8")
documents.extend(loader.load())
print(f"✅ 加载 MD: {file_path.name}")
return documents
def add_documents(self, documents: List[Document]):
"""添加文档到知识库"""
if not documents:
print("⚠️ 没有文档需要添加")
return
# 文本分块
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=CHUNK_SIZE,
chunk_overlap=CHUNK_OVERLAP,
separators=["\n\n", "\n", "。", ",", " ", ""]
)
chunks = text_splitter.split_documents(documents)
print(f"📄 文档分块: {len(documents)} 个文档 → {len(chunks)} 个片段")
# 添加到向量数据库
self.vectorstore.add_documents(chunks)
print(f"✅ 已添加到知识库")
def search(self, query: str, k: int = None) -> List[Document]:
"""语义搜索相关知识"""
k = k or RETRIEVAL_TOP_K
results = self.vectorstore.similarity_search(query, k=k)
return results
def get_retriever(self):
"""获取检索器(用于 LangChain)"""
return self.vectorstore.as_retriever(search_kwargs={"k": RETRIEVAL_TOP_K})
def init_knowledge_base():
"""初始化知识库(首次运行时使用)"""
print("🔧 初始化知识库...")
kb = KnowledgeBase()
# 加载知识库目录中的文档
documents = kb.load_documents()
if documents:
kb.add_documents(documents)
print(f"✅ 知识库初始化完成,共 {len(documents)} 个文档")
else:
print("⚠️ 没有找到文档,请在 knowledge_base/ 目录添加文档后重新运行")
return kb
# 全局知识库实例(单例模式)
_knowledge_base = None
def get_knowledge_base() -> KnowledgeBase:
"""获取知识库单例"""
global _knowledge_base
if _knowledge_base is None:
_knowledge_base = KnowledgeBase()
return _knowledge_base