RAG系统搭建深度学习笔记

def reciprocal_rank_fusion(results_list, k=60):
    """
    RRF（倒数排序融合）算法核心公式
    RRF_score = Σ 1/(k + rank_i)
    k通常取60，用于平滑处理
    """
    fused_scores = {}
    
    for results in results_list:
        for rank, doc in enumerate(results, 1):
            doc_id = doc['id']
            fused_scores[doc_id] = fused_scores.get(doc_id, 0) + 1 / (k + rank)
    
    sorted_docs = sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)
    return sorted_docs

from sentence_transformers import CrossEncoder

# 使用BGE-Reranker进行精排
reranker = CrossEncoder('BAAI/bge-reranker-base')

def rerank_documents(query, candidates, top_k=5):
    pairs = [(query, doc['text']) for doc in candidates]
    scores = reranker.predict(pairs)
    ranked = sorted(zip(scores, candidates), reverse=True)
    return ranked[:top_k]

def hyde_vector_mix(query, llm, embedder, alpha=0.7):
    """
    HyDE Vector Mix: 融合原始查询和假设答案的嵌入
    alpha: 混合比例，0.7表示70%假设答案 + 30%原始查询
    """
    # 1. LLM生成假设答案
    hypothetical = llm.invoke(f"请回答以下问题，生成一段假设性答案：{query}")
    
    # 2. 分别编码
    q_emb = embedder.embed(query)
    h_emb = embedder.embed(hypothetical.content)
    
    # 3. 加权融合
    mixed_emb = (1 - alpha) * q_emb + alpha * h_emb
    mixed_emb = mixed_emb / np.linalg.norm(mixed_emb)
    
    return mixed_emb

# langchain_rag.py - 完整RAG实现

import os
from dotenv import load_dotenv
from langchain_community.document_loaders import UnstructuredMarkdownLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_core.vectorstores import InMemoryVectorStore
from langchain_core.prompts import ChatPromptTemplate
from langchain_community.chat_models import ChatTongyi

# 1. 数据准备
loader = UnstructuredMarkdownLoader("your_document.md")
docs = loader.load()

# 2. 文本分块
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=400,
    chunk_overlap=50
)
chunks = text_splitter.split_documents(docs)

# 3. 向量索引构建
embeddings = HuggingFaceEmbeddings(
    model_name="BAAI/bge-small-zh-v1.5",
    model_kwargs={'device': 'cpu'}
)
vectorstore = InMemoryVectorStore(embeddings)
vectorstore.add_documents(chunks)

# 4. 检索函数
def retrieve_docs(query, k=3):
    return vectorstore.similarity_search(query, k=k)

# 5. RAG主流程
def rag_query(user_question):
    retrieved_docs = retrieve_docs(user_question, k=3)
    context = "\n\n".join(doc.page_content for doc in retrieved_docs)
    # ... 调用LLM生成回答
    return {"answer": answer, "sources": retrieved_docs}

# llamaindex_rag.py - LlamaIndex版本

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.embeddings.huggingface import HuggingFaceEmbedding

# 1. 配置嵌入模型
embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-zh-v1.5")

# 2. 加载文档
documents = SimpleDirectoryReader("./data").load_data()

# 3. 构建索引
index = VectorStoreIndex.from_documents(documents, embed_model=embed_model)

# 4. 创建查询引擎并查询
query_engine = index.as_query_engine(similarity_top_k=3)
response = query_engine.query("什么是强化学习？")
print(response)

# rag_demo.py - 完整可运行的RAG系统

class RAGSystem:
    """完整的RAG系统"""
    
    def __init__(self, persist_dir="./vector_store"):
        self.config = Config()
        self.vector_store = SimpleVectorStore(persist_dir)
        self.bm25 = BM25Retriever()
        self.embed_model = EmbeddingModel(self.config.EMBED_MODEL)
        self.llm = LLMGenerator(self.config.LLM_PROVIDER)
    
    def index_documents(self, documents):
        """索引文档"""
        # 文本分块
        splitter = TextSplitter(chunk_size=400, overlap=50)
        chunks = []
        for doc in documents:
            sub_chunks = splitter.split_text(doc.content)
            for chunk_text in sub_chunks:
                chunks.append(Document(content=chunk_text))
        
        # 向量化并存储
        embeddings = self.embed_model.embed([c.content for c in chunks])
        self.vector_store.add_documents(chunks, embeddings)
        self.bm25.index(chunks)
    
    def query(self, question):
        """RAG查询主函数"""
        # 1. 混合检索
        hybrid_retriever = HybridRetriever(self.vector_store, self.bm25)
        retrieved_docs = hybrid_retriever.retrieve(question, k=5)
        
        # 2. 组装上下文
        context = "\n\n".join([doc.content for doc in retrieved_docs])
        
        # 3. LLM生成
        answer = self.llm.generate(question, context)
        
        return {"answer": answer, "sources": retrieved_docs}