RAG\uff08Retrieval-Augmented Generation\uff0c\u68c0\u7d22\u589e\u5f3a\u751f\u6210\uff09\u662f\u5f53\u524d AI \u5e94\u7528\u6700\u70ed\u95e8\u7684\u6280\u672f\u4e4b\u4e00\u3002\u5b83\u89e3\u51b3\u4e86 LLM \u77e5\u8bc6\u622a\u6b62\u548c\u5e7b\u89c9\u95ee\u9898\uff0c\u8ba9 AI \u80fd\u591f\u57fa\u4e8e\u4f60\u7684\u79c1\u6709\u6570\u636e\u56de\u7b54\u95ee\u9898\u3002<\/p>\n
## \u4ec0\u4e48\u662f RAG\uff1f
\nRAG = \u68c0\u7d22 + \u751f\u6210\uff1a\u7528\u6237\u95ee\u9898 \u2192 \u68c0\u7d22\u76f8\u5173\u6587\u6863 \u2192 \u62fc\u63a5\u5230 Prompt \u2192 LLM \u751f\u6210\u56de\u7b54<\/p>\n
**\u6838\u5fc3\u4f18\u52bf\uff1a**
\n– \u77e5\u8bc6\u5b9e\u65f6\u66f4\u65b0\uff1a\u4e0d\u9700\u8981\u91cd\u65b0\u8bad\u7ec3\u6a21\u578b
\n– \u51cf\u5c11\u5e7b\u89c9\uff1a\u57fa\u4e8e\u771f\u5b9e\u6570\u636e\u56de\u7b54
\n– \u9886\u57df\u5b9a\u5236\uff1a\u9488\u5bf9\u7279\u5b9a\u4e1a\u52a1\u573a\u666f
\n– \u6570\u636e\u9690\u79c1\uff1a\u6570\u636e\u4e0d\u79bb\u5f00\u4f60\u7684\u63a7\u5236<\/p>\n
## \u6838\u5fc3\u7ec4\u4ef6<\/p>\n
### 1. \u6587\u6863\u52a0\u8f7d\u5668
\n“`python
\nfrom langchain.document_loaders import TextLoader, PDFLoader, WebBaseLoader<\/p>\n
loader = TextLoader(“docs\/intro.md”)
\ndocs = loader.load()
\n“`<\/p>\n
### 2. \u6587\u6863\u5206\u5272\u5668
\n“`python
\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter<\/p>\n
splitter = RecursiveCharacterTextSplitter(
\n chunk_size=1000,
\n chunk_overlap=200
\n)
\nsplits = splitter.split_documents(docs)
\n“`<\/p>\n
### 3. \u5d4c\u5165\u6a21\u578b
\n“`python
\nfrom langchain.embeddings import OpenAIEmbeddings
\nfrom langchain.embeddings import HuggingFaceEmbeddings<\/p>\n
embeddings = OpenAIEmbeddings()
\n# \u6216\u4f7f\u7528\u5f00\u6e90 Embeddings
\nembeddings = HuggingFaceEmbeddings(
\n model_name=”sentence-transformers\/all-MiniLM-L6-v2″
\n)
\n“`<\/p>\n
### 4. \u5411\u91cf\u6570\u636e\u5e93
\n\u652f\u6301 Chroma\u3001Pinecone\u3001Qdrant\u3001Weaviate\u3001FAISS \u7b49\u3002<\/p>\n
## \u5b9e\u6218\u9879\u76ee\uff1a\u4f01\u4e1a\u77e5\u8bc6\u5e93\u95ee\u7b54<\/p>\n
### \u9879\u76ee\u7ed3\u6784
\n“`
\nrag-knowledge-base\/
\n\u251c\u2500\u2500 config.py # \u914d\u7f6e
\n\u251c\u2500\u2500 document_loader.py # \u6587\u6863\u52a0\u8f7d
\n\u251c\u2500\u2500 document_chunker.py # \u6587\u6863\u5206\u5272
\n\u251c\u2500\u2500 vector_store.py # \u5411\u91cf\u5b58\u50a8
\n\u251c\u2500\u2500 qa_chain.py # \u95ee\u7b54\u94fe
\n\u2514\u2500\u2500 main.py # \u4e3b\u7a0b\u5e8f
\n“`<\/p>\n
### \u5b8c\u6574\u4ee3\u7801\u5b9e\u73b0<\/p>\n
**config.py\uff1a**
\n“`python
\nimport os
\nfrom dotenv import load_dotenv
\nload_dotenv()<\/p>\n
LLM_MODEL = os.getenv(“LLM_MODEL”, “gpt-4o”)
\nOPENAI_API_KEY = os.getenv(“OPENAI_API_KEY”)
\nEMBEDDING_MODEL = “text-embedding-3-small”
\nCHROMA_PERSIST_DIR = “.\/chroma_db”
\nCHUNK_SIZE = 1000
\nCHUNK_OVERLAP = 200
\nTOP_K = 3
\n“`<\/p>\n
**document_processor.py\uff1a**
\n“`python
\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter
\nfrom langchain.vectorstores import Chroma
\nfrom langchain.embeddings import OpenAIEmbeddings<\/p>\n
class DocumentProcessor:
\n def __init__(self, config):
\n self.embeddings = OpenAIEmbeddings(model=config.EMBEDDING_MODEL)
\n self.splitter = RecursiveCharacterTextSplitter(
\n chunk_size=config.CHUNK_SIZE,
\n chunk_overlap=config.CHUNK_OVERLAP
\n )<\/p>\n
def process_documents(self, documents):
\n return self.splitter.split_documents(documents)<\/p>\n
def create_vectorstore(self, documents, persist=True):
\n splits = self.process_documents(documents)
\n vectorstore = Chroma.from_documents(
\n documents=splits,
\n embedding=self.embeddings,
\n persist_directory=self.config.CHROMA_PERSIST_DIR if persist else None,
\n collection_name=”knowledge_base”
\n )
\n return vectorstore
\n“`<\/p>\n
## \u4f18\u5316\u6280\u5de7
\n– **\u6df7\u5408\u68c0\u7d22\uff1a** \u5411\u91cf\u68c0\u7d22 + BM25 \u5173\u952e\u8bcd\u68c0\u7d22
\n– **\u91cd\u6392\u5e8f\uff08Rerank\uff09\uff1a** \u4f7f\u7528 CohereRerank \u4f18\u5316\u7ed3\u679c
\n– **\u67e5\u8be2\u6269\u5c55\uff1a** \u751f\u6210\u591a\u4e2a\u76f8\u5173\u67e5\u8be2\u63d0\u9ad8\u53ec\u56de\u7387
\n– **\u7f13\u5b58\uff1a** Redis \u7f13\u5b58\u5e38\u89c1\u95ee\u9898<\/p>\n
## \u90e8\u7f72\u9009\u9879
\n– \u672c\u5730\u90e8\u7f72\uff1aStreamlit \/ Gradio
\n– \u4e91\u90e8\u7f72\uff1aVercel + Vercel AI SDK \/ FastAPI + Docker
\n– \u5fae\u670d\u52a1\uff1a\u6587\u6863\u5904\u7406\u3001\u5411\u91cf\u68c0\u7d22\u3001LLM \u63a8\u7406\u5206\u79bb<\/p>\n
## \u6210\u672c\u4f18\u5316
\n– \u9009\u62e9\u5408\u9002\u7684 Embedding \u6a21\u578b\uff08OpenAI text-embedding-3-small \u6700\u4fbf\u5b9c\uff0c\u6216\u4f7f\u7528\u5f00\u6e90\u6a21\u578b\uff09
\n– \u7f13\u5b58\u5e38\u89c1\u95ee\u9898
\n– \u6279\u91cf\u5904\u7406\u51cf\u5c11 API \u8c03\u7528
\n– \u6df7\u5408\u68c0\u7d22\u964d\u4f4e\u5bf9 LLM \u7684\u4f9d\u8d56<\/p>\n","protected":false},"excerpt":{"rendered":"
RAG\uff08Retrieval-Augmented Generation\uff0c\u68c0\u7d22\u589e\u5f3a\u751f\u6210\uff09\u662f\u5f53\u524d AI \u5e94\u7528\u6700\u70ed\u95e8\u7684 […]<\/p>\n","protected":false},"author":0,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"zakra_page_container_layout":"customizer","zakra_page_sidebar_layout":"customizer","zakra_remove_content_margin":false,"zakra_sidebar":"customizer","zakra_transparent_header":"customizer","zakra_logo":0,"zakra_main_header_style":"default","zakra_menu_item_color":"","zakra_menu_item_hover_color":"","zakra_menu_item_active_color":"","zakra_menu_active_style":"","zakra_page_header":true,"footnotes":""},"categories":[4],"tags":[],"class_list":["post-10","post","type-post","status-publish","format-standard","hentry","category-rag"],"_links":{"self":[{"href":"https:\/\/gmrea.com\/index.php?rest_route=\/wp\/v2\/posts\/10","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/gmrea.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/gmrea.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"replies":[{"embeddable":true,"href":"https:\/\/gmrea.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=10"}],"version-history":[{"count":0,"href":"https:\/\/gmrea.com\/index.php?rest_route=\/wp\/v2\/posts\/10\/revisions"}],"wp:attachment":[{"href":"https:\/\/gmrea.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=10"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/gmrea.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=10"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/gmrea.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=10"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}