feat: solution for 'Практическое задание: Агент с RAG-памятью'
This commit is contained in:
+2
-40
@@ -1,42 +1,4 @@
|
||||
```python
|
||||
"""
|
||||
Script to load documents from a directory into the vector store.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from vector_store import vector_store
|
||||
from langchain_text_splitter import RecursiveCharacterTextSplitter
|
||||
|
||||
# Chunking configuration
|
||||
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
|
||||
|
||||
|
||||
def load_documents_from_dir(directory: str):
|
||||
"""
|
||||
Load all supported text files from the given directory into the knowledge base.
|
||||
|
||||
Args:
|
||||
directory: Path to the directory containing documents.
|
||||
"""
|
||||
dir_path = Path(directory)
|
||||
for file_path in dir_path.rglob("*"):
|
||||
if file_path.is_file() and file_path.suffix.lower() in {".txt", ".md"}:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
title = file_path.stem
|
||||
chunks = splitter.split_text(content)
|
||||
vector_store.add_documents(chunks, [title] * len(chunks))
|
||||
print(f"Loaded {file_path} into knowledge base.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Load documents into the knowledge base.")
|
||||
parser.add_argument("directory", help="Path to directory with documents.")
|
||||
args = parser.parse_args()
|
||||
load_documents_from_dir(args.directory)
|
||||
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
main()
|
||||
Reference in New Issue
Block a user