19 lines
679 B
Python
19 lines
679 B
Python
import sys
|
|
from pathlib import Path
|
|
|
|
from utils import splitter, vector_store
|
|
from langchain_core.documents import Document
|
|
|
|
def load_documents(dir_path: str):
|
|
for file_path in Path(dir_path).rglob("*.txt"):
|
|
content = file_path.read_text(encoding="utf-8")
|
|
chunks = splitter.split_text(content)
|
|
docs = [Document(page_content=chunk, metadata={"title": file_path.name}) for chunk in chunks]
|
|
vector_store.add_documents(docs)
|
|
print(f"Added {len(docs)} chunks from {file_path}")
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 2:
|
|
print("Usage: python load_docs.py <directory>")
|
|
sys.exit(1)
|
|
load_documents(sys.argv[1]) |