diff --git a/load_documents.py b/load_documents.py index f5ba73a..1fe884a 100644 --- a/load_documents.py +++ b/load_documents.py @@ -1,14 +1,13 @@ -"""Utility script to load documents from a directory into the Qdrant vector store. - -The script walks through the specified directory, reads all .txt files, splits them into chunks using -`RecursiveCharacterTextSplitter`, embeds the chunks with `OllamaEmbeddings`, and stores them in the -local Qdrant collection via the helper functions defined in :mod:`main`. - -Usage: - python load_documents.py /path/to/docs - -The script prints the number of documents added. -""" +# Utility script to load documents from a directory into the Chroma vector store. +# +# The script walks through the specified directory, reads all .txt files, splits them into chunks using +# `RecursiveCharacterTextSplitter`, embeds the chunks with `OllamaEmbeddings`, and stores them in the +# local Chroma collection via the helper functions defined in :mod:`main`. +# +# Usage: +# python load_documents.py /path/to/docs +# +# The script prints the number of documents added. import os import sys @@ -48,7 +47,9 @@ def load_documents_from_dir(directory: str) -> int: title = file_path.stem documents = chunk_document(content, title) ids = [str(uuid4()) for _ in documents] - vector_store.add_documents(documents, ids=ids) + embeddings = embedding.embed_documents([doc.page_content for doc in documents]) + collection = vector_store.get_collection(name="rag_memory") + collection.add(ids=ids, documents=[doc.page_content for doc in documents], embeddings=embeddings, metadatas=[doc.metadata for doc in documents]) total_chunks += len(documents) return total_chunks