55 lines
1.9 KiB
Python
55 lines
1.9 KiB
Python
import os
|
|
from pathlib import Path
|
|
from typing import List
|
|
|
|
from langchain_ollama import OllamaEmbeddings
|
|
from langchain_chroma import Chroma
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
from langchain.docstore.document import Document
|
|
|
|
|
|
def create_vectorstore(persist_directory: str = "./chroma_db") -> Chroma:
|
|
"""Create or load a Chroma vector store backed by Ollama embeddings.
|
|
|
|
Parameters
|
|
----------
|
|
persist_directory: str
|
|
Directory where Chroma will persist its data.
|
|
|
|
Returns
|
|
-------
|
|
Chroma
|
|
A Chroma vector store instance.
|
|
"""
|
|
embeddings = OllamaEmbeddings(model="nomic-embed-text")
|
|
return Chroma(
|
|
persist_directory=persist_directory,
|
|
embedding_function=embeddings,
|
|
)
|
|
|
|
|
|
def load_documents(directory: str, vectorstore: Chroma, chunk_size: int = 1000, chunk_overlap: int = 200) -> None:
|
|
"""Load all .txt and .md files from *directory* into *vectorstore*.
|
|
|
|
The function reads files, splits them into chunks using
|
|
``RecursiveCharacterTextSplitter`` and adds the resulting
|
|
:class:`~langchain.docstore.document.Document` objects to the
|
|
vector store.
|
|
"""
|
|
splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
|
docs: List[Document] = []
|
|
for path in Path(directory).glob("**/*"):
|
|
if path.suffix.lower() not in {".txt", ".md"}:
|
|
continue
|
|
text = path.read_text(encoding="utf-8")
|
|
docs.extend(splitter.split_text(text))
|
|
# Convert list of strings to Document objects
|
|
documents = [Document(page_content=chunk, metadata={"source": str(p)}) for chunk in docs]
|
|
vectorstore.add_documents(documents)
|
|
|
|
# If this module is executed directly, load the default documents folder.
|
|
if __name__ == "__main__":
|
|
vs = create_vectorstore()
|
|
load_documents("documents", vs)
|
|
print("Vector store populated.")
|