Add src/chunker.py

src/chunker.py

@@ -0,0 +1,37 @@
+"""Utility for loading Markdown files into a Chroma vector store.
+
+The function `load_faq_to_chroma` reads all `.md` files from the data directory,
+chunks them with a recursive character splitter, and persists the embeddings
+using Ollama's `nomic-embed-text` model.
+"""
+
+from pathlib import Path
+from typing import List
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_ollama import OllamaEmbeddings
+from langchain_chroma import Chroma
+
+from .config import DATA_DIR, CHROMA_DIR
+
+def _load_markdown_files() -> List[str]:
+    """Return the text content of all Markdown files in DATA_DIR."""
+    texts: List[str] = []
+    for path in Path(DATA_DIR).glob("*.md"):
+        texts.append(path.read_text(encoding="utf-8"))
+    return texts
+
+def load_faq_to_chroma() -> Chroma:
+    """Load FAQ Markdown files into a persistent Chroma store.
+
+    Returns the Chroma instance for later use.
+    """
+    texts = _load_markdown_files()
+    # Simple chunking – 500 chars per chunk with 50 char overlap
+    splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    docs = splitter.split_text("\n\n".join(texts))
+
+    embeddings = OllamaEmbeddings(model="nomic-embed-text")
+    chroma = Chroma.from_texts(docs, embeddings, persist_directory=CHROMA_DIR)
+    chroma.persist()
+    return chroma