From 5ea484b37ad268834f1f1b67331d3ffa0e40f01f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=A0=D0=BE=D0=BC?=
 =?UTF-8?q?=D0=B0=D0=BD=D0=BE=D0=B2?= <kirilinka320@gmail.com>
Date: Thu, 4 Jun 2026 23:23:38 +0000
Subject: [PATCH] Add src/chunker.py

---
 src/chunker.py | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 src/chunker.py

diff --git a/src/chunker.py b/src/chunker.py
new file mode 100644
index 0000000..eab8a5e
--- /dev/null
+++ b/src/chunker.py
@@ -0,0 +1,37 @@
+"""Utility for loading Markdown files into a Chroma vector store.
+
+The function `load_faq_to_chroma` reads all `.md` files from the data directory,
+chunks them with a recursive character splitter, and persists the embeddings
+using Ollama's `nomic-embed-text` model.
+"""
+
+from pathlib import Path
+from typing import List
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_ollama import OllamaEmbeddings
+from langchain_chroma import Chroma
+
+from .config import DATA_DIR, CHROMA_DIR
+
+def _load_markdown_files() -> List[str]:
+    """Return the text content of all Markdown files in DATA_DIR."""
+    texts: List[str] = []
+    for path in Path(DATA_DIR).glob("*.md"):
+        texts.append(path.read_text(encoding="utf-8"))
+    return texts
+
+def load_faq_to_chroma() -> Chroma:
+    """Load FAQ Markdown files into a persistent Chroma store.
+
+    Returns the Chroma instance for later use.
+    """
+    texts = _load_markdown_files()
+    # Simple chunking – 500 chars per chunk with 50 char overlap
+    splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    docs = splitter.split_text("\n\n".join(texts))
+
+    embeddings = OllamaEmbeddings(model="nomic-embed-text")
+    chroma = Chroma.from_texts(docs, embeddings, persist_directory=CHROMA_DIR)
+    chroma.persist()
+    return chroma
\ No newline at end of file