From 3fdd844002a79d22338fca90726a7c72afcb2490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?= =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?= Date: Thu, 28 May 2026 09:26:55 +0000 Subject: [PATCH] add chunker.py --- chunker.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/chunker.py b/chunker.py index 3cedceb..ced3b4f 100644 --- a/chunker.py +++ b/chunker.py @@ -3,13 +3,19 @@ Chunking utilities for the RAG agent. Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100. """ +import os +from pathlib import Path +from typing import List from langchain_text_splitters import RecursiveCharacterTextSplitter -# Global splitter instance -splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) +# Default splitter configuration +CHUNK_SIZE = 500 +OVERLAP = 100 -def split_text(text: str) -> list[str]: +splitter = RecursiveCharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=OVERLAP) + +def split_text(text: str) -> List[str]: """Split a large string into chunks. Parameters @@ -24,8 +30,17 @@ def split_text(text: str) -> list[str]: """ return splitter.split_text(text) -# Example usage (not executed in tests) -if __name__ == "__main__": # pragma: no cover - sample = "\n".join([f"Line {i}" for i in range(1000)]) - chunks = split_text(sample) - print(f"Generated {len(chunks)} chunks") +# Helper to read all files from a directory and split them + +def load_and_split(directory: Path) -> List[tuple]: + """Load text files from *directory* and split into chunks. + + Returns a list of tuples (chunk, metadata). + Metadata contains the source file path. + """ + chunks = [] + for file_path in directory.rglob("*.txt"): + content = file_path.read_text(encoding="utf-8") + for chunk in split_text(content): + chunks.append((chunk, {"source": str(file_path)})) + return chunks