diff --git a/chunker.py b/chunker.py index 3cedceb..ced3b4f 100644 --- a/chunker.py +++ b/chunker.py @@ -3,13 +3,19 @@ Chunking utilities for the RAG agent. Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100. """ +import os +from pathlib import Path +from typing import List from langchain_text_splitters import RecursiveCharacterTextSplitter -# Global splitter instance -splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) +# Default splitter configuration +CHUNK_SIZE = 500 +OVERLAP = 100 -def split_text(text: str) -> list[str]: +splitter = RecursiveCharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=OVERLAP) + +def split_text(text: str) -> List[str]: """Split a large string into chunks. Parameters @@ -24,8 +30,17 @@ def split_text(text: str) -> list[str]: """ return splitter.split_text(text) -# Example usage (not executed in tests) -if __name__ == "__main__": # pragma: no cover - sample = "\n".join([f"Line {i}" for i in range(1000)]) - chunks = split_text(sample) - print(f"Generated {len(chunks)} chunks") +# Helper to read all files from a directory and split them + +def load_and_split(directory: Path) -> List[tuple]: + """Load text files from *directory* and split into chunks. + + Returns a list of tuples (chunk, metadata). + Metadata contains the source file path. + """ + chunks = [] + for file_path in directory.rglob("*.txt"): + content = file_path.read_text(encoding="utf-8") + for chunk in split_text(content): + chunks.append((chunk, {"source": str(file_path)})) + return chunks