diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..3cedceb --- /dev/null +++ b/chunker.py @@ -0,0 +1,31 @@ +""" +Chunking utilities for the RAG agent. + +Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100. +""" + +from langchain_text_splitters import RecursiveCharacterTextSplitter + +# Global splitter instance +splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) + +def split_text(text: str) -> list[str]: + """Split a large string into chunks. + + Parameters + ---------- + text: str + The raw document content. + + Returns + ------- + List[str] + A list of chunk strings. + """ + return splitter.split_text(text) + +# Example usage (not executed in tests) +if __name__ == "__main__": # pragma: no cover + sample = "\n".join([f"Line {i}" for i in range(1000)]) + chunks = split_text(sample) + print(f"Generated {len(chunks)} chunks")