diff --git a/chunker.py b/chunker.py index 07ad442..590ada9 100644 --- a/chunker.py +++ b/chunker.py @@ -1,34 +1,12 @@ """ Chunking utilities for the RAG agent. -Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and chunk_overlap=100. +Uses RecursiveCharacterTextSplitter from LangChain with a chunk size of 500 and an overlap of 100. """ - from langchain_text_splitters import RecursiveCharacterTextSplitter -# Default splitter configuration -DEFAULT_SPLITTER = RecursiveCharacterTextSplitter( +# Global splitter instance +chunker = RecursiveCharacterTextSplitter( chunk_size=500, chunk_overlap=100, ) - -def split_text(text: str): - """Split a large string into chunks. - - Parameters - ---------- - text: str - Text to split. - - Returns - ------- - list[str] - List of chunk strings. - """ - return DEFAULT_SPLITTER.split_text(text) - -# Example usage (not executed in tests) -if __name__ == "__main__": - sample = "\n".join([f"Line {i}" for i in range(1000)]) - chunks = split_text(sample) - print(f"Generated {len(chunks)} chunks") \ No newline at end of file