add chunker.py

chunker.py

@@ -1,34 +1,12 @@
 """
 Chunking utilities for the RAG agent.
 
-Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and chunk_overlap=100.
+Uses RecursiveCharacterTextSplitter from LangChain with a chunk size of 500 and an overlap of 100.
 """
-
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
-# Default splitter configuration
-DEFAULT_SPLITTER = RecursiveCharacterTextSplitter(
+# Global splitter instance
+chunker = RecursiveCharacterTextSplitter(
     chunk_size=500,
     chunk_overlap=100,
 )
-
-def split_text(text: str):
-    """Split a large string into chunks.
-
-    Parameters
-    ----------
-    text: str
-        Text to split.
-
-    Returns
-    -------
-    list[str]
-        List of chunk strings.
-    """
-    return DEFAULT_SPLITTER.split_text(text)
-
-# Example usage (not executed in tests)
-if __name__ == "__main__":
-    sample = "\n".join([f"Line {i}" for i in range(1000)])
-    chunks = split_text(sample)
-    print(f"Generated {len(chunks)} chunks")