add chunker.py

chunker.py

@@ -1,19 +1,26 @@
 """
-Chunking utilities using RecursiveCharacterTextSplitter.
+Chunking utilities for the RAG agent.
+
+This module provides a single function `get_text_splitter` that returns a
+:class:`langchain.text_splitter.RecursiveCharacterTextSplitter` configured to
+split documents into chunks of 500 characters with an overlap of 100.
+
+The splitter is used by :mod:`agent` when ingesting files.
 """
 
-from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 
-# Configure splitter: chunk size 500, overlap 100
-splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
 
-def split_text(text: str) -> list[str]:
-    """Split a large text into chunks.
+def get_text_splitter() -> RecursiveCharacterTextSplitter:
+    """Return a configured text splitter.
 
-    Args:
-        text: The raw document content.
-
-    Returns:
-        List of string chunks.
+    The splitter uses a chunk size of 500 characters and an overlap of 100
+    characters. These values are chosen to balance context length with the
+    ability to retrieve relevant passages during semantic search.
     """
-    return splitter.split_text(text)
+    return RecursiveCharacterTextSplitter(
+        chunk_size=500,
+        chunk_overlap=100,
+    )
+
+# End of chunker.py