Add chunker.py

chunker.py

@@ -0,0 +1,31 @@
+"""
+Chunking utilities for the RAG agent.
+
+Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100.
+"""
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+# Global splitter instance
+splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
+
+def split_text(text: str) -> list[str]:
+    """Split a large string into chunks.
+
+    Parameters
+    ----------
+    text: str
+        The raw document content.
+
+    Returns
+    -------
+    List[str]
+        A list of chunk strings.
+    """
+    return splitter.split_text(text)
+
+# Example usage (not executed in tests)
+if __name__ == "__main__":  # pragma: no cover
+    sample = "\n".join([f"Line {i}" for i in range(1000)])
+    chunks = split_text(sample)
+    print(f"Generated {len(chunks)} chunks")