From ade794dfd6cb87fdbe084c9316cc6e0c3b68e08a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?=
 =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?=
 <kirillkutlahmetov10@gmail.com>
Date: Thu, 28 May 2026 09:14:54 +0000
Subject: [PATCH] Add chunker.py

---
 chunker.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 chunker.py

diff --git a/chunker.py b/chunker.py
new file mode 100644
index 0000000..3cedceb
--- /dev/null
+++ b/chunker.py
@@ -0,0 +1,31 @@
+"""
+Chunking utilities for the RAG agent.
+
+Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100.
+"""
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+# Global splitter instance
+splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
+
+def split_text(text: str) -> list[str]:
+    """Split a large string into chunks.
+
+    Parameters
+    ----------
+    text: str
+        The raw document content.
+
+    Returns
+    -------
+    List[str]
+        A list of chunk strings.
+    """
+    return splitter.split_text(text)
+
+# Example usage (not executed in tests)
+if __name__ == "__main__":  # pragma: no cover
+    sample = "\n".join([f"Line {i}" for i in range(1000)])
+    chunks = split_text(sample)
+    print(f"Generated {len(chunks)} chunks")