From c7930543e70ea6075914179091f29a3e75a714c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?=
 =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?=
 <kirillkutlahmetov10@gmail.com>
Date: Thu, 28 May 2026 13:38:06 +0000
Subject: [PATCH] add chunker.py

---
 chunker.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 chunker.py

diff --git a/chunker.py b/chunker.py
new file mode 100644
index 0000000..78c2c17
--- /dev/null
+++ b/chunker.py
@@ -0,0 +1,22 @@
+"""Text chunking utilities for splitting documents before indexing."""
+from typing import List
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+
+
+def load_and_split(text: str, chunk_size: int = 500, chunk_overlap: int = 100) -> List[Document]:
+    """Split text into overlapping chunks suitable for vector store indexing.
+
+    Args:
+        text: raw input text
+        chunk_size: maximum size of each chunk in characters
+        chunk_overlap: number of characters to overlap between consecutive chunks
+
+    Returns:
+        list of Document objects
+    """
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+    )
+    return splitter.create_documents([text])