add chunker.py

chunker.py

@@ -0,0 +1,22 @@
+"""Text chunking utilities for splitting documents before indexing."""
+from typing import List
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+
+
+def load_and_split(text: str, chunk_size: int = 500, chunk_overlap: int = 100) -> List[Document]:
+    """Split text into overlapping chunks suitable for vector store indexing.
+
+    Args:
+        text: raw input text
+        chunk_size: maximum size of each chunk in characters
+        chunk_overlap: number of characters to overlap between consecutive chunks
+
+    Returns:
+        list of Document objects
+    """
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+    )
+    return splitter.create_documents([text])