Add chunker.py

chunker.py

@@ -0,0 +1,30 @@
+"""Utility module for chunking text.
+
+This module provides a helper function to split large text into smaller chunks suitable for
+embedding and storage in a vector database. It uses the
+`RecursiveCharacterTextSplitter` from the `langchain_text_splitters`
+package.
+"""
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+def chunk_text(text: str, chunk_size: int = 1000, chunk_overlap: int = 200) -> list[str]:
+    """Split *text* into smaller chunks.
+
+    Parameters
+    ----------
+    text: str
+        The raw text to split.
+    chunk_size: int, optional
+        Maximum size of each chunk in characters. Defaults to 1000.
+    chunk_overlap: int, optional
+        Number of overlapping characters between consecutive chunks.
+        Defaults to 200.
+
+    Returns
+    -------
+    list[str]
+        A list of text chunks.
+    """
+    splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    return splitter.split_text(text)