diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..11cee52 --- /dev/null +++ b/chunker.py @@ -0,0 +1,30 @@ +"""Utility module for chunking text. + +This module provides a helper function to split large text into smaller chunks suitable for +embedding and storage in a vector database. It uses the +`RecursiveCharacterTextSplitter` from the `langchain_text_splitters` +package. +""" + +from langchain_text_splitters import RecursiveCharacterTextSplitter + +def chunk_text(text: str, chunk_size: int = 1000, chunk_overlap: int = 200) -> list[str]: + """Split *text* into smaller chunks. + + Parameters + ---------- + text: str + The raw text to split. + chunk_size: int, optional + Maximum size of each chunk in characters. Defaults to 1000. + chunk_overlap: int, optional + Number of overlapping characters between consecutive chunks. + Defaults to 200. + + Returns + ------- + list[str] + A list of text chunks. + """ + splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap) + return splitter.split_text(text) \ No newline at end of file