diff --git a/chunker.py b/chunker.py index 5648121..1c402a9 100644 --- a/chunker.py +++ b/chunker.py @@ -1,17 +1,19 @@ """ -Chunking utilities for the RAG agent. - -Uses RecursiveCharacterTextSplitter from langchain.text_splitter to split documents into -chunks of 500 characters with an overlap of 100 characters. The splitter is exposed as a -singleton instance so that it can be reused across the project. +Chunking utilities using RecursiveCharacterTextSplitter. """ from langchain_text_splitters import RecursiveCharacterTextSplitter -# Singleton splitter configuration -CHUNKER = RecursiveCharacterTextSplitter( - chunk_size=500, - chunk_overlap=100, -) +# Configure splitter: chunk size 500, overlap 100 +splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) -__all__ = ["CHUNKER"] +def split_text(text: str) -> list[str]: + """Split a large text into chunks. + + Args: + text: The raw document content. + + Returns: + List of string chunks. + """ + return splitter.split_text(text)