diff --git a/chunker.py b/chunker.py index 82d8a8f..5648121 100644 --- a/chunker.py +++ b/chunker.py @@ -1,26 +1,17 @@ """ Chunking utilities for the RAG agent. -Uses RecursiveCharacterTextSplitter from langchain-text-splitters with a chunk size of 500 and overlap of 100. +Uses RecursiveCharacterTextSplitter from langchain.text_splitter to split documents into +chunks of 500 characters with an overlap of 100 characters. The splitter is exposed as a +singleton instance so that it can be reused across the project. """ from langchain_text_splitters import RecursiveCharacterTextSplitter -# Global splitter instance -splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) +# Singleton splitter configuration +CHUNKER = RecursiveCharacterTextSplitter( + chunk_size=500, + chunk_overlap=100, +) - -def split_document(text: str): - """Split a document into chunks. - - Parameters - ---------- - text: str - Full text of the document. - - Returns - ------- - list[str] - List of chunk strings. - """ - return splitter.split_text(text) +__all__ = ["CHUNKER"]