diff --git a/chunker.py b/chunker.py index 1c402a9..e99eb22 100644 --- a/chunker.py +++ b/chunker.py @@ -1,19 +1,26 @@ """ -Chunking utilities using RecursiveCharacterTextSplitter. +Chunking utilities for the RAG agent. + +This module provides a single function `get_text_splitter` that returns a +:class:`langchain.text_splitter.RecursiveCharacterTextSplitter` configured to +split documents into chunks of 500 characters with an overlap of 100. + +The splitter is used by :mod:`agent` when ingesting files. """ -from langchain_text_splitters import RecursiveCharacterTextSplitter +from langchain.text_splitter import RecursiveCharacterTextSplitter -# Configure splitter: chunk size 500, overlap 100 -splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) -def split_text(text: str) -> list[str]: - """Split a large text into chunks. +def get_text_splitter() -> RecursiveCharacterTextSplitter: + """Return a configured text splitter. - Args: - text: The raw document content. - - Returns: - List of string chunks. + The splitter uses a chunk size of 500 characters and an overlap of 100 + characters. These values are chosen to balance context length with the + ability to retrieve relevant passages during semantic search. """ - return splitter.split_text(text) + return RecursiveCharacterTextSplitter( + chunk_size=500, + chunk_overlap=100, + ) + +# End of chunker.py