diff --git a/chunker.py b/chunker.py deleted file mode 100644 index e99eb22..0000000 --- a/chunker.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Chunking utilities for the RAG agent. - -This module provides a single function `get_text_splitter` that returns a -:class:`langchain.text_splitter.RecursiveCharacterTextSplitter` configured to -split documents into chunks of 500 characters with an overlap of 100. - -The splitter is used by :mod:`agent` when ingesting files. -""" - -from langchain.text_splitter import RecursiveCharacterTextSplitter - - -def get_text_splitter() -> RecursiveCharacterTextSplitter: - """Return a configured text splitter. - - The splitter uses a chunk size of 500 characters and an overlap of 100 - characters. These values are chosen to balance context length with the - ability to retrieve relevant passages during semantic search. - """ - return RecursiveCharacterTextSplitter( - chunk_size=500, - chunk_overlap=100, - ) - -# End of chunker.py