From ea565bf41e460fba8199098713ac3a21f0a1747f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?= =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?= Date: Thu, 28 May 2026 13:06:10 +0000 Subject: [PATCH] add chunker.py --- chunker.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/chunker.py b/chunker.py index 1c402a9..e99eb22 100644 --- a/chunker.py +++ b/chunker.py @@ -1,19 +1,26 @@ """ -Chunking utilities using RecursiveCharacterTextSplitter. +Chunking utilities for the RAG agent. + +This module provides a single function `get_text_splitter` that returns a +:class:`langchain.text_splitter.RecursiveCharacterTextSplitter` configured to +split documents into chunks of 500 characters with an overlap of 100. + +The splitter is used by :mod:`agent` when ingesting files. """ -from langchain_text_splitters import RecursiveCharacterTextSplitter +from langchain.text_splitter import RecursiveCharacterTextSplitter -# Configure splitter: chunk size 500, overlap 100 -splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) -def split_text(text: str) -> list[str]: - """Split a large text into chunks. +def get_text_splitter() -> RecursiveCharacterTextSplitter: + """Return a configured text splitter. - Args: - text: The raw document content. - - Returns: - List of string chunks. + The splitter uses a chunk size of 500 characters and an overlap of 100 + characters. These values are chosen to balance context length with the + ability to retrieve relevant passages during semantic search. """ - return splitter.split_text(text) + return RecursiveCharacterTextSplitter( + chunk_size=500, + chunk_overlap=100, + ) + +# End of chunker.py