From 4db6ce6ac97abc7bc5668290537e9a39a4d8253e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?= =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?= Date: Thu, 28 May 2026 12:38:29 +0000 Subject: [PATCH] add chunker.py --- chunker.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/chunker.py b/chunker.py index 5648121..1c402a9 100644 --- a/chunker.py +++ b/chunker.py @@ -1,17 +1,19 @@ """ -Chunking utilities for the RAG agent. - -Uses RecursiveCharacterTextSplitter from langchain.text_splitter to split documents into -chunks of 500 characters with an overlap of 100 characters. The splitter is exposed as a -singleton instance so that it can be reused across the project. +Chunking utilities using RecursiveCharacterTextSplitter. """ from langchain_text_splitters import RecursiveCharacterTextSplitter -# Singleton splitter configuration -CHUNKER = RecursiveCharacterTextSplitter( - chunk_size=500, - chunk_overlap=100, -) +# Configure splitter: chunk size 500, overlap 100 +splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) -__all__ = ["CHUNKER"] +def split_text(text: str) -> list[str]: + """Split a large text into chunks. + + Args: + text: The raw document content. + + Returns: + List of string chunks. + """ + return splitter.split_text(text)