From 95c6d65bf42a8e662aa8085190a5961dfde0fd3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?= =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?= Date: Thu, 28 May 2026 10:30:55 +0000 Subject: [PATCH] add chunker.py --- chunker.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/chunker.py b/chunker.py index 82d8a8f..5648121 100644 --- a/chunker.py +++ b/chunker.py @@ -1,26 +1,17 @@ """ Chunking utilities for the RAG agent. -Uses RecursiveCharacterTextSplitter from langchain-text-splitters with a chunk size of 500 and overlap of 100. +Uses RecursiveCharacterTextSplitter from langchain.text_splitter to split documents into +chunks of 500 characters with an overlap of 100 characters. The splitter is exposed as a +singleton instance so that it can be reused across the project. """ from langchain_text_splitters import RecursiveCharacterTextSplitter -# Global splitter instance -splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) +# Singleton splitter configuration +CHUNKER = RecursiveCharacterTextSplitter( + chunk_size=500, + chunk_overlap=100, +) - -def split_document(text: str): - """Split a document into chunks. - - Parameters - ---------- - text: str - Full text of the document. - - Returns - ------- - list[str] - List of chunk strings. - """ - return splitter.split_text(text) +__all__ = ["CHUNKER"]