From e3238fabc4b8a9b6985e246b6aa7bbd624585686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?= =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?= Date: Thu, 28 May 2026 09:48:59 +0000 Subject: [PATCH] add chunker.py --- chunker.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/chunker.py b/chunker.py index 07ad442..590ada9 100644 --- a/chunker.py +++ b/chunker.py @@ -1,34 +1,12 @@ """ Chunking utilities for the RAG agent. -Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and chunk_overlap=100. +Uses RecursiveCharacterTextSplitter from LangChain with a chunk size of 500 and an overlap of 100. """ - from langchain_text_splitters import RecursiveCharacterTextSplitter -# Default splitter configuration -DEFAULT_SPLITTER = RecursiveCharacterTextSplitter( +# Global splitter instance +chunker = RecursiveCharacterTextSplitter( chunk_size=500, chunk_overlap=100, ) - -def split_text(text: str): - """Split a large string into chunks. - - Parameters - ---------- - text: str - Text to split. - - Returns - ------- - list[str] - List of chunk strings. - """ - return DEFAULT_SPLITTER.split_text(text) - -# Example usage (not executed in tests) -if __name__ == "__main__": - sample = "\n".join([f"Line {i}" for i in range(1000)]) - chunks = split_text(sample) - print(f"Generated {len(chunks)} chunks") \ No newline at end of file