From b7375bd16dc8045078b29e491396a8f9a8313342 Mon Sep 17 00:00:00 2001 From: balabanovan530 <175+balabanovan530@noreply.localhost> Date: Tue, 2 Jun 2026 06:31:42 +0000 Subject: [PATCH] Add splitter.py --- splitter.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 splitter.py diff --git a/splitter.py b/splitter.py new file mode 100644 index 0000000..c2cdcb0 --- /dev/null +++ b/splitter.py @@ -0,0 +1,8 @@ +from langchain.text_splitter import RecursiveCharacterTextSplitter + + +def chunk_text(text: str, chunk_size: int = 1000, chunk_overlap: int = 200, separators: list[str] | None = None) -> list[str]: + if separators is None: + separators = ["\n\n", "\n", " ", ""] + splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=separators) + return splitter.split_text(text)