From 7025c66ec053b77fb28bc6e16a8272c7cb1cc8a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=9A=D1=83=D1=82?= =?UTF-8?q?=D0=BB=D0=B0=D1=85=D0=BC=D0=B5=D1=82=D0=BE=D0=B2?= Date: Thu, 28 May 2026 09:35:30 +0000 Subject: [PATCH] Add chunker.py --- chunker.py | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/chunker.py b/chunker.py index ced3b4f..07ad442 100644 --- a/chunker.py +++ b/chunker.py @@ -1,46 +1,34 @@ """ Chunking utilities for the RAG agent. -Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100. +Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and chunk_overlap=100. """ -import os -from pathlib import Path -from typing import List from langchain_text_splitters import RecursiveCharacterTextSplitter # Default splitter configuration -CHUNK_SIZE = 500 -OVERLAP = 100 +DEFAULT_SPLITTER = RecursiveCharacterTextSplitter( + chunk_size=500, + chunk_overlap=100, +) -splitter = RecursiveCharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=OVERLAP) - -def split_text(text: str) -> List[str]: +def split_text(text: str): """Split a large string into chunks. Parameters ---------- text: str - The raw document content. + Text to split. Returns ------- - List[str] - A list of chunk strings. + list[str] + List of chunk strings. """ - return splitter.split_text(text) + return DEFAULT_SPLITTER.split_text(text) -# Helper to read all files from a directory and split them - -def load_and_split(directory: Path) -> List[tuple]: - """Load text files from *directory* and split into chunks. - - Returns a list of tuples (chunk, metadata). - Metadata contains the source file path. - """ - chunks = [] - for file_path in directory.rglob("*.txt"): - content = file_path.read_text(encoding="utf-8") - for chunk in split_text(content): - chunks.append((chunk, {"source": str(file_path)})) - return chunks +# Example usage (not executed in tests) +if __name__ == "__main__": + sample = "\n".join([f"Line {i}" for i in range(1000)]) + chunks = split_text(sample) + print(f"Generated {len(chunks)} chunks") \ No newline at end of file