diff --git a/chunker.py b/chunker.py index 590ada9..82d8a8f 100644 --- a/chunker.py +++ b/chunker.py @@ -1,12 +1,26 @@ """ Chunking utilities for the RAG agent. -Uses RecursiveCharacterTextSplitter from LangChain with a chunk size of 500 and an overlap of 100. +Uses RecursiveCharacterTextSplitter from langchain-text-splitters with a chunk size of 500 and overlap of 100. """ + from langchain_text_splitters import RecursiveCharacterTextSplitter # Global splitter instance -chunker = RecursiveCharacterTextSplitter( - chunk_size=500, - chunk_overlap=100, -) +splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100) + + +def split_document(text: str): + """Split a document into chunks. + + Parameters + ---------- + text: str + Full text of the document. + + Returns + ------- + list[str] + List of chunk strings. + """ + return splitter.split_text(text)