diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..78a3f00 --- /dev/null +++ b/chunker.py @@ -0,0 +1,11 @@ +"""Simple text splitter using RecursiveCharacterTextSplitter. +""" + +from langchain_text_splitters import RecursiveCharacterTextSplitter + +class Chunker: + def __init__(self, chunk_size: int = 500, chunk_overlap: int = 50): + self.splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap) + + def split(self, text: str) -> list[str]: + return self.splitter.split_text(text)