diff --git a/splitter.py b/splitter.py new file mode 100644 index 0000000..c2cdcb0 --- /dev/null +++ b/splitter.py @@ -0,0 +1,8 @@ +from langchain.text_splitter import RecursiveCharacterTextSplitter + + +def chunk_text(text: str, chunk_size: int = 1000, chunk_overlap: int = 200, separators: list[str] | None = None) -> list[str]: + if separators is None: + separators = ["\n\n", "\n", " ", ""] + splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=separators) + return splitter.split_text(text)