diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..78c2c17 --- /dev/null +++ b/chunker.py @@ -0,0 +1,22 @@ +"""Text chunking utilities for splitting documents before indexing.""" +from typing import List +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain_core.documents import Document + + +def load_and_split(text: str, chunk_size: int = 500, chunk_overlap: int = 100) -> List[Document]: + """Split text into overlapping chunks suitable for vector store indexing. + + Args: + text: raw input text + chunk_size: maximum size of each chunk in characters + chunk_overlap: number of characters to overlap between consecutive chunks + + Returns: + list of Document objects + """ + splitter = RecursiveCharacterTextSplitter( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + ) + return splitter.create_documents([text])