23 lines
790 B
Python
23 lines
790 B
Python
"""Text chunking utilities for splitting documents before indexing."""
|
|
from typing import List
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
from langchain_core.documents import Document
|
|
|
|
|
|
def load_and_split(text: str, chunk_size: int = 500, chunk_overlap: int = 100) -> List[Document]:
|
|
"""Split text into overlapping chunks suitable for vector store indexing.
|
|
|
|
Args:
|
|
text: raw input text
|
|
chunk_size: maximum size of each chunk in characters
|
|
chunk_overlap: number of characters to overlap between consecutive chunks
|
|
|
|
Returns:
|
|
list of Document objects
|
|
"""
|
|
splitter = RecursiveCharacterTextSplitter(
|
|
chunk_size=chunk_size,
|
|
chunk_overlap=chunk_overlap,
|
|
)
|
|
return splitter.create_documents([text])
|