20 lines
483 B
Python
20 lines
483 B
Python
"""
|
|
Chunking utilities using RecursiveCharacterTextSplitter.
|
|
"""
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
# Configure splitter: chunk size 500, overlap 100
|
|
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
|
|
|
|
def split_text(text: str) -> list[str]:
|
|
"""Split a large text into chunks.
|
|
|
|
Args:
|
|
text: The raw document content.
|
|
|
|
Returns:
|
|
List of string chunks.
|
|
"""
|
|
return splitter.split_text(text)
|