9 lines
425 B
Python
9 lines
425 B
Python
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
|
|
|
def chunk_text(text: str, chunk_size: int = 500, chunk_overlap: int = 100, separators: list[str] | None = None) -> list[str]:
|
|
if separators is None:
|
|
separators = ["\n\n", "\n", " ", ""]
|
|
splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=separators)
|
|
return splitter.split_text(text)
|