30 lines
982 B
Python
30 lines
982 B
Python
"""Utility module for chunking text.
|
|
|
|
This module provides a helper function to split large text into smaller chunks suitable for
|
|
embedding and storage in a vector database. It uses the
|
|
`RecursiveCharacterTextSplitter` from the `langchain_text_splitters`
|
|
package.
|
|
"""
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
def chunk_text(text: str, chunk_size: int = 1000, chunk_overlap: int = 200) -> list[str]:
|
|
"""Split *text* into smaller chunks.
|
|
|
|
Parameters
|
|
----------
|
|
text: str
|
|
The raw text to split.
|
|
chunk_size: int, optional
|
|
Maximum size of each chunk in characters. Defaults to 1000.
|
|
chunk_overlap: int, optional
|
|
Number of overlapping characters between consecutive chunks.
|
|
Defaults to 200.
|
|
|
|
Returns
|
|
-------
|
|
list[str]
|
|
A list of text chunks.
|
|
"""
|
|
splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
|
return splitter.split_text(text) |