27 lines
597 B
Python
27 lines
597 B
Python
"""
|
|
Chunking utilities for the RAG agent.
|
|
|
|
Uses RecursiveCharacterTextSplitter from langchain-text-splitters with a chunk size of 500 and overlap of 100.
|
|
"""
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
# Global splitter instance
|
|
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
|
|
|
|
|
|
def split_document(text: str):
|
|
"""Split a document into chunks.
|
|
|
|
Parameters
|
|
----------
|
|
text: str
|
|
Full text of the document.
|
|
|
|
Returns
|
|
-------
|
|
list[str]
|
|
List of chunk strings.
|
|
"""
|
|
return splitter.split_text(text)
|