34 lines
803 B
Python
34 lines
803 B
Python
"""
|
|
Chunking utilities for the RAG agent.
|
|
|
|
Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and chunk_overlap=100.
|
|
"""
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
# Default splitter configuration
|
|
DEFAULT_SPLITTER = RecursiveCharacterTextSplitter(
|
|
chunk_size=500,
|
|
chunk_overlap=100,
|
|
)
|
|
|
|
def split_text(text: str):
|
|
"""Split a large string into chunks.
|
|
|
|
Parameters
|
|
----------
|
|
text: str
|
|
Text to split.
|
|
|
|
Returns
|
|
-------
|
|
list[str]
|
|
List of chunk strings.
|
|
"""
|
|
return DEFAULT_SPLITTER.split_text(text)
|
|
|
|
# Example usage (not executed in tests)
|
|
if __name__ == "__main__":
|
|
sample = "\n".join([f"Line {i}" for i in range(1000)])
|
|
chunks = split_text(sample)
|
|
print(f"Generated {len(chunks)} chunks") |