32 lines
811 B
Python
32 lines
811 B
Python
"""
|
|
Chunking utilities for the RAG agent.
|
|
|
|
Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100.
|
|
"""
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
# Global splitter instance
|
|
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
|
|
|
|
def split_text(text: str) -> list[str]:
|
|
"""Split a large string into chunks.
|
|
|
|
Parameters
|
|
----------
|
|
text: str
|
|
The raw document content.
|
|
|
|
Returns
|
|
-------
|
|
List[str]
|
|
A list of chunk strings.
|
|
"""
|
|
return splitter.split_text(text)
|
|
|
|
# Example usage (not executed in tests)
|
|
if __name__ == "__main__": # pragma: no cover
|
|
sample = "\n".join([f"Line {i}" for i in range(1000)])
|
|
chunks = split_text(sample)
|
|
print(f"Generated {len(chunks)} chunks")
|