27 lines
846 B
Python
27 lines
846 B
Python
"""
|
|
Chunking utilities for the RAG agent.
|
|
|
|
This module provides a single function `get_text_splitter` that returns a
|
|
:class:`langchain.text_splitter.RecursiveCharacterTextSplitter` configured to
|
|
split documents into chunks of 500 characters with an overlap of 100.
|
|
|
|
The splitter is used by :mod:`agent` when ingesting files.
|
|
"""
|
|
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
|
|
|
def get_text_splitter() -> RecursiveCharacterTextSplitter:
|
|
"""Return a configured text splitter.
|
|
|
|
The splitter uses a chunk size of 500 characters and an overlap of 100
|
|
characters. These values are chosen to balance context length with the
|
|
ability to retrieve relevant passages during semantic search.
|
|
"""
|
|
return RecursiveCharacterTextSplitter(
|
|
chunk_size=500,
|
|
chunk_overlap=100,
|
|
)
|
|
|
|
# End of chunker.py
|