47 lines
1.2 KiB
Python
47 lines
1.2 KiB
Python
"""
|
|
Chunking utilities for the RAG agent.
|
|
|
|
Uses RecursiveCharacterTextSplitter from LangChain with chunk_size=500 and overlap=100.
|
|
"""
|
|
import os
|
|
from pathlib import Path
|
|
from typing import List
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
# Default splitter configuration
|
|
CHUNK_SIZE = 500
|
|
OVERLAP = 100
|
|
|
|
splitter = RecursiveCharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=OVERLAP)
|
|
|
|
def split_text(text: str) -> List[str]:
|
|
"""Split a large string into chunks.
|
|
|
|
Parameters
|
|
----------
|
|
text: str
|
|
The raw document content.
|
|
|
|
Returns
|
|
-------
|
|
List[str]
|
|
A list of chunk strings.
|
|
"""
|
|
return splitter.split_text(text)
|
|
|
|
# Helper to read all files from a directory and split them
|
|
|
|
def load_and_split(directory: Path) -> List[tuple]:
|
|
"""Load text files from *directory* and split into chunks.
|
|
|
|
Returns a list of tuples (chunk, metadata).
|
|
Metadata contains the source file path.
|
|
"""
|
|
chunks = []
|
|
for file_path in directory.rglob("*.txt"):
|
|
content = file_path.read_text(encoding="utf-8")
|
|
for chunk in split_text(content):
|
|
chunks.append((chunk, {"source": str(file_path)}))
|
|
return chunks
|