Add chunker.py
This commit is contained in:
+11
@@ -0,0 +1,11 @@
|
||||
"""Simple text splitter using RecursiveCharacterTextSplitter.
|
||||
"""
|
||||
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
|
||||
class Chunker:
|
||||
def __init__(self, chunk_size: int = 500, chunk_overlap: int = 50):
|
||||
self.splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
||||
|
||||
def split(self, text: str) -> list[str]:
|
||||
return self.splitter.split_text(text)
|
||||
Reference in New Issue
Block a user