Add chunker.py
This commit is contained in:
+11
@@ -0,0 +1,11 @@
|
|||||||
|
"""Simple text splitter using RecursiveCharacterTextSplitter.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||||
|
|
||||||
|
class Chunker:
|
||||||
|
def __init__(self, chunk_size: int = 500, chunk_overlap: int = 50):
|
||||||
|
self.splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
||||||
|
|
||||||
|
def split(self, text: str) -> list[str]:
|
||||||
|
return self.splitter.split_text(text)
|
||||||
Reference in New Issue
Block a user