Add chunker.py
This commit is contained in:
1 parent
e0b37d7b9f
commit
07c5b07617
1 file changed
+11
+11
@@ -0,0 +1,11 @@
|
|||||||
|
"""Simple text splitter using RecursiveCharacterTextSplitter.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||||
|
|
||||||
|
class Chunker:
|
||||||
|
def __init__(self, chunk_size: int = 500, chunk_overlap: int = 50):
|
||||||
|
self.splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
||||||
|
|
||||||
|
def split(self, text: str) -> list[str]:
|
||||||
|
return self.splitter.split_text(text)
|
||||||
Reference in new issue
Block a user