add chunker
This commit is contained in:
1 parent
d0b18dc236
commit
fdfcfa4b82
1 file changed
+7
@@ -0,0 +1,7 @@
|
||||
from langchain_text_splitter import RecursiveCharacterTextSplitter
|
||||
from langchain.schema import Document
|
||||
|
||||
def get_chunks(content: str, title: str):
|
||||
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
|
||||
texts = splitter.split_text(content)
|
||||
return [Document(page_content=t, metadata={"title": title, "content": t}) for t in texts]
|
||||
Reference in new issue
Block a user