add chunker
This commit is contained in:
1 parent
d0b18dc236
commit
fdfcfa4b82
1 file changed
+7
@@ -0,0 +1,7 @@
|
|||||||
|
from langchain_text_splitter import RecursiveCharacterTextSplitter
|
||||||
|
from langchain.schema import Document
|
||||||
|
|
||||||
|
def get_chunks(content: str, title: str):
|
||||||
|
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
|
||||||
|
texts = splitter.split_text(content)
|
||||||
|
return [Document(page_content=t, metadata={"title": title, "content": t}) for t in texts]
|
||||||
Reference in new issue
Block a user