add chunker
This commit is contained in:
@@ -0,0 +1,7 @@
|
|||||||
|
from langchain_text_splitter import RecursiveCharacterTextSplitter
|
||||||
|
from langchain.schema import Document
|
||||||
|
|
||||||
|
def get_chunks(content: str, title: str):
|
||||||
|
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
|
||||||
|
texts = splitter.split_text(content)
|
||||||
|
return [Document(page_content=t, metadata={"title": title, "content": t}) for t in texts]
|
||||||
Reference in New Issue
Block a user