add chunker
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
from langchain_text_splitter import RecursiveCharacterTextSplitter
|
||||
from langchain.schema import Document
|
||||
|
||||
def get_chunks(content: str, title: str):
|
||||
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
|
||||
texts = splitter.split_text(content)
|
||||
return [Document(page_content=t, metadata={"title": title, "content": t}) for t in texts]
|
||||
Reference in New Issue
Block a user