diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..f9876e6 --- /dev/null +++ b/chunker.py @@ -0,0 +1,7 @@ +from langchain_text_splitter import RecursiveCharacterTextSplitter +from langchain.schema import Document + +def get_chunks(content: str, title: str): + splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) + texts = splitter.split_text(content) + return [Document(page_content=t, metadata={"title": title, "content": t}) for t in texts]