diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..974a8a0 --- /dev/null +++ b/chunker.py @@ -0,0 +1,12 @@ +from langchain_text_splitter import RecursiveCharacterTextSplitter + +def get_chunks(content, title): + splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) + texts = splitter.split_text(content) + return [ + { + "page_content": t, + "metadata": {"title": title} + } + for t in texts + ]