add chunker

2026-05-15 08:39:58 +00:00
parent b4ba5bdec5
commit bd777ed02c
1 changed files with 12 additions and 0 deletions
@@ -0,0 +1,12 @@
+from langchain_text_splitter import RecursiveCharacterTextSplitter
+
+def get_chunks(content, title):
+    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    texts = splitter.split_text(content)
+    return [
+        {
+            "page_content": t,
+            "metadata": {"title": title}
+        }
+        for t in texts
+    ]