add chunker

2026-05-15 08:39:58 +00:00
parent b4ba5bdec5
commit bd777ed02c
1 changed files with 12 additions and 0 deletions
@@ -0,0 +1,12 @@
 from langchain_text_splitter import RecursiveCharacterTextSplitter
 def get_chunks(content, title):
    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
    texts = splitter.split_text(content)
    return [
        {
            "page_content": t,
            "metadata": {"title": title}
        }
        for t in texts
    ]