Add splitter.py

2026-06-02 06:31:42 +00:00
parent 2cc23af021
commit b7375bd16d
1 changed files with 8 additions and 0 deletions
@@ -0,0 +1,8 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+def chunk_text(text: str, chunk_size: int = 1000, chunk_overlap: int = 200, separators: list[str] | None = None) -> list[str]:
+    if separators is None:
+        separators = ["\n\n", "\n", " ", ""]
+    splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=separators)
+    return splitter.split_text(text)