Delete obsolete src/loader.py
This commit is contained in:
@@ -1,37 +0,0 @@
|
|||||||
"""Utility to load all text files from a directory into the knowledge base.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python -m src.loader /path/to/docs
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import pathlib
|
|
||||||
from typing import Iterable
|
|
||||||
|
|
||||||
from .vector_store import KnowledgeBase
|
|
||||||
|
|
||||||
def _iter_text_files(dir_path: pathlib.Path) -> Iterable[pathlib.Path]:
|
|
||||||
"""Yield all .txt files in *dir_path* recursively."""
|
|
||||||
for p in dir_path.rglob("*.txt"):
|
|
||||||
yield p
|
|
||||||
|
|
||||||
def load_directory(dir_path: pathlib.Path, kb: KnowledgeBase) -> None:
|
|
||||||
"""Load each text file into the knowledge base.
|
|
||||||
|
|
||||||
The file name (without extension) is used as the document title.
|
|
||||||
"""
|
|
||||||
for file_path in _iter_text_files(dir_path):
|
|
||||||
title = file_path.stem
|
|
||||||
content = file_path.read_text(encoding="utf-8")
|
|
||||||
kb.add_document(title=title, content=content)
|
|
||||||
print(f"Loaded {file_path} as '{title}'")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
parser = argparse.ArgumentParser(description="Load documents into the RAG knowledge base")
|
|
||||||
parser.add_argument("directory", type=str, help="Path to directory containing .txt files")
|
|
||||||
args = parser.parse_args()
|
|
||||||
kb = KnowledgeBase()
|
|
||||||
load_directory(pathlib.Path(args.directory), kb)
|
|
||||||
print("Loading complete.")
|
|
||||||
Reference in New Issue
Block a user