Delete obsolete src/loader.py
This commit is contained in:
@@ -1,37 +0,0 @@
|
||||
"""Utility to load all text files from a directory into the knowledge base.
|
||||
|
||||
Usage:
|
||||
python -m src.loader /path/to/docs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
from typing import Iterable
|
||||
|
||||
from .vector_store import KnowledgeBase
|
||||
|
||||
def _iter_text_files(dir_path: pathlib.Path) -> Iterable[pathlib.Path]:
|
||||
"""Yield all .txt files in *dir_path* recursively."""
|
||||
for p in dir_path.rglob("*.txt"):
|
||||
yield p
|
||||
|
||||
def load_directory(dir_path: pathlib.Path, kb: KnowledgeBase) -> None:
|
||||
"""Load each text file into the knowledge base.
|
||||
|
||||
The file name (without extension) is used as the document title.
|
||||
"""
|
||||
for file_path in _iter_text_files(dir_path):
|
||||
title = file_path.stem
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
kb.add_document(title=title, content=content)
|
||||
print(f"Loaded {file_path} as '{title}'")
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Load documents into the RAG knowledge base")
|
||||
parser.add_argument("directory", type=str, help="Path to directory containing .txt files")
|
||||
args = parser.parse_args()
|
||||
kb = KnowledgeBase()
|
||||
load_directory(pathlib.Path(args.directory), kb)
|
||||
print("Loading complete.")
|
||||
Reference in New Issue
Block a user