59 lines
1.7 KiB
Python
59 lines
1.7 KiB
Python
"""Векторное хранилище ChromaDB + эмбеддинги Ollama."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from langchain_chroma import Chroma
|
|
from langchain_core.documents import Document
|
|
from langchain_ollama import OllamaEmbeddings
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
COLLECTION_NAME = "knowledge_base"
|
|
CHROMA_PATH = os.getenv("CHROMA_PATH", "./chroma_data")
|
|
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
|
EMBED_MODEL = os.getenv("OLLAMA_EMBED_MODEL", "nomic-embed-text")
|
|
|
|
|
|
def get_embeddings() -> OllamaEmbeddings:
|
|
return OllamaEmbeddings(
|
|
model=EMBED_MODEL,
|
|
base_url=OLLAMA_BASE_URL,
|
|
)
|
|
|
|
|
|
def get_vector_store() -> Chroma:
|
|
Path(CHROMA_PATH).mkdir(parents=True, exist_ok=True)
|
|
return Chroma(
|
|
collection_name=COLLECTION_NAME,
|
|
embedding_function=get_embeddings(),
|
|
persist_directory=CHROMA_PATH,
|
|
)
|
|
|
|
|
|
def chunk_document(content: str, title: str) -> list[Document]:
|
|
splitter = RecursiveCharacterTextSplitter(
|
|
chunk_size=500,
|
|
chunk_overlap=100,
|
|
)
|
|
documents = splitter.create_documents(
|
|
texts=[content],
|
|
metadatas=[{"title": title}],
|
|
)
|
|
for index, doc in enumerate(documents):
|
|
doc.metadata["chunk_index"] = index
|
|
doc.metadata["source"] = title
|
|
return documents
|
|
|
|
|
|
def add_document_to_store(content: str, title: str) -> int:
|
|
store = get_vector_store()
|
|
chunks = chunk_document(content, title)
|
|
store.add_documents(chunks)
|
|
return len(chunks)
|
|
|
|
|
|
def search_store(query: str, max_results: int = 4) -> list[tuple[Document, float]]:
|
|
store = get_vector_store()
|
|
return store.similarity_search_with_score(query, k=max_results)
|