Add vector store module with Qdrant + Ollama embeddings
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Vector store module for RAG agent using Qdrant and Ollama embeddings."""
|
||||
|
||||
import uuid
|
||||
from typing import List, Optionl
|
||||
|
||||
from langchain_community.embeddings import OllamaEmbeddings
|
||||
from langchain_community.vectorstores import Qdrant
|
||||
from langchain_core.documents import Document
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import Distance, VectorParams
|
||||
|
||||
COLLECTION_NAME = "rag_knowledge_base"
|
||||
OLLAMA_BASE_URL = "http://localhost:11434"
|
||||
EMBED_MODEL = "nomic-embed-text"
|
||||
EMBED_DIMENSION = 768
|
||||
|
||||
def get_embeddings() -> OllamaEmbeddings:
|
||||
"""Create and return Ollama embeddings instance."""
|
||||
return OllamaEmbeddings(
|
||||
model=EMBED_MODEL
|
||||
base_url=OLLAMA_BASE_URL,
|
||||
)
|
||||
|
||||
|
||||
def get_qdrant_client() -> QdrantClient:
|
||||
"""Create and return Qdrant client (in-memory for local dev)."""
|
||||
return QdrantClient(":memory:")
|
||||
|
||||
|
||||
def create_collection(client: QdrantClient) -> None:
|
||||
"""Create Qdrant collection if doesn't exist."""
|
||||
existing = [c.name for c in client.get_collections().collections]
|
||||
if COLLECTION_NAME not in existing:
|
||||
client.create_collection(
|
||||
collection_name=COLLECTION_NAME,
|
||||
vectors_config=VectorParams(
|
||||
size=EMBED_DIMENSION,
|
||||
distance=Distance.COSINE,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_vector_store(client: Optional[QdrantClient] = None) -> Qdrant:
|
||||
"""Initialize and return Qdrant vector store."""
|
||||
if client is None:
|
||||
client = get_qdrant_client()
|
||||
create_collection(client)
|
||||
embeddings = get_embeddings()
|
||||
return Qdrant(
|
||||
client=client,
|
||||
collection_name=COLLECTION_NAME,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
|
||||
|
||||
def chunk_documents(
|
||||
documents: List[Document],
|
||||
chunk_size: int = 512,
|
||||
chunk_overlap: int = 50,
|
||||
) -> List[Document]:
|
||||
"""Split documents into chunks using RecursiveCharacterTextSplitter."""
|
||||
splitter = RecursiveCharacterTextSplitter(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
length_function=len,
|
||||
is_separator_regex=False,
|
||||
)
|
||||
return splitter.split_documents(documents)
|
||||
|
||||
|
||||
def add_documents_to_store(
|
||||
vector_store: Qdrant,
|
||||
documents: List[Document],
|
||||
) -> List[str]:
|
||||
"""Add documents to vector store, return list of IDs."""
|
||||
chunks = chunk_documents(documents)
|
||||
ids = [str(uuid.uuid4()) for _ in chunks]
|
||||
vector_store.add_documents(documents=chunks, ids=ids)
|
||||
return ids
|
||||
|
||||
|
||||
def search_store(
|
||||
vector_store: Qdrant,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
) -> List[Document]:
|
||||
"""Search vector store for relevant documents."""
|
||||
return vector_store.similarity_search(query, k=max_results)
|
||||
Reference in New Issue
Block a user