langchain-evolution-deck/examples/slide-118/script.py

"""
Slide 118: run_on_dataset + evaluator-ы
Section 5: Экосистема
Source: slides/section5-ecosystem/slide-08.js

Сгенерировано автоматически из slide-NN.js / sectionN.pptx.
Паттерн провайдера скопирован из
bro-js/agents/teacher/assistant/src/angry_teacher/llm.py.

Запуск:
  cd langchain-evolution-deck
  cp .env.example .env   # заполни OPENAI_API_KEY
  python examples/slide-118/script.py
"""
import os
from dotenv import load_dotenv
from langchain_openai import ChatOpenAI

load_dotenv()

llm = ChatOpenAI(
    # model="openrouter/free",
    # model="qwen/qwen3.5-35b-a3b",
    model="openai/gpt-oss-20b",
    # model="nvidia/nemotron-3-nano",
    # model="qwen/qwen3.5-9b",
    base_url="https://llm.brojs.ru/v1",
    # base_url="https://api.minimax.io/v1",
    # base_url="http://0.0.0.0:8090/v1",
    # base_url="https://openrouter.ai/api/v1",
    # api_key=os.getenv("MINIMAX_API_KEY"),
    api_key=(os.getenv("OPENAI_API_KEY") or "").strip() or None,
    temperature=0.7,
    stream_usage=True,
)


# === Code from slide 118 ===
from langsmith import Client
from langsmith.schemas import Example, Run

client = Client()

# 1. Target -- что прогоняем (chain, graph, agent, callable)
def target(inputs: dict) -> dict:
    return {"answer": my_chain.invoke(inputs["question"])}

# 2. Evaluator -- scoring function
def answer_match(run: Run, example: Example) -> dict:
    score = 1.0 if example.outputs["answer"] in run.outputs["answer"] else 0.0
    return {"key": "answer_match", "score": score}

# 3. Прогон: target на датасете + scoring
client.run_on_dataset(
    dataset_name="rag-qa-eval-v1",
    llm_or_chain_factory=target,
    evaluators=[answer_match],
)