petya
75601988c2
- Cover, TOC, 5 dividers, 3 recap slides - 5 sections (chains, langgraph, deepagents, openswe, ecosystem) - design-system.js with theme tokens + 9 helper functions - research/: timeline + sources + per-tech notes - final-compile.js + merge.js for rebuild pipeline - output/: langchain-evolution.pptx (2.3 MB) + langchain-evolution.pdf (1.1 MB) + 7 sample previews
59 lines
1.9 KiB
JavaScript
59 lines
1.9 KiB
JavaScript
// Slide 08: Evaluators + run_on_dataset -- off-line evaluation pipeline
|
|
// Code slide: custom evaluator + run_on_dataset.
|
|
|
|
const ds = require('./design-system');
|
|
|
|
function createSlide(pres, theme) {
|
|
const slide = pres.addSlide();
|
|
ds.helpers.slideBase(slide, pres, theme);
|
|
|
|
ds.helpers.addHeader(slide, pres, theme, {
|
|
eyebrow: 'STAGE 5: ECOSYSTEM',
|
|
section: 'Evaluators',
|
|
title: 'run_on_dataset + evaluator-ы',
|
|
sectionNumber: 5,
|
|
titleSize: 24,
|
|
});
|
|
|
|
ds.helpers.addCodeBlock(slide, pres, theme, {
|
|
x: 0.5, y: 1.45, w: 9.0, h: 2.85,
|
|
language: 'python',
|
|
fontSize: 10,
|
|
code: [
|
|
'from langsmith import Client',
|
|
'from langsmith.schemas import Example, Run',
|
|
'',
|
|
'client = Client()',
|
|
'',
|
|
'# 1. Target -- что прогоняем (chain, graph, agent, callable)',
|
|
'def target(inputs: dict) -> dict:',
|
|
' return {"answer": my_chain.invoke(inputs["question"])}',
|
|
'',
|
|
'# 2. Evaluator -- scoring function',
|
|
'def answer_match(run: Run, example: Example) -> dict:',
|
|
' score = 1.0 if example.outputs["answer"] in run.outputs["answer"] else 0.0',
|
|
' return {"key": "answer_match", "score": score}',
|
|
'',
|
|
'# 3. Прогон: target на датасете + scoring',
|
|
'client.run_on_dataset(',
|
|
' dataset_name="rag-qa-eval-v1",',
|
|
' llm_or_chain_factory=target,',
|
|
' evaluators=[answer_match],',
|
|
')',
|
|
].join('\n'),
|
|
});
|
|
|
|
ds.helpers.addCallout(slide, pres, theme, {
|
|
x: 0.5, y: 4.5, w: 9.0, h: 0.55,
|
|
kind: 'success',
|
|
text: 'Готовые evaluator-ы: llm_as_judge, exact_match, embedding_distance, cot_qa.',
|
|
});
|
|
|
|
ds.helpers.addSourceLine(slide, pres, theme, {
|
|
source: 'docs.smith.langchain.com Evaluation > Evaluator types',
|
|
});
|
|
ds.helpers.addPageNumber(slide, pres, theme, 8);
|
|
}
|
|
|
|
module.exports = { createSlide };
|