feat: solution for 'Повторный экзамен #2: Сравнительный обзор 3 сущностей (Tavily)'
This commit is contained in:
@@ -7,13 +7,13 @@
|
||||
EN
|
||||
Повторный экзамен #2: Сравнительный обзор 3 сущностей (Tavily)
|
||||
Зачёт
|
||||
Версия 6
|
||||
Версия 7
|
||||
Дедлайн сдачи: 31.08.2026
|
||||
|
||||
В работе
|
||||
|
||||
Требуется доработка
|
||||
|
||||
В вашем решении отсутствует упоминание и использование Qdrant, хотя это требование явно указано в задании. Пожалуйста, добавьте интеграцию с Qdrant или замените его на другой поддерживаемый вами векторный хранилище, чтобы решение соответствовало публичному стеку.
|
||||
В представленном решении отсутствует интеграция с Qdrant, как требуется в публичном стеке задания. Кроме того, не реализовано требуемое сравнение в виде markdown‑таблицы и явный вердикт. Пожалуйста, доработайте эти части, чтобы решение соответствовало требованиям.
|
||||
|
||||
Редактиров
|
||||
Редактиро
|
||||
+4
-12
@@ -1,24 +1,16 @@
|
||||
{
|
||||
"name": "tavily-qdrant-demo",
|
||||
"name": "entity-comparison",
|
||||
"version": "1.0.0",
|
||||
"description": "Demo project integrating Tavily API with Qdrant vector store",
|
||||
"description": "Compare three entities using Tavily, OpenAI embeddings, and Qdrant",
|
||||
"main": "src/index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/index.js"
|
||||
},
|
||||
"keywords": [
|
||||
"tavily",
|
||||
"qdrant",
|
||||
"vector",
|
||||
"search",
|
||||
"express"
|
||||
],
|
||||
"author": "Your Name",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@qdrant/js-client-rest": "^1.0.0",
|
||||
"axios": "^1.7.2",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.18.2"
|
||||
"openai": "^4.21.0"
|
||||
}
|
||||
}
|
||||
+95
-56
@@ -1,61 +1,100 @@
|
||||
const express = require('express');
|
||||
const { fetchEntityData } = require('./tavily');
|
||||
const {
|
||||
initClient,
|
||||
createCollection,
|
||||
upsertEmbeddings,
|
||||
searchEmbeddings
|
||||
} = require('./qdrant');
|
||||
require('dotenv').config();
|
||||
import dotenv from "dotenv";
|
||||
import { fetchSummary } from "./tavily.js";
|
||||
import { QdrantWrapper } from "./qdrant.js";
|
||||
import { OpenAI } from "openai";
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3000;
|
||||
dotenv.config();
|
||||
|
||||
// Middleware to parse JSON
|
||||
app.use(express.json());
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
||||
const openai = new OpenAI({ apiKey: OPENAI_API_KEY });
|
||||
|
||||
// Initialize Qdrant client and collection on startup
|
||||
(async () => {
|
||||
try {
|
||||
initClient();
|
||||
await createCollection();
|
||||
} catch (err) {
|
||||
console.error('Failed to initialize Qdrant:', err.message);
|
||||
const entities = [
|
||||
"Apple Inc.",
|
||||
"Microsoft Corporation",
|
||||
"Google LLC",
|
||||
];
|
||||
|
||||
async function getEmbedding(text) {
|
||||
const response = await openai.embeddings.create({
|
||||
model: "text-embedding-3-small",
|
||||
input: text,
|
||||
});
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
|
||||
function cosineSimilarity(vecA, vecB) {
|
||||
const dot = vecA.reduce((sum, a, i) => sum + a * vecB[i], 0);
|
||||
const normA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0));
|
||||
const normB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0));
|
||||
return dot / (normA * normB);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const qdrant = new QdrantWrapper();
|
||||
await qdrant.createCollection();
|
||||
|
||||
const entityData = {};
|
||||
|
||||
// Fetch summaries, embeddings and upsert
|
||||
for (const entity of entities) {
|
||||
console.log(`Processing ${entity}...`);
|
||||
const summary = await fetchSummary(entity);
|
||||
const embedding = await getEmbedding(summary);
|
||||
await qdrant.upsertEntity(entity, embedding, { name: entity, summary });
|
||||
entityData[entity] = { summary, embedding };
|
||||
}
|
||||
|
||||
// Compute pairwise similarities
|
||||
const similarities = {};
|
||||
for (const a of entities) {
|
||||
similarities[a] = {};
|
||||
for (const b of entities) {
|
||||
if (a === b) continue;
|
||||
const sim = cosineSimilarity(
|
||||
entityData[a].embedding,
|
||||
entityData[b].embedding
|
||||
);
|
||||
similarities[a][b] = sim.toFixed(4);
|
||||
}
|
||||
}
|
||||
|
||||
// Generate markdown table
|
||||
let markdown = "# Entity Comparison\n\n";
|
||||
markdown += "| Entity | Summary | Similarity to Apple | Similarity to Microsoft | Similarity to Google |\n";
|
||||
markdown += "|--------|---------|---------------------|------------------------|---------------------|\n";
|
||||
|
||||
for (const entity of entities) {
|
||||
const row = [
|
||||
entity,
|
||||
`"${entityData[entity].summary.replace(/\n/g, " ")}"`,
|
||||
similarities[entity]["Apple Inc."],
|
||||
similarities[entity]["Microsoft Corporation"],
|
||||
similarities[entity]["Google LLC"],
|
||||
];
|
||||
markdown += `| ${row.join(" | ")} |\n`;
|
||||
}
|
||||
|
||||
// Verdict
|
||||
let maxSim = -1;
|
||||
let pair = [];
|
||||
for (let i = 0; i < entities.length; i++) {
|
||||
for (let j = i + 1; j < entities.length; j++) {
|
||||
const a = entities[i];
|
||||
const b = entities[j];
|
||||
const sim = parseFloat(similarities[a][b]);
|
||||
if (sim > maxSim) {
|
||||
maxSim = sim;
|
||||
pair = [a, b];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
markdown += `\n**Verdict:** The entities with the highest similarity are **${pair[0]}** and **${pair[1]}** (similarity: ${maxSim.toFixed(4)}).\n`;
|
||||
|
||||
console.log(markdown);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
|
||||
// Route to fetch data for an entity and store embeddings
|
||||
app.get('/fetch/:entity', async (req, res) => {
|
||||
const entity = req.params.entity;
|
||||
try {
|
||||
const text = await fetchEntityData(entity);
|
||||
await upsertEmbeddings(entity, text);
|
||||
res.json({ status: 'success', entity, textLength: text.length });
|
||||
} catch (err) {
|
||||
res.status(500).json({ status: 'error', message: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Route to search embeddings
|
||||
app.get('/search', async (req, res) => {
|
||||
const query = req.query.q;
|
||||
if (!query) {
|
||||
return res.status(400).json({ status: 'error', message: 'Missing query parameter q' });
|
||||
}
|
||||
try {
|
||||
const results = await searchEmbeddings(query);
|
||||
res.json({ status: 'success', query, results });
|
||||
} catch (err) {
|
||||
res.status(500).json({ status: 'error', message: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Health check
|
||||
app.get('/', (req, res) => {
|
||||
res.send('Tavily-Qdrant Demo Server');
|
||||
});
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Server running on http://localhost:${PORT}`);
|
||||
});
|
||||
+33
-96
@@ -1,116 +1,53 @@
|
||||
const { QdrantClient } = require('@qdrant/js-client-rest');
|
||||
const axios = require('axios');
|
||||
require('dotenv').config();
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const QDRANT_URL = process.env.QDRANT_URL;
|
||||
const QDRANT_API_KEY = process.env.QDRANT_API_KEY;
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
||||
const VECTOR_SIZE = 1536; // OpenAI Ada embeddings size
|
||||
|
||||
const COLLECTION_NAME = 'entities';
|
||||
|
||||
let client = null;
|
||||
|
||||
// Initialize Qdrant client
|
||||
function initClient() {
|
||||
client = new QdrantClient({
|
||||
export class QdrantWrapper {
|
||||
constructor() {
|
||||
this.client = new QdrantClient({
|
||||
url: QDRANT_URL,
|
||||
apiKey: QDRANT_API_KEY
|
||||
apiKey: QDRANT_API_KEY,
|
||||
});
|
||||
}
|
||||
|
||||
// Create or recreate collection
|
||||
async function createCollection() {
|
||||
if (!client) initClient();
|
||||
try {
|
||||
await client.recreateCollection(COLLECTION_NAME, {
|
||||
vectors: {
|
||||
size: VECTOR_SIZE,
|
||||
distance: 'Cosine'
|
||||
this.collectionName = "entity_embeddings";
|
||||
}
|
||||
|
||||
async createCollection() {
|
||||
try {
|
||||
await this.client.createCollection(this.collectionName, {
|
||||
vectors: { size: 1536, distance: "Cosine" },
|
||||
});
|
||||
console.log(`Collection '${COLLECTION_NAME}' created/recreated.`);
|
||||
} catch (err) {
|
||||
console.error('Error creating collection:', err.message);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Get embedding from OpenAI
|
||||
async function getEmbedding(text) {
|
||||
try {
|
||||
const response = await axios.post(
|
||||
'https://api.openai.com/v1/embeddings',
|
||||
{
|
||||
input: text,
|
||||
model: 'text-embedding-ada-002'
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${OPENAI_API_KEY}`
|
||||
}
|
||||
}
|
||||
);
|
||||
if (response.data && response.data.data && response.data.data[0]) {
|
||||
return response.data.data[0].embedding;
|
||||
console.log(`Collection ${this.collectionName} created.`);
|
||||
} catch (e) {
|
||||
if (e.message.includes("already exists")) {
|
||||
console.log(`Collection ${this.collectionName} already exists.`);
|
||||
} else {
|
||||
throw new Error('No embedding returned');
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error getting embedding:', err.message);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Upsert embeddings for an entity
|
||||
async function upsertEmbeddings(entity, text) {
|
||||
if (!client) initClient();
|
||||
try {
|
||||
const vector = await getEmbedding(text);
|
||||
const point = {
|
||||
id: entity,
|
||||
async upsertEntity(id, vector, payload) {
|
||||
await this.client.upsert(this.collectionName, {
|
||||
points: [
|
||||
{
|
||||
id,
|
||||
vector,
|
||||
payload: {
|
||||
entity,
|
||||
text
|
||||
}
|
||||
};
|
||||
await client.upsertPoints(COLLECTION_NAME, {
|
||||
points: [point]
|
||||
payload,
|
||||
},
|
||||
],
|
||||
});
|
||||
console.log(`Upserted embeddings for entity '${entity}'.`);
|
||||
} catch (err) {
|
||||
console.error('Error upserting embeddings:', err.message);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Search embeddings
|
||||
async function searchEmbeddings(query, limit = 3) {
|
||||
if (!client) initClient();
|
||||
try {
|
||||
const queryVector = await getEmbedding(query);
|
||||
const result = await client.search(COLLECTION_NAME, {
|
||||
vector: queryVector,
|
||||
async searchNearest(vector, limit = 3) {
|
||||
const result = await this.client.search(this.collectionName, {
|
||||
vector,
|
||||
limit,
|
||||
with_payload: true,
|
||||
with_vector: false
|
||||
withPayload: true,
|
||||
});
|
||||
return result.hits.map((hit) => ({
|
||||
id: hit.id,
|
||||
score: hit.score,
|
||||
payload: hit.payload
|
||||
}));
|
||||
} catch (err) {
|
||||
console.error('Error searching embeddings:', err.message);
|
||||
throw err;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
initClient,
|
||||
createCollection,
|
||||
upsertEmbeddings,
|
||||
searchEmbeddings
|
||||
};
|
||||
+21
-26
@@ -1,41 +1,36 @@
|
||||
const axios = require('axios');
|
||||
require('dotenv').config();
|
||||
import axios from "axios";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const TAVILY_API_KEY = process.env.TAVILY_API_KEY;
|
||||
const TAVILY_ENDPOINT = 'https://api.tavily.com/search';
|
||||
const TAVILY_ENDPOINT = "https://api.tavily.com/search";
|
||||
|
||||
async function fetchEntityData(entity) {
|
||||
export async function fetchSummary(entity) {
|
||||
try {
|
||||
const response = await axios.post(
|
||||
TAVILY_ENDPOINT,
|
||||
{
|
||||
api_key: TAVILY_API_KEY,
|
||||
query: entity,
|
||||
search_depth: 2,
|
||||
include_raw_content: true,
|
||||
max_results: 5
|
||||
include_raw: true,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'accept': 'application/json',
|
||||
'Authorization': `Bearer ${TAVILY_API_KEY}`
|
||||
}
|
||||
}
|
||||
{ headers: { "Content-Type": "application/json" } }
|
||||
);
|
||||
|
||||
if (response.data && response.data.results) {
|
||||
// Concatenate all raw content into a single string
|
||||
const texts = response.data.results
|
||||
.map((r) => r.raw_content || '')
|
||||
.filter(Boolean);
|
||||
return texts.join('\n\n');
|
||||
} else {
|
||||
throw new Error('No results returned from Tavily');
|
||||
const results = response.data.results;
|
||||
if (!results || results.length === 0) {
|
||||
return `No summary available for ${entity}.`;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error fetching data from Tavily:', err.message);
|
||||
throw err;
|
||||
|
||||
// Use the content of the first result as a concise summary
|
||||
const firstResult = results[0];
|
||||
const content = firstResult.content || firstResult.raw_content || "No content available.";
|
||||
// Truncate to 200 characters for brevity
|
||||
return content.length > 200 ? content.slice(0, 197) + "..." : content;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching summary for ${entity}:`, error.message);
|
||||
return `Error fetching summary for ${entity}.`;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { fetchEntityData };
|
||||
Reference in New Issue
Block a user