Skip to content

Commit 252194b

Browse files
leoisadev1leoisadev1
committed
Merge main
2 parents 7777b52 + cea322e commit 252194b

32 files changed

Lines changed: 2293 additions & 207 deletions

‎apps/server/convex/_generated/api.d.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import type * as auth from "../auth.js";
1212
import type * as backgroundStream from "../backgroundStream.js";
13+
import type * as benchmarks from "../benchmarks.js";
1314
import type * as chats from "../chats.js";
1415
import type * as cleanupAction from "../cleanupAction.js";
1516
import type * as config_constants from "../config/constants.js";
@@ -23,6 +24,7 @@ import type * as lib_billingUtils from "../lib/billingUtils.js";
2324
import type * as lib_crypto from "../lib/crypto.js";
2425
import type * as lib_dbStats from "../lib/dbStats.js";
2526
import type * as lib_logger from "../lib/logger.js";
27+
import type * as lib_model_matching from "../lib/model_matching.js";
2628
import type * as lib_origins from "../lib/origins.js";
2729
import type * as lib_profiles from "../lib/profiles.js";
2830
import type * as lib_rateLimitUtils from "../lib/rateLimitUtils.js";
@@ -46,6 +48,7 @@ import type {
4648
declare const fullApi: ApiFromModules<{
4749
auth: typeof auth;
4850
backgroundStream: typeof backgroundStream;
51+
benchmarks: typeof benchmarks;
4952
chats: typeof chats;
5053
cleanupAction: typeof cleanupAction;
5154
"config/constants": typeof config_constants;
@@ -59,6 +62,7 @@ declare const fullApi: ApiFromModules<{
5962
"lib/crypto": typeof lib_crypto;
6063
"lib/dbStats": typeof lib_dbStats;
6164
"lib/logger": typeof lib_logger;
65+
"lib/model_matching": typeof lib_model_matching;
6266
"lib/origins": typeof lib_origins;
6367
"lib/profiles": typeof lib_profiles;
6468
"lib/rateLimitUtils": typeof lib_rateLimitUtils;

‎apps/server/convex/benchmarks.ts‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import { v } from "convex/values";
2+
import { internalAction, internalMutation, query } from "./_generated/server";
3+
import { internal } from "./_generated/api";
4+
import { buildMatchingMap, type AAModel } from "./lib/model_matching";
5+
6+
type AAModelsResponse = {
7+
data?: AAModel[];
8+
};
9+
10+
type OpenRouterModel = {
11+
id: string;
12+
};
13+
14+
type OpenRouterModelsResponse = {
15+
data?: OpenRouterModel[];
16+
};
17+
18+
const benchmarkValidator = v.object({
19+
openRouterModelId: v.string(),
20+
aaSlug: v.string(),
21+
aaCreatorName: v.string(),
22+
intelligenceIndex: v.optional(v.float64()),
23+
codingIndex: v.optional(v.float64()),
24+
mathIndex: v.optional(v.float64()),
25+
mmluPro: v.optional(v.float64()),
26+
gpqa: v.optional(v.float64()),
27+
scicode: v.optional(v.float64()),
28+
livecodebench: v.optional(v.float64()),
29+
math500: v.optional(v.float64()),
30+
aime: v.optional(v.float64()),
31+
});
32+
33+
export const fetchAndStoreBenchmarks = internalAction({
34+
args: {},
35+
handler: async (ctx) => {
36+
const apiKey = process.env.ARTIFICIAL_ANALYSIS_API_KEY;
37+
if (!apiKey) {
38+
console.warn("ARTIFICIAL_ANALYSIS_API_KEY is not set; skipping benchmark refresh");
39+
return;
40+
}
41+
42+
try {
43+
const openRouterResponse = await fetch("https://openrouter.ai/api/v1/models");
44+
if (!openRouterResponse.ok) {
45+
throw new Error(`Failed to fetch OpenRouter models: ${openRouterResponse.status}`);
46+
}
47+
const openRouterPayload = (await openRouterResponse.json()) as OpenRouterModelsResponse;
48+
const openRouterIds = (openRouterPayload.data ?? []).map((m) => m.id);
49+
console.log(`Fetched ${openRouterIds.length} OpenRouter model IDs`);
50+
51+
const response = await fetch("https://artificialanalysis.ai/api/v2/data/llms/models", {
52+
headers: {
53+
"x-api-key": apiKey,
54+
},
55+
});
56+
57+
if (!response.ok) {
58+
throw new Error(`Failed to fetch Artificial Analysis models: ${response.status}`);
59+
}
60+
61+
const payload = (await response.json()) as AAModelsResponse;
62+
const aaModels = Array.isArray(payload.data) ? payload.data : [];
63+
const matchingMap = buildMatchingMap(aaModels, openRouterIds);
64+
65+
const benchmarks = aaModels.flatMap((model) => {
66+
const openRouterModelId = matchingMap.get(model.slug);
67+
if (!openRouterModelId) return [];
68+
69+
const evaluations = model.evaluations ?? {};
70+
71+
return [{
72+
openRouterModelId,
73+
aaSlug: model.slug,
74+
aaCreatorName: model.model_creator.name,
75+
intelligenceIndex: evaluations.artificial_analysis_intelligence_index ?? undefined,
76+
codingIndex: evaluations.artificial_analysis_coding_index ?? undefined,
77+
mathIndex: evaluations.artificial_analysis_math_index ?? undefined,
78+
mmluPro: evaluations.mmlu_pro ?? undefined,
79+
gpqa: evaluations.gpqa ?? undefined,
80+
scicode: evaluations.scicode ?? undefined,
81+
livecodebench: evaluations.livecodebench ?? undefined,
82+
math500: evaluations.math_500 ?? undefined,
83+
aime: evaluations.aime ?? undefined,
84+
}];
85+
});
86+
87+
await ctx.runMutation((internal as any).benchmarks.storeBenchmarks, { benchmarks });
88+
} catch (error) {
89+
console.error("Failed to refresh Artificial Analysis benchmarks", error);
90+
}
91+
},
92+
});
93+
94+
export const storeBenchmarks = internalMutation({
95+
args: {
96+
benchmarks: v.array(benchmarkValidator),
97+
},
98+
handler: async (ctx, args) => {
99+
const lastUpdated = Date.now();
100+
101+
for (const benchmark of args.benchmarks) {
102+
const existing = await ctx.db
103+
.query("benchmarks")
104+
.withIndex("by_openrouter_id", (q) => q.eq("openRouterModelId", benchmark.openRouterModelId))
105+
.first();
106+
107+
if (existing) {
108+
await ctx.db.patch(existing._id, {
109+
...benchmark,
110+
lastUpdated,
111+
});
112+
continue;
113+
}
114+
115+
await ctx.db.insert("benchmarks", {
116+
...benchmark,
117+
lastUpdated,
118+
});
119+
}
120+
},
121+
});
122+
123+
export const getBenchmarkByOpenRouterId = query({
124+
args: {
125+
openRouterModelId: v.string(),
126+
},
127+
handler: async (ctx, args) => {
128+
return await ctx.db
129+
.query("benchmarks")
130+
.withIndex("by_openrouter_id", (q) => q.eq("openRouterModelId", args.openRouterModelId))
131+
.first();
132+
},
133+
});
134+
135+
export const getAllBenchmarks = query({
136+
args: {},
137+
handler: async (ctx) => {
138+
return await ctx.db.query("benchmarks").collect();
139+
},
140+
});
141+
142+

‎apps/server/convex/crons.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414

1515
import { cronJobs } from "convex/server";
16+
import { internal } from "./_generated/api";
1617
import { internalMutation } from "./_generated/server";
1718
import { v } from "convex/values";
1819
import { decrementStat, getStats, STAT_KEYS } from "./lib/dbStats";
@@ -22,7 +23,7 @@ const logger = createLogger("Cron");
2223

2324
const crons = cronJobs();
2425

25-
export default crons;
26+
crons.interval("refresh benchmarks", { hours: 8 }, (internal as any).benchmarks.fetchAndStoreBenchmarks);
2627

2728
/**
2829
* Cleanup soft-deleted records
@@ -240,3 +241,5 @@ export const generateDatabaseStats = internalMutation({
240241
}
241242
},
242243
});
244+
245+
export default crons;

‎apps/server/convex/lib/__tests__/model_matching.test.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { describe, expect, it } from "vitest";
2+
import {
3+
MANUAL_OVERRIDES,
4+
buildMatchingMap,
5+
normalizeSlug,
6+
type AAModel,
7+
} from "../model_matching";
8+
9+
function createAAModel(slug: string, creatorSlug: string): AAModel {
10+
return {
11+
slug,
12+
model_creator: {
13+
slug: creatorSlug,
14+
name: creatorSlug,
15+
},
16+
evaluations: {
17+
artificial_analysis_intelligence_index: 60,
18+
mmlu_pro: 0.7,
19+
},
20+
};
21+
}
22+
23+
describe("normalizeSlug", () => {
24+
it("normalizes casing and separators", () => {
25+
expect(normalizeSlug(" Claude_3.5 Sonnet ")).toBe("claude-3-5-sonnet");
26+
});
27+
28+
it("strips trailing numeric version suffixes", () => {
29+
expect(normalizeSlug("gemini-2.0-flash-001")).toBe("gemini-2-0-flash");
30+
expect(normalizeSlug("deepseek-v3-0324")).toBe("deepseek-v3");
31+
});
32+
});
33+
34+
describe("buildMatchingMap", () => {
35+
it.each([
36+
["claude-3-5-sonnet", "anthropic", "anthropic/claude-3.5-sonnet"],
37+
["deepseek-v3", "deepseek", "deepseek/deepseek-chat"],
38+
["llama-3-3-70b", "meta-llama", "meta-llama/llama-3.3-70b-instruct"],
39+
["gpt-4o", "openai", "openai/gpt-4o"],
40+
["gemini-2-5-pro", "google", "google/gemini-2.5-pro"],
41+
["grok-3", "x-ai", "x-ai/grok-3"],
42+
])("matches manual override for %s", (aaSlug, creatorSlug, expected) => {
43+
const map = buildMatchingMap(
44+
[createAAModel(aaSlug, creatorSlug)],
45+
[expected],
46+
);
47+
expect(map.get(aaSlug)).toBe(expected);
48+
});
49+
50+
it("matches with normalized fallback when exact slug does not exist", () => {
51+
const map = buildMatchingMap(
52+
[createAAModel("gemini-2-0-flash-001", "google")],
53+
["google/gemini-2.0-flash-001"],
54+
);
55+
expect(map.get("gemini-2-0-flash-001")).toBe("google/gemini-2.0-flash-001");
56+
});
57+
58+
it("returns no match for unknown models", () => {
59+
const map = buildMatchingMap(
60+
[createAAModel("totally-unknown-model", "unknown")],
61+
["openai/gpt-4o"],
62+
);
63+
expect(map.has("totally-unknown-model")).toBe(false);
64+
});
65+
66+
it("every manual override resolves when its target is in the id set", () => {
67+
const allTargetIds = [...new Set(Object.values(MANUAL_OVERRIDES))];
68+
const aaModels = Object.entries(MANUAL_OVERRIDES).map(([aaSlug, openRouterId]) =>
69+
createAAModel(aaSlug, openRouterId.split("/")[0] ?? ""),
70+
);
71+
72+
const map = buildMatchingMap(aaModels, allTargetIds);
73+
74+
for (const targetId of allTargetIds) {
75+
const matched = [...map.values()].includes(targetId);
76+
expect(matched, `Expected ${targetId} to be matched by at least one override`).toBe(true);
77+
}
78+
});
79+
80+
it("builds an AA slug to OpenRouter id map", () => {
81+
const aaModels: AAModel[] = [
82+
createAAModel("claude-3-5-sonnet", "anthropic"),
83+
createAAModel("gemini 2_5.flash", "google"),
84+
createAAModel("unlisted-model", "unknown"),
85+
];
86+
87+
const openRouterIds = [
88+
"anthropic/claude-3.5-sonnet",
89+
"google/gemini-2.5-flash",
90+
"openai/gpt-4o",
91+
];
92+
93+
const map = buildMatchingMap(aaModels, openRouterIds);
94+
95+
expect(map.get("claude-3-5-sonnet")).toBe("anthropic/claude-3.5-sonnet");
96+
expect(map.get("gemini 2_5.flash")).toBe("google/gemini-2.5-flash");
97+
expect(map.has("unlisted-model")).toBe(false);
98+
});
99+
100+
it("only matches against the provided OpenRouter ids", () => {
101+
const aaModels: AAModel[] = [createAAModel("claude-3-5-sonnet", "anthropic")];
102+
const openRouterIds = ["openai/gpt-4o"];
103+
104+
const map = buildMatchingMap(aaModels, openRouterIds);
105+
106+
expect(map.size).toBe(0);
107+
});
108+
});

0 commit comments

Comments
 (0)