WIP: Various updates for AI review

apps/server/convex/_generated/api.d.ts

@@ -10,6 +10,7 @@
 
 import type * as auth from "../auth.js";
 import type * as backgroundStream from "../backgroundStream.js";
+import type * as benchmarks from "../benchmarks.js";
 import type * as chats from "../chats.js";
 import type * as config_constants from "../config/constants.js";
 import type * as crons from "../crons.js";
@@ -22,6 +23,7 @@ import type * as lib_billingUtils from "../lib/billingUtils.js";
 import type * as lib_crypto from "../lib/crypto.js";
 import type * as lib_dbStats from "../lib/dbStats.js";
 import type * as lib_logger from "../lib/logger.js";
+import type * as lib_model_matching from "../lib/model_matching.js";
 import type * as lib_origins from "../lib/origins.js";
 import type * as lib_profiles from "../lib/profiles.js";
 import type * as lib_rateLimitUtils from "../lib/rateLimitUtils.js";
@@ -46,6 +48,7 @@ import type {
 declare const fullApi: ApiFromModules<{
   auth: typeof auth;
   backgroundStream: typeof backgroundStream;
+  benchmarks: typeof benchmarks;
   chats: typeof chats;
   "config/constants": typeof config_constants;
   crons: typeof crons;
@@ -58,6 +61,7 @@ declare const fullApi: ApiFromModules<{
   "lib/crypto": typeof lib_crypto;
   "lib/dbStats": typeof lib_dbStats;
   "lib/logger": typeof lib_logger;
+  "lib/model_matching": typeof lib_model_matching;
   "lib/origins": typeof lib_origins;
   "lib/profiles": typeof lib_profiles;
   "lib/rateLimitUtils": typeof lib_rateLimitUtils;

apps/server/convex/benchmarks.ts

@@ -1,12 +1,20 @@
 import { v } from "convex/values";
 import { internalAction, internalMutation, query } from "./_generated/server";
 import { internal } from "./_generated/api";
-import { buildMatchingMap, POPULAR_MODEL_IDS, type AAModel } from "./lib/model-matching";
+import { buildMatchingMap, type AAModel } from "./lib/model_matching";
 
 type AAModelsResponse = {
 	data?: AAModel[];
 };
 
+type OpenRouterModel = {
+	id: string;
+};
+
+type OpenRouterModelsResponse = {
+	data?: OpenRouterModel[];
+};
+
 const benchmarkValidator = v.object({
 	openRouterModelId: v.string(),
 	aaSlug: v.string(),
@@ -32,6 +40,14 @@ export const fetchAndStoreBenchmarks = internalAction({
 		}
 
 		try {
+			const openRouterResponse = await fetch("https://openrouter.ai/api/v1/models");
+			if (!openRouterResponse.ok) {
+				throw new Error(`Failed to fetch OpenRouter models: ${openRouterResponse.status}`);
+			}
+			const openRouterPayload = (await openRouterResponse.json()) as OpenRouterModelsResponse;
+			const openRouterIds = (openRouterPayload.data ?? []).map((m) => m.id);
+			console.log(`Fetched ${openRouterIds.length} OpenRouter model IDs`);
+
 			const response = await fetch("https://artificialanalysis.ai/api/v2/data/llms/models", {
 				headers: {
 					"x-api-key": apiKey,
@@ -44,7 +60,7 @@ export const fetchAndStoreBenchmarks = internalAction({
 
 			const payload = (await response.json()) as AAModelsResponse;
 			const aaModels = Array.isArray(payload.data) ? payload.data : [];
-			const matchingMap = buildMatchingMap(aaModels, [...POPULAR_MODEL_IDS]);
+			const matchingMap = buildMatchingMap(aaModels, openRouterIds);
 
 			const benchmarks = aaModels.flatMap((model) => {
 				const openRouterModelId = matchingMap.get(model.slug);
@@ -56,9 +72,9 @@ export const fetchAndStoreBenchmarks = internalAction({
 					openRouterModelId,
 					aaSlug: model.slug,
 					aaCreatorName: model.model_creator.name,
-					intelligenceIndex: evaluations.artificial_analysis_intelligence_index ?? undefined,
-					codingIndex: evaluations.coding_index ?? undefined,
-					mathIndex: evaluations.math_index ?? undefined,
+				intelligenceIndex: evaluations.artificial_analysis_intelligence_index ?? undefined,
+				codingIndex: evaluations.artificial_analysis_coding_index ?? undefined,
+				mathIndex: evaluations.artificial_analysis_math_index ?? undefined,
 					mmluPro: evaluations.mmlu_pro ?? undefined,
 					gpqa: evaluations.gpqa ?? undefined,
 					scicode: evaluations.scicode ?? undefined,
@@ -122,3 +138,5 @@ export const getAllBenchmarks = query({
 		return await ctx.db.query("benchmarks").collect();
 	},
 });
+
+

apps/server/convex/lib/__tests__/model-matching.test.ts → apps/server/convex/lib/__tests__/model_matching.test.ts

@@ -1,12 +1,10 @@
 import { describe, expect, it } from "vitest";
 import {
 	MANUAL_OVERRIDES,
-	POPULAR_MODEL_IDS,
 	buildMatchingMap,
-	matchAAtoOpenRouter,
 	normalizeSlug,
 	type AAModel,
-} from "../model-matching";
+} from "../model_matching";
 
 function createAAModel(slug: string, creatorSlug: string): AAModel {
 	return {
@@ -33,7 +31,7 @@ describe("normalizeSlug", () => {
 	});
 });
 
-describe("matchAAtoOpenRouter", () => {
+describe("buildMatchingMap", () => {
 	it.each([
 		["claude-3-5-sonnet", "anthropic", "anthropic/claude-3.5-sonnet"],
 		["deepseek-v3", "deepseek", "deepseek/deepseek-chat"],
@@ -42,34 +40,43 @@ describe("matchAAtoOpenRouter", () => {
 		["gemini-2-5-pro", "google", "google/gemini-2.5-pro"],
 		["grok-3", "x-ai", "x-ai/grok-3"],
 	])("matches manual override for %s", (aaSlug, creatorSlug, expected) => {
-		expect(matchAAtoOpenRouter(aaSlug, creatorSlug)).toBe(expected);
+		const map = buildMatchingMap(
+			[createAAModel(aaSlug, creatorSlug)],
+			[expected],
+		);
+		expect(map.get(aaSlug)).toBe(expected);
 	});
 
 	it("matches with normalized fallback when exact slug does not exist", () => {
-		expect(matchAAtoOpenRouter("gemini-2-0-flash-001", "google")).toBe("google/gemini-2.0-flash-001");
+		const map = buildMatchingMap(
+			[createAAModel("gemini-2-0-flash-001", "google")],
+			["google/gemini-2.0-flash-001"],
+		);
+		expect(map.get("gemini-2-0-flash-001")).toBe("google/gemini-2.0-flash-001");
 	});
 
-	it("returns null for unknown models instead of throwing", () => {
-		expect(() => matchAAtoOpenRouter("totally-unknown-model", "unknown")).not.toThrow();
-		expect(matchAAtoOpenRouter("totally-unknown-model", "unknown")).toBeNull();
+	it("returns no match for unknown models", () => {
+		const map = buildMatchingMap(
+			[createAAModel("totally-unknown-model", "unknown")],
+			["openai/gpt-4o"],
+		);
+		expect(map.has("totally-unknown-model")).toBe(false);
 	});
 
-	it("matches every popular model id", () => {
-		const matchedIds = new Set<string>();
+	it("every manual override resolves when its target is in the id set", () => {
+		const allTargetIds = [...new Set(Object.values(MANUAL_OVERRIDES))];
+		const aaModels = Object.entries(MANUAL_OVERRIDES).map(([aaSlug, openRouterId]) =>
+			createAAModel(aaSlug, openRouterId.split("/")[0] ?? ""),
+		);
 
-		for (const [aaSlug, openRouterId] of Object.entries(MANUAL_OVERRIDES)) {
-			const creatorSlug = openRouterId.split("/")[0] ?? "";
-			const result = matchAAtoOpenRouter(aaSlug, creatorSlug);
-			if (result) {
-				matchedIds.add(result);
-			}
-		}
+		const map = buildMatchingMap(aaModels, allTargetIds);
 
-		expect([...matchedIds].sort()).toEqual([...POPULAR_MODEL_IDS].sort());
+		for (const targetId of allTargetIds) {
+			const matched = [...map.values()].includes(targetId);
+			expect(matched, `Expected ${targetId} to be matched by at least one override`).toBe(true);
+		}
 	});
-});
 
-describe("buildMatchingMap", () => {
 	it("builds an AA slug to OpenRouter id map", () => {
 		const aaModels: AAModel[] = [
 			createAAModel("claude-3-5-sonnet", "anthropic"),

apps/server/convex/lib/model-matching.ts → apps/server/convex/lib/model_matching.ts

@@ -7,46 +7,62 @@ export interface AAModel {
 	evaluations: Record<string, number | null>;
 }
 
-export const POPULAR_MODEL_IDS = new Set([
-	"anthropic/claude-sonnet-4",
-	"anthropic/claude-3.5-sonnet",
-	"anthropic/claude-3.7-sonnet",
-	"anthropic/claude-3.5-haiku",
-	"openai/gpt-4o",
-	"openai/gpt-4o-mini",
-	"openai/gpt-4.1",
-	"openai/gpt-4.1-mini",
-	"openai/o3-mini",
-	"google/gemini-2.5-flash",
-	"google/gemini-2.5-pro",
-	"google/gemini-2.0-flash-001",
-	"deepseek/deepseek-chat",
-	"deepseek/deepseek-chat-v3.1",
-	"deepseek/deepseek-r1",
-	"meta-llama/llama-3.3-70b-instruct",
-	"x-ai/grok-3",
-	"mistralai/mistral-large-2411",
-	"qwen/qwen-2.5-72b-instruct",
-]);
-
 export const MANUAL_OVERRIDES: Record<string, string> = {
+	"claude-35-sonnet": "anthropic/claude-3.5-sonnet",
+	"claude-35-sonnet-june-24": "anthropic/claude-3.5-sonnet",
 	"claude-3-5-sonnet": "anthropic/claude-3.5-sonnet",
 	"claude-3-7-sonnet": "anthropic/claude-3.7-sonnet",
+	"claude-3-7-sonnet-thinking": "anthropic/claude-3.7-sonnet",
 	"claude-3-5-haiku": "anthropic/claude-3.5-haiku",
+	"claude-4-sonnet": "anthropic/claude-sonnet-4",
+	"claude-4-sonnet-thinking": "anthropic/claude-sonnet-4",
 	"claude-sonnet-4": "anthropic/claude-sonnet-4",
+	"claude-4-5-sonnet": "anthropic/claude-sonnet-4.5",
+	"claude-4-5-sonnet-thinking": "anthropic/claude-sonnet-4.5",
+	"claude-4-5-haiku": "anthropic/claude-haiku-4.5",
+	"claude-4-5-haiku-reasoning": "anthropic/claude-haiku-4.5",
+	"claude-opus-4-5": "anthropic/claude-opus-4.5",
+	"claude-opus-4-5-thinking": "anthropic/claude-opus-4.5",
+	"claude-opus-4-6": "anthropic/claude-opus-4.6",
+	"claude-opus-4-6-adaptive": "anthropic/claude-opus-4.6",
+	"claude-4-1-opus": "anthropic/claude-opus-4.1",
+	"claude-4-1-opus-thinking": "anthropic/claude-opus-4.1",
+	"claude-4-opus": "anthropic/claude-opus-4",
+	"claude-4-opus-thinking": "anthropic/claude-opus-4",
 	"deepseek-v3": "deepseek/deepseek-chat",
 	"deepseek-v3-0324": "deepseek/deepseek-chat-v3.1",
 	"deepseek-r1": "deepseek/deepseek-r1",
+	"deepseek-r1-0120": "deepseek/deepseek-r1",
+	"llama-3-3-instruct-70b": "meta-llama/llama-3.3-70b-instruct",
 	"llama-3-3-70b": "meta-llama/llama-3.3-70b-instruct",
+	"llama-3-1-instruct-405b": "meta-llama/llama-3.1-405b-instruct",
+	"llama-3-1-instruct-70b": "meta-llama/llama-3.1-70b-instruct",
+	"llama-3-1-instruct-8b": "meta-llama/llama-3.1-8b-instruct",
+	"llama-4-maverick": "meta-llama/llama-4-maverick",
+	"llama-4-scout": "meta-llama/llama-4-scout",
 	"gpt-4o": "openai/gpt-4o",
 	"gpt-4o-mini": "openai/gpt-4o-mini",
 	"o3-mini": "openai/o3-mini",
+	"o3": "openai/o3",
+	"o4-mini": "openai/o4-mini",
 	"grok-3": "x-ai/grok-3",
+	"grok-4": "x-ai/grok-4",
+	"mistral-large-2": "mistralai/mistral-large-2411",
+	"mistral-large-2407": "mistralai/mistral-large-2411",
 	"mistral-large-2411": "mistralai/mistral-large-2411",
+	"mistral-large-3": "mistralai/mistral-large-2512",
+	"mistral-medium-3": "mistralai/mistral-medium-3",
+	"mistral-medium-3-1": "mistralai/mistral-medium-3.1",
+	"mistral-small-3": "mistralai/mistral-small-24b-instruct-2501",
+	"mistral-small-3-1": "mistralai/mistral-small-24b-instruct-2501",
+	"mistral-small-3-2": "mistralai/mistral-small-24b-instruct-2501",
 	"qwen-2-5-72b-instruct": "qwen/qwen-2.5-72b-instruct",
 	"gemini-2-5-flash": "google/gemini-2.5-flash",
 	"gemini-2-5-pro": "google/gemini-2.5-pro",
 	"gemini-2-0-flash": "google/gemini-2.0-flash-001",
+	"gemini-3-flash": "google/gemini-3-flash-preview",
+	"gemini-3-flash-reasoning": "google/gemini-3-flash-preview",
+	"gemini-3-pro": "google/gemini-3-pro-preview",
 	"gpt-4-1": "openai/gpt-4.1",
 	"gpt-4-1-mini": "openai/gpt-4.1-mini",
 };
@@ -95,10 +111,6 @@ export function normalizeSlug(slug: string): string {
 	return normalized.replace(/(?:-\d{3,4})+$/, "");
 }
 
-export function matchAAtoOpenRouter(aaSlug: string, aaCreatorSlug: string): string | null {
-	return matchAAtoOpenRouterWithIds(aaSlug, aaCreatorSlug, POPULAR_MODEL_IDS);
-}
-
 export function buildMatchingMap(
 	aaModels: AAModel[],
 	openRouterIds: string[],

apps/web/src/components/circular-progress.tsx

@@ -93,7 +93,8 @@ export function CircularProgress({
 				y={center}
 				textAnchor="middle"
 				dominantBaseline="central"
-				className={cn("font-semibold", colorClass.replace("text-", "fill-"))}
+				fill="currentColor"
+				className={cn("font-semibold", colorClass)}
 				style={{ fontSize: `${fontSize}px` }}
 			>
 				{`${Math.round(clampedValue)}%`}