Merge pull request #164 from vercel-labs/jude/agent-eval-pinnable-judge

[agent-eval] Pin the agentic judge to a fixed agent and model

Author: gaojude

.changeset/pinnable-judge.md

@@ -0,0 +1,5 @@
+---
+"@vercel/agent-eval": minor
+---
+
+Pin the agentic LLM judge to a fixed agent + model via `ExperimentConfig.judge`. By default the `expect(environment|transcript)` matchers still self-grade with the codegen agent+model; setting `judge: { agent?, model }` grades every run with one fixed judge — the apples-to-apples choice for cross-model comparisons (judge quality no longer varies with the model under test, and a model never grades itself). When `judge.agent` names a different agent, its CLI is installed in the sandbox and its key is resolved from its own env var (falling back to `VERCEL_OIDC_TOKEN`). Pinning is reflected in the eval fingerprint, so pinned runs don't reuse self-graded cached results.

README.md

@@ -200,7 +200,22 @@ Two matchers, on either subject:
 
 You supply only the **criterion** string; the framework owns the judge prompt and the verdict contract. On failure the assertion message carries the judge's reasoning, e.g. `[judge:environment] FAIL (score 0.42): product list is a Client Component`, so a failed judge clause is distinguishable from a failed deterministic test or a crash.
 
-The judge uses the same agent and model as the run under test. Because each assertion is a real agent run, it costs time and tokens — keep criteria focused.
+By default the judge uses the **same agent and model** as the run under test (self-grading). Because each assertion is a real agent run, it costs time and tokens — keep criteria focused.
+
+**Pin the judge** to grade every run with one fixed agent + model — the apples-to-apples choice when comparing models, since the judge quality no longer varies with the model under test (and a model never grades itself):
+
+```typescript
+const config: ExperimentConfig = {
+  agent: 'codex',
+  model: 'gpt-5.4',
+  // Grade with a fixed Claude judge regardless of the model under test.
+  judge: { agent: 'vercel-ai-gateway/claude-code', model: 'claude-opus-4-8' },
+};
+```
+
+- `judge.model` is required (pinning the model is the point).
+- `judge.agent` is optional and defaults to the codegen agent — omit it to keep the same harness and only pin the model. When it names a different agent, that agent's CLI is installed in the sandbox automatically and its key is resolved from its own env var (falling back to `VERCEL_OIDC_TOKEN`).
+- Pinning changes the eval fingerprint, so a pinned run won't reuse self-graded cached results.
 
 > **Note**: requires `validation: 'vitest'` (the default). The framework gives the eval process the run's credentials automatically so the judge can call the agent CLI in-sandbox.
 
@@ -279,6 +294,10 @@ const config: ExperimentConfig = {
   // 'changed' - copy only files modified by the agent
   // 'all' - copy the entire project including original fixture files
   copyFiles: 'changed',
+
+  // Pin the agentic LLM judge (see "Agentic LLM judge" above). Omit to self-grade
+  // with the codegen agent+model. `model` required; `agent` defaults to codegen.
+  judge: { agent: 'vercel-ai-gateway/claude-code', model: 'claude-opus-4-8' },
 };
 
 export default config;

packages/agent-eval/src/cli.ts

@@ -17,6 +17,7 @@ import { Dashboard, createConsoleProgressHandler } from './lib/dashboard.js';
 import type { ProgressEvent, Classification } from './lib/types.js';
 import { initProject, getPostInitInstructions } from './lib/init.js';
 import { getAgent } from './lib/agents/index.js';
+import { resolveAgentApiKey } from './lib/agents/shared.js';
 import { getSandboxBackendInfo } from './lib/sandbox.js';
 import { computeFingerprint } from './lib/fingerprint.js';
 import { scanReusableResults } from './lib/results.js';
@@ -147,7 +148,7 @@ async function runExperimentCommand(configInput: string, options: { dry?: boolea
     // Get the agent to check for required API key
     const agent = getAgent(config.agent);
     const apiKeyEnvVar = agent.getApiKeyEnvVar();
-    const apiKey = process.env[apiKeyEnvVar] ?? process.env.VERCEL_OIDC_TOKEN;
+    const apiKey = resolveAgentApiKey(agent.getApiKeyEnvVar);
     if (!apiKey) {
       console.error(chalk.red(`${apiKeyEnvVar} (or VERCEL_OIDC_TOKEN) environment variable is required`));
       console.error(chalk.gray(`Get your API key at: https://vercel.com/dashboard -> AI Gateway`));
@@ -474,7 +475,7 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
 
         const agent = getAgent(config.agent);
         const apiKeyEnvVar = agent.getApiKeyEnvVar();
-        const apiKey = process.env[apiKeyEnvVar] ?? process.env.VERCEL_OIDC_TOKEN;
+        const apiKey = resolveAgentApiKey(agent.getApiKeyEnvVar);
         if (!apiKey) {
           console.error(chalk.red(`${apiKeyEnvVar} (or VERCEL_OIDC_TOKEN) not set, skipping ${baseExperimentName}`));
           return;

packages/agent-eval/src/integration.test.ts

@@ -386,6 +386,38 @@ test('environment judge (false)', async () => {
       expect(result.result.status).toBe('failed');
       expect(result.outputContent?.eval ?? '').toContain('[judge:environment] FAIL');
     }, 900000);
+
+    it('PASSES with a judge pinned to a different model than codegen', async () => {
+      // Codegen runs sonnet; the judge is pinned to a fixed model (opus). This is the
+      // cross-model dashboard config — grade every model with one fixed judge. Proves
+      // the pinned model flows: judge-config.model = the pinned model, not codegen's.
+      writeJudgeFixture(
+        'judge-pinned',
+        `
+import { test, expect } from 'vitest';
+import { environment } from '@vercel/agent-eval/eval';
+
+test('environment judge (pinned, true)', async () => {
+  await expect(environment).toSatisfyCriterion(
+    'exports a greet() function that returns a non-empty greeting string'
+  );
+});
+`
+      );
+      const fixture = loadFixture(TEST_DIR, 'judge-pinned');
+      const result = await runSingleEval(fixture, {
+        agent: 'vercel-ai-gateway/claude-code',
+        model: 'sonnet',
+        timeout: 900,
+        apiKey: process.env.AI_GATEWAY_API_KEY!,
+        scripts: [],
+        judge: { model: 'claude-opus-4-8' },
+      });
+      if (result.result.status === 'failed') {
+        console.error('judge-pinned eval output:\n', result.outputContent?.eval);
+      }
+      expect(result.result.status).toBe('passed');
+    }, 900000);
   });
 
   describe.skipIf(!hasAnthropicCredentials)('Claude Code (Direct API) sandbox execution', () => {

packages/agent-eval/src/lib/agents/claude-code/agent.ts

@@ -84,5 +84,6 @@ export function createClaudeCodeAgent({ useVercelAiGateway }: { useVercelAiGatew
       return definition.defaultModel;
     },
     run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
+    definition,
   };
 }

packages/agent-eval/src/lib/agents/codex/agent.ts

@@ -237,5 +237,6 @@ export function createCodexAgent({ useVercelAiGateway }: { useVercelAiGateway: b
       return definition.defaultModel;
     },
     run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
+    definition,
   };
 }

packages/agent-eval/src/lib/agents/cursor/agent.ts

@@ -92,5 +92,6 @@ export function createCursorAgent(): Agent {
       return definition.defaultModel;
     },
     run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
+    definition,
   };
 }

packages/agent-eval/src/lib/agents/eval-helper.mjs

@@ -119,8 +119,9 @@ function readJudgeConfig() {
   try {
     return JSON.parse(readFileSync(JUDGE_CONFIG_PATH, 'utf8'));
   } catch {
-    // No config (e.g. run outside the orchestrator) → let the agent CLI default.
-    return { model: null, extra: null };
+    // No config (e.g. run outside the orchestrator) → reuse the codegen runner and
+    // let the agent CLI pick its default model.
+    return { runnerPath: RUNNER_PATH, model: null, extra: null };
   }
 }
 
@@ -136,9 +137,12 @@ function runJudge(subject, criterion, opts = {}) {
   const resultPath = `${JUDGE_IO_DIR}/${id}-result.json`;
   const cfg = readJudgeConfig();
 
-  // Same AgentRunInput contract run.mjs already understands. Reuse the codegen
-  // model + host-computed extra (e.g. codex's resolved model/effort) so the judge
-  // is the SAME agent. No secrets here — the key rides in process.env (inherited).
+  // Same AgentRunInput contract the runner already understands. The judge model +
+  // host-computed extra come from judge-config.json: by default they match the
+  // codegen run (self-grade); when the experiment pins a judge, runnerPath points at
+  // the pinned agent's judge-run.mjs and model is the pinned model. No secrets here —
+  // the key rides in process.env (inherited from the orchestrator's validation env).
+  const runnerPath = cfg.runnerPath ?? RUNNER_PATH;
   const input = {
     prompt: buildJudgePrompt(subject, criterion, verdictPath, opts),
     model: cfg.model ?? undefined,
@@ -147,7 +151,7 @@ function runJudge(subject, criterion, opts = {}) {
     extra: cfg.extra ?? undefined,
   };
 
-  const res = spawnSync('node', [RUNNER_PATH, JSON.stringify(input)], {
+  const res = spawnSync('node', [runnerPath, JSON.stringify(input)], {
     cwd: process.cwd(),
     env: process.env,
     encoding: 'utf8',

packages/agent-eval/src/lib/agents/gemini/agent.ts

@@ -75,5 +75,6 @@ export function createGeminiAgent(): Agent {
       return definition.defaultModel;
     },
     run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
+    definition,
   };
 }

packages/agent-eval/src/lib/agents/opencode/agent.ts

@@ -207,5 +207,6 @@ export function createOpenCodeAgent(): Agent {
       return definition.defaultModel;
     },
     run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
+    definition,
   };
 }