feat: accept array of models in experiment

Author: paoloricciuti

src/cli.ts

@@ -101,8 +101,19 @@ async function runExperimentCommand(configInput: string, options: { dry?: boolea
       console.log(chalk.green(`  - ${name}`));
     }
 
-    console.log(chalk.blue(`\nRunning ${evalNames.length} eval(s) x ${config.runs} run(s) = ${evalNames.length * config.runs} total runs`));
-    console.log(chalk.blue(`Agent: ${config.agent}, Model: ${config.model}, Timeout: ${config.timeout}s, Early Exit: ${config.earlyExit}`));
+	const models = Array.isArray(config.model) ? config.model : [config.model];
+
+    // Show info for all models
+    const totalRunsPerModel = evalNames.length * config.runs;
+    const totalRuns = totalRunsPerModel * models.length;
+
+    if (models.length > 1) {
+      console.log(chalk.blue(`\nRunning ${evalNames.length} eval(s) x ${config.runs} run(s) x ${models.length} model(s) = ${totalRuns} total runs`));
+      console.log(chalk.blue(`Agent: ${config.agent}, Models: ${models.join(', ')}, Timeout: ${config.timeout}s, Early Exit: ${config.earlyExit}`));
+    } else {
+      console.log(chalk.blue(`\nRunning ${evalNames.length} eval(s) x ${config.runs} run(s) = ${totalRuns} total runs`));
+      console.log(chalk.blue(`Agent: ${config.agent}, Model: ${models[0]}, Timeout: ${config.timeout}s, Early Exit: ${config.earlyExit}`));
+    }
 
     // Show which sandbox backend will be used
     const sandboxInfo = getSandboxBackendInfo({ backend: config.sandbox });
@@ -127,23 +138,44 @@ async function runExperimentCommand(configInput: string, options: { dry?: boolea
     const selectedFixtures = fixtures.filter((f) => evalNames.includes(f.name));
 
     // Get experiment name from config file
-    const experimentName = basename(configPath, '.ts').replace(/\.js$/, '');
+    const baseExperimentName = basename(configPath, '.ts').replace(/\.js$/, '');
     const resultsDir = resolve(process.cwd(), 'results');
 
     console.log(chalk.blue('\nStarting experiment...'));
 
-    // Run the experiment
-    const results = await runExperiment({
-      config,
-      fixtures: selectedFixtures,
-      apiKey,
-      resultsDir,
-      experimentName,
-      onProgress: (msg) => console.log(msg),
-    });
+    // Run experiments for each model
+    let allPassed = true;
+    for (const model of models) {
+      // Create a config for this specific model
+      const modelConfig = { ...config, model };
+
+      // Include model in experiment name when multiple models are specified
+      const experimentName = models.length > 1
+        ? `${baseExperimentName}/${model}`
+        : baseExperimentName;
+
+      if (models.length > 1) {
+        console.log(chalk.blue(`\n--- Running with model: ${model} ---`));
+      }
+
+      // Run the experiment
+      const results = await runExperiment({
+        config: modelConfig,
+        fixtures: selectedFixtures,
+        apiKey,
+        resultsDir,
+        experimentName,
+        onProgress: (msg) => console.log(msg),
+      });
+
+      // Check if this experiment passed
+      const experimentPassed = results.evals.every((e) => e.passedRuns === e.totalRuns);
+      if (!experimentPassed) {
+        allPassed = false;
+      }
+    }
 
     // Exit with appropriate code
-    const allPassed = results.evals.every((e) => e.passedRuns === e.totalRuns);
     process.exit(allPassed ? 0 : 1);
   } catch (error) {
     if (error instanceof Error) {

src/lib/config.test.ts

@@ -25,6 +25,14 @@ describe('validateConfig', () => {
     expect(() => validateConfig(config)).not.toThrow();
   });
 
+  it('accepts array of models', () => {
+    const config = {
+      agent: 'claude-code',
+      model: ['opus', 'sonnet', 'haiku'],
+    };
+    expect(() => validateConfig(config)).not.toThrow();
+  });
+
   it('accepts function evals filter', () => {
     const config = {
       agent: 'claude-code',

src/lib/config.ts

@@ -34,7 +34,7 @@ const experimentConfigSchema = z.object({
     'codex',
     'vercel-ai-gateway/opencode',
   ]),
-  model: z.string().optional(),
+  model: z.union([z.string(), z.array(z.string())]).optional(),
   evals: z
     .union([z.string(), z.array(z.string()), z.function().args(z.string()).returns(z.boolean())])
     .optional(),

src/lib/runner.ts

@@ -10,6 +10,7 @@ import type {
   EvalRunData,
   EvalSummary,
   ExperimentResults,
+  RunnableExperimentConfig,
 } from './types.js';
 import { getAgent } from './agents/index.js';
 import {
@@ -27,7 +28,7 @@ import {
  */
 export interface RunExperimentOptions {
   /** Resolved experiment configuration */
-  config: ResolvedExperimentConfig;
+  config: RunnableExperimentConfig;
   /** Fixtures to run */
   fixtures: EvalFixture[];
   /** API key for the agent */
@@ -246,30 +247,44 @@ export async function runExperiment(
 /**
  * Run a single eval (for testing/debugging).
  */
-export async function runSingleEval(
+export async function runSingleEval<T extends ResolvedExperimentConfig['model']>(
   fixture: EvalFixture,
   options: {
     agent?: ResolvedExperimentConfig['agent'];
-    model: ResolvedExperimentConfig['model'];
+    model: T;
     timeout: number;
     apiKey: string;
     setup?: ResolvedExperimentConfig['setup'];
     scripts?: string[];
     sandbox?: ResolvedExperimentConfig['sandbox'];
     verbose?: boolean;
   }
-): Promise<EvalRunData> {
+): Promise<T extends Array<unknown> ? EvalRunData[] : EvalRunData> {
   const agent = getAgent(options.agent ?? 'vercel-ai-gateway/claude-code');
 
-  const agentResult = await agent.run(fixture.path, {
-    prompt: fixture.prompt,
-    model: options.model,
-    timeout: options.timeout * 1000,
-    apiKey: options.apiKey,
-    setup: options.setup,
-    scripts: options.scripts,
-    sandbox: options.sandbox,
-  });
+  const models: string[] = Array.isArray(options.model) ? options.model : [options.model];
+
+  const results: EvalRunData[] = [];
+
+  for (const model of models) {
+
+	const agentResult = await agent.run(fixture.path, {
+		prompt: fixture.prompt,
+		model,
+		timeout: options.timeout * 1000,
+		apiKey: options.apiKey,
+		setup: options.setup,
+		scripts: options.scripts,
+		sandbox: options.sandbox,
+	});
+
+    results.push(agentResultToEvalRunData(agentResult));
+  }
+
+  // TODO: remove this on the next major and return an array directly...it's just here to prevent breaking changes
+  if(!Array.isArray(options.model)) {
+	return results[0] as T extends Array<unknown> ? EvalRunData[] : EvalRunData;
+  }
 
-  return agentResultToEvalRunData(agentResult);
+  return results as T extends Array<unknown> ? EvalRunData[] : EvalRunData;
 }

src/lib/types.ts

@@ -61,8 +61,10 @@ export interface ExperimentConfig {
   /** Which AI agent to use */
   agent: AgentType;
 
-  /** Which AI model the agent should use. Default is agent-specific: 'opus' for claude-code, 'openai/gpt-5.2-codex' for codex */
-  model?: ModelTier;
+  /** Which AI model the agent should use. Can be a single model or array of models to test.
+   * If an array is provided, the experiment will run on each model.
+   * Default is agent-specific: 'opus' for claude-code, 'openai/gpt-5.2-codex' for codex */
+  model?: ModelTier | ModelTier[];
 
   /** Which evals to run. Can be a string, array, or filter function. @default '*' (all evals) */
   evals?: string | string[] | EvalFilter;
@@ -90,6 +92,21 @@ export interface ExperimentConfig {
  * Resolved experiment config with all defaults applied.
  */
 export interface ResolvedExperimentConfig {
+  agent: AgentType;
+  model: ModelTier | ModelTier[];
+  evals: string | string[] | EvalFilter;
+  runs: number;
+  earlyExit: boolean;
+  scripts: string[];
+  timeout: number;
+  setup?: SetupFunction;
+  sandbox: SandboxBackend | 'auto';
+}
+
+/**
+ * Resolved experiment config with all defaults applied.
+ */
+export interface RunnableExperimentConfig {
   agent: AgentType;
   model: ModelTier;
   evals: string | string[] | EvalFilter;