eval.yaml

name: Run Benchmark Batch

on:
  workflow_dispatch:
    inputs:
      model:
        description: 'Model to evaluate (see run_batch.py MODELS dict)'
        required: true
        type: string
      start:
        description: 'Start task index (inclusive)'
        required: true
        type: number
      end:
        description: 'End task index (exclusive)'
        required: true
        type: number
      parallel:
        description: 'Max concurrent tasks'
        required: false
        default: '3'
        type: string
      tracking_id:
        description: 'UUID for orchestrator to match results'
        required: true
        type: string
      run_start:
        description: 'Run start timestamp for aggregation'
        required: true
        type: string

jobs:
  run-batch:
    runs-on: ubuntu-latest
    timeout-minutes: 120

    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "latest"

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync

      - name: Run benchmark batch
        env:
          BROWSER_USE_SETUP_LOGGING: "false"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }}
        run: |
          uv run python run_batch.py \
            --model "${{ inputs.model }}" \
            --start ${{ inputs.start }} \
            --end ${{ inputs.end }} \
            --parallel ${{ inputs.parallel }} \
            --tracking-id "${{ inputs.tracking_id }}" \
            --run-start "${{ inputs.run_start }}" \
            --output result.json

      - name: Upload result
        uses: actions/upload-artifact@v4
        with:
          name: batch-${{ inputs.tracking_id }}
          path: result.json
          retention-days: 30