Skip to content

Commit fa35f25

Browse files
MagMuellerMagMueller
authored
Default V2 to BrowserCode with Cloud and local runs (#38)
* Make BrowserCode V2 the GitHub Actions entry point * Support local Chrome and external V2 runs with optional Laminar * Use explicit task axis for the Actions matrix * Continuously validate Actions syntax and local Chrome * Limit test workflow token to read-only contents * Preserve complete BrowserCode preflight output across process exit
1 parent bdaa3d2 commit fa35f25

16 files changed

Lines changed: 2924 additions & 88 deletions

‎.env.example‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
# Required for Cloud browsers or --fetch-use; not required for local Chrome alone
12
BROWSER_USE_API_KEY=
2-
# Required for BU Bench V2's default findings judge (gpt-5.6-luna, xhigh)
3+
# Required for the default Luna executor and V2 findings judge
34
OPENAI_API_KEY=
45
# Required for the legacy V1 / Stealth binary judge and Gemini executor models
56
GOOGLE_API_KEY=
@@ -23,3 +24,7 @@ ONKERNEL_API_KEY=
2324
REBROWSER_API_KEY=
2425
STEEL_API_KEY=
2526
DRIVER_API_KEY=
27+
28+
# Optional: save scores and text/tool traces in your private Laminar project
29+
# Install with: uv sync --frozen --extra laminar
30+
LMNR_PROJECT_API_KEY=

‎.github/workflows/run-benchmark.yml‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
name: Run BU Bench V2
2+
run-name: BU200 ${{ inputs.browser }} · ${{ inputs.model }} · ${{ inputs.tasks }} tasks
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
browser:
8+
description: Browser provider
9+
type: choice
10+
options: [browser-use-cloud, local_headless]
11+
default: browser-use-cloud
12+
model:
13+
description: BrowserCode provider/model
14+
type: string
15+
default: openai/gpt-6-luna
16+
reasoning:
17+
description: Agent reasoning effort
18+
type: choice
19+
options: [none, minimal, low, medium, high, xhigh, max]
20+
default: low
21+
tasks:
22+
description: Number of tasks (all 200 by default)
23+
type: number
24+
default: 200
25+
task_ids:
26+
description: Optional exact task IDs separated by spaces
27+
type: string
28+
task_timeout:
29+
description: Seconds per task
30+
type: number
31+
default: 3600
32+
max_parallel:
33+
description: Maximum concurrent task runners
34+
type: number
35+
default: 12
36+
fetch_use:
37+
description: Enable BrowserCode fetch service (requires Browser Use key)
38+
type: boolean
39+
default: true
40+
41+
permissions:
42+
contents: read
43+
44+
env:
45+
BROWSER_USE_SETUP_LOGGING: "false"
46+
47+
jobs:
48+
prepare:
49+
runs-on: ubuntu-latest
50+
outputs:
51+
task_ids: ${{ steps.tasks.outputs.task_ids }}
52+
evaluation_id: ${{ steps.tasks.outputs.evaluation_id }}
53+
steps:
54+
- uses: actions/checkout@v4
55+
- uses: astral-sh/setup-uv@v6
56+
- run: uv sync --frozen --python 3.12 --extra laminar
57+
- name: Validate inputs and select tasks
58+
id: tasks
59+
env:
60+
TASK_IDS: ${{ inputs.task_ids }}
61+
TASK_COUNT: ${{ inputs.tasks }}
62+
MAX_PARALLEL: ${{ inputs.max_parallel }}
63+
TASK_TIMEOUT: ${{ inputs.task_timeout }}
64+
EVAL_MODEL: ${{ inputs.model }}
65+
EVAL_BROWSER: ${{ inputs.browser }}
66+
LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_PROJECT_API_KEY }}
67+
run: |
68+
python3 - <<'PY'
69+
import os
70+
assert 1 <= int(os.environ["MAX_PARALLEL"]) <= 50
71+
assert 1 <= int(os.environ["TASK_TIMEOUT"]) <= 14400
72+
PY
73+
uv run python bcode_results.py prepare --tasks "$TASK_COUNT" --task-ids "$TASK_IDS"
74+
75+
evaluate:
76+
needs: prepare
77+
strategy:
78+
fail-fast: false
79+
max-parallel: ${{ fromJSON(format('{0}', inputs.max_parallel)) }}
80+
matrix:
81+
task_id: ${{ fromJSON(needs.prepare.outputs.task_ids) }}
82+
runs-on: ubuntu-latest
83+
timeout-minutes: 260
84+
env:
85+
BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }}
86+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
87+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
88+
GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
89+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
90+
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
91+
LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_PROJECT_API_KEY }}
92+
BENCHMARK_LAMINAR_EVAL_ID: ${{ needs.prepare.outputs.evaluation_id }}
93+
steps:
94+
- uses: actions/checkout@v4
95+
- uses: astral-sh/setup-uv@v6
96+
- run: uv sync --frozen --python 3.12 --extra laminar
97+
- name: Install BrowserCode
98+
run: |
99+
curl -fsSL https://bcode.sh/install | bash -s -- --version 0.1.20 --no-modify-path
100+
- name: Execute and judge task
101+
env:
102+
TASK_ID: ${{ matrix.task_id }}
103+
EVAL_MODEL: ${{ inputs.model }}
104+
REASONING: ${{ inputs.reasoning }}
105+
BROWSER: ${{ inputs.browser }}
106+
TASK_TIMEOUT: ${{ inputs.task_timeout }}
107+
FETCH_USE: ${{ inputs.fetch_use }}
108+
run: |
109+
fetch_flag=--no-fetch-use
110+
if [ "$FETCH_USE" = true ]; then fetch_flag=--fetch-use; fi
111+
uv run python run_eval.py \
112+
--task-ids "$TASK_ID" --parallel 1 \
113+
--model "$EVAL_MODEL" --agent-reasoning "$REASONING" \
114+
--browser "$BROWSER" "$fetch_flag" \
115+
--task-timeout "$TASK_TIMEOUT" --output-dir run_data
116+
- name: Export public-safe configuration
117+
if: ${{ always() }}
118+
run: uv run python bcode_results.py export
119+
- name: Upload scores and configuration
120+
if: ${{ always() }}
121+
uses: actions/upload-artifact@v4
122+
with:
123+
name: score-${{ matrix.task_id }}
124+
path: |
125+
run_data/**/public-result.json
126+
run_data/**/public-config.json
127+
if-no-files-found: error
128+
retention-days: 30
129+
- name: Retain full evidence in private repositories
130+
if: ${{ always() && github.event.repository.private }}
131+
uses: actions/upload-artifact@v4
132+
with:
133+
name: evidence-${{ matrix.task_id }}
134+
path: |
135+
run_data
136+
!run_data/**/browser-profile/**
137+
!run_data/**/state/**
138+
!run_data/**/preflight-state/**
139+
if-no-files-found: warn
140+
retention-days: 14
141+
142+
aggregate:
143+
if: ${{ always() && needs.prepare.result == 'success' }}
144+
needs: [prepare, evaluate]
145+
runs-on: ubuntu-latest
146+
steps:
147+
- uses: actions/checkout@v4
148+
- uses: astral-sh/setup-uv@v6
149+
- run: uv sync --frozen --python 3.12
150+
- uses: actions/download-artifact@v4
151+
with:
152+
pattern: score-*
153+
path: artifacts
154+
- name: Check every task and aggregate scores
155+
env:
156+
EXPECTED_TASK_IDS: ${{ needs.prepare.outputs.task_ids }}
157+
run: uv run python bcode_results.py aggregate --root artifacts
158+
- name: Upload aggregate results
159+
if: ${{ always() }}
160+
uses: actions/upload-artifact@v4
161+
with:
162+
name: bu-bench-v2-aggregate
163+
path: aggregate-results.json
164+
retention-days: 30

‎.github/workflows/test.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: Test V2 runner
2+
on:
3+
pull_request:
4+
push:
5+
branches: [main]
6+
permissions:
7+
contents: read
8+
jobs:
9+
test:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
13+
- uses: astral-sh/setup-uv@v6
14+
- run: uv sync --frozen --python 3.12
15+
- run: BCODE_TEST_CHROME="$(command -v google-chrome)" uv run python -m unittest discover -s tests
16+
- run: uv run ruff check bcode_runner.py bcode_eval.py bcode_results.py tests/test_bcode_runner.py tests/test_bcode_binary.py
17+
- run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7
18+
- run: |
19+
"$(go env GOPATH)/bin/actionlint" .github/workflows/run-benchmark.yml

0 commit comments

Comments
 (0)