Repository navigation
[agent-cli] every caller that quotes a failed eas run reads it with readEasFailure #594
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # @ref llp/0002-testing-and-evals.plan.md | |
| name: Expo Agent CLI | |
| on: | |
| workflow_dispatch: {} | |
| push: | |
| branches: [main] | |
| paths: | |
| - .github/workflows/agent-cli.yml | |
| - packages/@expo/agent-cli/** | |
| - bun.lock | |
| - package.json | |
| - bunfig.toml | |
| pull_request: | |
| paths: | |
| - .github/workflows/agent-cli.yml | |
| - packages/@expo/agent-cli/** | |
| - bun.lock | |
| - package.json | |
| - bunfig.toml | |
| schedule: | |
| # Weekly, Monday 15:00 UTC — only the report-only tier 1 job runs on this trigger. | |
| - cron: 0 15 * * 1 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Tier 0 of LLP 0002: deterministic, model-free, runs on every PR. | |
| tier0-linux: | |
| if: github.event_name != 'schedule' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: 👀 Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: 🥟 Setup bun | |
| uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version: 1.3.14 | |
| - name: ⬢ Setup Node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 24 | |
| - name: 📦 Install dependencies | |
| run: bun install --ignore-scripts --frozen-lockfile | |
| - name: 🛠 Build @expo/agent-cli | |
| run: bun run build | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🔎 Typecheck @expo/agent-cli | |
| run: bun run typecheck | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🚨 Lint @expo/agent-cli | |
| run: bun run lint | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🧪 Test @expo/agent-cli | |
| run: bun run test | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🧪 Eval adapter tests | |
| run: bun run test:eval-harness | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🎯 Tier 0 evals | |
| run: bun evals/run.mjs --tier 0 | |
| working-directory: packages/@expo/agent-cli | |
| # E2E is the long pole of tier 0 (~320s run as one step), so it is sharded across parallel | |
| # jobs instead. A size-balanced sequencer (see e2e/vitest.config.ts) splits the 31 files so no | |
| # shard gets both big ones; each builds its own ncc bundle because the suite spawns `bin/cli.js`. | |
| # Both OSes run here — the win32 shards are what exercise the skills linker's directory-junction | |
| # path (nothing else does). | |
| tier0-e2e: | |
| if: github.event_name != 'schedule' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-24.04, windows-2022] | |
| shard: [1, 2, 3] | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: 👀 Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: 🥟 Setup bun | |
| uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version: 1.3.14 | |
| - name: ⬢ Setup Node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 24 | |
| - name: 📦 Install dependencies | |
| run: bun install --ignore-scripts --frozen-lockfile | |
| - name: 🛠 Build @expo/agent-cli | |
| run: bun run build | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🧪 E2E Test shard ${{ matrix.shard }}/3 | |
| run: bun run test:e2e -- --shard=${{ matrix.shard }}/3 | |
| working-directory: packages/@expo/agent-cli | |
| # The iOS half of the installed-app check only runs where `hostPlatforms()` asks for iOS: macOS. | |
| # One runner, two files — the stub `xcrun` (simctl and devicectl) sits first on PATH, so no real | |
| # simulator or phone is touched. Kept apart from the shards because macOS minutes cost ten times | |
| # a Linux minute and these two files are the only ones that need the platform. | |
| tier0-e2e-macos: | |
| if: github.event_name != 'schedule' | |
| runs-on: macos-15 | |
| timeout-minutes: 20 | |
| steps: | |
| - name: 👀 Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: 🥟 Setup bun | |
| uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version: 1.3.14 | |
| - name: ⬢ Setup Node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 24 | |
| - name: 📦 Install dependencies | |
| run: bun install --ignore-scripts --frozen-lockfile | |
| - name: 🛠 Build @expo/agent-cli | |
| run: bun run build | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🧪 E2E installed-app check on macOS | |
| run: bun run test:e2e -- status-installed-test installed-readers-test | |
| working-directory: packages/@expo/agent-cli | |
| # Windows unit coverage: `resolveSpawnTarget` and the win32 branches of the path helpers. | |
| # The win32 e2e run — including the skills linker's directory-junction path, which nothing | |
| # else exercises — is sharded in `tier0-e2e` above. | |
| tier0-windows: | |
| if: github.event_name != 'schedule' | |
| runs-on: windows-2022 | |
| timeout-minutes: 45 | |
| steps: | |
| - name: 👀 Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: 🥟 Setup bun | |
| uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version: 1.3.14 | |
| - name: ⬢ Setup Node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 24 | |
| - name: 📦 Install dependencies | |
| run: bun install --ignore-scripts --frozen-lockfile | |
| - name: 🛠 Build @expo/agent-cli | |
| run: bun run build | |
| working-directory: packages/@expo/agent-cli | |
| - name: 🧪 Test @expo/agent-cli | |
| run: bun run test | |
| working-directory: packages/@expo/agent-cli | |
| # Short real agent integration tests. Every relevant PR runs the entire Tier 1 suite. | |
| # Advisory until repeated CI evidence establishes reliability; individual results stay visible. | |
| tier1-agent-eval: | |
| if: >- | |
| github.event_name == 'pull_request' || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'schedule' && github.repository == 'expo/expo-agent-cli') | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| continue-on-error: true | |
| timeout-minutes: 30 | |
| env: | |
| AGENT_CLI_EVAL_MODEL: qwen3:8b | |
| AGENT_CLI_EVAL_MODEL_DIGEST: 500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version: 1.3.14 | |
| - name: Setup Node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 24 | |
| - name: Install locked dependencies (including the real Expo fixture) | |
| run: bun install --ignore-scripts --frozen-lockfile | |
| - name: Build CLI | |
| run: bun run --filter @expo/agent-cli build | |
| - name: Install Ollama | |
| run: curl -fsSL https://ollama.com/install.sh | sh | |
| env: | |
| OLLAMA_VERSION: 0.32.15 | |
| - name: Start Ollama and pull the model | |
| run: | | |
| ollama serve > "$RUNNER_TEMP/ollama.log" 2>&1 & | |
| for _ in $(seq 1 30); do | |
| curl -fsS http://127.0.0.1:11434/api/version && break | |
| sleep 1 | |
| done | |
| curl -fsS http://127.0.0.1:11434/api/version | |
| ollama pull "$AGENT_CLI_EVAL_MODEL" | |
| - name: Agent integration evals | |
| id: evals | |
| run: bun run test:evals | |
| working-directory: packages/@expo/agent-cli | |
| - name: Summarize eval evidence | |
| if: always() | |
| run: bun evals/summarize.ts | |
| working-directory: packages/@expo/agent-cli | |
| - name: Preserve eval evidence | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: tier1-evidence-${{ github.run_id }}-${{ github.run_attempt }} | |
| include-hidden-files: true | |
| retention-days: 14 | |
| path: | | |
| packages/@expo/agent-cli/evals/.artifacts/ | |
| ${{ runner.temp }}/ollama.log |