Publish the eval observatory and preserve previous editions #84
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # CI for the co-evolution toolkit (v1.3 Phase 2; closes audit F-4). | |
| # | |
| # The matrix is the cross-platform contract: ubuntu (GNU userland, bash 5), | |
| # macos (BSD userland, stock bash 3.2 — the canary for every GNU-ism), and | |
| # windows (Git Bash/MINGW, the project's original home). All suites are | |
| # hermetic: agent CLIs are PATH-stubbed, no network, no LLM spend. | |
| name: ci | |
| on: | |
| push: | |
| branches: [master] | |
| paths-ignore: ['docs/**', '.planning/**', 'README.md', 'notesforhumans.md', 'llms.txt', 'CITATION.cff', 'LICENSE'] | |
| pull_request: | |
| paths-ignore: ['docs/**', '.planning/**', 'README.md', 'notesforhumans.md', 'llms.txt', 'CITATION.cff', 'LICENSE'] | |
| env: | |
| YQ_VERSION: v4.44.3 | |
| jobs: | |
| observatory: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| - name: Check score fidelity, archive preservation and current build | |
| run: python benchmarks/site/tests/test_observatory.py | |
| - name: Check score inclusion and repository denominators | |
| run: node --test benchmarks/site/tests/test-observatory-data.cjs | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # yq is pinned: `releases/latest` silently re-versions the toolchain under | |
| # us, so a green run yesterday says nothing about today's. | |
| - name: Cache mikefarah yq | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local/bin | |
| key: yq-${{ matrix.os }}-${{ env.YQ_VERSION }} | |
| - name: Install mikefarah yq | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$HOME/.local/bin" | |
| case "$(uname -s)" in | |
| Linux) asset=yq_linux_amd64 ;; | |
| Darwin) asset=yq_darwin_arm64; [[ "$(uname -m)" == x86_64 ]] && asset=yq_darwin_amd64 ;; | |
| MINGW*|MSYS*|CYGWIN*) asset=yq_windows_amd64.exe ;; | |
| *) echo "unsupported runner OS"; exit 1 ;; | |
| esac | |
| dest="$HOME/.local/bin/yq" | |
| [[ "$asset" == *.exe ]] && dest="$HOME/.local/bin/yq.exe" | |
| if [[ ! -x "$dest" ]]; then | |
| curl -sSL -o "$dest" \ | |
| "https://github.com/mikefarah/yq/releases/download/$YQ_VERSION/$asset" | |
| chmod +x "$dest" | |
| fi | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Doctor (diagnostic only — agent CLIs are stubbed in tests) | |
| shell: bash | |
| run: bash scripts/doctor.sh || true | |
| - name: Show environment | |
| shell: bash | |
| run: | | |
| bash --version | head -1 | |
| uname -a | |
| yq --version | |
| jq --version | |
| - name: Run full test suite | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # macOS ships bash 3.2, which has no `wait -n`; run-all.sh would fall | |
| # back to serial anyway, so ask for it directly and keep the log clean. | |
| jobs_flag="--jobs 4" | |
| case "$(uname -s)" in | |
| Darwin) jobs_flag="" ;; | |
| esac | |
| bash tests/run-all.sh $jobs_flag --ledger "$RUNNER_TEMP/run-ledger" | |
| # The ledger holds a .log and a .result (with elapsed seconds) per suite — | |
| # the only record of where a slow or failed run spent its time. | |
| - name: Upload run ledger | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: run-ledger-${{ matrix.os }} | |
| path: ${{ runner.temp }}/run-ledger | |
| if-no-files-found: warn | |
| mcp: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| - name: MCP package test (vendor + build + hermetic smoke) | |
| shell: bash | |
| run: cd mcp && npm ci && npm test |