fixes to livebooks #45
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| comment: | |
| description: "Optional comment to attach to the benchmark run" | |
| required: false | |
| default: "" | |
| # Prevent concurrent benchmark runs so results are comparable. | |
| concurrency: | |
| group: benchmarks | |
| cancel-in-progress: false | |
| jobs: | |
| bench: | |
| name: Run benchmarks and publish | |
| runs-on: ubuntu-latest | |
| # Required for github-action-benchmark to push results to gh-pages. | |
| permissions: | |
| contents: write | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" | |
| steps: | |
| - uses: actions/checkout@v6.0.3 | |
| - name: Set up Elixir | |
| uses: erlef/setup-beam@v1.24.0 | |
| with: | |
| elixir-version: "1.20" | |
| otp-version: "29" | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| - name: Cache Rust build | |
| uses: actions/cache@v5.0.5 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| native/ex_arrow_native/target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('native/ex_arrow_native/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Cache Mix build | |
| uses: actions/cache@v5.0.5 | |
| with: | |
| path: | | |
| deps | |
| _build | |
| key: ${{ runner.os }}-mix-bench-${{ hashFiles('mix.lock') }}-${{ hashFiles('native/ex_arrow_native/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-mix-bench- | |
| - name: Install dependencies | |
| env: | |
| EX_ARROW_BUILD: "1" | |
| run: mix deps.get | |
| - name: Compile | |
| env: | |
| EX_ARROW_BUILD: "1" | |
| run: mix compile --warnings-as-errors | |
| - name: Create bench output directory | |
| run: mkdir -p bench/output | |
| # Run each benchmark in :dev env (Benchee is a :dev dep). | |
| # We use --no-halt so the VM stays alive through multi-script runs. | |
| - name: Run IPC read benchmark | |
| env: | |
| EX_ARROW_BUILD: "1" | |
| MIX_ENV: dev | |
| run: mix run bench/ipc_read_bench.exs | |
| - name: Run IPC write benchmark | |
| env: | |
| EX_ARROW_BUILD: "1" | |
| MIX_ENV: dev | |
| run: mix run bench/ipc_write_bench.exs | |
| - name: Run Flight benchmark | |
| env: | |
| EX_ARROW_BUILD: "1" | |
| MIX_ENV: dev | |
| run: mix run bench/flight_bench.exs | |
| - name: Run ADBC stream benchmark | |
| env: | |
| EX_ARROW_BUILD: "1" | |
| MIX_ENV: dev | |
| run: mix run bench/adbc_bench.exs | |
| - name: Run pipeline benchmark | |
| env: | |
| EX_ARROW_BUILD: "1" | |
| MIX_ENV: dev | |
| run: mix run bench/pipeline_bench.exs | |
| # Merge all individual JSON results into one file for the benchmark action. | |
| # benchee_json v1.0+ outputs a JSON list of scenario objects; older builds | |
| # used {"suite": {"name": {...}}}. Both formats are handled here. | |
| - name: Merge JSON results | |
| run: | | |
| python3 - <<'EOF' | |
| import json, glob, os | |
| def extract_scenarios(data, suite_name): | |
| """Normalise benchee_json output into a flat list of (name, avg_ns) pairs.""" | |
| results = [] | |
| # Format A: list of scenario objects (benchee_json >= 1.0) | |
| if isinstance(data, list): | |
| for scenario in data: | |
| if not isinstance(scenario, dict): | |
| continue | |
| name = scenario.get("name", "unknown") | |
| avg = (scenario | |
| .get("run_time_data", {}) | |
| .get("statistics", {}) | |
| .get("average")) | |
| if avg is not None: | |
| results.append((f"[{suite_name}] {name}", avg)) | |
| # Format B: {"suite": {"scenario_name": {"run_time_data": {...}}}} | |
| elif isinstance(data, dict): | |
| for name, stats in data.get("suite", {}).items(): | |
| if not isinstance(stats, dict): | |
| continue | |
| avg = (stats | |
| .get("run_time_data", {}) | |
| .get("statistics", {}) | |
| .get("average")) | |
| if avg is not None: | |
| results.append((f"[{suite_name}] {name}", avg)) | |
| return results | |
| merged = [] | |
| for path in sorted(glob.glob("bench/output/*.json")): | |
| # Skip the merged output file to avoid processing our own output. | |
| if os.path.basename(path) == "merged.json": | |
| continue | |
| try: | |
| with open(path) as f: | |
| data = json.load(f) | |
| except Exception as e: | |
| print(f"Skipping {path}: {e}") | |
| continue | |
| suite_name = os.path.splitext(os.path.basename(path))[0] | |
| for name, avg in extract_scenarios(data, suite_name): | |
| merged.append({"name": name, "unit": "ns/op", "value": round(avg)}) | |
| out = "bench/output/merged.json" | |
| with open(out, "w") as f: | |
| json.dump(merged, f, indent=2) | |
| print(f"Wrote {len(merged)} entries to {out}") | |
| if not merged: | |
| print("WARNING: no benchmark entries found — check bench/output/*.json") | |
| EOF | |
| # git-action-benchmark commits to gh-pages; git requires a user identity. | |
| - name: Set up git user for benchmark commit | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| # Publish benchmark results as a GitHub Pages comment using | |
| # benchmark-action/github-action-benchmark. | |
| # Results are stored in the gh-pages branch under bench/data.js so they | |
| # persist across runs and can be viewed as a trend chart. | |
| # auto-push: true handles the gh-pages push via github-token — no | |
| # separate push step or GH_PAGES_TOKEN secret is required. | |
| - name: Store benchmark results | |
| uses: benchmark-action/github-action-benchmark@v1.22.1 | |
| with: | |
| name: ExArrow Benchmark Suite | |
| tool: customSmallerIsBetter | |
| output-file-path: bench/output/merged.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| # Push results to the gh-pages branch so they are visible at | |
| # https://<owner>.github.io/<repo>/dev/bench/ | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: dev/bench | |
| # Alert in a PR comment if performance regresses by more than 20%. | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| fail-on-alert: false | |
| comment-always: false | |
| # Upload HTML reports as a workflow artifact (7-day retention). | |
| - name: Upload HTML reports | |
| uses: actions/upload-artifact@v7.0.1 | |
| with: | |
| name: benchmark-html-${{ github.sha }} | |
| path: bench/output/*.html | |
| retention-days: 7 |