Benchmarks #87
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| workflow_run: | |
| workflows: [CI] | |
| types: [completed] | |
| branches: [master] | |
| workflow_dispatch: | |
| concurrency: | |
| group: benchmark | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.workflow_run.conclusion == 'success' | |
| outputs: | |
| should_run: ${{ steps.check.outputs.should_run }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| fetch-depth: 0 | |
| - name: Check for benchmark-relevant changes | |
| id: check | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then | |
| echo "Manual trigger, running benchmarks" | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| LAST_SHA=$(curl -fsS https://raw.githubusercontent.com/Dtronix/Quarry-benchmarks/gh-pages/dev/bench/runs/runs.json 2>/dev/null \ | |
| | jq -r '.[0].sha // empty' 2>/dev/null || echo "") | |
| if [ -z "$LAST_SHA" ]; then | |
| echo "No previous benchmark run found, running benchmarks" | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if ! git cat-file -e "$LAST_SHA" 2>/dev/null; then | |
| echo "Previous benchmark SHA $LAST_SHA not in history, running benchmarks" | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| CHANGED=$(git diff --name-only "$LAST_SHA" HEAD) | |
| echo "Changed files since $LAST_SHA:" | |
| echo "$CHANGED" | |
| if echo "$CHANGED" | grep -qE '^(src/Quarry/|src/Quarry\.Shared/|src/Quarry\.Generator/|src/Quarry\.Benchmarks/|src/Quarry\.Benchmarks\.Reporter/|\.github/workflows/benchmark\.yml|scripts/benchmark-pages/)'; then | |
| echo "Benchmark-relevant changes detected, running benchmarks" | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "No benchmark-relevant changes, skipping" | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| benchmark: | |
| needs: check | |
| if: needs.check.outputs.should_run == 'true' | |
| runs-on: debian-benchmark | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| fetch-depth: 0 | |
| - name: Setup .NET 10 | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Run benchmarks | |
| run: | | |
| dotnet run --project src/Quarry.Benchmarks -c Release \ | |
| -- --filter '*' --artifacts BenchmarkDotNet.Artifacts | |
| - name: Merge benchmark results | |
| run: | | |
| jq -s '{ | |
| Title: "Quarry Benchmarks", | |
| HostEnvironmentInfo: .[0].HostEnvironmentInfo, | |
| Benchmarks: [.[].Benchmarks[] | select(.Statistics != null)] | |
| }' BenchmarkDotNet.Artifacts/results/*-report-full.json \ | |
| > BenchmarkDotNet.Artifacts/combined-results.json | |
| - name: Generate merged HTML report | |
| run: | | |
| COMMIT_SHA=$(git log -1 --format='%H' HEAD) | |
| COMMIT_MSG=$(git log -1 --format='%s' HEAD) | |
| COMMIT_DATE=$(git log -1 --format='%aI' HEAD) | |
| mkdir -p BenchmarkDotNet.Artifacts/reports | |
| dotnet run --project src/Quarry.Benchmarks.Reporter -c Release -- \ | |
| BenchmarkDotNet.Artifacts/combined-results.json \ | |
| BenchmarkDotNet.Artifacts/reports/report.html \ | |
| "$COMMIT_SHA" "$COMMIT_MSG" "$COMMIT_DATE" | |
| - name: Ensure gh-pages branch exists | |
| run: | | |
| TMP_DIR=$(mktemp -d) | |
| cd "$TMP_DIR" | |
| git clone "https://x-access-token:${TOKEN}@github.com/Dtronix/Quarry-benchmarks.git" repo | |
| cd repo | |
| if git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then | |
| echo "gh-pages branch already exists" | |
| else | |
| echo "Creating gh-pages branch" | |
| git config user.name "github-action-benchmark" | |
| git config user.email "github@users.noreply.github.com" | |
| git checkout --orphan gh-pages | |
| git rm -rf . 2>/dev/null || true | |
| echo "# Quarry Benchmarks" > README.md | |
| git add README.md | |
| git commit -m "Initialize gh-pages branch" | |
| git push origin gh-pages | |
| fi | |
| cd / | |
| rm -rf "$TMP_DIR" | |
| env: | |
| TOKEN: ${{ secrets.BENCHMARK_TOKEN }} | |
| - name: Publish benchmark run (data.js, report, pages, manifest) | |
| run: | | |
| COMMIT_SHA=$(git log -1 --format='%H' HEAD) | |
| COMMIT_MSG=$(git log -1 --format='%s' HEAD) | |
| COMMIT_DATE_ISO=$(git log -1 --format='%aI' HEAD) | |
| COMMIT_DATE_PREFIX=$(date -d "$COMMIT_DATE_ISO" -u +%Y-%m-%d) | |
| SHORT_SHA=${COMMIT_SHA:0:8} | |
| REPORT_NAME="${COMMIT_DATE_PREFIX}-${SHORT_SHA}.html" | |
| rm -rf benchmark-store | |
| git clone --branch gh-pages \ | |
| "https://x-access-token:${TOKEN}@github.com/Dtronix/Quarry-benchmarks.git" \ | |
| benchmark-store | |
| cd benchmark-store | |
| git config user.name "github-action-benchmark" | |
| git config user.email "github@users.noreply.github.com" | |
| # ── Build the new data.js entry ───────────────────────────────── | |
| # Commit metadata comes from the GitHub API so we get the same | |
| # {name, email, username} shape the old action produced. | |
| COMMIT_OBJ=$(curl -fsS \ | |
| -H "Authorization: Bearer ${TOKEN}" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "https://api.github.com/repos/${GITHUB_REPOSITORY}/commits/${COMMIT_SHA}" | \ | |
| jq '{ | |
| author: ({name: .commit.author.name, email: .commit.author.email} | |
| + (if .author and .author.login then {username: .author.login} else {} end)), | |
| committer: ({name: .commit.committer.name, email: .commit.committer.email} | |
| + (if .committer and .committer.login then {username: .committer.login} else {} end)), | |
| id: .sha, | |
| message: .commit.message, | |
| timestamp: .commit.author.date, | |
| url: .html_url | |
| }') | |
| # Build the new run entry. Filters to Quarry_* methods and pulls | |
| # both Mean (time) and BytesAllocatedPerOperation (memory) in one | |
| # pass from the BenchmarkDotNet JSON. | |
| NEW_ENTRY=$(jq -n \ | |
| --argjson commit "$COMMIT_OBJ" \ | |
| --slurpfile bdn ../BenchmarkDotNet.Artifacts/combined-results.json \ | |
| '{ | |
| commit: $commit, | |
| date: (now * 1000 | floor), | |
| tool: "benchmarkdotnet", | |
| benches: [ | |
| $bdn[0].Benchmarks[] | |
| | select(.Method | startswith("Quarry_")) | |
| | { | |
| name: .FullName, | |
| value: .Statistics.Mean, | |
| unit: "ns", | |
| range: ("± " + (.Statistics.StandardDeviation | tostring)), | |
| allocated: (.Memory.BytesAllocatedPerOperation // 0) | |
| } | |
| ] | |
| }') | |
| # ── Merge into existing data.js, or seed a fresh file ────────── | |
| mkdir -p dev/bench/runs | |
| DATA_JS=dev/bench/data.js | |
| if [ ! -f "$DATA_JS" ]; then | |
| printf 'window.BENCHMARK_DATA = %s\n' \ | |
| '{"lastUpdate":0,"repoUrl":"https://github.com/Dtronix/Quarry","entries":{"Quarry Benchmarks":[]}}' \ | |
| > "$DATA_JS" | |
| fi | |
| # Strip the `window.BENCHMARK_DATA = ` wrapper (and trailing `;` if present). | |
| EXISTING_JSON=$(sed -e '1s|^window\.BENCHMARK_DATA *= *||' -e '$s|;[[:space:]]*$||' "$DATA_JS") | |
| PATCHED_JSON=$(jq --argjson entry "$NEW_ENTRY" \ | |
| '.lastUpdate = (now * 1000 | floor) | |
| | .entries["Quarry Benchmarks"] += [$entry] | |
| | .entries["Quarry Benchmarks"] |= (if length > 500 then .[length - 500:] else . end)' \ | |
| <<<"$EXISTING_JSON") | |
| { printf 'window.BENCHMARK_DATA = '; printf '%s' "$PATCHED_JSON"; } > "$DATA_JS" | |
| # ── Copy per-run HTML report and static front-end pages ──────── | |
| cp ../BenchmarkDotNet.Artifacts/reports/report.html "dev/bench/runs/${REPORT_NAME}" | |
| cp ../scripts/benchmark-pages/landing.html index.html | |
| cp ../scripts/benchmark-pages/README.md README.md | |
| cp ../scripts/benchmark-pages/dashboard.html dev/bench/index.html | |
| cp ../scripts/benchmark-pages/runs.html dev/bench/runs/index.html | |
| # ── Update runs manifest ─────────────────────────────────────── | |
| MANIFEST=dev/bench/runs/runs.json | |
| if [ ! -f "$MANIFEST" ]; then echo '[]' > "$MANIFEST"; fi | |
| jq --arg sha "$COMMIT_SHA" \ | |
| --arg msg "$COMMIT_MSG" \ | |
| --arg date "$COMMIT_DATE_ISO" \ | |
| --arg file "$REPORT_NAME" \ | |
| '. + [{sha: $sha, message: $msg, date: $date, file: $file}] | |
| | sort_by(.date) | reverse | .[0:500]' \ | |
| "$MANIFEST" > "$MANIFEST.tmp" | |
| mv "$MANIFEST.tmp" "$MANIFEST" | |
| # Trim old report files beyond 500 | |
| KEEP_FILES=$(jq -r '.[].file' "$MANIFEST" | sort -u) | |
| for f in dev/bench/runs/*.html; do | |
| BASENAME=$(basename "$f") | |
| if [ "$BASENAME" = "index.html" ]; then continue; fi | |
| if ! grep -qx "$BASENAME" <<<"$KEEP_FILES"; then | |
| echo "Removing old report: $BASENAME" | |
| rm -f "$f" | |
| fi | |
| done | |
| git add -A | |
| if git diff --cached --quiet; then | |
| echo "No changes to push" | |
| else | |
| git commit -m "Add report for ${SHORT_SHA}" | |
| git push origin gh-pages | |
| fi | |
| env: | |
| TOKEN: ${{ secrets.BENCHMARK_TOKEN }} |