Skip to content

Commit 8f11cc8

Browse files
sirbrilligsirbrillig
authored
Support single-file sizing and multiple candidate branches in benchmark.sh (#116)
* Support single-file sizing and multiple candidate branches in benchmark.sh Adds LINES_PER_FILE to size each generated file, and CANDIDATES to compare multiple branches against trunk in one hyperfine run. Worktrees are now created with --detach so trunk can be benchmarked even when checked out in the main worktree. * Add CACHE_MODE to benchmark.sh for cold/hot/partial cache scenarios Adds --cache via CACHE_MODE=hot and a partial-cache mode that dirties file1.php before each iteration so the modified-side cache misses while the unmodified-side cache (keyed on git revision) still hits. Each benchmark also clears the cache via hyperfine --setup so branches start fresh.
1 parent a369fce commit 8f11cc8

1 file changed

Lines changed: 108 additions & 54 deletions

File tree

‎benchmark.sh‎

Lines changed: 108 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,72 @@
11
#!/usr/bin/env bash
2-
# benchmark.sh — compare the current branch vs trunk
2+
# benchmark.sh — compare candidate branches vs trunk
33
#
44
# Usage:
55
# ./benchmark.sh
66
#
77
# Environment variables (all optional):
8-
# N_FILES=10 number of changed PHP files to scan
9-
# RUNS=10 hyperfine measurement runs
10-
# WARMUP=2 hyperfine warmup runs
8+
# N_FILES=10 number of changed PHP files to scan
9+
# LINES_PER_FILE=14 approximate target line count per file (committed version)
10+
# RUNS=10 hyperfine measurement runs
11+
# WARMUP=2 hyperfine warmup runs
12+
# CANDIDATES="<branches>" space-separated branches to compare against trunk.
13+
# Defaults to the currently checked-out branch.
14+
# Example: CANDIDATES="two-batch-phpcs-invocations batch-phpcs-invocations"
1115
#
1216
# Artifacts left in the repo root (not tracked by git):
13-
# .bench-trunk/ git worktree for trunk (reused on subsequent runs)
14-
# benchmark-results.md hyperfine markdown export
17+
# .bench-trunk/, .bench-<branch>/ git worktrees (reused on subsequent runs)
18+
# benchmark-results.md hyperfine markdown export
1519

1620
set -euo pipefail
1721

1822
# ── Paths ──────────────────────────────────────────────────────────────────
1923
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
20-
CURRENT_BIN="$SCRIPT_DIR/bin/phpcs-changed"
2124
TRUNK_WORKTREE="$SCRIPT_DIR/.bench-trunk"
2225
TRUNK_BIN="$TRUNK_WORKTREE/bin/phpcs-changed"
2326
PHPCS="$SCRIPT_DIR/vendor/bin/phpcs"
2427
RESULTS_FILE="$SCRIPT_DIR/benchmark-results.md"
2528

2629
# ── Tuneable ───────────────────────────────────────────────────────────────
2730
N_FILES=${N_FILES:-10}
31+
LINES_PER_FILE=${LINES_PER_FILE:-14}
2832
RUNS=${RUNS:-10}
2933
WARMUP=${WARMUP:-2}
34+
CURRENT_BRANCH_DEFAULT="$(git -C "$SCRIPT_DIR" branch --show-current)"
35+
CANDIDATES=${CANDIDATES:-$CURRENT_BRANCH_DEFAULT}
36+
# CACHE_MODE: cold (no --cache), hot (--cache, warmups prime everything),
37+
# partial (--cache, but modified side is dirtied each iteration so only the
38+
# unmodified side hits).
39+
CACHE_MODE=${CACHE_MODE:-cold}
3040

3141
# ── Sanity checks ─────────────────────────────────────────────────────────
3242
command -v hyperfine >/dev/null 2>&1 \
3343
|| { echo "ERROR: hyperfine not found. Install with: brew install hyperfine"; exit 1; }
3444
[ -f "$PHPCS" ] \
3545
|| { echo "ERROR: phpcs not found at $PHPCS. Run: composer install"; exit 1; }
3646

37-
# ── 1. Trunk worktree ─────────────────────────────────────────────────────
38-
if [ ! -d "$TRUNK_WORKTREE" ]; then
39-
echo "→ Creating trunk worktree at $TRUNK_WORKTREE …"
40-
git -C "$SCRIPT_DIR" worktree add "$TRUNK_WORKTREE" trunk
41-
else
42-
echo "→ Trunk worktree already exists."
43-
fi
47+
# ── 1. Worktrees ──────────────────────────────────────────────────────────
48+
ensure_worktree () {
49+
local branch="$1"
50+
local path="$2"
51+
if [ ! -d "$path" ]; then
52+
echo "→ Creating worktree for '$branch' at $path …"
53+
# --detach lets us add a worktree even if the branch is checked out elsewhere.
54+
git -C "$SCRIPT_DIR" worktree add --detach "$path" "$branch"
55+
else
56+
echo "→ Worktree for '$branch' already exists at $path."
57+
fi
58+
}
59+
60+
ensure_worktree "trunk" "$TRUNK_WORKTREE"
61+
62+
CANDIDATE_PATHS=()
63+
CANDIDATE_NAMES=()
64+
for branch in $CANDIDATES; do
65+
path="$SCRIPT_DIR/.bench-${branch}"
66+
ensure_worktree "$branch" "$path"
67+
CANDIDATE_PATHS+=("$path")
68+
CANDIDATE_NAMES+=("$branch")
69+
done
4470

4571
# ── 2. Ephemeral benchmark git repo ───────────────────────────────────────
4672
BENCH_REPO="$(mktemp -d)"
@@ -50,39 +76,42 @@ git -C "$BENCH_REPO" init -q
5076
git -C "$BENCH_REPO" config user.email "bench@example.com"
5177
git -C "$BENCH_REPO" config user.name "Benchmark"
5278

53-
# Create initial PHP files with PSR2 violations (uppercase TRUE / FALSE).
54-
# Both the committed and staged versions have violations so both builds must
55-
# scan both file versions — this is the worst-case and ensures a fair comparison.
56-
echo "→ Preparing $N_FILES PHP test files …"
79+
# Each method block adds 6 lines and one violation (TRUE/FALSE).
80+
# Class scaffold (header + closer) costs ~5 lines, so derive block count from target.
81+
BLOCKS_PER_FILE=$(( (LINES_PER_FILE - 5) / 6 ))
82+
[ "$BLOCKS_PER_FILE" -lt 2 ] && BLOCKS_PER_FILE=2
83+
84+
# Alternate violations across blocks for variety.
85+
VIOLATIONS=(TRUE FALSE)
86+
87+
echo "→ Preparing $N_FILES PHP test files (~$LINES_PER_FILE lines, $BLOCKS_PER_FILE method blocks each) …"
5788
FILES=()
5889
for i in $(seq 1 "$N_FILES"); do
5990
fname="file${i}.php"
60-
61-
# Committed (HEAD) version ─ existing violations: TRUE, FALSE
62-
cat > "$BENCH_REPO/$fname" << PHPEOF
63-
<?php
64-
class Foo${i}
65-
{
66-
public function check(): bool
67-
{
68-
\$a = TRUE;
69-
return \$a;
70-
}
71-
72-
public function invert(): bool
73-
{
74-
\$b = FALSE;
75-
return !\$b;
76-
}
77-
}
78-
PHPEOF
91+
fpath="$BENCH_REPO/$fname"
92+
93+
{
94+
echo "<?php"
95+
echo "class Foo${i}"
96+
echo "{"
97+
for b in $(seq 1 "$BLOCKS_PER_FILE"); do
98+
violation=${VIOLATIONS[$(( (b - 1) % ${#VIOLATIONS[@]} ))]}
99+
printf ' public function method%d(): bool\n' "$b"
100+
echo ' {'
101+
printf ' $v = %s;\n' "$violation"
102+
echo ' return $v;'
103+
echo ' }'
104+
echo ''
105+
done
106+
echo "}"
107+
} > "$fpath"
79108

80109
FILES+=("$fname")
81110
git -C "$BENCH_REPO" add "$fname"
82111
done
83112
git -C "$BENCH_REPO" commit -q -m "initial commit"
84113

85-
# Staged version ─ add a new violation to each file
114+
# Staged version ─ add a new violation (NULL) to each file
86115
for i in $(seq 1 "$N_FILES"); do
87116
fname="file${i}.php"
88117
cat >> "$BENCH_REPO/$fname" << PHPEOF
@@ -96,31 +125,56 @@ PHPEOF
96125
git -C "$BENCH_REPO" add "$fname"
97126
done
98127

128+
ACTUAL_LINES=$(wc -l < "$BENCH_REPO/file1.php" | tr -d ' ')
129+
echo "→ Actual line count per file (staged): $ACTUAL_LINES"
130+
99131
# ── 3. Run hyperfine ──────────────────────────────────────────────────────
100132
FILES_STR="${FILES[*]}"
101-
CURRENT_BRANCH="$(git -C "$SCRIPT_DIR" branch --show-current)"
102-
CURRENT_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD)"
103133
TRUNK_SHA="$(git -C "$TRUNK_WORKTREE" rev-parse --short HEAD)"
104134

105-
CURRENT_CMD="php '$CURRENT_BIN' --git-staged --phpcs-path='$PHPCS' --standard=PSR2 --always-exit-zero $FILES_STR"
106-
TRUNK_CMD="php '$TRUNK_BIN' --git-staged --phpcs-path='$PHPCS' --standard=PSR2 --always-exit-zero $FILES_STR"
135+
case "$CACHE_MODE" in
136+
cold) CACHE_FLAG="" ;;
137+
hot) CACHE_FLAG="--cache" ;;
138+
partial) CACHE_FLAG="--cache" ;;
139+
*) echo "ERROR: CACHE_MODE must be cold, hot, or partial"; exit 1 ;;
140+
esac
141+
142+
HF_ARGS=(--warmup "$WARMUP" --runs "$RUNS")
143+
144+
# Ensure each benchmark starts from a clean cache so the first warmup populates
145+
# it consistently rather than inheriting the previous branch's cache file.
146+
HF_ARGS+=(--setup "rm -f $BENCH_REPO/.phpcs-changed-cache")
147+
148+
# In partial mode, dirty file1.php before each iteration so the modified-side
149+
# cache misses while the unmodified-side cache (keyed on git revision) still hits.
150+
if [ "$CACHE_MODE" = "partial" ]; then
151+
HF_ARGS+=(--prepare "echo '// noise' >> $BENCH_REPO/file1.php && git -C $BENCH_REPO add file1.php")
152+
fi
153+
154+
# Trunk baseline first so it appears at the top of the table.
155+
TRUNK_CMD="php '$TRUNK_BIN' --git-staged --phpcs-path='$PHPCS' --standard=PSR2 --always-exit-zero $CACHE_FLAG $FILES_STR"
156+
HF_ARGS+=(-n "trunk ($TRUNK_SHA)" "$TRUNK_CMD")
157+
158+
for idx in "${!CANDIDATE_PATHS[@]}"; do
159+
path="${CANDIDATE_PATHS[$idx]}"
160+
name="${CANDIDATE_NAMES[$idx]}"
161+
sha="$(git -C "$path" rev-parse --short HEAD)"
162+
bin="$path/bin/phpcs-changed"
163+
cmd="php '$bin' --git-staged --phpcs-path='$PHPCS' --standard=PSR2 --always-exit-zero $CACHE_FLAG $FILES_STR"
164+
HF_ARGS+=(-n "$name ($sha)" "$cmd")
165+
done
107166

108167
echo ""
109-
printf "Benchmark: %d staged PHP files, --standard=PSR2\n" "$N_FILES"
110-
printf " %-30s %s\n" "$CURRENT_BRANCH" "$CURRENT_SHA"
111-
printf " %-30s %s\n" "trunk" "$TRUNK_SHA"
168+
printf "Benchmark: %d staged PHP files, ~%d lines/file, cache=%s, --standard=PSR2\n" "$N_FILES" "$ACTUAL_LINES" "$CACHE_MODE"
169+
printf " baseline: trunk (%s)\n" "$TRUNK_SHA"
170+
for idx in "${!CANDIDATE_NAMES[@]}"; do
171+
path="${CANDIDATE_PATHS[$idx]}"
172+
printf " candidate: %s (%s)\n" "${CANDIDATE_NAMES[$idx]}" "$(git -C "$path" rev-parse --short HEAD)"
173+
done
112174
echo ""
113175

114176
cd "$BENCH_REPO"
115-
# shellcheck disable=SC2086
116-
hyperfine \
117-
--warmup "$WARMUP" \
118-
--runs "$RUNS" \
119-
-n "$CURRENT_BRANCH ($CURRENT_SHA)" \
120-
-n "trunk ($TRUNK_SHA)" \
121-
"$CURRENT_CMD" \
122-
"$TRUNK_CMD" \
123-
--export-markdown "$RESULTS_FILE"
177+
hyperfine "${HF_ARGS[@]}" --export-markdown "$RESULTS_FILE"
124178

125179
echo ""
126180
echo "Results written to $RESULTS_FILE"

0 commit comments

Comments
 (0)