Skip to content

Commit 2b3ebf2

Browse files
web3dev1337claude
web3dev1337
and
claude
committed
feat: deterministic A/B benchmark + headless client fixes
- Fix SwiftShader: use --enable-unsafe-swiftshader (old flag deprecated) - Fix HTTPS certs: ignoreHTTPSErrors + CDP Security.setIgnoreCertificateErrors - Fix fetch: patch targetAddressSpace (Chrome PNA API not available) - Fix warmCert: pre-accept self-signed HTTPS cert before client navigation - Add player entity spawning in perf-harness (was spectator-only) - Add set_camera, throttle_cpu, walk_player (sendMovement via network packets) - Add wait_for_entities action for entity load synchronization - Add stress-walkthrough preset: 200 idle entities, deterministic positions - Expose __HYTOPIA_GAME__ on window for headless camera/input control - Fix BaselineComparer null-safety for operations + nested format support A/B results (PR #2 blob shadows, 16 visible entities, SwiftShader): - Frame time: 83ms → 152ms (+82% FAIL) - Max draw calls: 25 → 82 (+228%) - Max triangles: 263k → 584k (+121%) - Bug found: TransparentSortData missing on shadow meshes (error spam) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 4e716cc commit 2b3ebf2

8 files changed

Lines changed: 544 additions & 4 deletions

File tree

‎client/src/core/PerfBridge.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ export default class PerfBridge {
2020
};
2121

2222
(window as any).__HYTOPIA_PERF__ = perf;
23+
24+
// Expose game instance for headless client control (camera, input)
25+
(window as any).__HYTOPIA_GAME__ = game;
2326
}
2427

2528
private _snapshot() {

‎packages/perf-tools/perf-results/walkthrough-baseline.json‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
{
2+
"timestamp": "2026-03-06T05:23:50.026Z",
3+
"scenario": "stress-walkthrough",
4+
"durationMs": 94602,
5+
"baseline": {
6+
"avgTickMs": 0.5391012166361615,
7+
"maxTickMs": 17.94537400000263,
8+
"p95TickMs": 1.0751982444444568,
9+
"p99TickMs": 3.3784996888897796,
10+
"ticksOverBudgetPct": 0.006376105191566539,
11+
"avgMemoryMb": 66.61536475287544,
12+
"avgFps": 12.755555555555556,
13+
"client": {
14+
"avgFps": 12.755555555555556,
15+
"minFps": 8,
16+
"avgFrameTimeMs": 83.55111111071375,
17+
"avgDrawCalls": 25,
18+
"maxDrawCalls": 25,
19+
"avgTriangles": 263769,
20+
"maxTriangles": 263769,
21+
"avgGeometries": 68,
22+
"avgEntities": 16,
23+
"avgVisibleChunks": 30,
24+
"avgUsedMemoryMb": 152.89682466718887
25+
},
26+
"operations": {
27+
"entities_tick": {
28+
"avgMs": 0.12939661985288534,
29+
"p95Ms": 0.2722047333333952
30+
},
31+
"physics_step": {
32+
"avgMs": 0.1328637783304221,
33+
"p95Ms": 0.23616346666741367
34+
},
35+
"physics_cleanup": {
36+
"avgMs": 0.005769868592307142,
37+
"p95Ms": 0.007910844444510682
38+
},
39+
"simulation_step": {
40+
"avgMs": 0.14639699397135983,
41+
"p95Ms": 0.2683269555539785
42+
},
43+
"entities_emit_updates": {
44+
"avgMs": 0.22037241875926325,
45+
"p95Ms": 0.46857908888980293
46+
},
47+
"send_all_packets": {
48+
"avgMs": 0.005150242018657108,
49+
"p95Ms": 0.005713133334615527
50+
},
51+
"network_synchronize_cleanup": {
52+
"avgMs": 0.002902808306821984,
53+
"p95Ms": 0.004927022222141709
54+
},
55+
"network_synchronize": {
56+
"avgMs": 0.02167050894632742,
57+
"p95Ms": 0.025265599999369847
58+
},
59+
"world_tick": {
60+
"avgMs": 0.5284478310519916,
61+
"p95Ms": 1.256651333332411
62+
},
63+
"ticker_tick": {
64+
"avgMs": 0.6083506673469862,
65+
"p95Ms": 1.5245236666667754
66+
}
67+
},
68+
"network": {
69+
"totalBytesSent": 0,
70+
"totalBytesReceived": 0,
71+
"maxConnectedPlayers": 0,
72+
"avgBytesSentPerSecond": 0,
73+
"maxBytesSentPerSecond": 0,
74+
"avgBytesReceivedPerSecond": 0,
75+
"maxBytesReceivedPerSecond": 0,
76+
"avgPacketsSentPerSecond": 0,
77+
"maxPacketsSentPerSecond": 0,
78+
"avgPacketsReceivedPerSecond": 0,
79+
"maxPacketsReceivedPerSecond": 0,
80+
"avgSerializationMs": 0,
81+
"compressionCountTotal": 0
82+
}
83+
},
84+
"phases": [
85+
{
86+
"name": "spawn-entities",
87+
"durationMs": 283,
88+
"collected": false
89+
},
90+
{
91+
"name": "wait-for-world",
92+
"durationMs": 8504,
93+
"collected": false
94+
},
95+
{
96+
"name": "walkthrough",
97+
"durationMs": 77149,
98+
"collected": true
99+
}
100+
],
101+
"metrics": {
102+
"tickReportCount": 0,
103+
"spikeCount": 0,
104+
"serverSnapshotCount": 45,
105+
"clientSnapshotCount": 45
106+
}
107+
}

‎packages/perf-tools/perf-results/walkthrough-blob-shadows.json‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
{
2+
"timestamp": "2026-03-06T05:25:52.669Z",
3+
"scenario": "stress-walkthrough",
4+
"durationMs": 93623,
5+
"baseline": {
6+
"avgTickMs": 0.4463646582871844,
7+
"maxTickMs": 9.024755000005825,
8+
"p95TickMs": 0.920134800000071,
9+
"p99TickMs": 1.746577422222577,
10+
"ticksOverBudgetPct": 0,
11+
"avgMemoryMb": 43.593108622233075,
12+
"avgFps": 23.727272727272727,
13+
"client": {
14+
"avgFps": 23.727272727272727,
15+
"minFps": 0,
16+
"avgFrameTimeMs": 152.19772727347234,
17+
"avgDrawCalls": 17.75,
18+
"maxDrawCalls": 82,
19+
"avgTriangles": 155875.47727272726,
20+
"maxTriangles": 584747,
21+
"avgGeometries": 63,
22+
"avgEntities": 15.545454545454545,
23+
"avgVisibleChunks": 64.06818181818181,
24+
"avgUsedMemoryMb": 220.8743019104004
25+
},
26+
"operations": {
27+
"entities_tick": {
28+
"avgMs": 0.12082475065899774,
29+
"p95Ms": 0.212825199998608
30+
},
31+
"physics_step": {
32+
"avgMs": 0.12011465469852715,
33+
"p95Ms": 0.1786375777785401
34+
},
35+
"physics_cleanup": {
36+
"avgMs": 0.006925774357740608,
37+
"p95Ms": 0.007451844444989951
38+
},
39+
"simulation_step": {
40+
"avgMs": 0.13415876547377642,
41+
"p95Ms": 0.20245715555631452
42+
},
43+
"entities_emit_updates": {
44+
"avgMs": 0.15868798814720403,
45+
"p95Ms": 0.2651034888884169
46+
},
47+
"world_tick": {
48+
"avgMs": 0.44518506949526604,
49+
"p95Ms": 0.8249018000006294
50+
},
51+
"ticker_tick": {
52+
"avgMs": 0.5173913805869171,
53+
"p95Ms": 0.9369601777788679
54+
},
55+
"send_all_packets": {
56+
"avgMs": 0.0055072860737605586,
57+
"p95Ms": 0.00635031110970077
58+
},
59+
"network_synchronize_cleanup": {
60+
"avgMs": 0.005068940954863205,
61+
"p95Ms": 0.0048210888887600355
62+
},
63+
"network_synchronize": {
64+
"avgMs": 0.02610076192554791,
65+
"p95Ms": 0.02581884444480238
66+
}
67+
},
68+
"network": {
69+
"totalBytesSent": 0,
70+
"totalBytesReceived": 0,
71+
"maxConnectedPlayers": 0,
72+
"avgBytesSentPerSecond": 0,
73+
"maxBytesSentPerSecond": 0,
74+
"avgBytesReceivedPerSecond": 0,
75+
"maxBytesReceivedPerSecond": 0,
76+
"avgPacketsSentPerSecond": 0,
77+
"maxPacketsSentPerSecond": 0,
78+
"avgPacketsReceivedPerSecond": 0,
79+
"maxPacketsReceivedPerSecond": 0,
80+
"avgSerializationMs": 0,
81+
"compressionCountTotal": 0
82+
}
83+
},
84+
"phases": [
85+
{
86+
"name": "spawn-entities",
87+
"durationMs": 232,
88+
"collected": false
89+
},
90+
{
91+
"name": "wait-for-world",
92+
"durationMs": 5576,
93+
"collected": false
94+
},
95+
{
96+
"name": "walkthrough",
97+
"durationMs": 79685,
98+
"collected": true
99+
}
100+
],
101+
"metrics": {
102+
"tickReportCount": 0,
103+
"spikeCount": 0,
104+
"serverSnapshotCount": 45,
105+
"clientSnapshotCount": 44
106+
}
107+
}

‎packages/perf-tools/src/presets/stress-walkthrough.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
name: "stress-walkthrough"
2+
description: "Deterministic stress test: 200 idle entities at fixed positions, player walks through them on a repeatable path"
3+
warmupMs: 5000
4+
phases:
5+
- name: spawn-entities
6+
actions:
7+
- type: load_map
8+
mapPath: "assets/maps/boilerplate-small.json"
9+
# All bots use idle behavior = they stay at their deterministic spawn positions
10+
# (circle around 0,10,0 with radius based on sqrt(count))
11+
- type: spawn_bots
12+
count: 200
13+
behavior: idle
14+
- name: wait-for-world
15+
actions:
16+
- type: wait_for_entities
17+
count: 10
18+
durationMs: 30000
19+
# Let entities fully load and settle
20+
- type: wait
21+
durationMs: 5000
22+
- name: walkthrough
23+
collect: true
24+
duration: 45s
25+
actions:
26+
# Face toward entity cluster center (yaw=0 = looking along +Z toward origin)
27+
- type: set_camera
28+
yaw: 0
29+
pitch: 0.3
30+
durationMs: 500
31+
# Walk forward into the entity cluster
32+
- type: walk_player
33+
durationMs: 5000
34+
options: { key: w }
35+
# Turn right 90 degrees and strafe through entities
36+
- type: set_camera
37+
yaw: -1.57
38+
pitch: 0.3
39+
durationMs: 300
40+
- type: walk_player
41+
durationMs: 4000
42+
options: { key: w }
43+
# Turn to face back through entities (180 from start)
44+
- type: set_camera
45+
yaw: 3.14
46+
pitch: 0.3
47+
durationMs: 300
48+
- type: walk_player
49+
durationMs: 5000
50+
options: { key: w }
51+
# Turn left 90 and walk through again
52+
- type: set_camera
53+
yaw: 1.57
54+
pitch: 0.3
55+
durationMs: 300
56+
- type: walk_player
57+
durationMs: 4000
58+
options: { key: w }
59+
# Face original direction, walk forward again
60+
- type: set_camera
61+
yaw: 0
62+
pitch: 0.3
63+
durationMs: 300
64+
- type: walk_player
65+
durationMs: 5000
66+
options: { key: w }
67+
thresholds:
68+
tick_duration_ms:
69+
avg: 12
70+
p95: 15
71+
p99: 20
72+
memory_mb:
73+
max: 512
74+
client:
75+
fps_avg: 15
76+
draw_calls_max: 200
77+
triangles_max: 1000000
78+
frame_time_ms_max: 200

‎packages/perf-tools/src/runners/BenchmarkRunner.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,73 @@ export default class BenchmarkRunner {
278278
case 'disconnect_clients':
279279
await this._disconnectWsClients(action.count);
280280
break;
281+
case 'wait_for_entities':
282+
if (this._headlessClient) {
283+
const minEntities = action.count ?? 1;
284+
const timeout = action.durationMs ?? 30000;
285+
286+
this._log(`[bench] Waiting for ${minEntities}+ entities (timeout ${timeout}ms)`);
287+
288+
const start = Date.now();
289+
290+
while (Date.now() - start < timeout) {
291+
const snap = await this._headlessClient.collectClientMetrics();
292+
293+
if (snap?.entities && snap.entities.count >= minEntities) {
294+
this._log(`[bench] Got ${snap.entities.count} entities`);
295+
break;
296+
}
297+
298+
await new Promise(r => setTimeout(r, 1000));
299+
}
300+
}
301+
break;
302+
case 'walk_player':
303+
if (this._headlessClient) {
304+
const key = (action.options?.key as string) ?? 'w';
305+
const dur = action.durationMs ?? 3000;
306+
307+
this._log(`[bench] Walking player: key=${key} for ${dur}ms`);
308+
309+
try {
310+
await this._headlessClient.sendMovement(key, dur);
311+
} catch {
312+
this._log('[bench] walk_player: input failed (non-fatal)');
313+
}
314+
}
315+
break;
316+
case 'set_camera':
317+
if (this._headlessClient) {
318+
const yaw = action.yaw ?? 0;
319+
const pitch = action.pitch ?? -0.3;
320+
321+
this._log(`[bench] Setting camera: yaw=${yaw} pitch=${pitch}`);
322+
323+
try {
324+
if (action.position) {
325+
await this._headlessClient.setCameraPosition(action.position.x, action.position.y, action.position.z);
326+
}
327+
328+
await this._headlessClient.lookAt(yaw, pitch);
329+
await new Promise(r => setTimeout(r, action.durationMs ?? 500));
330+
} catch {
331+
this._log('[bench] set_camera: failed (non-fatal)');
332+
}
333+
}
334+
break;
335+
case 'throttle_cpu':
336+
if (this._headlessClient) {
337+
const rate = action.rate ?? 1;
338+
339+
this._log(`[bench] CPU throttle: ${rate}x`);
340+
341+
try {
342+
await this._headlessClient.setCpuThrottle(rate);
343+
} catch {
344+
this._log('[bench] throttle_cpu: failed (non-fatal)');
345+
}
346+
}
347+
break;
281348
case 'custom':
282349
throw new Error(`Action not supported yet: ${action.type}`);
283350
}

0 commit comments

Comments
 (0)