Skip to content

Commit 560e9ae

Browse files
committed
test: fix Android observability scenario to Android contracts
The full:observability-artifacts scenario (#1482/#1484) had never executed end-to-end: both nightly Android Full Emulator Suite runs since merge died on adb infra before the suite ran, and a live run fails deterministically at its first perf assertion. Fixing that revealed four more latent failures, each written against iOS or remote-daemon behavior the Android live run does not have. Validated with two consecutive green full-tier runs on a dedicated Pixel 9 Pro XL emulator. - perf metrics: assert totalPssKb (Android's required meminfo field) instead of the Apple-only residentMemoryKb. - presses: reveal the Quick-actions card with scroll steps before pressing home-open-catalog/home-open-settings — Android snapshots only contain on-screen nodes — and restore scroll top before waiting on the home title, since scroll position persists across tab switches. - batch get: target id="dismiss-notice" (a node that owns its text); Android resolves the home-title container to a child's text (the subtitle), unlike iOS's container label. - events: run an explicit snapshot so the timeline assertion holds when the scenario runs standalone under AGENT_DEVICE_ANDROID_E2E_SCENARIOS. - artifacts: assert the local-client contract — trace-log tracked, downloadable, consumed; screen-recording inventory entries only exist for remote clients (artifacts without a client localPath are never tracked). Also stop consuming the download response body in the assert message before arrayBuffer() reads it.
1 parent 14b71c9 commit 560e9ae

1 file changed

Lines changed: 20 additions & 15 deletions

File tree

test/integration/android-emulator-e2e/live-observability-scenario.ts

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ async function assertMetrics(context: LiveContext): Promise<void> {
3737
assert.equal(metrics?.startup?.available, true, JSON.stringify(perf.json));
3838
assert.ok(Number(metrics?.startup?.lastDurationMs) > 0, JSON.stringify(perf.json));
3939
assert.equal(metrics?.memory?.available, true, JSON.stringify(perf.json));
40-
assert.ok(Number(metrics?.memory?.residentMemoryKb) > 0, JSON.stringify(perf.json));
40+
assert.ok(Number(metrics?.memory?.totalPssKb) > 0, JSON.stringify(perf.json));
4141
assert.equal(metrics?.cpu?.available, true, JSON.stringify(perf.json));
4242
assert.ok(Number.isFinite(Number(metrics?.cpu?.usagePercent)), JSON.stringify(perf.json));
4343
verifyCommand(
4444
context,
4545
C.perf,
46-
'Android startup, resident memory, and CPU metrics are typed and numeric',
46+
'Android startup, PSS memory, and CPU metrics are typed and numeric',
4747
);
4848
}
4949

@@ -65,6 +65,7 @@ async function assertLogs(context: LiveContext): Promise<void> {
6565

6666
async function assertTraceAndRecording(context: LiveContext): Promise<void> {
6767
const tracePath = path.join(context.artifactDir, 'fixture.adtrace');
68+
await runStep(context, 'reveal Android quick actions before trace', ['scroll', 'down', '0.7']);
6869
await runStep(context, 'start Android interaction trace', ['trace', 'start', tracePath]);
6970
await runStep(context, 'trace Android visible mutation', ['press', 'id="home-open-catalog"']);
7071
await runStep(context, 'stop Android interaction trace', ['trace', 'stop', tracePath]);
@@ -88,6 +89,12 @@ async function assertTraceAndRecording(context: LiveContext): Promise<void> {
8889
'click',
8990
'label="Home"',
9091
]);
92+
await runStep(context, 'restore Android fixture home scroll top', ['scroll', 'top']);
93+
await runStep(context, 'reveal Android quick actions before recording', [
94+
'scroll',
95+
'down',
96+
'0.7',
97+
]);
9198
await runStep(context, 'record Android visible mutation', ['press', 'id="home-open-settings"']);
9299
await assertWaitText(context, 'Settings');
93100
await runStep(context, 'stop short Android screen recording', ['record', 'stop']);
@@ -97,14 +104,15 @@ async function assertTraceAndRecording(context: LiveContext): Promise<void> {
97104

98105
async function assertBatchAndEvents(context: LiveContext): Promise<void> {
99106
await runStep(context, 'return to Android fixture home before batch', ['click', 'label="Home"']);
107+
await runStep(context, 'restore Android fixture home title before batch', ['scroll', 'top']);
100108
await assertWaitText(context, 'Agent Device Tester');
101109
const batch = await runStep(context, 'run Android nested semantic read batch', [
102110
'batch',
103111
'--steps',
104112
JSON.stringify([
105113
{
106114
command: 'get',
107-
input: { format: 'text', target: { kind: 'selector', selector: 'id="home-title"' } },
115+
input: { format: 'text', target: { kind: 'selector', selector: 'id="dismiss-notice"' } },
108116
},
109117
{
110118
command: 'is',
@@ -119,11 +127,12 @@ async function assertBatchAndEvents(context: LiveContext): Promise<void> {
119127
);
120128
assert.equal(results.length, 2, JSON.stringify(batch.json));
121129
assert.equal(results[0]?.command, 'get', JSON.stringify(batch.json));
122-
assert.equal(results[0]?.data?.text, 'Agent Device Tester', JSON.stringify(batch.json));
130+
assert.equal(results[0]?.data?.text, 'Dismiss notice', JSON.stringify(batch.json));
123131
assert.equal(results[1]?.command, 'is', JSON.stringify(batch.json));
124132
assert.equal(results[1]?.data?.pass, true, JSON.stringify(batch.json));
125133
verifyCommand(context, C.batch, 'Android batch retains nested get and is evidence');
126134

135+
await runStep(context, 'capture Android snapshot for the event timeline', ['snapshot']);
127136
const timeline = await collectPagedEventTimeline(async (cursor) => {
128137
const result = await runStep(
129138
context,
@@ -150,15 +159,9 @@ async function assertArtifactInventory(context: LiveContext): Promise<void> {
150159
const inventory = await runStep(context, 'list Android daemon artifacts', ['artifacts']);
151160
const artifacts = inventory.json?.data?.artifacts;
152161
assert.ok(Array.isArray(artifacts), JSON.stringify(inventory.json));
153-
for (const type of ['screen-recording', 'trace-log']) {
154-
assert.ok(
155-
artifacts.some(
156-
(artifact: { artifactType?: unknown; sizeBytes?: unknown }) =>
157-
artifact.artifactType === type && Number(artifact.sizeBytes) > 0,
158-
),
159-
`artifact inventory missing non-empty ${type}: ${JSON.stringify(artifacts)}`,
160-
);
161-
}
162+
// Recordings are absent by design for local clients: record stop hands the MP4
163+
// to the caller's path directly (asserted above), and only remote clients get a
164+
// downloadable inventory entry. The trace is tracked unconditionally.
162165
const trace = artifacts.find(
163166
(artifact: { artifactType?: unknown; id?: unknown; sizeBytes?: unknown }) =>
164167
artifact.artifactType === 'trace-log' &&
@@ -186,7 +189,7 @@ async function assertArtifactInventory(context: LiveContext): Promise<void> {
186189
verifyCommand(
187190
context,
188191
C.artifacts,
189-
'daemon inventory lists non-empty recording and trace artifacts, and a trace download consumes its entry',
192+
'daemon inventory lists a non-empty trace artifact and its download consumes the entry',
190193
);
191194
}
192195

@@ -204,6 +207,8 @@ async function downloadDaemonArtifact(
204207
`http://127.0.0.1:${daemon.httpPort}/artifacts/${encodeURIComponent(artifactId)}`,
205208
{ headers: { authorization: `Bearer ${daemon.token}` } },
206209
);
207-
assert.equal(response.status, 200, `artifact download failed: ${await response.text()}`);
210+
if (response.status !== 200) {
211+
assert.fail(`artifact download failed: ${response.status} ${await response.text()}`);
212+
}
208213
return response.arrayBuffer();
209214
}

0 commit comments

Comments
 (0)