fix: harden Android nightly runtime evidence

Author: thymikee

.github/actions/setup-fixture-app/action.yml

@@ -36,6 +36,10 @@ inputs:
     description: 'How long to poll for a concurrently produced artifact before building inline'
     required: false
     default: '0'
+  require-artifact:
+    description: 'Fail instead of building inline when no trusted same-head artifact is available'
+    required: false
+    default: 'false'
 outputs:
   app-path:
     description: 'Path to the ready .app bundle or APK'
@@ -95,7 +99,7 @@ runs:
         bash "$GITHUB_ACTION_PATH/fetch-artifact.sh" \
           "${{ inputs.platform }}" "$DEST" "${{ inputs.wait-for-artifact-seconds }}" \
           "${{ github.repository }}" "${{ github.event.pull_request.head.sha || github.sha }}" \
-          "$GITHUB_ACTION_PATH"
+          "$GITHUB_ACTION_PATH" "${{ inputs.require-artifact }}"
 
     # Fallback only: the producer builds the same way. `--device generic` avoids
     # needing a booted simulator; Release embeds the bundle so no Metro is needed.

.github/actions/setup-fixture-app/fetch-artifact.sh

@@ -8,6 +8,7 @@ WAIT_SECONDS="${3:?wait duration is required}"
 REPOSITORY="${4:?repository is required}"
 EXPECTED_HEAD_SHA="${5:?expected head SHA is required}"
 ACTION_PATH="${6:?action path is required}"
+REQUIRE_ARTIFACT="${7:-false}"
 
 NAME="$(sh "$ACTION_PATH/resolve-artifact-name.sh" "$PLATFORM")"
 TRUSTED_ARTIFACT="$ACTION_PATH/trusted-artifact.mjs"
@@ -18,6 +19,13 @@ case "$WAIT_SECONDS" in
     exit 1
     ;;
 esac
+case "$REQUIRE_ARTIFACT" in
+  true|false) ;;
+  *)
+    echo "::error::require-artifact must be true or false."
+    exit 1
+    ;;
+esac
 
 # A lookup failure (API outage, auth, transient 5xx) must not fail the caller
 # or trigger a pointless wait: it takes the inline-build path.
@@ -89,11 +97,19 @@ if [ -n "$ART_ID" ]; then
     SOURCE=artifact
     echo "restored $NAME from the build cache"
   else
+    if [ "$REQUIRE_ARTIFACT" = true ]; then
+      echo "::error::Could not restore required fixture artifact $NAME."
+      exit 1
+    fi
     echo "::warning::Could not restore $NAME; building inline."
     rm -rf "$DEST"/*
   fi
   rm -rf "$STAGE"
 else
+  if [ "$REQUIRE_ARTIFACT" = true ]; then
+    echo "::error::Required fixture artifact $NAME is unavailable for this exact head."
+    exit 1
+  fi
   echo "$NAME not in the cache after the configured wait; building inline."
 fi
 echo "source=$SOURCE" >> "$GITHUB_OUTPUT"

.github/workflows/replays-nightly.yml

@@ -93,6 +93,8 @@ jobs:
     name: Android Full Emulator Suite
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    env:
+      AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/android-e2e-state
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -112,6 +114,7 @@ jobs:
         with:
           platform: android
           install: 'false'
+          require-artifact: 'true'
 
       - name: Mark Android emulator setup complete
         id: android-setup
@@ -137,7 +140,7 @@ jobs:
             AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --test test/integration/smoke-android-emulator.test.ts
             FULL_FINISHED_AT="$(date +%s)"
             COVERAGE_REPORT="$(find test/artifacts/android-emulator -name coverage-report.json -print | tail -n 1)"
-            node --input-type=module -e 'import fs from "node:fs"; const report = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); console.log(`Android full scenario body: ${report.timings.scenarioBodyDurationMs}ms`); for (const scenario of report.timings.scenarios) console.log(`  ${scenario.id}: ${scenario.durationMs}ms`);' "$COVERAGE_REPORT"
+            node --input-type=module -e 'import fs from "node:fs"; const report = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); console.log(`Android full scenario body: ${report.timings.runDurationMs}ms`); for (const scenario of report.timings.scenarios) console.log(`  ${scenario.id}: ${scenario.durationMs}ms`);' "$COVERAGE_REPORT"
             echo "Android setup before emulator: $((BOOT_FINISHED_AT - ${{ steps.android-setup.outputs.seconds }}))s"
             echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s"
             node --experimental-strip-types src/bin.ts test test/integration/replays/android/01-settings.ad --retries 2 --artifacts-dir test/artifacts/replays-android-settings --report-junit test/artifacts/replays-android-settings.junit.xml
@@ -148,7 +151,7 @@ jobs:
         uses: ./.github/actions/upload-agent-device-artifacts
         with:
           artifact-name: replay-nightly-android-artifacts
-          agent-state-dir: ${{ steps.android-replay-host.outputs.agent-state-dir }}
+          agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }}
 
   nightly-ios:
     name: iOS Replay Suite

examples/test-app/app.config.js

@@ -25,7 +25,7 @@ module.exports = {
         {
           microphonePermission:
             'Allow Agent Device Tester to exercise microphone permission recovery.',
-          recordAudioAndroid: false,
+          recordAudioAndroid: true,
         },
       ],
     ],

packages/contracts/src/recording.ts

@@ -46,6 +46,7 @@ export type TraceCommandResult =
   | {
       trace: 'stopped';
       outPath: string;
+      artifacts: DaemonArtifact[];
     };
 
 /**

src/daemon/handlers/record-trace.ts

@@ -63,7 +63,23 @@ export async function handleRecordTraceCommands(params: {
     }
     session.trace = undefined;
     recordSessionAction(sessionStore, session, req, command, { action: 'stop', outPath });
-    return { ok: true, data: { trace: 'stopped', outPath } satisfies TraceCommandResult };
+    const clientOutPath = req.meta?.clientArtifactPaths?.outPath ?? outPath;
+    return {
+      ok: true,
+      data: {
+        trace: 'stopped',
+        outPath,
+        artifacts: [
+          {
+            field: 'outPath',
+            artifactType: 'trace-log',
+            path: outPath,
+            localPath: clientOutPath,
+            fileName: path.basename(clientOutPath),
+          },
+        ],
+      } satisfies TraceCommandResult,
+    };
   }
 
   return null;

test/integration/android-emulator-e2e/live-lifecycle-scenario.ts

@@ -23,22 +23,13 @@ export async function assertLifecycleAndSystem(context: LiveContext): Promise<vo
 }
 
 async function assertPermissionRecovery(context: LiveContext): Promise<void> {
-  await runStep(context, 'reset microphone permission before runtime prompt', [
-    'settings',
-    'permission',
-    'reset',
-    'microphone',
-  ]);
-  await runStep(context, 'request Android microphone permission in fixture', [
-    'click',
-    'id="automation-request-microphone"',
-  ]);
-  const prompt = await runStep(context, 'inspect Android microphone permission prompt', [
-    'alert',
-    'get',
-  ]);
-  assert.equal(prompt.json?.data?.alert?.source, 'permission', JSON.stringify(prompt.json));
-  await runStep(context, 'accept Android microphone permission prompt', ['alert', 'accept']);
+  await resetAndRequestMicrophonePermission(context, 'accept');
+  await assertElementText(context, 'id="automation-microphone-permission"', 'granted');
+
+  await resetAndRequestMicrophonePermission(context, 'deny');
+  await assertElementText(context, 'id="automation-microphone-permission"', 'denied');
+
+  await resetAndRequestMicrophonePermission(context, 'accept after denial');
   await assertElementText(context, 'id="automation-microphone-permission"', 'granted');
 
   await runStep(context, 'revoke Android microphone permission', [
@@ -54,15 +45,42 @@ async function assertPermissionRecovery(context: LiveContext): Promise<void> {
   verifyCommand(
     context,
     C.settings,
-    'grant and deny runtime permission transitions are fixture-visible',
+    'prompt accept, prompt deny, and adb revoke transitions are fixture-visible',
   );
   verifyBehavior(
     context,
     'runtime-permission-recovery',
-    'fixture observed prompt acceptance then the app-active readback after pm revoke',
+    'fixture observed native prompt accept and deny, then app-active readback after pm revoke',
   );
 }
 
+async function resetAndRequestMicrophonePermission(
+  context: LiveContext,
+  action: 'accept' | 'deny' | 'accept after denial',
+): Promise<void> {
+  await runStep(context, `reset microphone permission before ${action}`, [
+    'settings',
+    'permission',
+    'reset',
+    'microphone',
+  ]);
+  await runStep(context, `scroll to microphone permission control before ${action}`, [
+    'scroll',
+    'bottom',
+  ]);
+  await runStep(context, `request Android microphone permission for ${action}`, [
+    'click',
+    'id="automation-request-microphone"',
+  ]);
+  const prompt = await runStep(context, `inspect Android microphone prompt for ${action}`, [
+    'alert',
+    'get',
+  ]);
+  assert.equal(prompt.json?.data?.alert?.source, 'permission', JSON.stringify(prompt.json));
+  const alertAction = action === 'deny' ? 'dismiss' : 'accept';
+  await runStep(context, `${action} Android microphone permission prompt`, ['alert', alertAction]);
+}
+
 async function assertPushBroadcast(context: LiveContext): Promise<void> {
   const payload = JSON.stringify({ extras: { count: 7, source: 'android-nightly' } });
   const pushed = await runStep(context, 'broadcast typed Android fixture push', [
@@ -72,6 +90,7 @@ async function assertPushBroadcast(context: LiveContext): Promise<void> {
   ]);
   assert.equal(pushed.json?.data?.package, context.appId, JSON.stringify(pushed.json));
   assert.equal(pushed.json?.data?.extrasCount, 2, JSON.stringify(pushed.json));
+  await runStep(context, 'scroll to Android push receiver state', ['scroll', 'bottom']);
   await runStep(context, 'refresh fixture push receiver state', [
     'click',
     'id="automation-refresh-push-broadcast"',

test/integration/android-emulator-e2e/live-observability-scenario.ts

@@ -6,12 +6,7 @@ import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts';
 import { readDaemonInfo } from '../../../src/daemon/client/daemon-client-metadata.ts';
 import { resolveDaemonPaths } from '../../../src/daemon/config.ts';
 import { collectPagedEventTimeline, type EventTimelinePage } from './event-timeline.ts';
-import {
-  assertJsonContains,
-  assertMp4File,
-  assertNonEmptyFile,
-  assertWaitText,
-} from './live-assertions.ts';
+import { assertMp4File, assertNonEmptyFile, assertWaitText } from './live-assertions.ts';
 import { type LiveContext, runStep, verifyCommand } from './live-harness.ts';
 
 const C = PUBLIC_COMMANDS;
@@ -114,12 +109,16 @@ async function assertBatchAndEvents(context: LiveContext): Promise<void> {
       },
     ]),
   ]);
-  assertJsonContains(
-    batch,
-    'Agent Device Tester',
-    'batch should contain nested Android get result',
+  const results = batch.json?.data?.results;
+  assert.ok(
+    Array.isArray(results),
+    `batch must retain nested results: ${JSON.stringify(batch.json)}`,
   );
-  assertJsonContains(batch, '"pass":true', 'batch should contain nested Android is result');
+  assert.equal(results.length, 2, JSON.stringify(batch.json));
+  assert.equal(results[0]?.command, 'get', JSON.stringify(batch.json));
+  assert.equal(results[0]?.data?.text, 'Agent Device Tester', JSON.stringify(batch.json));
+  assert.equal(results[1]?.command, 'is', JSON.stringify(batch.json));
+  assert.equal(results[1]?.data?.pass, true, JSON.stringify(batch.json));
   verifyCommand(context, C.batch, 'Android batch retains nested get and is evidence');
 
   const timeline = await collectPagedEventTimeline(async (cursor) => {
@@ -158,13 +157,13 @@ async function assertArtifactInventory(context: LiveContext): Promise<void> {
     );
   }
   const trace = artifacts.find(
-    (artifact: { artifactId?: unknown; artifactType?: unknown; sizeBytes?: unknown }) =>
+    (artifact: { artifactType?: unknown; id?: unknown; sizeBytes?: unknown }) =>
       artifact.artifactType === 'trace-log' &&
-      typeof artifact.artifactId === 'string' &&
+      typeof artifact.id === 'string' &&
       Number(artifact.sizeBytes) > 0,
-  ) as { artifactId: string; sizeBytes: number } | undefined;
+  ) as { id: string; sizeBytes: number } | undefined;
   assert.ok(trace, `trace artifact is not downloadable: ${JSON.stringify(artifacts)}`);
-  const downloaded = await downloadDaemonArtifact(context, trace.artifactId);
+  const downloaded = await downloadDaemonArtifact(context, trace.id);
   assert.equal(
     downloaded.byteLength,
     trace.sizeBytes,
@@ -177,9 +176,7 @@ async function assertArtifactInventory(context: LiveContext): Promise<void> {
   const remaining = afterDownload.json?.data?.artifacts;
   assert.ok(Array.isArray(remaining), JSON.stringify(afterDownload.json));
   assert.equal(
-    remaining.some(
-      (artifact: { artifactId?: unknown }) => artifact.artifactId === trace.artifactId,
-    ),
+    remaining.some((artifact: { id?: unknown }) => artifact.id === trace.id),
     false,
     'artifact download must consume its inventory entry',
   );