refactor(replay-test): extract packages/replay-test (#1478 P3b)

package.json

@@ -145,7 +145,7 @@
     "check:unit": "pnpm check:contention-retry && pnpm test:unit && pnpm test:smoke",
     "check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit",
     "prepack": "pnpm check:mcp-metadata && pnpm package:npm",
-    "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/maestro packages/provider-webdriver packages/provider-limrun && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json",
+    "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/maestro packages/replay-test packages/provider-webdriver packages/provider-limrun && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json",
     "test-app:install": "pnpm install --dir examples/test-app",
     "test-app:start": "pnpm --dir examples/test-app start",
     "test-app:ios": "pnpm --dir examples/test-app ios",
@@ -250,6 +250,7 @@
     "@agent-device/maestro": "workspace:*",
     "@agent-device/provider-limrun": "workspace:*",
     "@agent-device/provider-webdriver": "workspace:*",
+    "@agent-device/replay-test": "workspace:*",
     "@agent-device/xml": "workspace:*",
     "@chenglou/freerange": "^0.0.1",
     "@stryker-mutator/core": "9.6.1",

packages/replay-test/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@agent-device/replay-test",
+  "version": "0.0.0",
+  "private": true,
+  "sideEffects": false,
+  "type": "module",
+  "description": "Private format-neutral replay-test scheduler for agent-device.",
+  "dependencies": {
+    "@agent-device/contracts": "workspace:*",
+    "@agent-device/kernel": "workspace:*"
+  },
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  }
+}

packages/replay-test/src/index.ts

@@ -0,0 +1,47 @@
+/**
+ * The replay-test package façade (#1478 P3).
+ *
+ * One function plus the values crossing its seam. Everything else — scheduling, retries,
+ * sharding distribution, attempt identity, timeout policy, finalization/cleanup ordering, and
+ * result aggregation — is private to `internal/`.
+ *
+ * The scheduler is format-agnostic: it imports neither engine, and a source reaches it only as
+ * a `ReplayTestManifest`. Host authority arrives as narrow capabilities on
+ * `ReplayTestRuntimeDependencies`; none of them hands over a daemon request, a session store,
+ * mutable session state, or an engine.
+ */
+export { runReplayTestSuite } from './internal/session-test.ts';
+
+export type {
+  ReplayTestAttemptError,
+  ReplayTestAttemptFailed,
+  ReplayTestAttemptOutcome,
+  ReplayTestAttemptPassed,
+  ReplayTestAttemptStep,
+  ReplayTestAttemptStepSink,
+  ReplayTestBindAttemptCancellation,
+  ReplayTestAttemptCancellation,
+  ReplayTestCleanupSession,
+  ReplayTestDiscoverSources,
+  ReplayTestEmitDiagnostic,
+  ReplayTestEmitProgress,
+  ReplayTestExecutionDependencies,
+  ReplayTestFinalizeAttempt,
+  ReplayTestIsCanceled,
+  ReplayTestManifest,
+  ReplayTestPlatform,
+  ReplayTestRunReplay,
+  ReplayTestRunReplayParams,
+  ReplayTestRuntimeDependencies,
+  ReplayTestSource,
+  ReplayTestSuiteOutcome,
+  ReplayTestSuiteRequest,
+  ReplayTestTarget,
+} from './internal/session-test-types.ts';
+
+export type {
+  ReplayTestResolveShardTargets,
+  ReplayTestShardContext,
+  ReplayTestShardMode,
+  ReplayTestShardTarget,
+} from './internal/session-test-sharding.ts';

src/daemon/handlers/__tests__/session-test-artifacts.test.ts → packages/replay-test/src/internal/__tests__/session-test-artifacts.test.ts

@@ -7,8 +7,20 @@ import {
   materializeReplayTestAttemptArtifacts,
   prepareReplayTestAttemptArtifacts,
 } from '../session-test-artifacts.ts';
-import { toReplayTestAttemptOutcome } from '../session-test-outcome.ts';
-import type { DaemonResponse } from '../../types.ts';
+import type { ReplayTestAttemptOutcome } from '../session-test-types.ts';
+
+// Building outcomes from a DaemonResponse is the adapter's job and is pinned on that side; a
+// package test states the neutral outcome directly (#1478 P3b).
+const passedOutcome = (
+  overrides: Partial<Extract<ReplayTestAttemptOutcome, { status: 'passed' }>> = {},
+): ReplayTestAttemptOutcome => ({
+  status: 'passed',
+  replayed: 1,
+  healed: 0,
+  warnings: [],
+  artifactPaths: [],
+  ...overrides,
+});
 
 test('materializeReplayTestAttemptArtifacts writes replay and result manifests for passing attempts', () => {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-artifacts-pass-'));
@@ -19,16 +31,8 @@ test('materializeReplayTestAttemptArtifacts writes replay and result manifests f
   fs.writeFileSync(screenshotPath, 'png');
 
   prepareReplayTestAttemptArtifacts(replayPath, attemptDir);
-  const response: DaemonResponse = {
-    ok: true,
-    data: {
-      replayed: 4,
-      healed: 1,
-      artifactPaths: [screenshotPath],
-    },
-  };
   materializeReplayTestAttemptArtifacts({
-    outcome: toReplayTestAttemptOutcome(response),
+    outcome: passedOutcome({ replayed: 4, healed: 1, artifactPaths: [screenshotPath] }),
     filePath: replayPath,
     sessionName: 'default:test:suite:1',
     attempts: 1,
@@ -70,21 +74,19 @@ test('materializeReplayTestAttemptArtifacts writes failure manifest and copies l
   fs.writeFileSync(logPath, 'log');
 
   prepareReplayTestAttemptArtifacts(replayPath, attemptDir);
-  const response: DaemonResponse = {
-    ok: false,
-    error: {
-      code: 'COMMAND_FAILED',
-      message: 'TIMEOUT after 5000ms',
-      hint: 'Replay test timeouts are cooperative.',
-      logPath,
-      details: {
-        reason: 'timeout',
-        artifactPaths: [screenshotPath],
+  materializeReplayTestAttemptArtifacts({
+    outcome: {
+      status: 'failed',
+      error: {
+        code: 'COMMAND_FAILED',
+        message: 'Replay test timed out',
+        hint: 'Replay test timeouts are cooperative.',
+        logPath,
+        details: { reason: 'timeout', artifactPaths: [screenshotPath] },
       },
+      artifactPaths: [screenshotPath],
+      infrastructure: false,
     },
-  };
-  materializeReplayTestAttemptArtifacts({
-    outcome: toReplayTestAttemptOutcome(response),
     filePath: replayPath,
     sessionName: 'default:test:suite:2',
     attempts: 2,

packages/replay-test/src/internal/__tests__/session-test-discovery.test.ts

@@ -0,0 +1,77 @@
+import { test } from 'vitest';
+import assert from 'node:assert/strict';
+import { AppError } from '@agent-device/kernel/errors';
+import { discoverReplayTestEntries } from '../session-test-discovery.ts';
+import type { ReplayTestManifest, ReplayTestSource } from '../session-test-types.ts';
+
+// Scheduler-owned discovery policy (#1478 P3b): which sources a --platform filter runs, which
+// it skips and with what message, and rejecting a suite that matched nothing. Inspection is the
+// host's, so these drive fake sources rather than the filesystem — the ordering and routing
+// behavior they used to share a file with is pinned host-side.
+const declared = (value: 'ios' | 'android'): ReplayTestManifest => ({
+  device: { platform: { kind: 'declared', value } },
+});
+const unspecified: ReplayTestManifest = { device: { platform: { kind: 'unspecified' } } };
+const callerBound: ReplayTestManifest = { device: { platform: { kind: 'caller-bound' } } };
+
+const sourcesOf =
+  (...entries: ReplayTestSource[]) =>
+  () =>
+    entries;
+
+test('platform filter skips sources that declared no platform', () => {
+  const entries = discoverReplayTestEntries({
+    inputs: ['suite'],
+    platformFilter: 'android',
+    discoverSources: sourcesOf(
+      { path: '01-untyped.ad', manifest: unspecified },
+      { path: '02-android.ad', manifest: declared('android') },
+    ),
+  });
+
+  const untyped = entries.find((entry) => entry.path === '01-untyped.ad');
+  assert.equal(untyped?.kind, 'skip');
+  if (untyped?.kind === 'skip') {
+    assert.match(untyped.message, /missing platform metadata for --platform android/);
+  }
+  assert.equal(entries.find((entry) => entry.path === '02-android.ad')?.kind, 'run');
+});
+
+test('platform filter runs caller-bound sources, which take their platform from the invocation', () => {
+  const entries = discoverReplayTestEntries({
+    inputs: ['suite'],
+    platformFilter: 'android',
+    discoverSources: sourcesOf({ path: '01-flow.yaml', manifest: callerBound }),
+  });
+
+  assert.equal(entries.length, 1);
+  assert.equal(entries[0]?.kind, 'run');
+});
+
+test('platform filter drops declared sources that do not match, without a skip entry', () => {
+  const entries = discoverReplayTestEntries({
+    inputs: ['suite'],
+    platformFilter: 'android',
+    discoverSources: sourcesOf(
+      { path: '01-ios.ad', manifest: declared('ios') },
+      { path: '02-android.ad', manifest: declared('android') },
+    ),
+  });
+
+  assert.deepEqual(
+    entries.map((entry) => entry.path),
+    ['02-android.ad'],
+  );
+});
+
+test('a suite that matched nothing after filtering is rejected', () => {
+  assert.throws(
+    () =>
+      discoverReplayTestEntries({
+        inputs: ['suite'],
+        platformFilter: 'android',
+        discoverSources: sourcesOf({ path: '01-ios.ad', manifest: declared('ios') }),
+      }),
+    (error: unknown) => error instanceof AppError && /No replay tests matched/.test(error.message),
+  );
+});

src/daemon/handlers/__tests__/session-test-runtime.test.ts → packages/replay-test/src/internal/__tests__/session-test-runtime.test.ts

@@ -2,10 +2,36 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { afterEach, expect, test, vi } from 'vitest';
-import { isRequestCanceled } from '../../../request/cancel.ts';
+
 import { runReplayTestAttempt } from '../session-test-runtime.ts';
+
 import type { ReplayTestAttemptOutcome } from '../session-test-types.ts';
 
+// What the scheduler owes its host around cancellation (#1478 P3b): cancel exactly once when
+// an attempt times out, and always release when it settles. How the daemon then maps that onto
+// its request registry is the adapter's contract, pinned in
+// `src/daemon/handlers/__tests__/session-replay-cancellation.test.ts`.
+const cancellations: Array<{ attemptId: string; canceled: number; released: number }> = [];
+
+function trackCancellation() {
+  cancellations.length = 0;
+  return {
+    emitDiagnostic: () => {},
+    bindAttemptCancellation: ({ attemptId }: { attemptId: string }) => {
+      const record = { attemptId, canceled: 0, released: 0 };
+      cancellations.push(record);
+      return {
+        cancel: () => {
+          record.canceled += 1;
+        },
+        release: () => {
+          record.released += 1;
+        },
+      };
+    },
+  };
+}
+
 const PASSED: ReplayTestAttemptOutcome = {
   status: 'passed',
   replayed: 1,
@@ -57,6 +83,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se
     runReplay: async () => await replayPromise,
     finalizeAttempt,
     cleanupSession,
+    ...trackCancellation(),
   });
 
   await vi.advanceTimersByTimeAsync(10);
@@ -80,7 +107,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se
     }),
   );
   expect(lifecycleEvents).toEqual(['finalize', 'cleanup']);
-  expect(isRequestCanceled('req-timeout-open')).toBe(true);
+  expect(cancellations[0]?.canceled).toBe(1);
   // #1478 P3: the second cleanup is strictly deferred until the abandoned replay settles, so
   // the attempt returns having cleaned up exactly once. P3 keeps this orchestration inside
   // replay-test while the cleanup effect itself stays in the daemon adapter.
@@ -94,7 +121,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se
   });
   await replaySettled;
   await vi.waitFor(() => {
-    expect(isRequestCanceled('req-timeout-open')).toBe(false);
+    expect(cancellations[0]?.released).toBe(1);
   });
   await vi.waitFor(() => {
     expect(cleanupSession).toHaveBeenCalledTimes(2);
@@ -118,6 +145,7 @@ test('runReplayTestAttempt keeps a passing replay passed when finalization fails
       infrastructure: false,
     }),
     cleanupSession,
+    ...trackCancellation(),
   });
 
   expect(result.status).toBe('passed');
@@ -149,6 +177,7 @@ test('runReplayTestAttempt finalizes before cleanup and records that order in th
     cleanupSession: async () => {
       lifecycleEvents.push('cleanup');
     },
+    ...trackCancellation(),
   });
 
   expect(result.status).toBe('passed');
@@ -188,6 +217,7 @@ test('runReplayTestAttempt cleans up once when a timed-out replay settles inside
       return undefined;
     },
     cleanupSession,
+    ...trackCancellation(),
   });
 
   await vi.advanceTimersByTimeAsync(10);
@@ -225,6 +255,7 @@ test('runReplayTestAttempt cleans up without a finalizer and adds no finalizatio
     requestId: 'req-no-finalizer',
     runReplay: async () => PASSED,
     cleanupSession,
+    ...trackCancellation(),
   });
 
   expect(result.status).toBe('passed');

src/daemon/handlers/session-test-artifacts.ts → packages/replay-test/src/internal/session-test-artifacts.ts

@@ -1,8 +1,6 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { isMaestroYamlPath } from '../../replay/format.ts';
-import type { ReplayTestAttemptOutcome } from './session-test-types.ts';
-import { SessionStore } from '../session-store.ts';
+import type { ReplayTestAttemptOutcome } from '@agent-device/replay-test';
 
 const DEFAULT_TEST_ARTIFACTS_ROOT = '.agent-device/test-artifacts';
 
@@ -12,7 +10,9 @@ export function resolveReplayTestArtifactsDir(params: {
   suiteInvocationId: string;
 }): string {
   const { artifactsDir, cwd, suiteInvocationId } = params;
-  const resolvedRoot = SessionStore.expandHome(artifactsDir ?? DEFAULT_TEST_ARTIFACTS_ROOT, cwd);
+  // `artifactsDir` arrives already home-expanded: resolving `~` and the caller's cwd is host
+  // work, done once when the daemon builds the suite request.
+  const resolvedRoot = path.resolve(cwd ?? '.', artifactsDir ?? DEFAULT_TEST_ARTIFACTS_ROOT);
   return path.join(resolvedRoot, suiteInvocationId);
 }
 
@@ -107,7 +107,10 @@ function copyReplaySourceFile(filePath: string, attemptArtifactsDir: string): vo
   const genericReplayPath = path.join(attemptArtifactsDir, 'replay.ad');
   fs.copyFileSync(filePath, genericReplayPath);
 
-  if (!isMaestroYamlPath(filePath)) return;
+  // Sources that are not native `.ad` keep their original filename alongside `replay.ad`, so
+  // CI artifacts point back at the flow the author wrote. This is artifact naming, not format
+  // routing — it asks what the file is called, never which engine will run it.
+  if (path.extname(filePath).toLowerCase() === '.ad') return;
   // Keep replay.ad for existing artifact consumers, and preserve the original
   // Maestro filename so CI artifacts point back to the source flow.
   const originalReplayPath = path.join(attemptArtifactsDir, path.basename(filePath));