diff --git a/package.json b/package.json index 05f983ad61..167fc274d9 100644 --- a/package.json +++ b/package.json @@ -145,7 +145,7 @@ "check:unit": "pnpm check:contention-retry && pnpm test:unit && pnpm test:smoke", "check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit", "prepack": "pnpm check:mcp-metadata && pnpm package:npm", - "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/maestro packages/provider-webdriver packages/provider-limrun && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json", + "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/maestro packages/replay-test packages/provider-webdriver packages/provider-limrun && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json", "test-app:install": "pnpm install --dir examples/test-app", "test-app:start": "pnpm --dir examples/test-app start", "test-app:ios": "pnpm --dir examples/test-app ios", @@ -250,6 +250,7 @@ "@agent-device/maestro": "workspace:*", "@agent-device/provider-limrun": "workspace:*", "@agent-device/provider-webdriver": "workspace:*", + "@agent-device/replay-test": "workspace:*", "@agent-device/xml": "workspace:*", "@chenglou/freerange": "^0.0.1", "@stryker-mutator/core": "9.6.1", diff --git a/packages/replay-test/package.json b/packages/replay-test/package.json new file mode 100644 index 0000000000..50e9c2e273 --- /dev/null +++ b/packages/replay-test/package.json @@ -0,0 +1,18 @@ +{ + "name": "@agent-device/replay-test", + "version": "0.0.0", + "private": true, + "sideEffects": false, + "type": "module", + "description": "Private format-neutral replay-test scheduler for agent-device.", + "dependencies": { + "@agent-device/contracts": "workspace:*", + "@agent-device/kernel": "workspace:*" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + } +} diff --git a/packages/replay-test/src/index.ts b/packages/replay-test/src/index.ts new file mode 100644 index 0000000000..b0841b137d --- /dev/null +++ b/packages/replay-test/src/index.ts @@ -0,0 +1,47 @@ +/** + * The replay-test package façade (#1478 P3). + * + * One function plus the values crossing its seam. Everything else — scheduling, retries, + * sharding distribution, attempt identity, timeout policy, finalization/cleanup ordering, and + * result aggregation — is private to `internal/`. + * + * The scheduler is format-agnostic: it imports neither engine, and a source reaches it only as + * a `ReplayTestManifest`. Host authority arrives as narrow capabilities on + * `ReplayTestRuntimeDependencies`; none of them hands over a daemon request, a session store, + * mutable session state, or an engine. + */ +export { runReplayTestSuite } from './internal/session-test.ts'; + +export type { + ReplayTestAttemptError, + ReplayTestAttemptFailed, + ReplayTestAttemptOutcome, + ReplayTestAttemptPassed, + ReplayTestAttemptStep, + ReplayTestAttemptStepSink, + ReplayTestBindAttemptCancellation, + ReplayTestAttemptCancellation, + ReplayTestCleanupSession, + ReplayTestDiscoverSources, + ReplayTestEmitDiagnostic, + ReplayTestEmitProgress, + ReplayTestExecutionDependencies, + ReplayTestFinalizeAttempt, + ReplayTestIsCanceled, + ReplayTestManifest, + ReplayTestPlatform, + ReplayTestRunReplay, + ReplayTestRunReplayParams, + ReplayTestRuntimeDependencies, + ReplayTestSource, + ReplayTestSuiteOutcome, + ReplayTestSuiteRequest, + ReplayTestTarget, +} from './internal/session-test-types.ts'; + +export type { + ReplayTestResolveShardTargets, + ReplayTestShardContext, + ReplayTestShardMode, + ReplayTestShardTarget, +} from './internal/session-test-sharding.ts'; diff --git a/src/daemon/handlers/__tests__/session-test-artifacts.test.ts b/packages/replay-test/src/internal/__tests__/session-test-artifacts.test.ts similarity index 80% rename from src/daemon/handlers/__tests__/session-test-artifacts.test.ts rename to packages/replay-test/src/internal/__tests__/session-test-artifacts.test.ts index 36beb140eb..c74b0e9abd 100644 --- a/src/daemon/handlers/__tests__/session-test-artifacts.test.ts +++ b/packages/replay-test/src/internal/__tests__/session-test-artifacts.test.ts @@ -7,8 +7,20 @@ import { materializeReplayTestAttemptArtifacts, prepareReplayTestAttemptArtifacts, } from '../session-test-artifacts.ts'; -import { toReplayTestAttemptOutcome } from '../session-test-outcome.ts'; -import type { DaemonResponse } from '../../types.ts'; +import type { ReplayTestAttemptOutcome } from '../session-test-types.ts'; + +// Building outcomes from a DaemonResponse is the adapter's job and is pinned on that side; a +// package test states the neutral outcome directly (#1478 P3b). +const passedOutcome = ( + overrides: Partial> = {}, +): ReplayTestAttemptOutcome => ({ + status: 'passed', + replayed: 1, + healed: 0, + warnings: [], + artifactPaths: [], + ...overrides, +}); test('materializeReplayTestAttemptArtifacts writes replay and result manifests for passing attempts', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-artifacts-pass-')); @@ -19,16 +31,8 @@ test('materializeReplayTestAttemptArtifacts writes replay and result manifests f fs.writeFileSync(screenshotPath, 'png'); prepareReplayTestAttemptArtifacts(replayPath, attemptDir); - const response: DaemonResponse = { - ok: true, - data: { - replayed: 4, - healed: 1, - artifactPaths: [screenshotPath], - }, - }; materializeReplayTestAttemptArtifacts({ - outcome: toReplayTestAttemptOutcome(response), + outcome: passedOutcome({ replayed: 4, healed: 1, artifactPaths: [screenshotPath] }), filePath: replayPath, sessionName: 'default:test:suite:1', attempts: 1, @@ -70,21 +74,19 @@ test('materializeReplayTestAttemptArtifacts writes failure manifest and copies l fs.writeFileSync(logPath, 'log'); prepareReplayTestAttemptArtifacts(replayPath, attemptDir); - const response: DaemonResponse = { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: 'TIMEOUT after 5000ms', - hint: 'Replay test timeouts are cooperative.', - logPath, - details: { - reason: 'timeout', - artifactPaths: [screenshotPath], + materializeReplayTestAttemptArtifacts({ + outcome: { + status: 'failed', + error: { + code: 'COMMAND_FAILED', + message: 'Replay test timed out', + hint: 'Replay test timeouts are cooperative.', + logPath, + details: { reason: 'timeout', artifactPaths: [screenshotPath] }, }, + artifactPaths: [screenshotPath], + infrastructure: false, }, - }; - materializeReplayTestAttemptArtifacts({ - outcome: toReplayTestAttemptOutcome(response), filePath: replayPath, sessionName: 'default:test:suite:2', attempts: 2, diff --git a/packages/replay-test/src/internal/__tests__/session-test-discovery.test.ts b/packages/replay-test/src/internal/__tests__/session-test-discovery.test.ts new file mode 100644 index 0000000000..ebc2eff1b9 --- /dev/null +++ b/packages/replay-test/src/internal/__tests__/session-test-discovery.test.ts @@ -0,0 +1,77 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { discoverReplayTestEntries } from '../session-test-discovery.ts'; +import type { ReplayTestManifest, ReplayTestSource } from '../session-test-types.ts'; + +// Scheduler-owned discovery policy (#1478 P3b): which sources a --platform filter runs, which +// it skips and with what message, and rejecting a suite that matched nothing. Inspection is the +// host's, so these drive fake sources rather than the filesystem — the ordering and routing +// behavior they used to share a file with is pinned host-side. +const declared = (value: 'ios' | 'android'): ReplayTestManifest => ({ + device: { platform: { kind: 'declared', value } }, +}); +const unspecified: ReplayTestManifest = { device: { platform: { kind: 'unspecified' } } }; +const callerBound: ReplayTestManifest = { device: { platform: { kind: 'caller-bound' } } }; + +const sourcesOf = + (...entries: ReplayTestSource[]) => + () => + entries; + +test('platform filter skips sources that declared no platform', () => { + const entries = discoverReplayTestEntries({ + inputs: ['suite'], + platformFilter: 'android', + discoverSources: sourcesOf( + { path: '01-untyped.ad', manifest: unspecified }, + { path: '02-android.ad', manifest: declared('android') }, + ), + }); + + const untyped = entries.find((entry) => entry.path === '01-untyped.ad'); + assert.equal(untyped?.kind, 'skip'); + if (untyped?.kind === 'skip') { + assert.match(untyped.message, /missing platform metadata for --platform android/); + } + assert.equal(entries.find((entry) => entry.path === '02-android.ad')?.kind, 'run'); +}); + +test('platform filter runs caller-bound sources, which take their platform from the invocation', () => { + const entries = discoverReplayTestEntries({ + inputs: ['suite'], + platformFilter: 'android', + discoverSources: sourcesOf({ path: '01-flow.yaml', manifest: callerBound }), + }); + + assert.equal(entries.length, 1); + assert.equal(entries[0]?.kind, 'run'); +}); + +test('platform filter drops declared sources that do not match, without a skip entry', () => { + const entries = discoverReplayTestEntries({ + inputs: ['suite'], + platformFilter: 'android', + discoverSources: sourcesOf( + { path: '01-ios.ad', manifest: declared('ios') }, + { path: '02-android.ad', manifest: declared('android') }, + ), + }); + + assert.deepEqual( + entries.map((entry) => entry.path), + ['02-android.ad'], + ); +}); + +test('a suite that matched nothing after filtering is rejected', () => { + assert.throws( + () => + discoverReplayTestEntries({ + inputs: ['suite'], + platformFilter: 'android', + discoverSources: sourcesOf({ path: '01-ios.ad', manifest: declared('ios') }), + }), + (error: unknown) => error instanceof AppError && /No replay tests matched/.test(error.message), + ); +}); diff --git a/src/daemon/handlers/__tests__/session-test-runtime.test.ts b/packages/replay-test/src/internal/__tests__/session-test-runtime.test.ts similarity index 88% rename from src/daemon/handlers/__tests__/session-test-runtime.test.ts rename to packages/replay-test/src/internal/__tests__/session-test-runtime.test.ts index 86ee3ac114..1a45bb36f5 100644 --- a/src/daemon/handlers/__tests__/session-test-runtime.test.ts +++ b/packages/replay-test/src/internal/__tests__/session-test-runtime.test.ts @@ -2,10 +2,36 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; -import { isRequestCanceled } from '../../../request/cancel.ts'; + import { runReplayTestAttempt } from '../session-test-runtime.ts'; + import type { ReplayTestAttemptOutcome } from '../session-test-types.ts'; +// What the scheduler owes its host around cancellation (#1478 P3b): cancel exactly once when +// an attempt times out, and always release when it settles. How the daemon then maps that onto +// its request registry is the adapter's contract, pinned in +// `src/daemon/handlers/__tests__/session-replay-cancellation.test.ts`. +const cancellations: Array<{ attemptId: string; canceled: number; released: number }> = []; + +function trackCancellation() { + cancellations.length = 0; + return { + emitDiagnostic: () => {}, + bindAttemptCancellation: ({ attemptId }: { attemptId: string }) => { + const record = { attemptId, canceled: 0, released: 0 }; + cancellations.push(record); + return { + cancel: () => { + record.canceled += 1; + }, + release: () => { + record.released += 1; + }, + }; + }, + }; +} + const PASSED: ReplayTestAttemptOutcome = { status: 'passed', replayed: 1, @@ -57,6 +83,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se runReplay: async () => await replayPromise, finalizeAttempt, cleanupSession, + ...trackCancellation(), }); await vi.advanceTimersByTimeAsync(10); @@ -80,7 +107,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se }), ); expect(lifecycleEvents).toEqual(['finalize', 'cleanup']); - expect(isRequestCanceled('req-timeout-open')).toBe(true); + expect(cancellations[0]?.canceled).toBe(1); // #1478 P3: the second cleanup is strictly deferred until the abandoned replay settles, so // the attempt returns having cleaned up exactly once. P3 keeps this orchestration inside // replay-test while the cleanup effect itself stays in the daemon adapter. @@ -94,7 +121,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se }); await replaySettled; await vi.waitFor(() => { - expect(isRequestCanceled('req-timeout-open')).toBe(false); + expect(cancellations[0]?.released).toBe(1); }); await vi.waitFor(() => { expect(cleanupSession).toHaveBeenCalledTimes(2); @@ -118,6 +145,7 @@ test('runReplayTestAttempt keeps a passing replay passed when finalization fails infrastructure: false, }), cleanupSession, + ...trackCancellation(), }); expect(result.status).toBe('passed'); @@ -149,6 +177,7 @@ test('runReplayTestAttempt finalizes before cleanup and records that order in th cleanupSession: async () => { lifecycleEvents.push('cleanup'); }, + ...trackCancellation(), }); expect(result.status).toBe('passed'); @@ -188,6 +217,7 @@ test('runReplayTestAttempt cleans up once when a timed-out replay settles inside return undefined; }, cleanupSession, + ...trackCancellation(), }); await vi.advanceTimersByTimeAsync(10); @@ -225,6 +255,7 @@ test('runReplayTestAttempt cleans up without a finalizer and adds no finalizatio requestId: 'req-no-finalizer', runReplay: async () => PASSED, cleanupSession, + ...trackCancellation(), }); expect(result.status).toBe('passed'); diff --git a/src/daemon/handlers/session-test-artifacts.ts b/packages/replay-test/src/internal/session-test-artifacts.ts similarity index 87% rename from src/daemon/handlers/session-test-artifacts.ts rename to packages/replay-test/src/internal/session-test-artifacts.ts index a439ef6f33..92d360bfe7 100644 --- a/src/daemon/handlers/session-test-artifacts.ts +++ b/packages/replay-test/src/internal/session-test-artifacts.ts @@ -1,8 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; -import { isMaestroYamlPath } from '../../replay/format.ts'; -import type { ReplayTestAttemptOutcome } from './session-test-types.ts'; -import { SessionStore } from '../session-store.ts'; +import type { ReplayTestAttemptOutcome } from '@agent-device/replay-test'; const DEFAULT_TEST_ARTIFACTS_ROOT = '.agent-device/test-artifacts'; @@ -12,7 +10,9 @@ export function resolveReplayTestArtifactsDir(params: { suiteInvocationId: string; }): string { const { artifactsDir, cwd, suiteInvocationId } = params; - const resolvedRoot = SessionStore.expandHome(artifactsDir ?? DEFAULT_TEST_ARTIFACTS_ROOT, cwd); + // `artifactsDir` arrives already home-expanded: resolving `~` and the caller's cwd is host + // work, done once when the daemon builds the suite request. + const resolvedRoot = path.resolve(cwd ?? '.', artifactsDir ?? DEFAULT_TEST_ARTIFACTS_ROOT); return path.join(resolvedRoot, suiteInvocationId); } @@ -107,7 +107,10 @@ function copyReplaySourceFile(filePath: string, attemptArtifactsDir: string): vo const genericReplayPath = path.join(attemptArtifactsDir, 'replay.ad'); fs.copyFileSync(filePath, genericReplayPath); - if (!isMaestroYamlPath(filePath)) return; + // Sources that are not native `.ad` keep their original filename alongside `replay.ad`, so + // CI artifacts point back at the flow the author wrote. This is artifact naming, not format + // routing — it asks what the file is called, never which engine will run it. + if (path.extname(filePath).toLowerCase() === '.ad') return; // Keep replay.ad for existing artifact consumers, and preserve the original // Maestro filename so CI artifacts point back to the source flow. const originalReplayPath = path.join(attemptArtifactsDir, path.basename(filePath)); diff --git a/src/daemon/handlers/session-test-attempt.ts b/packages/replay-test/src/internal/session-test-attempt.ts similarity index 94% rename from src/daemon/handlers/session-test-attempt.ts rename to packages/replay-test/src/internal/session-test-attempt.ts index a9d82ac404..2086bd5a77 100644 --- a/src/daemon/handlers/session-test-attempt.ts +++ b/packages/replay-test/src/internal/session-test-attempt.ts @@ -1,5 +1,4 @@ import path from 'node:path'; -import { emitRequestProgress } from '../../request/progress.ts'; import type { ReplaySuiteTestFailed, ReplaySuiteTestResult } from '@agent-device/contracts/replay'; import type { ReplayTestProgressEvent } from '@agent-device/contracts/progress'; import { @@ -15,10 +14,9 @@ import { import { runReplayTestAttempt } from './session-test-runtime.ts'; import type { ReplayTestAttemptOutcome, - ReplayTestRuntimeDependencies, + ReplayTestExecutionDependencies, } from './session-test-types.ts'; import type { ReplayTestShardContext } from './session-test-sharding.ts'; -import { isRequestCanceled } from '../../request/cancel.ts'; type ReplayTestCaseResult = Extract; type ReplayTestAttemptFailure = NonNullable< @@ -59,7 +57,7 @@ type ReplayTestCaseParams = { suiteIndex: number; suiteTotal: number; shard?: ReplayTestShardContext; -} & ReplayTestRuntimeDependencies; +} & ReplayTestExecutionDependencies; type ReplayTestCaseContext = { testStartedAt: number; @@ -108,7 +106,7 @@ async function runReplayTestCaseAttempts( }; for (let attemptIndex = 0; attemptIndex <= params.retries; attemptIndex += 1) { - if (isRequestCanceled(params.requestId)) break; + if (params.isCanceled()) break; const attempt = await runSingleReplayTestAttempt(params, context, attemptIndex); updateReplayTestCaseOutcome(outcome, attempt); if (shouldStopReplayTestAttempts(params, attempt.outcome, attemptIndex)) break; @@ -171,12 +169,17 @@ async function runSingleReplayTestAttempt( requestId: attemptRequestId, parentRequestId: requestId, timeoutMs, - platform: entry.metadata.platform, - target: entry.metadata.target, + // The scheduler only ever names a declared platform; caller-bound and unspecified both + // mean "do not pin one on the nested request" (#1478 P3b). + platform: + entry.manifest.device.platform.kind === 'declared' + ? entry.manifest.device.platform.value + : undefined, + target: entry.manifest.device.target, artifactsDir: attemptArtifactsDir, shard, onStep: (step) => { - emitRequestProgress({ + params.emitProgress({ type: 'replay-test', ...attemptProgress, status: 'progress', @@ -189,6 +192,8 @@ async function runSingleReplayTestAttempt( runReplay: params.runReplay, cleanupSession: params.cleanupSession, finalizeAttempt: params.finalizeAttempt, + emitDiagnostic: params.emitDiagnostic, + bindAttemptCancellation: params.bindAttemptCancellation, }); const durationMs = Date.now() - startedAt; materializeReplayTestAttemptArtifacts({ @@ -223,7 +228,7 @@ function emitReplayTestStartProgress( ): void { const { entry, sessionName, suiteInvocationId, caseIndex, suiteIndex, suiteTotal, shard } = params; - emitRequestProgress({ + params.emitProgress({ type: 'replay-test', file: entry.path, title: entry.title, @@ -260,7 +265,7 @@ function shouldStopReplayTestAttempts( ): boolean { return ( outcome.status === 'passed' || - isRequestCanceled(params.requestId) || + params.isCanceled() || outcome.infrastructure || attemptIndex >= params.retries ); @@ -272,7 +277,7 @@ function emitReplayTestRetryProgress( attempt: ReplayTestAttemptResult, ): void { if (attempt.outcome.status === 'passed') return; - emitRequestProgress({ + params.emitProgress({ type: 'replay-test', file: params.entry.path, title: params.entry.title, @@ -312,7 +317,7 @@ function buildReplayTestPassedResult( const { entry, suiteIndex, suiteTotal, shard } = params; const attemptOutcome = outcome.finalOutcome; if (attemptOutcome?.status !== 'passed') throw new Error('Expected passing replay test outcome.'); - emitRequestProgress({ + params.emitProgress({ type: 'replay-test', file: entry.path, title: entry.title, @@ -358,7 +363,7 @@ function buildReplayTestFailedResult( attemptOutcome?.status === 'failed' ? attemptOutcome.error : { code: 'COMMAND_FAILED', message: 'Unknown replay test failure' }; - emitRequestProgress({ + params.emitProgress({ type: 'replay-test', file: entry.path, title: entry.title, diff --git a/src/daemon/handlers/session-test-discovery.ts b/packages/replay-test/src/internal/session-test-discovery.ts similarity index 58% rename from src/daemon/handlers/session-test-discovery.ts rename to packages/replay-test/src/internal/session-test-discovery.ts index d8950247e1..0d5b2783f9 100644 --- a/src/daemon/handlers/session-test-discovery.ts +++ b/packages/replay-test/src/internal/session-test-discovery.ts @@ -1,12 +1,11 @@ -import fs from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { isApplePlatform, type PlatformSelector } from '@agent-device/kernel/device'; -import { inspectMaestroFlow } from '@agent-device/maestro'; -import { resolveRequestTrackingId } from '../../request/cancel.ts'; -import { resolveReplayFormat } from '../../replay/format.ts'; -import { readReplayScriptMetadata, type ReplayScriptMetadata } from '../../replay/script.ts'; -import { discoverReplaySourcePaths } from '../replay-source-discovery.ts'; +import type { + ReplayTestDiscoverSources, + ReplayTestManifest, + ReplayTestPlatform, +} from './session-test-types.ts'; const MAX_REPLAY_TEST_RETRIES = 3; @@ -15,7 +14,7 @@ export type ReplayTestDiscoveryEntry = kind: 'run'; path: string; title?: string; - metadata: ReplayScriptMetadata; + manifest: ReplayTestManifest; } | { kind: 'skip'; @@ -26,46 +25,51 @@ export type ReplayTestDiscoveryEntry = export type ReplayTestRunEntry = Extract; +/** + * Applies discovery policy to host-inspected sources (#1478 P3b). + * + * Inspection belongs to the host, which has the engines. This is the neutral half: which + * sources a `--platform` filter runs, which it skips and with what message, and rejecting a + * suite that matched nothing. + */ export function discoverReplayTestEntries(params: { inputs: string[]; cwd?: string; platformFilter?: PlatformSelector; - replayBackend?: string; + discoverSources: ReplayTestDiscoverSources; }): ReplayTestDiscoveryEntry[] { - const { inputs, cwd, platformFilter, replayBackend } = params; - const resolvedCwd = cwd ?? process.cwd(); - const filePaths = discoverReplaySourcePaths({ - inputs, - cwd: resolvedCwd, - replayBackend, - }); + const { inputs, cwd, platformFilter, discoverSources } = params; + const sources = discoverSources({ inputs, cwd }); const entries: ReplayTestDiscoveryEntry[] = []; - for (const filePath of filePaths) { - const script = fs.readFileSync(filePath, 'utf8'); - const metadata = readReplayScriptMetadata(script); - const title = readReplayTestTitle(script, filePath, replayBackend); + for (const source of sources) { + const { path: filePath, manifest } = source; + const run = { kind: 'run', path: filePath, title: manifest.title, manifest } as const; if (!platformFilter) { - entries.push({ kind: 'run', path: filePath, title, metadata }); + entries.push(run); continue; } - if (!metadata.platform) { - if (resolveReplayFormat(filePath, replayBackend) === 'maestro') { - entries.push({ kind: 'run', path: filePath, title, metadata }); - } else { - entries.push({ - kind: 'skip', - path: filePath, - reason: 'skipped-by-filter', - message: `missing platform metadata for --platform ${platformFilter}`, - }); - } + const declared = manifest.device.platform; + // A caller-bound source takes its platform from the invocation, so a filter never skips + // it for lacking declared metadata; an unspecified one declared nothing and is skipped + // with the message the suite result has always carried. + if (declared.kind === 'caller-bound') { + entries.push(run); continue; } - if (!matchesPlatformFilter(platformFilter, metadata.platform)) { + if (declared.kind === 'unspecified') { + entries.push({ + kind: 'skip', + path: filePath, + reason: 'skipped-by-filter', + message: `missing platform metadata for --platform ${platformFilter}`, + }); continue; } - entries.push({ kind: 'run', path: filePath, title, metadata }); + if (!matchesPlatformFilter(platformFilter, declared.value)) { + continue; + } + entries.push(run); } const runnableCount = entries.filter((entry) => entry.kind === 'run').length; @@ -111,11 +115,15 @@ export function buildReplayTestAttemptRequestId(params: { shardIndex?: number; }): string { const { requestId, suiteInvocationId, filePath, caseIndex, attemptIndex, shardIndex } = params; - const shardPart = shardIndex === undefined ? '' : `:shard:${shardIndex + 1}`; - return resolveRequestTrackingId( - `${requestId ?? suiteInvocationId}${shardPart}:test:${caseIndex + 1}:${path.basename(filePath)}:attempt:${attemptIndex + 1}`, - suiteInvocationId, - ); + return [ + requestId ?? suiteInvocationId, + ...(shardIndex === undefined ? [] : ['shard', shardIndex + 1]), + 'test', + caseIndex + 1, + path.basename(filePath), + 'attempt', + attemptIndex + 1, + ].join(':'); } export function resolveReplayTestTimeout( @@ -134,17 +142,7 @@ export function resolveReplayTestRetries( return Math.max(0, Math.min(MAX_REPLAY_TEST_RETRIES, resolved)); } -function readReplayTestTitle( - script: string, - filePath: string, - replayBackend: string | undefined, -): string | undefined { - return resolveReplayFormat(filePath, replayBackend) === 'maestro' - ? inspectMaestroFlow(script, filePath).name - : undefined; -} - -function matchesPlatformFilter(filter: PlatformSelector, candidate: PlatformSelector): boolean { +function matchesPlatformFilter(filter: PlatformSelector, candidate: ReplayTestPlatform): boolean { if (filter === 'apple') { return isApplePlatform(candidate); } diff --git a/src/daemon/handlers/session-test-runtime.ts b/packages/replay-test/src/internal/session-test-runtime.ts similarity index 87% rename from src/daemon/handlers/session-test-runtime.ts rename to packages/replay-test/src/internal/session-test-runtime.ts index f0be12c772..7c5373a625 100644 --- a/src/daemon/handlers/session-test-runtime.ts +++ b/packages/replay-test/src/internal/session-test-runtime.ts @@ -1,21 +1,16 @@ import fs from 'node:fs'; import path from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; -import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { normalizeError } from '@agent-device/kernel/errors'; -import { - clearRequestCanceled, - getRequestSignal, - markRequestCanceled, - registerRequestAbort, -} from '../../request/cancel.ts'; -import type { ReplayScriptMetadata } from '../../replay/script.ts'; import { replayTestAttemptFailure, type ReplayTestAttemptFailed, type ReplayTestAttemptOutcome, type ReplayTestAttemptStepSink, + type ReplayTestEmitDiagnostic, + type ReplayTestPlatform, type ReplayTestRunReplayParams, + type ReplayTestTarget, type ReplayTestRuntimeDependencies, } from './session-test-types.ts'; @@ -31,12 +26,21 @@ export async function runReplayTestAttempt( requestId: string; parentRequestId?: string; timeoutMs?: number; - platform?: ReplayScriptMetadata['platform']; - target?: ReplayScriptMetadata['target']; + platform?: ReplayTestPlatform; + target?: ReplayTestTarget; artifactsDir?: string; shard?: ReplayTestRunReplayParams['shard']; onStep?: ReplayTestAttemptStepSink; - } & ReplayTestRuntimeDependencies, + // Only the capabilities this attempt actually exercises. It never publishes progress, so + // it is not handed `emitProgress` — authority narrows across every hop (#1478 P3b). + } & Pick< + ReplayTestRuntimeDependencies, + | 'runReplay' + | 'cleanupSession' + | 'finalizeAttempt' + | 'emitDiagnostic' + | 'bindAttemptCancellation' + >, ): Promise { const { filePath, @@ -52,9 +56,13 @@ export async function runReplayTestAttempt( runReplay, cleanupSession, finalizeAttempt, + emitDiagnostic, + bindAttemptCancellation, } = params; - registerRequestAbort(requestId); - const clearParentAbortRelay = relayReplayTestAbortFromParent(requestId, parentRequestId); + const cancellation = bindAttemptCancellation({ + attemptId: requestId, + parentAttemptId: parentRequestId, + }); const artifactPaths = new Set(); let timeoutHandle: ReturnType | undefined; let timedOut = false; @@ -79,13 +87,13 @@ export async function runReplayTestAttempt( artifactsDir, artifactPaths, tracePath, + appendTimingEvent: (event) => appendReplayTestTimingEvent(tracePath, event), shard, onStep, }) .catch((error) => replayTestAttemptFailure({ error: normalizeError(error) })) .finally(() => { - clearParentAbortRelay(); - clearRequestCanceled(requestId); + cancellation.release(); }); try { @@ -96,7 +104,7 @@ export async function runReplayTestAttempt( new Promise((resolve) => { timeoutHandle = setTimeout(() => { timedOut = true; - markRequestCanceled(requestId); + cancellation.cancel(); resolve(createReplayTestTimeoutOutcome(timeoutMs, [...artifactPaths])); }, timeoutMs); }), @@ -131,6 +139,7 @@ export async function runReplayTestAttempt( cleanupSession, sessionName, requestId, + emitDiagnostic, }); } } @@ -140,6 +149,7 @@ export async function runReplayTestAttempt( artifactPaths, artifactsDir, tracePath, + emitDiagnostic, }); if (outcome?.status === 'passed' && finalizeFailure) { outcome = appendReplayTestWarning( @@ -190,27 +200,6 @@ export async function runReplayTestAttempt( ); } -function relayReplayTestAbortFromParent( - requestId: string, - parentRequestId: string | undefined, -): () => void { - if (!parentRequestId || parentRequestId === requestId) return () => {}; - const parentSignal = getRequestSignal(parentRequestId); - if (!parentSignal) return () => {}; - - const cancelRequest = () => { - markRequestCanceled(requestId); - }; - if (parentSignal.aborted) { - cancelRequest(); - return () => {}; - } - parentSignal.addEventListener('abort', cancelRequest, { once: true }); - return () => { - parentSignal.removeEventListener('abort', cancelRequest); - }; -} - async function waitForReplayAfterTimeout( replayPromise: Promise, ): Promise { @@ -225,8 +214,9 @@ async function cleanupSessionAfterLateReplay(params: { cleanupSession: ReplayTestRuntimeDependencies['cleanupSession']; sessionName: string; requestId: string; + emitDiagnostic: ReplayTestEmitDiagnostic; }): Promise { - const { replayPromise, cleanupSession, sessionName, requestId } = params; + const { replayPromise, cleanupSession, sessionName, requestId, emitDiagnostic } = params; try { await replayPromise; } finally { @@ -253,8 +243,10 @@ async function finalizeReplayTestAttempt(params: { artifactPaths: Set; artifactsDir?: string; tracePath?: string; + emitDiagnostic: ReplayTestEmitDiagnostic; }): Promise { - const { finalizeAttempt, sessionName, artifactPaths, artifactsDir, tracePath } = params; + const { finalizeAttempt, sessionName, artifactPaths, artifactsDir, tracePath, emitDiagnostic } = + params; if (!finalizeAttempt) return undefined; const finalizeStartedAt = Date.now(); appendReplayTestTimingEvent(tracePath, { @@ -268,6 +260,7 @@ async function finalizeReplayTestAttempt(params: { artifactPaths, artifactsDir, tracePath, + appendTimingEvent: (event) => appendReplayTestTimingEvent(tracePath, event), }); appendReplayTestTimingEvent(tracePath, { type: 'replay_test_finalize_stop', @@ -334,8 +327,8 @@ function prepareReplayTestTimingTrace(params: { sessionName: string; requestId: string; timeoutMs?: number; - platform?: ReplayScriptMetadata['platform']; - target?: ReplayScriptMetadata['target']; + platform?: ReplayTestPlatform; + target?: ReplayTestTarget; }): string | undefined { const { artifactsDir, diff --git a/packages/replay-test/src/internal/session-test-sharding.ts b/packages/replay-test/src/internal/session-test-sharding.ts new file mode 100644 index 0000000000..6cb81ff6cd --- /dev/null +++ b/packages/replay-test/src/internal/session-test-sharding.ts @@ -0,0 +1,69 @@ +import type { ReplayTestPlatform, ReplayTestTarget } from './session-test-types.ts'; + +export type ReplayTestShardMode = 'all' | 'split'; + +/** + * The device a shard runs against, in neutral vocabulary. + * + * The scheduler reads `id`/`name` for session labels and progress metadata. `platform` and + * `target` are here because the host needs them to bind a nested request, and both are already + * neutral kernel types — so `DeviceInfo` itself never crosses into scheduling. + */ +export type ReplayTestShardTarget = Readonly<{ + id: string; + name: string; + platform: ReplayTestPlatform; + target?: ReplayTestTarget; +}>; + +export type ReplayTestShardContext = { + shardIndex: number; + shardCount: number; + device: ReplayTestShardTarget; +}; + +/** + * Resolves the devices a sharded run will use (#1478 P3b). + * + * Enumerating inventory, applying allowlists and simulator set paths, and rejecting a run with + * too few devices are all host concerns. The scheduler decides how many shards there are and + * which entries go to each; it never enumerates hardware. + */ +export type ReplayTestResolveShardTargets = ( + shardCount: number, +) => Promise; + +export type ReplayTestShardPlan = { + mode: ReplayTestShardMode; + shardCount: number; + total: number; + shards: Array; +}; + +export async function buildReplayTestShardPlan( + mode: Readonly<{ kind: ReplayTestShardMode; count: number }> | undefined, + runnableEntries: TEntry[], + skippedCount: number, + resolveShardTargets: ReplayTestResolveShardTargets, +): Promise | undefined> { + if (!mode) return undefined; + if (runnableEntries.length === 0) return undefined; + + const devices = await resolveShardTargets(mode.count); + return { + mode: mode.kind, + shardCount: mode.count, + total: + skippedCount + + (mode.kind === 'all' ? runnableEntries.length * mode.count : runnableEntries.length), + shards: devices.map((device, index) => ({ + shardIndex: index, + shardCount: mode.count, + device, + entries: + mode.kind === 'all' + ? runnableEntries + : runnableEntries.filter((_entry, entryIndex) => entryIndex % mode.count === index), + })), + }; +} diff --git a/packages/replay-test/src/internal/session-test-types.ts b/packages/replay-test/src/internal/session-test-types.ts new file mode 100644 index 0000000000..eb68efe0ee --- /dev/null +++ b/packages/replay-test/src/internal/session-test-types.ts @@ -0,0 +1,289 @@ +import type { ReplaySuiteResult, ReplaySuiteTestFailed } from '@agent-device/contracts/replay'; +import type { SnapshotDiagnosticsSummary } from '@agent-device/contracts/capture'; +import type { + ReplayTestProgressEvent, + ReplayTestSuiteProgressEvent, +} from '@agent-device/contracts/progress'; +import type { DeviceTarget, PlatformSelector } from '@agent-device/kernel/device'; +import type { + ReplayTestResolveShardTargets, + ReplayTestShardContext, + ReplayTestShardMode, +} from './session-test-sharding.ts'; + +/** + * The device vocabulary a scheduler may name, sourced from the neutral kernel rather than + * through `replay/script.ts` (#1478 P3b). These resolve to exactly what the `.ad` metadata + * types resolved to — `Exclude` and `DeviceTarget` — so the manifest + * shape is unchanged; only the import direction is. A format-neutral scheduler must not name + * an engine module, and P5 relocates that engine into `packages/ad-replay` regardless. + */ +export type ReplayTestPlatform = Exclude; +export type ReplayTestTarget = DeviceTarget; + +/** + * One suite run, in vocabulary the scheduler owns (#1478 P3b). + * + * The façade takes this instead of a `DaemonRequest`. Every field is one the scheduler + * actually reads; the adapter translates flags and meta into it and translates the result + * back to a daemon response. Nothing here names a transport, a command, or a session store. + * + * `replayBackend` is absent by design: it selects an engine, and the adapter has already + * applied it when building the source-discovery and shard-target capabilities. + */ +export type ReplayTestSuiteRequest = Readonly<{ + /** Paths and globs to expand. Never empty; the adapter rejects an empty invocation. */ + inputs: readonly string[]; + cwd?: string; + /** Correlation id for attempt identity and the suite invocation id. */ + requestId?: string; + /** Base session name that attempt sessions derive from. */ + sessionName: string; + platformFilter?: PlatformSelector; + artifactsDir?: string; + failFast?: boolean; + retries?: number; + timeoutMs?: number; + shard?: Readonly<{ mode: ReplayTestShardMode; count: number }>; +}>; + +/** + * What a suite run produces. Nothing throws across the façade: an invalid invocation or a + * discovery failure resolves as `failed` with ADR 0010 error fields, which the adapter maps to + * a daemon error response exactly as the in-handler `errorResponse` calls used to. + */ +export type ReplayTestSuiteOutcome = + | Readonly<{ status: 'completed'; data: ReplaySuiteResult }> + | Readonly<{ status: 'failed'; error: Readonly<{ code: string; message: string }> }>; + +/** + * Everything the scheduler may know about one discovered source (#1478 P3b). + * + * Inspecting a source needs an engine, so the host does it and the scheduler receives this. + * + * The fields are exactly the four the scheduler consumes (platform, target, retries, + * timeoutMs) plus the title reporters display. Nothing else belongs here without a + * demonstrated scheduler or reporter call site — no source format, app ID, environment, + * Maestro tags/steps, digest, includes, config, or source path. + */ +export type ReplayTestManifest = Readonly<{ + title?: string; + device: Readonly<{ + /** + * How the source determines its platform. This is what replaced the scheduler's old + * `resolveReplayFormat(...) === 'maestro'` check: it needs to know whether a missing + * platform means "the caller supplies it" or "this source declared nothing", never which + * engine produced it. + * + * - `declared` — the source names a platform, and a `--platform` filter compares against it; + * - `caller-bound` — the format leaves platform to the caller (Maestro), so a filter runs it; + * - `unspecified` — the source declared none, so a filter skips it as unmatched. + */ + platform: + | Readonly<{ kind: 'declared'; value: ReplayTestPlatform }> + | Readonly<{ kind: 'caller-bound' }> + | Readonly<{ kind: 'unspecified' }>; + target?: ReplayTestTarget; + }>; + attemptDefaults?: Readonly<{ + timeoutMs?: number; + retries?: number; + }>; +}>; + +/** One source the host found and inspected, ready for scheduler filtering policy. */ +export type ReplayTestSource = Readonly<{ + path: string; + manifest: ReplayTestManifest; +}>; + +/** + * Expands the caller's inputs and inspects each source (#1478 P3b). + * + * Path expansion, file reading, format routing, and per-engine inspection are all host work. + * The scheduler keeps discovery *policy* — platform filtering, run/skip classification, and + * the "no replay tests matched" error — which is the part that is genuinely format-neutral. + */ +export type ReplayTestDiscoverSources = (params: { + inputs: string[]; + cwd?: string; +}) => readonly ReplayTestSource[]; + +/** + * One execution step an engine reports while an attempt runs (#1478 P3, finding 1). + * + * Step payloads originate below the attempt boundary, inside engine execution, and used to + * reach the reporter through a request-global `AsyncLocalStorage` seeded per attempt. The + * scheduler now hands each attempt a narrow sink instead, so step progress is an explicit + * per-attempt port with two real adapters (native `.ad` and Maestro) rather than ambient + * request state the scheduler cannot see. + */ +export type ReplayTestAttemptStep = { + index: number; + total: number; + command?: string; + value?: string; +}; + +export type ReplayTestAttemptStepSink = (step: ReplayTestAttemptStep) => void; + +/** + * ADR 0010 error fields exactly as the public suite result publishes them. This is the + * neutral wire error, not `DaemonResponse`: the scheduler never sees a daemon response shape. + */ +export type ReplayTestAttemptError = ReplaySuiteTestFailed['error']; + +export type ReplayTestAttemptPassed = { + status: 'passed'; + replayed: number; + healed: number; + warnings: readonly string[]; + artifactPaths: readonly string[]; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; +}; + +export type ReplayTestAttemptFailed = { + status: 'failed'; + error: ReplayTestAttemptError; + artifactPaths: readonly string[]; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; + /** + * The host's verdict that this failure is environmental (device/runner/boot) rather than a + * test failure, so retrying and continuing the suite cannot help. Classification needs + * platform boot-diagnostic vocabulary, which the scheduler must not import, so the host + * tags the outcome and the scheduler only reads the tag. + */ + infrastructure: boolean; +}; + +/** Every expected attempt state resolves as a tagged outcome; nothing throws across the seam. */ +export type ReplayTestAttemptOutcome = ReplayTestAttemptPassed | ReplayTestAttemptFailed; + +export type ReplayTestRunReplayParams = { + filePath: string; + sessionName: string; + platform?: ReplayTestPlatform; + target?: ReplayTestTarget; + requestId?: string; + artifactsDir?: string; + artifactPaths?: Set; + tracePath?: string; + /** + * Appends one event to this attempt's timing trace. The host records video lifecycle events + * into the same trace the scheduler writes; handing it this closure keeps the trace format + * private to the package instead of exporting a writer from the façade, and scopes the + * authority to exactly this attempt's trace. + */ + appendTimingEvent: (event: Record) => void; + shard?: ReplayTestShardContext; + onStep?: ReplayTestAttemptStepSink; +}; + +export type ReplayTestRunReplay = ( + params: ReplayTestRunReplayParams, +) => Promise; + +export type ReplayTestCleanupSession = (sessionName: string) => Promise; + +/** + * Runs after the attempt settles and before cleanup. Returns a failure outcome when + * finalization itself failed, or `undefined` when there was nothing to finalize. + */ +export type ReplayTestFinalizeAttempt = (params: { + sessionName: string; + artifactPaths: Set; + artifactsDir?: string; + tracePath?: string; + /** Same per-attempt trace appender the run receives; finalization records video events too. */ + appendTimingEvent: (event: Record) => void; +}) => Promise; + +/** + * Publishes one reporter-facing progress event (#1478 P3b). + * + * The host owns the request-global sink and injects this; the scheduler holds no ambient + * authority to publish. Deliberately narrower than `RequestProgressSink`: the scheduler emits + * suite and per-test events, never `CommandProgressEvent`, so it is not handed the ability to. + */ +export type ReplayTestEmitProgress = ( + event: ReplayTestSuiteProgressEvent | ReplayTestProgressEvent, +) => void; + +/** + * Whether the suite this scheduler is running has been canceled (#1478 P3b). + * + * The host binds this to its own request, so the scheduler asks a question it is entitled to + * ask and learns nothing about how cancellation is tracked or keyed. + */ +export type ReplayTestIsCanceled = () => boolean; + +/** + * One operational diagnostic (#1478 P3b). `emitDiagnostic` reads a request-global + * `AsyncLocalStorage` scope, so the scheduler receives the narrow publish capability instead + * of the module. The level set is spelled out rather than imported so the vocabulary crossing + * the seam stays neutral. + */ +export type ReplayTestEmitDiagnostic = (event: { + level?: 'info' | 'warn' | 'error' | 'debug'; + phase: string; + durationMs?: number; + data?: Record; +}) => void; + +/** + * Cancellation binding for one attempt (#1478 P3b). + * + * The brief gives the daemon adapter the job of mapping an engine-neutral attempt id to + * daemon request identifiers and *binding cancellation*. The scheduler still owns timeout + * policy — it decides when an attempt has run too long — so it needs to say "stop this + * attempt" and "I am done with it", and nothing more. Registering the abort, relaying a + * parent request's abort, and clearing the registry entry are all host concerns behind this. + */ +export type ReplayTestAttemptCancellation = { + /** Signal the running attempt to stop. The scheduler calls this on timeout. */ + cancel: () => void; + /** Release whatever the host bound for this attempt. Always called once it settles. */ + release: () => void; +}; + +export type ReplayTestBindAttemptCancellation = (params: { + attemptId: string; + parentAttemptId?: string; +}) => ReplayTestAttemptCancellation; + +export type ReplayTestRuntimeDependencies = { + runReplay: ReplayTestRunReplay; + cleanupSession: ReplayTestCleanupSession; + finalizeAttempt?: ReplayTestFinalizeAttempt; + emitProgress: ReplayTestEmitProgress; + isCanceled: ReplayTestIsCanceled; + emitDiagnostic: ReplayTestEmitDiagnostic; + bindAttemptCancellation: ReplayTestBindAttemptCancellation; + discoverSources: ReplayTestDiscoverSources; + resolveShardTargets: ReplayTestResolveShardTargets; +}; + +/** + * What attempt execution needs. Discovery and shard resolution happen once, in the suite entry + * point, so nothing below it is handed the ability to enumerate sources or bind devices. + */ +export type ReplayTestExecutionDependencies = Omit< + ReplayTestRuntimeDependencies, + 'discoverSources' | 'resolveShardTargets' +>; + +/** Neutral failure outcome helper; keeps timeout/unknown construction in one place. */ +export function replayTestAttemptFailure(params: { + error: ReplayTestAttemptError; + artifactPaths?: readonly string[]; + infrastructure?: boolean; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; +}): ReplayTestAttemptFailed { + return { + status: 'failed', + error: params.error, + artifactPaths: params.artifactPaths ?? [], + infrastructure: params.infrastructure ?? false, + ...(params.snapshotDiagnostics ? { snapshotDiagnostics: params.snapshotDiagnostics } : {}), + }; +} diff --git a/src/daemon/handlers/session-test.ts b/packages/replay-test/src/internal/session-test.ts similarity index 68% rename from src/daemon/handlers/session-test.ts rename to packages/replay-test/src/internal/session-test.ts index e9a1b9c37a..8e3051accf 100644 --- a/src/daemon/handlers/session-test.ts +++ b/packages/replay-test/src/internal/session-test.ts @@ -1,13 +1,10 @@ import { asAppError, normalizeError } from '@agent-device/kernel/errors'; -import { errorResponse } from './response.ts'; -import type { DaemonRequest, DaemonResponse } from '../types.ts'; import type { ReplaySuiteResult, ReplaySuiteTestFailed, ReplaySuiteTestResult, } from '@agent-device/contracts/replay'; import { resolveReplayTestArtifactsDir } from './session-test-artifacts.ts'; -import { emitRequestProgress } from '../../request/progress.ts'; import { buildReplayTestInvocationId, discoverReplayTestEntries, @@ -16,13 +13,21 @@ import { resolveReplayTestTimeout, } from './session-test-discovery.ts'; import { runReplayTestCase, type ReplayTestCaseReport } from './session-test-attempt.ts'; -import type { ReplayTestRuntimeDependencies } from './session-test-types.ts'; +import type { + ReplayTestEmitProgress, + ReplayTestIsCanceled, + ReplayTestExecutionDependencies, + ReplayTestRuntimeDependencies, + ReplayTestSuiteOutcome, + ReplayTestSuiteRequest, + ReplayTestDiscoverSources, +} from './session-test-types.ts'; import { buildReplayTestShardPlan, + type ReplayTestResolveShardTargets, type ReplayTestShardContext, type ReplayTestShardPlan, } from './session-test-sharding.ts'; -import { isRequestCanceled } from '../../request/cancel.ts'; import { mergeSnapshotDiagnostics } from '@agent-device/contracts/capture'; type ReplayTestEntry = ReturnType[number]; @@ -42,23 +47,38 @@ type ReplayTestSuitePlan = { export async function runReplayTestSuite( params: { - req: DaemonRequest; - sessionName: string; + request: ReplayTestSuiteRequest; } & ReplayTestRuntimeDependencies, -): Promise { - const { req, sessionName, runReplay, cleanupSession, finalizeAttempt } = params; - if ((req.positionals?.length ?? 0) === 0) { - return errorResponse('INVALID_ARGS', 'test requires at least one path or glob'); +): Promise { + const { + request, + runReplay, + cleanupSession, + finalizeAttempt, + emitProgress, + isCanceled, + emitDiagnostic, + bindAttemptCancellation, + discoverSources, + resolveShardTargets, + } = params; + const sessionName = request.sessionName; + if (request.inputs.length === 0) { + return { + status: 'failed', + error: { code: 'INVALID_ARGS', message: 'test requires at least one path or glob' }, + }; } try { const suiteStartedAt = Date.now(); - const plan = await prepareReplayTestSuitePlan(req); - emitReplayTestSuiteStart(plan); + const plan = await prepareReplayTestSuitePlan(request, discoverSources, resolveShardTargets); + emitReplayTestSuiteStart(plan, emitProgress); const results: ReplaySuiteTestResult[] = plan.shardPlan ? emitSkippedReplayTestResults({ entries: plan.entries, total: plan.total, + emitProgress, }) : []; @@ -68,14 +88,18 @@ export async function runReplayTestSuite( shards: plan.shardPlan.shards, sessionName, suiteInvocationId: plan.suiteInvocationId, - cwd: req.meta?.cwd, - requestId: req.meta?.requestId, - flags: req.flags, + cwd: request.cwd, + requestId: request.requestId, + request, suiteArtifactsDir: plan.suiteArtifactsDir, suiteTotal: plan.total, runReplay, cleanupSession, finalizeAttempt, + emitProgress, + isCanceled, + emitDiagnostic, + bindAttemptCancellation, })), ); } else { @@ -84,48 +108,64 @@ export async function runReplayTestSuite( discoveryEntries: plan.entries, sessionName, suiteInvocationId: plan.suiteInvocationId, - cwd: req.meta?.cwd, - requestId: req.meta?.requestId, - flags: req.flags, + cwd: request.cwd, + requestId: request.requestId, + request, suiteArtifactsDir: plan.suiteArtifactsDir, suiteTotal: plan.total, runReplay, cleanupSession, finalizeAttempt, + emitProgress, + isCanceled, + emitDiagnostic, + bindAttemptCancellation, })), ); } const data = summarizeReplayTestResults(plan.total, results, Date.now() - suiteStartedAt); - return { ok: true, data }; + return { status: 'completed', data }; } catch (err) { const appErr = asAppError(err); - return errorResponse(appErr.code, appErr.message); + return { status: 'failed', error: { code: appErr.code, message: appErr.message } }; } } -async function prepareReplayTestSuitePlan(req: DaemonRequest): Promise { - const entries = discoverReplayTestSuiteEntries(req); +async function prepareReplayTestSuitePlan( + request: ReplayTestSuiteRequest, + discoverSources: ReplayTestDiscoverSources, + resolveShardTargets: ReplayTestResolveShardTargets, +): Promise { + const entries = discoverReplayTestSuiteEntries(request, discoverSources); const runnable = runnableReplayTestEntries(entries); const skippedCount = entries.length - runnable.length; - const suiteInvocationId = buildReplayTestInvocationId(req.meta?.requestId); - const shardPlan = await buildReplayTestShardPlan(req.flags, runnable, skippedCount); + const suiteInvocationId = buildReplayTestInvocationId(request.requestId); + const shardPlan = await buildReplayTestShardPlan( + request.shard ? { kind: request.shard.mode, count: request.shard.count } : undefined, + runnable, + skippedCount, + resolveShardTargets, + ); return { entries, runnable, shardPlan, suiteInvocationId, - suiteArtifactsDir: replayTestSuiteArtifactsDir(req, suiteInvocationId), + suiteArtifactsDir: replayTestSuiteArtifactsDir(request, suiteInvocationId), total: shardPlan?.total ?? entries.length, }; } -function discoverReplayTestSuiteEntries(req: DaemonRequest): ReplayTestEntry[] { +function discoverReplayTestSuiteEntries( + request: ReplayTestSuiteRequest, + discoverSources: ReplayTestDiscoverSources, +): ReplayTestEntry[] { return discoverReplayTestEntries({ - inputs: req.positionals ?? [], - cwd: req.meta?.cwd, - platformFilter: req.flags?.platform, - replayBackend: req.flags?.replayBackend, + inputs: [...request.inputs], + cwd: request.cwd, + platformFilter: request.platformFilter, + discoverSources, }); } @@ -135,16 +175,22 @@ function runnableReplayTestEntries(entries: ReplayTestEntry[]): ReplayTestQueued ); } -function replayTestSuiteArtifactsDir(req: DaemonRequest, suiteInvocationId: string): string { +function replayTestSuiteArtifactsDir( + request: ReplayTestSuiteRequest, + suiteInvocationId: string, +): string { return resolveReplayTestArtifactsDir({ - artifactsDir: typeof req.flags?.artifactsDir === 'string' ? req.flags.artifactsDir : undefined, - cwd: req.meta?.cwd, + artifactsDir: typeof request.artifactsDir === 'string' ? request.artifactsDir : undefined, + cwd: request.cwd, suiteInvocationId, }); } -function emitReplayTestSuiteStart(plan: ReplayTestSuitePlan): void { - emitRequestProgress({ +function emitReplayTestSuiteStart( + plan: ReplayTestSuitePlan, + emitProgress: ReplayTestEmitProgress, +): void { + emitProgress({ type: 'replay-test-suite', status: 'start', total: plan.total, @@ -159,12 +205,13 @@ function emitReplayTestSuiteStart(plan: ReplayTestSuitePlan): void { function emitSkippedReplayTestResults(params: { entries: ReplayTestEntry[]; total: number; + emitProgress: ReplayTestEmitProgress; }): ReplaySuiteTestResult[] { const { entries, total } = params; const results: ReplaySuiteTestResult[] = []; for (const [entryIndex, entry] of entries.entries()) { if (entry.kind !== 'skip') continue; - emitRequestProgress({ + params.emitProgress({ type: 'replay-test', file: entry.path, status: 'skip', @@ -190,10 +237,10 @@ async function runReplayTestShards( suiteInvocationId: string; cwd?: string; requestId?: string; - flags: DaemonRequest['flags']; + request: ReplayTestSuiteRequest; suiteArtifactsDir: string; suiteTotal: number; - } & ReplayTestRuntimeDependencies, + } & ReplayTestExecutionDependencies, ): Promise { const settled = await Promise.allSettled( params.shards.map(async (shard) => await runReplayTestShard({ ...params, shard })), @@ -239,10 +286,10 @@ async function runReplayTestShard( suiteInvocationId: string; cwd?: string; requestId?: string; - flags: DaemonRequest['flags']; + request: ReplayTestSuiteRequest; suiteArtifactsDir: string; suiteTotal: number; - } & ReplayTestRuntimeDependencies, + } & ReplayTestExecutionDependencies, ): Promise { const { shard, sessionName } = params; return await runReplayTestEntries({ @@ -267,10 +314,10 @@ async function runReplayTestEntriesInDiscoveryOrder( suiteInvocationId: string; cwd?: string; requestId?: string; - flags: DaemonRequest['flags']; + request: ReplayTestSuiteRequest; suiteArtifactsDir: string; suiteTotal: number; - } & ReplayTestRuntimeDependencies, + } & ReplayTestExecutionDependencies, ): Promise { const { discoveryEntries, @@ -278,19 +325,23 @@ async function runReplayTestEntriesInDiscoveryOrder( suiteInvocationId, cwd, requestId, - flags, + request, suiteArtifactsDir, suiteTotal, runReplay, cleanupSession, finalizeAttempt, + emitProgress, + isCanceled, + emitDiagnostic, + bindAttemptCancellation, } = params; const results: ReplaySuiteTestResult[] = []; let executed = 0; for (const [entryIndex, entry] of discoveryEntries.entries()) { - if (isRequestCanceled(requestId)) break; + if (isCanceled()) break; if (entry.kind === 'skip') { - emitRequestProgress({ + emitProgress({ type: 'replay-test', file: entry.path, status: 'skip', @@ -315,17 +366,24 @@ async function runReplayTestEntriesInDiscoveryOrder( caseIndex: executed - 1, cwd, requestId, - retries: resolveReplayTestRetries(flags?.retries, entry.metadata.retries), - timeoutMs: resolveReplayTestTimeout(flags?.timeoutMs, entry.metadata.timeoutMs), + retries: resolveReplayTestRetries(request.retries, entry.manifest.attemptDefaults?.retries), + timeoutMs: resolveReplayTestTimeout( + request.timeoutMs, + entry.manifest.attemptDefaults?.timeoutMs, + ), suiteArtifactsDir, suiteIndex: entryIndex + 1, suiteTotal, runReplay, cleanupSession, finalizeAttempt, + emitProgress, + isCanceled, + emitDiagnostic, + bindAttemptCancellation, }); results.push(report.result); - if (shouldStopReplayTestExecution(report, flags, requestId)) break; + if (shouldStopReplayTestExecution(report, request, isCanceled)) break; } return results; } @@ -337,11 +395,11 @@ async function runReplayTestEntries( suiteInvocationId: string; cwd?: string; requestId?: string; - flags: DaemonRequest['flags']; + request: ReplayTestSuiteRequest; suiteArtifactsDir: string; suiteTotal: number; shard?: ReplayTestShardContext; - } & ReplayTestRuntimeDependencies, + } & ReplayTestExecutionDependencies, ): Promise { const { entries, @@ -349,17 +407,21 @@ async function runReplayTestEntries( suiteInvocationId, cwd, requestId, - flags, + request, suiteArtifactsDir, suiteTotal, shard, runReplay, cleanupSession, finalizeAttempt, + emitProgress, + isCanceled, + emitDiagnostic, + bindAttemptCancellation, } = params; const results: ReplaySuiteTestResult[] = []; for (const [entryIndex, queued] of entries.entries()) { - if (isRequestCanceled(requestId)) break; + if (isCanceled()) break; const { entry, suiteIndex } = queued; const report = await runReplayTestCase({ entry, @@ -368,8 +430,11 @@ async function runReplayTestEntries( caseIndex: entryIndex, cwd, requestId, - retries: resolveReplayTestRetries(flags?.retries, entry.metadata.retries), - timeoutMs: resolveReplayTestTimeout(flags?.timeoutMs, entry.metadata.timeoutMs), + retries: resolveReplayTestRetries(request.retries, entry.manifest.attemptDefaults?.retries), + timeoutMs: resolveReplayTestTimeout( + request.timeoutMs, + entry.manifest.attemptDefaults?.timeoutMs, + ), suiteArtifactsDir, suiteIndex, suiteTotal, @@ -377,21 +442,25 @@ async function runReplayTestEntries( runReplay, cleanupSession, finalizeAttempt, + emitProgress, + isCanceled, + emitDiagnostic, + bindAttemptCancellation, }); results.push(report.result); - if (shouldStopReplayTestExecution(report, flags, requestId)) break; + if (shouldStopReplayTestExecution(report, request, isCanceled)) break; } return results; } function shouldStopReplayTestExecution( report: ReplayTestCaseReport, - flags: DaemonRequest['flags'], - requestId: string | undefined, + request: ReplayTestSuiteRequest, + isCanceled: ReplayTestIsCanceled, ): boolean { return ( - isRequestCanceled(requestId) || - (flags?.failFast === true && report.result.status === 'failed') || + isCanceled() || + (request.failFast === true && report.result.status === 'failed') || report.infrastructure ); } diff --git a/packages/replay-test/tsconfig.json b/packages/replay-test/tsconfig.json new file mode 100644 index 0000000000..935c871a4d --- /dev/null +++ b/packages/replay-test/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, + "noEmit": false, + "emitDeclarationOnly": true, + "declaration": true, + "declarationDir": "./dist-types", + "rootDir": "./src" + }, + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c6ff235038..2a17ade5bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,6 +39,9 @@ importers: '@agent-device/provider-webdriver': specifier: workspace:* version: link:packages/provider-webdriver + '@agent-device/replay-test': + specifier: workspace:* + version: link:packages/replay-test '@agent-device/xml': specifier: workspace:* version: link:packages/xml @@ -132,16 +135,25 @@ importers: specifier: workspace:* version: link:../xml + packages/replay-test: + dependencies: + '@agent-device/contracts': + specifier: workspace:* + version: link:../contracts + '@agent-device/kernel': + specifier: workspace:* + version: link:../kernel + packages/xml: {} website: devDependencies: '@callstack/rspress-preset': specifier: ^0.6.6 - version: 0.6.6(@rsbuild/core@2.0.11)(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + version: 0.6.6(@rsbuild/core@2.0.11)(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))(react-dom@19.2.7(react@19.2.7))(react@19.2.7) '@rspress/core': specifier: ^2.0.12 - version: 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2) + version: 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0) packages: @@ -3187,7 +3199,7 @@ snapshots: '@babel/code-frame': 7.29.7 '@babel/generator': 7.29.7 '@babel/helper-compilation-targets': 7.29.7 - '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helpers': 7.29.7 '@babel/parser': 7.29.7 '@babel/template': 7.29.7 @@ -3222,14 +3234,14 @@ snapshots: lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/helper-create-class-features-plugin@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.29.7 - '@babel/helper-member-expression-to-functions': 7.29.7 + '@babel/helper-member-expression-to-functions': 7.29.7(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.29.7 - '@babel/helper-replace-supers': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) - '@babel/helper-skip-transparent-expression-wrappers': 7.29.7 + '@babel/helper-replace-supers': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.29.7(supports-color@7.2.0) '@babel/traverse': 7.29.7(supports-color@7.2.0) semver: 6.3.1 transitivePeerDependencies: @@ -3237,24 +3249,24 @@ snapshots: '@babel/helper-globals@7.29.7': {} - '@babel/helper-member-expression-to-functions@7.29.7': + '@babel/helper-member-expression-to-functions@7.29.7(supports-color@7.2.0)': dependencies: '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-imports@7.29.7': + '@babel/helper-module-imports@7.29.7(supports-color@7.2.0)': dependencies: '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) - '@babel/helper-module-imports': 7.29.7 + '@babel/helper-module-imports': 7.29.7(supports-color@7.2.0) '@babel/helper-validator-identifier': 7.29.7 '@babel/traverse': 7.29.7(supports-color@7.2.0) transitivePeerDependencies: @@ -3266,16 +3278,16 @@ snapshots: '@babel/helper-plugin-utils@7.29.7': {} - '@babel/helper-replace-supers@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/helper-replace-supers@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) - '@babel/helper-member-expression-to-functions': 7.29.7 + '@babel/helper-member-expression-to-functions': 7.29.7(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.29.7 '@babel/traverse': 7.29.7(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/helper-skip-transparent-expression-wrappers@7.29.7': + '@babel/helper-skip-transparent-expression-wrappers@7.29.7(supports-color@7.2.0)': dependencies: '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 @@ -3305,10 +3317,10 @@ snapshots: dependencies: '@babel/types': 7.29.7 - '@babel/plugin-proposal-decorators@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/plugin-proposal-decorators@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) - '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 '@babel/plugin-syntax-decorators': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) transitivePeerDependencies: @@ -3329,7 +3341,7 @@ snapshots: '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-transform-destructuring@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/plugin-transform-destructuring@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 @@ -3337,41 +3349,41 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/plugin-transform-explicit-resource-management@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/plugin-transform-explicit-resource-management@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-transform-destructuring': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/plugin-transform-destructuring': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/plugin-transform-modules-commonjs@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/plugin-transform-modules-commonjs@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) - '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/plugin-transform-typescript@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/plugin-transform-typescript@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.29.7 - '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/helper-skip-transparent-expression-wrappers': 7.29.7 + '@babel/helper-skip-transparent-expression-wrappers': 7.29.7(supports-color@7.2.0) '@babel/plugin-syntax-typescript': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) transitivePeerDependencies: - supports-color - '@babel/preset-typescript@7.28.5(@babel/core@7.29.7(supports-color@7.2.0))': + '@babel/preset-typescript@7.28.5(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 '@babel/helper-validator-option': 7.29.7 '@babel/plugin-syntax-jsx': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) - '@babel/plugin-transform-modules-commonjs': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) - '@babel/plugin-transform-typescript': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/plugin-transform-modules-commonjs': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/plugin-transform-typescript': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -3405,11 +3417,11 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@callstack/rspress-preset@0.6.6(@rsbuild/core@2.0.11)(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + '@callstack/rspress-preset@0.6.6(@rsbuild/core@2.0.11)(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': dependencies: - '@callstack/rspress-theme': 0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2) - '@rspress/plugin-sitemap': 2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2)) + '@callstack/rspress-theme': 0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0) + '@rspress/plugin-sitemap': 2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0)) '@vercel/analytics': 2.0.1(react@19.2.7) rsbuild-plugin-open-graph: 1.1.2(@rsbuild/core@2.0.11) zod: 4.3.6 @@ -3425,9 +3437,9 @@ snapshots: - vue - vue-router - '@callstack/rspress-theme@0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + '@callstack/rspress-theme@0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': dependencies: - '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2) + '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0) react: 19.2.7 react-dom: 19.2.7(react@19.2.7) @@ -3642,7 +3654,7 @@ snapshots: - supports-color - utf-8-validate - '@mdx-js/mdx@3.1.1': + '@mdx-js/mdx@3.1.1(supports-color@7.2.0)': dependencies: '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.5 @@ -3654,14 +3666,14 @@ snapshots: estree-util-is-identifier-name: 3.0.0 estree-util-scope: 1.0.0 estree-walker: 3.0.3 - hast-util-to-jsx-runtime: 2.3.6 + hast-util-to-jsx-runtime: 2.3.6(supports-color@7.2.0) markdown-extensions: 2.0.0 recma-build-jsx: 1.0.0 recma-jsx: 1.0.1(acorn@8.16.0) recma-stringify: 1.0.0 - rehype-recma: 1.0.0 - remark-mdx: 3.1.1 - remark-parse: 11.0.0 + rehype-recma: 1.0.0(supports-color@7.2.0) + remark-mdx: 3.1.1(supports-color@7.2.0) + remark-parse: 11.0.0(supports-color@7.2.0) remark-rehype: 11.1.2 source-map: 0.7.6 unified: 11.0.5 @@ -4060,9 +4072,9 @@ snapshots: optionalDependencies: '@rspack/core': 2.0.6(@swc/helpers@0.5.23) - '@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2)': + '@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@mdx-js/mdx': 3.1.1 + '@mdx-js/mdx': 3.1.1(supports-color@7.2.0) '@mdx-js/react': 3.1.1(@types/react@19.2.13)(react@19.2.7) '@rsbuild/core': 2.0.11 '@rsbuild/plugin-react': 2.0.0(@rsbuild/core@2.0.11)(@rspack/core@2.0.6(@swc/helpers@0.5.23)) @@ -4075,9 +4087,9 @@ snapshots: copy-to-clipboard: 3.3.3 flexsearch: 0.8.212 hast-util-heading-rank: 3.0.0 - hast-util-to-jsx-runtime: 2.3.6 - mdast-util-mdx: 3.0.0 - mdast-util-mdxjs-esm: 2.0.1 + hast-util-to-jsx-runtime: 2.3.6(supports-color@7.2.0) + mdast-util-mdx: 3.0.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) medium-zoom: 1.1.0 nprogress: 0.2.0 react: 19.2.7 @@ -4088,11 +4100,11 @@ snapshots: react-router-dom: 7.18.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7) rehype-external-links: 3.0.0 rehype-raw: 7.0.0 - remark-cjk-friendly: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5) - remark-cjk-friendly-gfm-strikethrough: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5) - remark-gfm: 4.0.1 - remark-mdx: 3.1.1 - remark-parse: 11.0.0 + remark-cjk-friendly: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5) + remark-cjk-friendly-gfm-strikethrough: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5) + remark-gfm: 4.0.1(supports-color@7.2.0) + remark-mdx: 3.1.1(supports-color@7.2.0) + remark-parse: 11.0.0(supports-color@7.2.0) remark-stringify: 11.0.0 scroll-into-view-if-needed: 3.1.0 shiki: 4.0.2 @@ -4110,9 +4122,9 @@ snapshots: - micromark-util-types - supports-color - '@rspress/plugin-sitemap@2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))': + '@rspress/plugin-sitemap@2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))': dependencies: - '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2) + '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0) '@rspress/shared@2.0.12': dependencies: @@ -4222,9 +4234,9 @@ snapshots: '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/generator': 7.29.7 '@babel/parser': 7.29.3 - '@babel/plugin-proposal-decorators': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) - '@babel/plugin-transform-explicit-resource-management': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) - '@babel/preset-typescript': 7.28.5(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/plugin-proposal-decorators': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/plugin-transform-explicit-resource-management': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/preset-typescript': 7.28.5(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@stryker-mutator/api': 9.6.1 '@stryker-mutator/util': 9.6.1 angular-html-parser: 10.4.0 @@ -4855,7 +4867,7 @@ snapshots: web-namespaces: 2.0.1 zwitch: 2.0.4 - hast-util-to-estree@3.1.3: + hast-util-to-estree@3.1.3(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.5 @@ -4865,9 +4877,9 @@ snapshots: estree-util-attach-comments: 3.0.0 estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) property-information: 7.1.0 space-separated-tokens: 2.0.2 style-to-js: 1.1.21 @@ -4890,7 +4902,7 @@ snapshots: stringify-entities: 4.0.4 zwitch: 2.0.4 - hast-util-to-jsx-runtime@2.3.6: + hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/hast': 3.0.4 @@ -4899,9 +4911,9 @@ snapshots: devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) property-information: 7.1.0 space-separated-tokens: 2.0.2 style-to-js: 1.1.21 @@ -5097,14 +5109,14 @@ snapshots: unist-util-is: 6.0.1 unist-util-visit-parents: 6.0.2 - mdast-util-from-markdown@2.0.3: + mdast-util-from-markdown@2.0.3(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 '@types/unist': 3.0.3 decode-named-character-reference: 1.3.0 devlop: 1.1.0 mdast-util-to-string: 4.0.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-util-decode-numeric-character-reference: 2.0.2 micromark-util-decode-string: 2.0.1 micromark-util-normalize-identifier: 2.0.1 @@ -5122,67 +5134,67 @@ snapshots: mdast-util-find-and-replace: 3.0.2 micromark-util-character: 2.1.1 - mdast-util-gfm-footnote@2.1.0: + mdast-util-gfm-footnote@2.1.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 micromark-util-normalize-identifier: 2.0.1 transitivePeerDependencies: - supports-color - mdast-util-gfm-strikethrough@2.0.0: + mdast-util-gfm-strikethrough@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-table@2.0.0: + mdast-util-gfm-table@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 markdown-table: 3.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-task-list-item@2.0.0: + mdast-util-gfm-task-list-item@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm@3.1.0: + mdast-util-gfm@3.1.0(supports-color@7.2.0): dependencies: - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-gfm-autolink-literal: 2.0.1 - mdast-util-gfm-footnote: 2.1.0 - mdast-util-gfm-strikethrough: 2.0.0 - mdast-util-gfm-table: 2.0.0 - mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-gfm-footnote: 2.1.0(supports-color@7.2.0) + mdast-util-gfm-strikethrough: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-table: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-task-list-item: 2.0.0(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-expression@2.0.1: + mdast-util-mdx-expression@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-jsx@3.2.0: + mdast-util-mdx-jsx@3.2.0(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 @@ -5190,7 +5202,7 @@ snapshots: '@types/unist': 3.0.3 ccount: 2.0.1 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 parse-entities: 4.0.2 stringify-entities: 4.0.4 @@ -5199,23 +5211,23 @@ snapshots: transitivePeerDependencies: - supports-color - mdast-util-mdx@3.0.0: + mdast-util-mdx@3.0.0(supports-color@7.2.0): dependencies: - mdast-util-from-markdown: 2.0.3 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdxjs-esm@2.0.1: + mdast-util-mdxjs-esm@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color @@ -5274,11 +5286,11 @@ snapshots: micromark-util-symbol: 2.0.1 micromark-util-types: 2.0.2 - micromark-extension-cjk-friendly-gfm-strikethrough@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2): + micromark-extension-cjk-friendly-gfm-strikethrough@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)): dependencies: devlop: 1.1.0 get-east-asian-width: 1.5.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-extension-cjk-friendly-util: 3.0.1(micromark-util-types@2.0.2) micromark-util-character: 2.1.1 micromark-util-chunked: 2.0.1 @@ -5295,10 +5307,10 @@ snapshots: optionalDependencies: micromark-util-types: 2.0.2 - micromark-extension-cjk-friendly@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2): + micromark-extension-cjk-friendly@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)): dependencies: devlop: 1.1.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-extension-cjk-friendly-util: 3.0.1(micromark-util-types@2.0.2) micromark-util-chunked: 2.0.1 micromark-util-resolve-all: 2.0.1 @@ -5529,7 +5541,7 @@ snapshots: micromark-util-types@2.0.2: {} - micromark@4.0.2: + micromark@4.0.2(supports-color@7.2.0): dependencies: '@types/debug': 4.1.13 debug: 4.4.3(supports-color@7.2.0) @@ -5812,17 +5824,17 @@ snapshots: hast-util-raw: 9.1.0 vfile: 6.0.3 - rehype-recma@1.0.0: + rehype-recma@1.0.0(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/hast': 3.0.4 - hast-util-to-estree: 3.1.3 + hast-util-to-estree: 3.1.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color - remark-cjk-friendly-gfm-strikethrough@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5): + remark-cjk-friendly-gfm-strikethrough@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5): dependencies: - micromark-extension-cjk-friendly-gfm-strikethrough: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2) + micromark-extension-cjk-friendly-gfm-strikethrough: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)) unified: 11.0.5 optionalDependencies: '@types/mdast': 4.0.4 @@ -5830,9 +5842,9 @@ snapshots: - micromark - micromark-util-types - remark-cjk-friendly@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5): + remark-cjk-friendly@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5): dependencies: - micromark-extension-cjk-friendly: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2) + micromark-extension-cjk-friendly: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)) unified: 11.0.5 optionalDependencies: '@types/mdast': 4.0.4 @@ -5840,28 +5852,28 @@ snapshots: - micromark - micromark-util-types - remark-gfm@4.0.1: + remark-gfm@4.0.1(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-gfm: 3.1.0 + mdast-util-gfm: 3.1.0(supports-color@7.2.0) micromark-extension-gfm: 3.0.0 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-stringify: 11.0.0 unified: 11.0.5 transitivePeerDependencies: - supports-color - remark-mdx@3.1.1: + remark-mdx@3.1.1(supports-color@7.2.0): dependencies: - mdast-util-mdx: 3.0.0 + mdast-util-mdx: 3.0.0(supports-color@7.2.0) micromark-extension-mdxjs: 3.0.0 transitivePeerDependencies: - supports-color - remark-parse@11.0.0: + remark-parse@11.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) micromark-util-types: 2.0.2 unified: 11.0.5 transitivePeerDependencies: diff --git a/scripts/layering/daemon-modularity.test.ts b/scripts/layering/daemon-modularity.test.ts index 563dc49bf4..488dec41d7 100644 --- a/scripts/layering/daemon-modularity.test.ts +++ b/scripts/layering/daemon-modularity.test.ts @@ -79,7 +79,10 @@ test('external daemon/types.ts importer membership changes require the baseline test('planned logical modules start with zero forbidden imports', () => { const edges = resolveImportEdges( new Map([ - ['src/replay/test/scheduler.ts', "import type { Device } from '../../platforms/device.ts';"], + [ + 'packages/replay-test/src/internal/scheduler.ts', + "import type { Device } from '../../../../src/platforms/device.ts';", + ], ['src/platforms/device.ts', 'export type Device = { id: string };'], ]), ); @@ -93,11 +96,11 @@ test('replay-test rejects request-global and engine-internal imports', () => { const edges = resolveImportEdges( new Map([ [ - 'src/replay/test/scheduler.ts', + 'packages/replay-test/src/internal/scheduler.ts', [ - "import { emitRequestProgress } from '../../request/progress.ts';", - "import { readReplayScriptMetadata } from '../script.ts';", - "import { parseMaestroProgram } from '../../compat/maestro/program-ir-parser.ts';", + "import { emitRequestProgress } from '../../../../src/request/progress.ts';", + "import { readReplayScriptMetadata } from '../../../../src/replay/script.ts';", + "import { parseMaestroProgram } from '../../../../src/compat/maestro/program-ir-parser.ts';", ].join('\n'), ], ['src/request/progress.ts', 'export function emitRequestProgress() {}'], @@ -117,11 +120,14 @@ test('replay-test rejects request-global and engine-internal imports', () => { ); }); -test('replay-test may still import its own files inside the wider replay engine root', () => { +test('replay-test may still import its own files inside the package', () => { const edges = resolveImportEdges( new Map([ - ['src/replay/test/reporting.ts', "import { spec } from './reporters/spec.ts';"], - ['src/replay/test/reporters/spec.ts', 'export const spec = 1;'], + [ + 'packages/replay-test/src/internal/reporting.ts', + "import { spec } from './reporters/spec.ts';", + ], + ['packages/replay-test/src/internal/reporters/spec.ts', 'export const spec = 1;'], ]), ); diff --git a/scripts/layering/daemon-modularity.ts b/scripts/layering/daemon-modularity.ts index 0b129348fe..e67af83f4a 100644 --- a/scripts/layering/daemon-modularity.ts +++ b/scripts/layering/daemon-modularity.ts @@ -72,7 +72,7 @@ export const LOGICAL_MODULE_POLICIES: readonly LogicalModulePolicy[] = [ // remaining roots are engine internals — reaching into either is how a scheduler quietly // acquires daemon authority or an engine-specific value shape. name: 'replay-test', - roots: ['src/replay/test/'], + roots: ['packages/replay-test/src/'], forbiddenTargetRoots: [ 'src/daemon/', 'src/platforms/', @@ -80,7 +80,7 @@ export const LOGICAL_MODULE_POLICIES: readonly LogicalModulePolicy[] = [ 'src/request/', 'src/replay/', 'src/compat/', - 'src/maestro/', + 'packages/maestro/', 'src/ad-replay/', ], }, @@ -91,7 +91,7 @@ const ENGINE_FILE_PREFIXES = [ 'packages/maestro/src/', 'src/replay/', 'src/daemon/handlers/session-replay', - 'src/daemon/handlers/session-test', + 'packages/replay-test/src/', ] as const; export function checkDaemonModularityRatchets( diff --git a/scripts/layering/model.ts b/scripts/layering/model.ts index b3db75591b..8f6b0b7302 100644 --- a/scripts/layering/model.ts +++ b/scripts/layering/model.ts @@ -36,6 +36,7 @@ const TARGET_DAG_RANK = new Map([ ['platforms', 1], ['recording', 1], ['replay', 1], + ['replay-test', 1], ['request', 1], ['screenshot-diff', 1], ['selectors', 1], diff --git a/src/daemon/handlers/__tests__/session-replay-cancellation.test.ts b/src/daemon/handlers/__tests__/session-replay-cancellation.test.ts new file mode 100644 index 0000000000..401513e503 --- /dev/null +++ b/src/daemon/handlers/__tests__/session-replay-cancellation.test.ts @@ -0,0 +1,79 @@ +import { expect, test } from 'vitest'; +import { + clearRequestCanceled, + getRequestSignal, + isRequestCanceled, + markRequestCanceled, + registerRequestAbort, +} from '../../../request/cancel.ts'; +import { bindReplayTestAttemptCancellation } from '../session-replay.ts'; + +// The daemon half of the replay-test cancellation seam (#1478 P3b). The scheduler only says +// "cancel" and "release"; everything here — registry entries, the parent-abort relay, and +// cleanup — is the adapter's, and used to live inside the attempt runtime. The scheduler's own +// obligation (cancel once on timeout, always release) is pinned in the package's runtime tests. + +test('cancel marks the attempt canceled and release clears the registry entry', () => { + const cancellation = bindReplayTestAttemptCancellation({ attemptId: 'attempt-1' }); + expect(isRequestCanceled('attempt-1')).toBe(false); + + cancellation.cancel(); + expect(isRequestCanceled('attempt-1')).toBe(true); + + cancellation.release(); + expect(isRequestCanceled('attempt-1')).toBe(false); +}); + +test('a parent abort cancels the in-flight attempt so a canceled suite stops', () => { + registerRequestAbort('suite-1'); + const cancellation = bindReplayTestAttemptCancellation({ + attemptId: 'attempt-2', + parentAttemptId: 'suite-1', + }); + + markRequestCanceled('suite-1'); + expect(isRequestCanceled('attempt-2')).toBe(true); + + cancellation.release(); + clearRequestCanceled('suite-1'); +}); + +test('a parent already aborted cancels the attempt at bind time', () => { + registerRequestAbort('suite-2'); + markRequestCanceled('suite-2'); + + const cancellation = bindReplayTestAttemptCancellation({ + attemptId: 'attempt-3', + parentAttemptId: 'suite-2', + }); + expect(isRequestCanceled('attempt-3')).toBe(true); + + cancellation.release(); + clearRequestCanceled('suite-2'); +}); + +test('release detaches the relay so a later parent abort cannot cancel a settled attempt', () => { + registerRequestAbort('suite-3'); + const cancellation = bindReplayTestAttemptCancellation({ + attemptId: 'attempt-4', + parentAttemptId: 'suite-3', + }); + + cancellation.release(); + markRequestCanceled('suite-3'); + expect(isRequestCanceled('attempt-4')).toBe(false); + + clearRequestCanceled('suite-3'); +}); + +test('an attempt that is its own parent binds no relay', () => { + // Binding registers the attempt itself, so there is no separate parent to relay from. + const cancellation = bindReplayTestAttemptCancellation({ + attemptId: 'attempt-5', + parentAttemptId: 'attempt-5', + }); + + expect(getRequestSignal('attempt-5')?.aborted).toBe(false); + cancellation.release(); + expect(isRequestCanceled('attempt-5')).toBe(false); +}); diff --git a/src/daemon/handlers/__tests__/session-test-runner.test.ts b/src/daemon/handlers/__tests__/session-test-runner.test.ts index 0ce55813e6..f56a9c3ede 100644 --- a/src/daemon/handlers/__tests__/session-test-runner.test.ts +++ b/src/daemon/handlers/__tests__/session-test-runner.test.ts @@ -1,5 +1,5 @@ -import { test, expect } from 'vitest'; -import * as fs from 'node:fs'; +import { test, expect, vi } from 'vitest'; +import fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { clearRequestCanceled, markRequestCanceled } from '../../../request/cancel.ts'; @@ -123,26 +123,56 @@ test('test filters replay scripts by context platform and skips untyped files', test('test binds each replay script to its declared platform metadata', async () => { const sessionStore = makeSessionStore(); const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-platforms-')); - fs.writeFileSync(path.join(root, '01-android.ad'), 'context platform=android\nopen "Demo"\n'); - fs.writeFileSync(path.join(root, '02-ios.ad'), 'context platform=ios\nopen "Settings"\n'); + const scripts = ['01-android.ad', '02-ios.ad']; + fs.writeFileSync(path.join(root, scripts[0]!), 'context platform=android\nopen "Demo"\n'); + fs.writeFileSync(path.join(root, scripts[1]!), 'context platform=ios\nopen "Settings"\n'); + + // Directory expansion deliberately PRESERVES filesystem order to match Maestro — only glob + // expansion sorts, and `preserves Maestro directory filesystem order` pins that. So this test + // cannot assume two freshly written files enumerate in creation order; on filesystems where + // they do not, the platform-to-script binding it asserts appears reversed. Pinning the + // enumeration keeps the subject (each script binds to ITS declared platform, and session + // numbering follows discovery order) without depending on the host filesystem. + // Only this suite's own directory is intercepted, and the spy is restored in a `finally`. + // A mock that asserted on its argument, or that leaked past a throw, would fire from inside + // `fs` for whatever else shares the worker — which surfaces as a worker crash with no failed + // test rather than a readable assertion. + const realOpendirSync = fs.opendirSync; + const opendirSync = vi.spyOn(fs, 'opendirSync').mockImplementation(((directory, ...rest) => { + if (directory !== root) return realOpendirSync(directory, ...(rest as [])); + let index = 0; + return { + readSync: () => { + const name = scripts[index++]; + if (!name) return null; + return { name, isDirectory: () => false, isFile: () => true } as fs.Dirent; + }, + closeSync: () => {}, + } as fs.Dir; + }) as typeof fs.opendirSync); const invoked: DaemonRequest[] = []; - const response = await handleSessionCommands({ - req: { - token: 't', - session: 'default', - command: 'test', - positionals: [root], - meta: { cwd: root, requestId: 'suite-platforms' }, - }, - sessionName: 'default', - logPath: path.join(os.tmpdir(), 'daemon.log'), - sessionStore, - invoke: async (req) => { - invoked.push(req); - return { ok: true, data: {} }; - }, - }); + let response; + try { + response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-platforms' }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async (req) => { + invoked.push(req); + return { ok: true, data: {} }; + }, + }); + } finally { + opendirSync.mockRestore(); + } expect(response?.ok).toBeTruthy(); expect(invoked.map((req) => req.flags?.platform)).toEqual(['android', 'ios']); diff --git a/src/daemon/handlers/__tests__/session-test-discovery.test.ts b/src/daemon/handlers/__tests__/session-test-source-discovery.test.ts similarity index 64% rename from src/daemon/handlers/__tests__/session-test-discovery.test.ts rename to src/daemon/handlers/__tests__/session-test-source-discovery.test.ts index f6057a6be1..aaf219ad13 100644 --- a/src/daemon/handlers/__tests__/session-test-discovery.test.ts +++ b/src/daemon/handlers/__tests__/session-test-source-discovery.test.ts @@ -4,16 +4,20 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { discoverReplayTestEntries } from '../session-test-discovery.ts'; +import { buildReplayTestSourceDiscovery } from '../session-test-source-discovery.ts'; -test('discoverReplayTestEntries discovers nested .ad suites through native DFS traversal', () => { +// Path expansion, traversal ordering and format routing are host work (#1478 P3b). These pin +// the inspection capability directly; scheduler filtering policy is pinned in the package. +const discoverSources = (replayBackend?: string) => buildReplayTestSourceDiscovery(replayBackend); + +test('replay-test source discovery discovers nested .ad suites through native DFS traversal', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-')); const nested = path.join(root, 'nested'); fs.mkdirSync(nested, { recursive: true }); fs.writeFileSync(path.join(nested, '02-second.ad'), 'context platform=android\nopen "Second"\n'); fs.writeFileSync(path.join(root, '01-first.ad'), 'context platform=ios\nopen "First"\n'); - const entries = discoverReplayTestEntries({ inputs: [root], cwd: root }); + const entries = discoverSources()({ inputs: [root], cwd: root }); assert.deepEqual( new Set(entries.map((entry) => entry.path)), @@ -21,40 +25,7 @@ test('discoverReplayTestEntries discovers nested .ad suites through native DFS t ); }); -test('discoverReplayTestEntries skips untyped scripts when platform filter is set', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-filter-')); - fs.writeFileSync(path.join(root, '01-untyped.ad'), 'open "Demo"\n'); - fs.writeFileSync(path.join(root, '02-android.ad'), 'context platform=android\nopen "Demo"\n'); - - const entries = discoverReplayTestEntries({ - inputs: [root], - cwd: root, - platformFilter: 'android', - }); - - const untyped = entries.find((entry) => path.basename(entry.path) === '01-untyped.ad'); - const android = entries.find((entry) => path.basename(entry.path) === '02-android.ad'); - assert.equal(untyped?.kind, 'skip'); - assert.equal(android?.kind, 'run'); - if (untyped?.kind === 'skip') { - assert.match(untyped.message, /missing platform metadata for --platform android/); - } -}); - -test('discoverReplayTestEntries rejects empty post-filter suites', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-empty-')); - fs.writeFileSync(path.join(root, '01-ios.ad'), 'context platform=ios\nopen "Settings"\n'); - - assert.throws( - () => discoverReplayTestEntries({ inputs: [root], cwd: root, platformFilter: 'android' }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - error.message === 'No replay tests matched for --platform android.', - ); -}); - -test('discoverReplayTestEntries includes Maestro yaml flows for Maestro test suites', () => { +test('replay-test source discovery includes Maestro yaml flows for Maestro test suites', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-')); fs.writeFileSync( path.join(root, '01-flow.yaml'), @@ -63,28 +34,25 @@ test('discoverReplayTestEntries includes Maestro yaml flows for Maestro test sui fs.writeFileSync(path.join(root, '02-flow.yml'), 'appId: demo\n---\n- launchApp\n'); fs.writeFileSync(path.join(root, '03-flow.ad'), 'open "Demo"\n'); - const entries = discoverReplayTestEntries({ - inputs: [root], - cwd: root, - platformFilter: 'android', - replayBackend: 'maestro', - }); + const entries = discoverSources('maestro')({ inputs: [root], cwd: root }); assert.deepEqual( new Set(entries.map((entry) => path.basename(entry.path))), new Set(['01-flow.yaml', '02-flow.yml', '03-flow.ad']), ); - assert.equal(entries.find((entry) => path.basename(entry.path) === '01-flow.yaml')?.kind, 'run'); - assert.equal(entries.find((entry) => path.basename(entry.path) === '02-flow.yml')?.kind, 'run'); - assert.equal(entries.find((entry) => path.basename(entry.path) === '03-flow.ad')?.kind, 'skip'); + // Maestro sources take their platform from the invocation; a native source in Maestro mode + // is still native and declares none. What a --platform filter then does with each is pinned + // in the package's policy tests. + const platformKind = (basename: string) => + entries.find((entry) => path.basename(entry.path) === basename)?.manifest.device.platform.kind; + assert.equal(platformKind('01-flow.yaml'), 'caller-bound'); + assert.equal(platformKind('02-flow.yml'), 'caller-bound'); + assert.equal(platformKind('03-flow.ad'), 'unspecified'); const namedFlow = entries.find((entry) => path.basename(entry.path) === '01-flow.yaml'); - assert.equal(namedFlow?.kind, 'run'); - if (namedFlow?.kind === 'run') { - assert.equal(namedFlow.title, 'Bottom Tabs - Dynamic'); - } + assert.equal(namedFlow?.manifest.title, 'Bottom Tabs - Dynamic'); }); -test('discoverReplayTestEntries preserves Maestro directory filesystem order', () => { +test('replay-test source discovery preserves Maestro directory filesystem order', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-sort-')); const flowFiles = ['10-legacy.ad', '30-zeta.yaml', '05-compat.ad', '20-beta.yml']; for (const fileName of flowFiles) { @@ -110,11 +78,7 @@ test('discoverReplayTestEntries preserves Maestro directory filesystem order', ( }); try { - const entries = discoverReplayTestEntries({ - inputs: [root], - cwd: root, - replayBackend: 'maestro', - }); + const entries = discoverSources('maestro')({ inputs: [root], cwd: root }); assert.deepEqual( entries.map((entry) => path.basename(entry.path)), @@ -125,7 +89,7 @@ test('discoverReplayTestEntries preserves Maestro directory filesystem order', ( } }); -test('discoverReplayTestEntries preserves Maestro nested directory DFS order', () => { +test('replay-test source discovery preserves Maestro nested directory DFS order', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-dfs-')); const nested = path.join(root, 'nested'); fs.mkdirSync(nested, { recursive: true }); @@ -161,11 +125,7 @@ test('discoverReplayTestEntries preserves Maestro nested directory DFS order', ( }); try { - const entries = discoverReplayTestEntries({ - inputs: [root], - cwd: root, - replayBackend: 'maestro', - }); + const entries = discoverSources('maestro')({ inputs: [root], cwd: root }); assert.deepEqual( entries.map((entry) => path.relative(root, entry.path)), @@ -176,18 +136,14 @@ test('discoverReplayTestEntries preserves Maestro nested directory DFS order', ( } }); -test('discoverReplayTestEntries preserves explicit Maestro file order', () => { +test('replay-test source discovery preserves explicit Maestro file order', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-order-')); const second = path.join(root, '02-second.yaml'); const first = path.join(root, '01-first.yaml'); fs.writeFileSync(first, 'appId: demo\n---\n- launchApp\n'); fs.writeFileSync(second, 'appId: demo\n---\n- launchApp\n'); - const entries = discoverReplayTestEntries({ - inputs: [second, first], - cwd: root, - replayBackend: 'maestro', - }); + const entries = discoverSources('maestro')({ inputs: [second, first], cwd: root }); assert.deepEqual( entries.map((entry) => path.basename(entry.path)), @@ -195,7 +151,7 @@ test('discoverReplayTestEntries preserves explicit Maestro file order', () => { ); }); -test('discoverReplayTestEntries orders Maestro file inputs before expanded flows', () => { +test('replay-test source discovery orders Maestro file inputs before expanded flows', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-files-')); const suite = path.join(root, 'suite'); const globSuite = path.join(root, 'glob-suite'); @@ -206,10 +162,9 @@ test('discoverReplayTestEntries orders Maestro file inputs before expanded flows fs.writeFileSync(path.join(suite, '01-directory.yaml'), 'appId: demo\n---\n- launchApp\n'); fs.writeFileSync(path.join(globSuite, '02-glob.yaml'), 'appId: demo\n---\n- launchApp\n'); - const entries = discoverReplayTestEntries({ + const entries = discoverSources('maestro')({ inputs: [suite, path.join(globSuite, '*.yaml'), explicit], cwd: root, - replayBackend: 'maestro', }); assert.deepEqual( @@ -218,16 +173,15 @@ test('discoverReplayTestEntries orders Maestro file inputs before expanded flows ); }); -test('discoverReplayTestEntries de-duplicates overlapping Maestro file and glob inputs', () => { +test('replay-test source discovery de-duplicates overlapping Maestro file and glob inputs', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-overlap-')); const explicit = path.join(root, '02-explicit.yaml'); fs.writeFileSync(explicit, 'appId: demo\n---\n- launchApp\n'); fs.writeFileSync(path.join(root, '01-expanded.yaml'), 'appId: demo\n---\n- launchApp\n'); - const entries = discoverReplayTestEntries({ + const entries = discoverSources('maestro')({ inputs: [explicit, path.join(root, '*.yaml')], cwd: root, - replayBackend: 'maestro', }); assert.deepEqual( @@ -236,17 +190,16 @@ test('discoverReplayTestEntries de-duplicates overlapping Maestro file and glob ); }); -test('discoverReplayTestEntries sorts mixed Maestro glob matches by YAML-first compatibility order', () => { +test('replay-test source discovery sorts mixed Maestro glob matches by YAML-first compatibility order', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-mixed-glob-')); fs.writeFileSync(path.join(root, '20-zeta.yaml'), 'appId: demo\n---\n- launchApp\n'); fs.writeFileSync(path.join(root, '10-alpha.yml'), 'appId: demo\n---\n- launchApp\n'); fs.writeFileSync(path.join(root, '00-native.ad'), 'open "Demo"\n'); fs.writeFileSync(path.join(root, '30-native.ad'), 'open "Demo"\n'); - const entries = discoverReplayTestEntries({ + const entries = discoverSources('maestro')({ inputs: [path.join(root, '*.{yaml,yml,ad}')], cwd: root, - replayBackend: 'maestro', }); assert.deepEqual( @@ -255,13 +208,13 @@ test('discoverReplayTestEntries sorts mixed Maestro glob matches by YAML-first c ); }); -test('discoverReplayTestEntries rejects YAML without explicit Maestro routing', () => { +test('replay-test source discovery rejects YAML without explicit Maestro routing', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-yaml-route-')); const flowPath = path.join(root, 'flow.yaml'); fs.writeFileSync(flowPath, 'appId: demo\n---\n- launchApp\n'); assert.throws( - () => discoverReplayTestEntries({ inputs: [flowPath], cwd: root }), + () => discoverSources()({ inputs: [flowPath], cwd: root }), (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS' && diff --git a/src/daemon/handlers/session-replay-maestro-observer.ts b/src/daemon/handlers/session-replay-maestro-observer.ts index d5776de6ae..92658549e9 100644 --- a/src/daemon/handlers/session-replay-maestro-observer.ts +++ b/src/daemon/handlers/session-replay-maestro-observer.ts @@ -5,7 +5,7 @@ import type { MaestroFailedAction, } from '@agent-device/maestro'; import { AppError } from '@agent-device/kernel/errors'; -import type { ReplayTestAttemptStepSink } from './session-test-types.ts'; +import type { ReplayTestAttemptStepSink } from '@agent-device/replay-test'; import { stripUndefined } from '../../utils/parsing.ts'; import { appendReplayTraceEvent } from './session-replay-trace.ts'; diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index ce084c9c54..fc100b2d0e 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -28,7 +28,7 @@ import { SessionStore } from '../session-store.ts'; import { errorResponse } from './response.ts'; import { buildReplayBuiltinVars } from './session-replay-vars.ts'; import { createMaestroReplayObserver } from './session-replay-maestro-observer.ts'; -import type { ReplayTestAttemptStepSink } from './session-test-types.ts'; +import type { ReplayTestAttemptStepSink } from '@agent-device/replay-test'; import { buildTypedMaestroReplayErrorResponse, buildTypedMaestroSuccessResponse, diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index ba0f79bd34..2f78e9213d 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -57,7 +57,7 @@ import { } from './session-replay-target-verification.ts'; import { buildReplayBuiltinVars } from './session-replay-vars.ts'; import { runTypedMaestroReplayFile } from './session-replay-maestro-runtime.ts'; -import type { ReplayTestAttemptStep, ReplayTestAttemptStepSink } from './session-test-types.ts'; +import type { ReplayTestAttemptStep, ReplayTestAttemptStepSink } from '@agent-device/replay-test'; import { getRequestSignal } from '../../request/cancel.ts'; /** Per-run invariants for a single replay step (ADR 0012 step 4 verify + dispatch + guard). */ diff --git a/src/daemon/handlers/session-replay-video-recording.ts b/src/daemon/handlers/session-replay-video-recording.ts index a64904e5c0..1682802183 100644 --- a/src/daemon/handlers/session-replay-video-recording.ts +++ b/src/daemon/handlers/session-replay-video-recording.ts @@ -4,7 +4,6 @@ import type { SessionStore } from '../session-store.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { sleep } from '../../utils/timeouts.ts'; import { handleRecordCommand } from './record-trace-recording.ts'; -import { appendReplayTestTimingEvent } from './session-test-runtime.ts'; import { collectReplayActionArtifactPaths } from './session-replay-runtime-artifacts.ts'; import { defaultRecordingPath, @@ -30,12 +29,13 @@ type ReplayTestVideoRecordingParams = { sessionStore: SessionStore; artifactsDir: string | undefined; tracePath: string | undefined; + appendTimingEvent: (event: Record) => void; }; export async function startReplayTestVideoRecordingIfReady( params: ReplayTestVideoRecordingParams, ): Promise { - const { req, sessionName, logPath, sessionStore, artifactsDir, tracePath } = params; + const { req, sessionName, logPath, sessionStore, artifactsDir, appendTimingEvent } = params; if (req.flags?.recordVideo !== true) return undefined; const activeSession = sessionStore.get(sessionName); if (!activeSession || activeSession.recording) return undefined; @@ -44,7 +44,7 @@ export async function startReplayTestVideoRecordingIfReady( const videoPath = artifactsDir ? path.join(artifactsDir, `recording${extension}`) : defaultRecordingPath(activeSession.device.platform); - appendVideoTimingEvent(tracePath, { + appendVideoTimingEvent(appendTimingEvent, { type: 'video_recording_start', session: sessionName, videoPath, @@ -67,7 +67,7 @@ export async function startReplayTestVideoRecordingIfReady( logPath, }); if (!startResponse.ok) { - appendVideoTimingEvent(tracePath, { + appendVideoTimingEvent(appendTimingEvent, { type: 'video_recording_start_failed', session: sessionName, videoPath, @@ -78,7 +78,7 @@ export async function startReplayTestVideoRecordingIfReady( const prerollStartedAt = Date.now(); await sleep(REPLAY_TEST_VIDEO_RECORDING_PREROLL_MS); - appendVideoTimingEvent(tracePath, { + appendVideoTimingEvent(appendTimingEvent, { type: 'video_preroll_done', session: sessionName, durationMs: Date.now() - prerollStartedAt, @@ -97,11 +97,11 @@ export async function finalizeReplayTestVideoRecording( artifactPaths: Set; }, ): Promise { - const { req, sessionName, logPath, sessionStore, tracePath, artifactPaths } = params; + const { req, sessionName, logPath, sessionStore, artifactPaths, appendTimingEvent } = params; if (req.flags?.recordVideo !== true) return undefined; if (!sessionStore.get(sessionName)?.recording) return undefined; - appendVideoTimingEvent(tracePath, { + appendVideoTimingEvent(appendTimingEvent, { type: 'video_tail_start', session: sessionName, requestedDurationMs: REPLAY_TEST_VIDEO_RECORDING_TAIL_MS, @@ -123,7 +123,7 @@ export async function finalizeReplayTestVideoRecording( logPath, }); collectReplayActionArtifactPaths(stopResponse).forEach((entry) => artifactPaths.add(entry)); - appendVideoTimingEvent(tracePath, { + appendVideoTimingEvent(appendTimingEvent, { type: 'video_recording_stop', session: sessionName, ok: stopResponse.ok, @@ -144,11 +144,8 @@ export async function finalizeReplayTestVideoRecording( } function appendVideoTimingEvent( - tracePath: string | undefined, + appendTimingEvent: ReplayTestVideoRecordingParams['appendTimingEvent'], event: Record, ): void { - appendReplayTestTimingEvent(tracePath, { - ...event, - ts: new Date().toISOString(), - }); + appendTimingEvent({ ...event, ts: new Date().toISOString() }); } diff --git a/src/daemon/handlers/session-replay.ts b/src/daemon/handlers/session-replay.ts index 46cac94d1f..73c7dfb815 100644 --- a/src/daemon/handlers/session-replay.ts +++ b/src/daemon/handlers/session-replay.ts @@ -1,13 +1,33 @@ import type { CommandFlags } from '../../core/dispatch.ts'; +import type { ReplayScriptMetadata } from '../../replay/script.ts'; import type { DaemonInvokeFn, DaemonRequest, DaemonResponse } from '../types.ts'; import { SessionStore } from '../session-store.ts'; -import { runReplayTestSuite } from './session-test.ts'; +import { runReplayTestSuite } from '@agent-device/replay-test'; import { handleCloseCommand } from './session-close.ts'; import { runReplayScriptFile } from './session-replay-runtime.ts'; import { collectReplayActionArtifactPaths } from './session-replay-runtime-artifacts.ts'; import { errorResponse } from './response.ts'; -import type { ReplayScriptMetadata } from '../../replay/script.ts'; -import { buildReplayTestShardFlags, type ReplayTestShardContext } from './session-test-sharding.ts'; +import { asAppError } from '@agent-device/kernel/errors'; +import { emitRequestProgress } from '../../request/progress.ts'; +import { + clearRequestCanceled, + getRequestSignal, + isRequestCanceled, + markRequestCanceled, + registerRequestAbort, +} from '../../request/cancel.ts'; +import { emitDiagnostic } from '../../utils/diagnostics.ts'; +import type { + ReplayTestBindAttemptCancellation, + ReplayTestShardContext, + ReplayTestSuiteRequest, +} from '@agent-device/replay-test'; +import { buildReplayTestSourceDiscovery } from './session-test-source-discovery.ts'; +import { + buildReplayTestShardFlags, + buildReplayTestShardTargetResolver, + readReplayTestShardSelection, +} from './session-test-shard-devices.ts'; import { toReplayTestAttemptOutcome, toReplayTestFinalizeFailure } from './session-test-outcome.ts'; import type { LeaseRegistry } from '../lease-registry.ts'; import { @@ -16,6 +36,50 @@ import { startReplayTestVideoRecordingIfReady, } from './session-replay-video-recording.ts'; +/** + * Binds one replay-test attempt to daemon request cancellation (#1478 P3b). + * + * The scheduler owns timeout policy and says only "cancel this attempt" / "release it". Every + * registry interaction — registering the abort, relaying the parent request's abort so a + * canceled suite stops its in-flight attempt, and clearing the entry — is host work and lives + * here, next to the rest of the daemon adapter. + */ +export const bindReplayTestAttemptCancellation: ReplayTestBindAttemptCancellation = ({ + attemptId, + parentAttemptId, +}) => { + registerRequestAbort(attemptId); + const clearParentRelay = relayReplayTestAbortFromParent(attemptId, parentAttemptId); + return { + cancel: () => markRequestCanceled(attemptId), + release: () => { + clearParentRelay(); + clearRequestCanceled(attemptId); + }, + }; +}; + +function relayReplayTestAbortFromParent( + requestId: string, + parentRequestId: string | undefined, +): () => void { + if (!parentRequestId || parentRequestId === requestId) return () => {}; + const parentSignal = getRequestSignal(parentRequestId); + if (!parentSignal) return () => {}; + + const cancelRequest = () => { + markRequestCanceled(requestId); + }; + if (parentSignal.aborted) { + cancelRequest(); + return () => {}; + } + parentSignal.addEventListener('abort', cancelRequest, { once: true }); + return () => { + parentSignal.removeEventListener('abort', cancelRequest); + }; +} + export function buildNestedReplayFlags(params: { parentFlags: CommandFlags | undefined; platform: ReplayScriptMetadata['platform'] | undefined; @@ -92,9 +156,24 @@ export async function handleSessionReplayCommands(params: { 'test does not support --save-script; the agent-supervised repair loop is replay-only. Repair the failing script directly with replay --save-script.', ); } - return await runReplayTestSuite({ - req, - sessionName, + // Translating flags can reject them (mutually exclusive or non-positive shard counts). + // That rejection has always surfaced as an INVALID_ARGS response, so it is caught here + // rather than escaping the handler now that translation happens before the suite runs. + let suiteRequest: ReplayTestSuiteRequest; + try { + suiteRequest = toReplayTestSuiteRequest(req, sessionName); + } catch (err) { + const appErr = asAppError(err); + return errorResponse(appErr.code, appErr.message); + } + const outcome = await runReplayTestSuite({ + request: suiteRequest, + // The host owns the request-global progress sink; the scheduler receives only the + // narrow emit capability (#1478 P3b). + emitProgress: emitRequestProgress, + isCanceled: () => isRequestCanceled(req.meta?.requestId), + emitDiagnostic, + bindAttemptCancellation: bindReplayTestAttemptCancellation, runReplay: async ({ filePath, sessionName: testSessionName, @@ -104,6 +183,7 @@ export async function handleSessionReplayCommands(params: { artifactsDir, artifactPaths, tracePath, + appendTimingEvent, shard, onStep, }) => { @@ -128,6 +208,7 @@ export async function handleSessionReplayCommands(params: { sessionStore, artifactsDir, tracePath, + appendTimingEvent, }; const openLifecycle = buildReplayTestVideoOpenLifecycle(videoRecordingParams); const replayResponse = await runReplayScriptFile({ @@ -169,6 +250,7 @@ export async function handleSessionReplayCommands(params: { artifactPaths, artifactsDir, tracePath, + appendTimingEvent, }) => toReplayTestFinalizeFailure( await finalizeReplayTestVideoRecording({ @@ -178,9 +260,12 @@ export async function handleSessionReplayCommands(params: { sessionStore, artifactsDir, tracePath, + appendTimingEvent, artifactPaths, }), ), + discoverSources: buildReplayTestSourceDiscovery(req.flags?.replayBackend), + resolveShardTargets: buildReplayTestShardTargetResolver(req.flags), cleanupSession: async (testSessionName) => { if (!sessionStore.get(testSessionName)) return; await handleCloseCommand({ @@ -199,7 +284,43 @@ export async function handleSessionReplayCommands(params: { }); }, }); + return outcome.status === 'completed' + ? { ok: true, data: outcome.data } + : errorResponse(outcome.error.code, outcome.error.message); } return null; } + +/** + * Translates a daemon `test` request into the scheduler's neutral request (#1478 P3b). + * + * `replayBackend` is deliberately not carried across: it selects an engine, and it has already + * been applied here when building the source-discovery and shard-target capabilities. + */ +function toReplayTestSuiteRequest(req: DaemonRequest, sessionName: string): ReplayTestSuiteRequest { + const flags = req.flags ?? {}; + const cwd = req.meta?.cwd; + const artifactsDir = stringFlag(flags.artifactsDir); + return { + inputs: req.positionals ?? [], + sessionName, + cwd, + requestId: req.meta?.requestId, + platformFilter: flags.platform, + artifactsDir: + artifactsDir === undefined ? undefined : SessionStore.expandHome(artifactsDir, cwd), + failFast: flags.failFast === true, + retries: numberFlag(flags.retries), + timeoutMs: numberFlag(flags.timeoutMs), + shard: readReplayTestShardSelection(flags), + }; +} + +function numberFlag(value: unknown): number | undefined { + return typeof value === 'number' ? value : undefined; +} + +function stringFlag(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} diff --git a/src/daemon/handlers/session-test-outcome.ts b/src/daemon/handlers/session-test-outcome.ts index d4b68d5a09..64fda0ab9c 100644 --- a/src/daemon/handlers/session-test-outcome.ts +++ b/src/daemon/handlers/session-test-outcome.ts @@ -1,7 +1,7 @@ import { readSnapshotDiagnosticsSummary } from '@agent-device/contracts/capture'; import type { DaemonResponse } from '../types.ts'; import { isReplayInfrastructureFailure } from './session-test-infrastructure.ts'; -import type { ReplayTestAttemptFailed, ReplayTestAttemptOutcome } from './session-test-types.ts'; +import type { ReplayTestAttemptFailed, ReplayTestAttemptOutcome } from '@agent-device/replay-test'; /** * The one place a `DaemonResponse` becomes a neutral replay-test attempt outcome (#1478 P3). diff --git a/src/daemon/handlers/session-test-sharding.ts b/src/daemon/handlers/session-test-shard-devices.ts similarity index 73% rename from src/daemon/handlers/session-test-sharding.ts rename to src/daemon/handlers/session-test-shard-devices.ts index 967802ad32..7156e41d67 100644 --- a/src/daemon/handlers/session-test-sharding.ts +++ b/src/daemon/handlers/session-test-shard-devices.ts @@ -11,47 +11,44 @@ import { } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { CommandFlags } from '../../core/dispatch.ts'; - -export type ReplayTestShardMode = 'all' | 'split'; - -export type ReplayTestShardContext = { - shardIndex: number; - shardCount: number; - device: DeviceInfo; -}; - -export type ReplayTestShardPlan = { - mode: ReplayTestShardMode; - shardCount: number; - total: number; - shards: Array; -}; - -export async function buildReplayTestShardPlan( +import type { + ReplayTestShardMode, + ReplayTestResolveShardTargets, + ReplayTestShardContext, + ReplayTestShardTarget, +} from '@agent-device/replay-test'; + +/** + * The daemon adapter's shard-device binding (#1478 P3b). + * + * Inventory enumeration, allowlists, simulator set paths, explicit `--device` selectors, and + * the too-few-devices error live here. The scheduler decides how many shards exist and which + * entries each one runs; it never enumerates hardware. + */ +export function buildReplayTestShardTargetResolver( flags: CommandFlags | undefined, - runnableEntries: TEntry[], - skippedCount: number, -): Promise | undefined> { - const mode = readReplayTestShardMode(flags); - if (!mode) return undefined; - if (runnableEntries.length === 0) return undefined; - - const devices = await resolveReplayTestShardDevices(flags, mode.count); +): ReplayTestResolveShardTargets { + return async (shardCount) => { + const devices = await resolveReplayTestShardDevices(flags, shardCount); + return devices.map(toReplayTestShardTarget); + }; +} + +function toReplayTestShardTarget(device: DeviceInfo): ReplayTestShardTarget { + // Implicit selection already filters to mobile; an explicit `--device` selector could in + // principle name a web target, which is not a shardable device. Reject it here rather than + // widening the neutral vocabulary to carry a platform the scheduler can never run. + if (device.platform === 'web' || !isMobilePlatform(device)) { + throw new AppError( + 'INVALID_ARGS', + `test sharding requires a mobile device; ${device.id} is ${device.platform}`, + ); + } return { - mode: mode.kind, - shardCount: mode.count, - total: - skippedCount + - (mode.kind === 'all' ? runnableEntries.length * mode.count : runnableEntries.length), - shards: devices.map((device, index) => ({ - shardIndex: index, - shardCount: mode.count, - device, - entries: - mode.kind === 'all' - ? runnableEntries - : runnableEntries.filter((_entry, entryIndex) => entryIndex % mode.count === index), - })), + id: device.id, + name: device.name, + platform: device.platform, + ...(device.target !== undefined ? { target: device.target } : {}), }; } @@ -77,27 +74,6 @@ export function buildReplayTestShardFlags( : { ...base, udid: shard.device.id }; } -function readReplayTestShardMode( - flags: CommandFlags | undefined, -): { kind: ReplayTestShardMode; count: number } | undefined { - const shardAll = readPositiveShardCount(flags?.shardAll, '--shard-all'); - const shardSplit = readPositiveShardCount(flags?.shardSplit, '--shard-split'); - if (shardAll !== undefined && shardSplit !== undefined) { - throw new AppError('INVALID_ARGS', '--shard-all and --shard-split are mutually exclusive'); - } - if (shardAll !== undefined) return { kind: 'all', count: shardAll }; - if (shardSplit !== undefined) return { kind: 'split', count: shardSplit }; - return undefined; -} - -function readPositiveShardCount(value: unknown, flagName: string): number | undefined { - if (value === undefined) return undefined; - if (typeof value !== 'number' || !Number.isInteger(value) || value < 1) { - throw new AppError('INVALID_ARGS', `${flagName} requires a positive integer`); - } - return value; -} - async function resolveReplayTestShardDevices( flags: CommandFlags | undefined, shardCount: number, @@ -197,3 +173,29 @@ function normalizeDeviceName(value: string): string { function compareShardDevices(a: DeviceInfo, b: DeviceInfo): number { return a.id.localeCompare(b.id); } + +/** + * Reads the `--shard-all` / `--shard-split` selection from daemon flags (#1478 P3b). + * + * Flag parsing is host work; the scheduler receives a resolved mode and count. + */ +export function readReplayTestShardSelection( + flags: CommandFlags | undefined, +): { mode: ReplayTestShardMode; count: number } | undefined { + const shardAll = readPositiveShardCount(flags?.shardAll, '--shard-all'); + const shardSplit = readPositiveShardCount(flags?.shardSplit, '--shard-split'); + if (shardAll !== undefined && shardSplit !== undefined) { + throw new AppError('INVALID_ARGS', '--shard-all and --shard-split are mutually exclusive'); + } + if (shardAll !== undefined) return { mode: 'all', count: shardAll }; + if (shardSplit !== undefined) return { mode: 'split', count: shardSplit }; + return undefined; +} + +function readPositiveShardCount(value: unknown, flagName: string): number | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'number' || !Number.isInteger(value) || value < 1) { + throw new AppError('INVALID_ARGS', `${flagName} requires a positive integer`); + } + return value; +} diff --git a/src/daemon/handlers/session-test-source-discovery.ts b/src/daemon/handlers/session-test-source-discovery.ts new file mode 100644 index 0000000000..1d0d3f4254 --- /dev/null +++ b/src/daemon/handlers/session-test-source-discovery.ts @@ -0,0 +1,63 @@ +import fs from 'node:fs'; +import { inspectMaestroFlow } from '@agent-device/maestro'; +import { resolveReplayFormat } from '../../replay/format.ts'; +import { readReplayScriptMetadata } from '../../replay/script.ts'; +import { discoverReplaySourcePaths } from '../replay-source-discovery.ts'; +import type { + ReplayTestDiscoverSources, + ReplayTestManifest, + ReplayTestSource, +} from '@agent-device/replay-test'; + +/** + * The daemon adapter's source-inspection capability (#1478 P3b). + * + * Path expansion, file reading, format routing, and per-engine inspection live here; the + * scheduler receives neutral manifests and keeps discovery policy. + * + * This is the one place that knows a source can be `.ad` or Maestro. That knowledge converts + * into the manifest's platform tag and then disappears: `caller-bound` is what Maestro looks + * like from the scheduler's side, and nothing downstream can recover the format from it. + */ +export function buildReplayTestSourceDiscovery( + replayBackend: string | undefined, +): ReplayTestDiscoverSources { + return ({ inputs, cwd }) => { + const resolvedCwd = cwd ?? process.cwd(); + const filePaths = discoverReplaySourcePaths({ inputs, cwd: resolvedCwd, replayBackend }); + return filePaths.map((filePath) => inspectReplayTestSource(filePath, replayBackend)); + }; +} + +function inspectReplayTestSource( + filePath: string, + replayBackend: string | undefined, +): ReplayTestSource { + const script = fs.readFileSync(filePath, 'utf8'); + const isMaestro = resolveReplayFormat(filePath, replayBackend) === 'maestro'; + const metadata = readReplayScriptMetadata(script); + const manifest: ReplayTestManifest = { + ...(isMaestro ? { title: inspectMaestroFlow(script, filePath).name } : {}), + device: { + // A declared platform wins for either format. Without one, Maestro takes its platform + // from the invocation (`caller-bound`) while a native source has simply declared none + // (`unspecified`) — which is exactly the distinction the old + // `resolveReplayFormat(...) === 'maestro'` branch was making inside the filter. + platform: metadata.platform + ? { kind: 'declared', value: metadata.platform } + : isMaestro + ? { kind: 'caller-bound' } + : { kind: 'unspecified' }, + ...(metadata.target !== undefined ? { target: metadata.target } : {}), + }, + ...(metadata.timeoutMs !== undefined || metadata.retries !== undefined + ? { + attemptDefaults: { + ...(metadata.timeoutMs !== undefined ? { timeoutMs: metadata.timeoutMs } : {}), + ...(metadata.retries !== undefined ? { retries: metadata.retries } : {}), + }, + } + : {}), + }; + return { path: filePath, manifest }; +} diff --git a/src/daemon/handlers/session-test-types.ts b/src/daemon/handlers/session-test-types.ts deleted file mode 100644 index 47d1d22d8f..0000000000 --- a/src/daemon/handlers/session-test-types.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { ReplaySuiteTestFailed } from '@agent-device/contracts/replay'; -import type { SnapshotDiagnosticsSummary } from '@agent-device/contracts/capture'; -import type { ReplayScriptMetadata } from '../../replay/script.ts'; -import type { ReplayTestShardContext } from './session-test-sharding.ts'; - -/** - * One execution step an engine reports while an attempt runs (#1478 P3, finding 1). - * - * Step payloads originate below the attempt boundary, inside engine execution, and used to - * reach the reporter through a request-global `AsyncLocalStorage` seeded per attempt. The - * scheduler now hands each attempt a narrow sink instead, so step progress is an explicit - * per-attempt port with two real adapters (native `.ad` and Maestro) rather than ambient - * request state the scheduler cannot see. - */ -export type ReplayTestAttemptStep = { - index: number; - total: number; - command?: string; - value?: string; -}; - -export type ReplayTestAttemptStepSink = (step: ReplayTestAttemptStep) => void; - -/** - * ADR 0010 error fields exactly as the public suite result publishes them. This is the - * neutral wire error, not `DaemonResponse`: the scheduler never sees a daemon response shape. - */ -export type ReplayTestAttemptError = ReplaySuiteTestFailed['error']; - -export type ReplayTestAttemptPassed = { - status: 'passed'; - replayed: number; - healed: number; - warnings: readonly string[]; - artifactPaths: readonly string[]; - snapshotDiagnostics?: SnapshotDiagnosticsSummary; -}; - -export type ReplayTestAttemptFailed = { - status: 'failed'; - error: ReplayTestAttemptError; - artifactPaths: readonly string[]; - snapshotDiagnostics?: SnapshotDiagnosticsSummary; - /** - * The host's verdict that this failure is environmental (device/runner/boot) rather than a - * test failure, so retrying and continuing the suite cannot help. Classification needs - * platform boot-diagnostic vocabulary, which the scheduler must not import, so the host - * tags the outcome and the scheduler only reads the tag. - */ - infrastructure: boolean; -}; - -/** Every expected attempt state resolves as a tagged outcome; nothing throws across the seam. */ -export type ReplayTestAttemptOutcome = ReplayTestAttemptPassed | ReplayTestAttemptFailed; - -export type ReplayTestRunReplayParams = { - filePath: string; - sessionName: string; - platform?: ReplayScriptMetadata['platform']; - target?: ReplayScriptMetadata['target']; - requestId?: string; - artifactsDir?: string; - artifactPaths?: Set; - tracePath?: string; - shard?: ReplayTestShardContext; - onStep?: ReplayTestAttemptStepSink; -}; - -export type ReplayTestRunReplay = ( - params: ReplayTestRunReplayParams, -) => Promise; - -export type ReplayTestCleanupSession = (sessionName: string) => Promise; - -/** - * Runs after the attempt settles and before cleanup. Returns a failure outcome when - * finalization itself failed, or `undefined` when there was nothing to finalize. - */ -export type ReplayTestFinalizeAttempt = (params: { - sessionName: string; - artifactPaths: Set; - artifactsDir?: string; - tracePath?: string; -}) => Promise; - -export type ReplayTestRuntimeDependencies = { - runReplay: ReplayTestRunReplay; - cleanupSession: ReplayTestCleanupSession; - finalizeAttempt?: ReplayTestFinalizeAttempt; -}; - -/** Neutral failure outcome helper; keeps timeout/unknown construction in one place. */ -export function replayTestAttemptFailure(params: { - error: ReplayTestAttemptError; - artifactPaths?: readonly string[]; - infrastructure?: boolean; - snapshotDiagnostics?: SnapshotDiagnosticsSummary; -}): ReplayTestAttemptFailed { - return { - status: 'failed', - error: params.error, - artifactPaths: params.artifactPaths ?? [], - infrastructure: params.infrastructure ?? false, - ...(params.snapshotDiagnostics ? { snapshotDiagnostics: params.snapshotDiagnostics } : {}), - }; -} diff --git a/test/integration/android-emulator-e2e/live-replay-scenarios.ts b/test/integration/android-emulator-e2e/live-replay-scenarios.ts index b58781086b..3cf082af5b 100644 --- a/test/integration/android-emulator-e2e/live-replay-scenarios.ts +++ b/test/integration/android-emulator-e2e/live-replay-scenarios.ts @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; -import { assertNonEmptyFile } from './live-assertions.ts'; import { type LiveContext, runStep, @@ -14,8 +13,8 @@ import { assertReplayCommands, readReplayCommands, replayAttemptTimeoutMs, - replaySuiteHostTimeoutMs, } from '../live-device-e2e/replay-evidence.ts'; +import { runLiveReplayTestSuite } from '../live-device-e2e/replay-suite.ts'; const C = PUBLIC_COMMANDS; @@ -77,39 +76,13 @@ export async function assertFixtureReplays(context: LiveContext): Promise const checkoutReplay = path.resolve('examples/test-app/replays/checkout-form-android.ad'); const gestureReplay = path.resolve('examples/test-app/replays/gesture-lab-android.ad'); - const junitPath = path.join(context.artifactDir, 'fixture-replays.junit.xml'); - const suiteArtifacts = path.join(context.artifactDir, 'fixture-replays'); - const suite = await runStep( + const { commandsByScript } = await runLiveReplayTestSuite({ context, - 'run Android fixture suite without retries', - [ - 'test', - checkoutReplay, - gestureReplay, - '--artifacts-dir', - suiteArtifacts, - '--report-junit', - junitPath, - ], - { timeoutMs: replaySuiteHostTimeoutMs([checkoutReplay, gestureReplay], 0) }, - ); - assert.equal(suite.json?.data?.failed, 0, JSON.stringify(suite.json)); - assert.equal(suite.json?.data?.passed, 2, JSON.stringify(suite.json)); - assertNonEmptyFile(junitPath, 'fixture JUnit'); - for (const replayPath of [checkoutReplay, gestureReplay]) { - const result = ( - suite.json?.data?.tests as - | Array<{ file?: unknown; replayed?: unknown; status?: unknown }> - | undefined - )?.find((entry) => path.resolve(String(entry.file)) === replayPath); - assert.equal(result?.status, 'passed', JSON.stringify(suite.json)); - assert.equal( - result?.replayed, - readReplayCommands(replayPath).length, - JSON.stringify(suite.json), - ); - } - assertReplayCommands(gestureReplay, readReplayCommands(gestureReplay), [C.gesture]); + runStep, + step: 'run Android fixture suite without retries', + scripts: [checkoutReplay, gestureReplay], + }); + assertReplayCommands(gestureReplay, commandsByScript.get(gestureReplay) ?? [], [C.gesture]); verifyCommand(context, C.test, 'retry-free Android fixture suite emits non-empty JUnit evidence'); verifyNestedReplayCommand( context, diff --git a/test/integration/ios-simulator-e2e/live-replay-scenarios.ts b/test/integration/ios-simulator-e2e/live-replay-scenarios.ts index 3a57193026..cdf39d768a 100644 --- a/test/integration/ios-simulator-e2e/live-replay-scenarios.ts +++ b/test/integration/ios-simulator-e2e/live-replay-scenarios.ts @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; -import { assertNonEmptyFile } from './live-assertions.ts'; import { type LiveContext, runStep, @@ -14,8 +13,8 @@ import { assertReplayCommands, readReplayCommands, replayAttemptTimeoutMs, - replaySuiteHostTimeoutMs, } from '../live-device-e2e/replay-evidence.ts'; +import { runLiveReplayTestSuite } from '../live-device-e2e/replay-suite.ts'; const C = PUBLIC_COMMANDS; @@ -50,47 +49,23 @@ export async function assertFixtureReplays(context: LiveContext): Promise 'replay proved down, bottom footer, up, and top rediscovery in one fixture journey', ); - const junitPath = path.join(context.artifactDir, 'fixture-replays.junit.xml'); - const suiteArtifacts = path.join(context.artifactDir, 'fixture-replays'); const checkoutReplay = path.resolve( 'test/integration/replays/ios/fixture/02-checkout-release.ad', ); const gestureReplay = path.resolve('examples/test-app/replays/gesture-lab.ad'); - const suiteRetries = 2; - const suite = await runStep( + const { commandsByScript } = await runLiveReplayTestSuite({ context, - 'run fixture suite through public test command', - [ - 'test', - checkoutReplay, - gestureReplay, - '--retries', - String(suiteRetries), - '--artifacts-dir', - suiteArtifacts, - '--report-junit', - junitPath, - ], - { - timeoutMs: replaySuiteHostTimeoutMs([checkoutReplay, gestureReplay], suiteRetries), - }, - ); - assert.equal(suite.json?.data?.failed, 0, JSON.stringify(suite.json)); - assert.equal(suite.json?.data?.passed, 2, JSON.stringify(suite.json)); - const suiteTests = Array.isArray(suite.json?.data?.tests) ? suite.json.data.tests : []; + runStep, + step: 'run fixture suite through public test command', + scripts: [checkoutReplay, gestureReplay], + retries: 2, + }); for (const [replayPath, expectedCommands] of [ [checkoutReplay, [C.swipe]], [gestureReplay, [C.gesture]], ] as const) { - const commands = readReplayCommands(replayPath); - const result = suiteTests.find( - (entry: { file?: unknown }) => path.resolve(String(entry.file)) === replayPath, - ); - assert.equal(result?.status, 'passed', JSON.stringify(suite.json)); - assert.equal(result?.replayed, commands.length, JSON.stringify(result)); - assertReplayCommands(replayPath, commands, expectedCommands); + assertReplayCommands(replayPath, commandsByScript.get(replayPath) ?? [], expectedCommands); } - assertNonEmptyFile(junitPath, 'fixture JUnit'); verifyNestedReplayCommand( context, C.gesture, diff --git a/test/integration/live-device-e2e/replay-suite.ts b/test/integration/live-device-e2e/replay-suite.ts new file mode 100644 index 0000000000..a502ccff17 --- /dev/null +++ b/test/integration/live-device-e2e/replay-suite.ts @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { assertNonEmptyFile } from './assertions.ts'; +import { readReplayCommands, replaySuiteHostTimeoutMs } from './replay-evidence.ts'; + +/** + * The live `test`-suite harness, shared by the iOS and Android journeys (#1478 P3). + * + * Both platforms invoked the public `test` command and then re-derived the same value-contract + * assertions by hand: suite totals, per-script status, replay counts, and non-empty JUnit. Those + * are claims about the published suite result, identical on every platform, and they drifted + * apart in small ways — one iterated with `readReplayCommands` inline, the other cast + * `data.tests` at the call site. + * + * This owns exactly that boundary: invocation, JUnit validation, per-script status, and replay + * counts. Everything a platform actually differs on stays with the caller — which scripts run, + * the retry policy, which commands each script is expected to exercise, and the behavioral + * claims recorded as evidence. It takes the caller's `runStep` rather than binding a context + * type, so it is not a platform-configured runner and cannot template a platform's journey. + */ + +type LiveSuiteStepResult = { + json?: { data?: Record } | undefined; +}; + +type LiveSuiteRunStep = ( + context: TContext, + step: string, + args: string[], + stepOptions?: { timeoutMs?: number }, +) => Promise; + +type LiveSuiteTestEntry = { + file?: unknown; + status?: unknown; + replayed?: unknown; +}; + +export type LiveReplaySuiteRun = { + /** The raw step result, for platform-specific assertions the shared contract does not cover. */ + suite: LiveSuiteStepResult; + junitPath: string; + /** Commands read from each script, keyed by resolved path, so callers need not re-read them. */ + commandsByScript: ReadonlyMap; +}; + +export async function runLiveReplayTestSuite(params: { + context: TContext; + runStep: LiveSuiteRunStep; + step: string; + scripts: readonly string[]; + /** Omitted or 0 runs without `--retries`, which is itself a platform claim worth keeping. */ + retries?: number; + artifactName?: string; +}): Promise { + const { context, runStep, step, scripts, retries = 0, artifactName = 'fixture-replays' } = params; + const resolvedScripts = scripts.map((script) => path.resolve(script)); + const junitPath = path.join(context.artifactDir, `${artifactName}.junit.xml`); + const suiteArtifacts = path.join(context.artifactDir, artifactName); + + const suite = await runStep( + context, + step, + [ + 'test', + ...resolvedScripts, + ...(retries > 0 ? ['--retries', String(retries)] : []), + '--artifacts-dir', + suiteArtifacts, + '--report-junit', + junitPath, + ], + { timeoutMs: replaySuiteHostTimeoutMs(resolvedScripts, retries) }, + ); + + const evidence = () => JSON.stringify(suite.json); + assert.equal(suite.json?.data?.failed, 0, evidence()); + assert.equal(suite.json?.data?.passed, resolvedScripts.length, evidence()); + + const tests = Array.isArray(suite.json?.data?.tests) + ? (suite.json.data.tests as LiveSuiteTestEntry[]) + : []; + const commandsByScript = new Map(); + for (const script of resolvedScripts) { + const result = tests.find((entry) => path.resolve(String(entry.file)) === script); + const commands = readReplayCommands(script); + commandsByScript.set(script, commands); + assert.equal(result?.status, 'passed', evidence()); + assert.equal(result?.replayed, commands.length, evidence()); + } + + assertNonEmptyFile(junitPath, 'fixture JUnit'); + return { suite, junitPath, commandsByScript }; +}