test(replay-test): cover failFast, retry exhaustion, plan-prep failure, JUnit escaping

src/daemon/handlers/__tests__/session-test-suite.test.ts

@@ -102,6 +102,88 @@ test('test does not retry infrastructure startup failures and stops the suite',
   expect(tests[0]?.attempts).toBe(1);
 });
 
+test('test --fail-fast stops the suite after the first failure and leaves the rest notRun', async () => {
+  const sessionStore = makeSessionStore();
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-fail-fast-'));
+  const firstPath = path.join(root, '01-first.ad');
+  const secondPath = path.join(root, '02-second.ad');
+  const thirdPath = path.join(root, '03-third.ad');
+  fs.writeFileSync(firstPath, 'context platform=ios\nopen "Demo"\n');
+  fs.writeFileSync(secondPath, 'context platform=ios\nopen "Demo"\n');
+  fs.writeFileSync(thirdPath, 'context platform=ios\nopen "Demo"\n');
+
+  const invoked: DaemonRequest[] = [];
+  const response = await handleSessionCommands({
+    req: {
+      token: 't',
+      session: 'default',
+      command: 'test',
+      // Explicit files (not a directory) so discovery preserves input order — directory scans
+      // use native DFS order, which is not guaranteed to be lexicographic.
+      positionals: [firstPath, secondPath, thirdPath],
+      meta: { cwd: root, requestId: 'suite-fail-fast' },
+      flags: { failFast: true },
+    },
+    sessionName: 'default',
+    logPath: path.join(os.tmpdir(), 'daemon.log'),
+    sessionStore,
+    invoke: async (req) => {
+      invoked.push(req);
+      return {
+        ok: false,
+        error: { code: 'COMMAND_FAILED', message: 'assertion failed' },
+      };
+    },
+  });
+
+  const data = expectOkData(response);
+  expect(invoked.length).toBe(1);
+  expect(data.total).toBe(3);
+  expect(data.executed).toBe(1);
+  expect(data.failed).toBe(1);
+  expect(data.notRun).toBe(2);
+  const tests = data.tests as Array<Record<string, unknown>>;
+  expect(tests).toHaveLength(1);
+  expect(tests[0]?.file).toBe(firstPath);
+});
+
+test('test surfaces a suite-level failure when a source fails to parse', async () => {
+  const sessionStore = makeSessionStore();
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-bad-source-'));
+  // Malformed env directive (missing "="): readReplayScriptMetadata throws during source
+  // discovery, before any entry is runnable. The suite-level try/catch in session-test.ts must
+  // convert that throw into a failed outcome rather than letting it escape the handler.
+  fs.writeFileSync(
+    path.join(root, '01-malformed.ad'),
+    'env BROKEN\ncontext platform=ios\nopen "Demo"\n',
+  );
+
+  const invoked: DaemonRequest[] = [];
+  const response = await handleSessionCommands({
+    req: {
+      token: 't',
+      session: 'default',
+      command: 'test',
+      positionals: [root],
+      meta: { cwd: root, requestId: 'suite-bad-source' },
+      flags: {},
+    },
+    sessionName: 'default',
+    logPath: path.join(os.tmpdir(), 'daemon.log'),
+    sessionStore,
+    invoke: async (req) => {
+      invoked.push(req);
+      return { ok: true, data: { replayed: 1, healed: 0 } };
+    },
+  });
+
+  expect(invoked.length).toBe(0);
+  expect(response?.ok).toBe(false);
+  if (response?.ok !== false) throw new Error('Expected failed daemon response.');
+  expect(response.error.code).toBe('INVALID_ARGS');
+  expect(response.error.message).toMatch(/Invalid env directive on line 1/);
+});
+
 test('test discovers Maestro YAML suites when replay backend is set', async () => {
   const sessionStore = makeSessionStore();
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-maestro-'));
@@ -244,6 +326,43 @@ test('test emits progress when attempts retry and pass', async () => {
   });
 });
 
+test('test stops retrying after maxAttempts when every attempt fails', async () => {
+  const sessionStore = makeSessionStore();
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-retry-exhaust-'));
+  fs.writeFileSync(path.join(root, '01-always-fails.ad'), 'context platform=ios\nopen "Demo"\n');
+
+  let attemptCount = 0;
+  const response = await handleSessionCommands({
+    req: {
+      token: 't',
+      session: 'default',
+      command: 'test',
+      positionals: [root],
+      meta: { cwd: root, requestId: 'suite-retry-exhaust' },
+      flags: { retries: 2 },
+    },
+    sessionName: 'default',
+    logPath: path.join(os.tmpdir(), 'daemon.log'),
+    sessionStore,
+    invoke: async () => {
+      attemptCount += 1;
+      return {
+        ok: false,
+        error: { code: 'COMMAND_FAILED', message: `attempt ${attemptCount} failed` },
+      };
+    },
+  });
+
+  const data = expectOkData(response);
+  // maxAttempts = retries + 1 = 3. The loop bound is attemptIndex <= retries, so a fix-off-by-one
+  // in that bound would run either 2 or 4 attempts instead of exactly 3.
+  expect(attemptCount).toBe(3);
+  expect(data.failed).toBe(1);
+  const tests = data.tests as Array<Record<string, unknown>>;
+  expect(tests[0]?.attempts).toBe(3);
+  expect(tests[0]?.status).toBe('failed');
+});
+
 test('test emits skip progress without synthetic duration', async () => {
   const sessionStore = makeSessionStore();
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-skip-progress-'));

src/replay/test/reporters/__tests__/junit.test.ts

@@ -0,0 +1,109 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { test } from 'vitest';
+import type { ReplaySuiteResult } from '@agent-device/contracts/replay';
+import { parseXmlDocumentSync, type XmlNode } from '@agent-device/xml';
+import { createJunitReplayTestReporter } from '../junit.ts';
+import type { ReplayTestReporterContext } from '../types.ts';
+
+const context: ReplayTestReporterContext = {
+  stdout: { isTTY: false, write() {} },
+  stderr: { isTTY: false, write() {} },
+};
+
+// Characters that are individually significant to an XML parser: `<` opens a tag, `&` starts an
+// entity reference, `"` closes an attribute value, and a raw newline inside an attribute must
+// survive as a literal character rather than breaking the attribute boundary.
+const TRICKY_TITLE = 'Sign in <required> & "quoted"\nsecond line';
+const TRICKY_MESSAGE = 'Expected <button id="ok"> & none found\nsecond line';
+
+function writeSuiteAndParse(suite: ReplaySuiteResult): XmlNode[] {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-junit-reporter-'));
+  const reportPath = path.join(dir, 'report.xml');
+  const reporter = createJunitReplayTestReporter(reportPath);
+  reporter.onSuiteEnd?.(suite, context);
+  const xml = fs.readFileSync(reportPath, 'utf8');
+  return parseXmlDocumentSync(xml);
+}
+
+function findChild(node: XmlNode, name: string): XmlNode | undefined {
+  return node.children.find((child) => child.name === name);
+}
+
+test('buildReplayJunitXml escapes tricky failure title/message and round-trips through the XML parser', () => {
+  const suite: ReplaySuiteResult = {
+    total: 1,
+    executed: 1,
+    passed: 0,
+    failed: 1,
+    skipped: 0,
+    notRun: 0,
+    durationMs: 1500,
+    failures: [],
+    tests: [
+      {
+        file: '/tmp/flows/login.ad',
+        title: TRICKY_TITLE,
+        session: 'default',
+        status: 'failed',
+        durationMs: 1500,
+        attempts: 1,
+        error: { code: 'COMMAND_FAILED', message: TRICKY_MESSAGE },
+      },
+    ],
+  };
+  suite.failures = suite.tests.filter((result) => result.status === 'failed');
+
+  // Well-formed XML: a broken escape would make this throw instead of returning nodes.
+  const nodes = writeSuiteAndParse(suite);
+
+  const testsuites = nodes[0];
+  assert.ok(testsuites);
+  assert.equal(testsuites.name, 'testsuites');
+  const testsuite = findChild(testsuites, 'testsuite');
+  assert.ok(testsuite);
+  const testcase = findChild(testsuite, 'testcase');
+  assert.ok(testcase);
+
+  // Attribute round-trip: the raw title survives escaping into an attribute value and decoding
+  // back out, newline included.
+  assert.equal(testcase.attributes.name, TRICKY_TITLE);
+  assert.equal(testcase.attributes.file, '/tmp/flows/login.ad');
+
+  const failure = findChild(testcase, 'failure');
+  assert.ok(failure);
+  assert.equal(failure.attributes.message, TRICKY_MESSAGE);
+  // The failure body opens with the raw error message (buildFailureDetails' first line).
+  assert.ok(failure.text?.startsWith(TRICKY_MESSAGE));
+});
+
+test('buildReplayJunitXml escapes tricky skip message', () => {
+  const suite: ReplaySuiteResult = {
+    total: 1,
+    executed: 0,
+    passed: 0,
+    failed: 0,
+    skipped: 1,
+    notRun: 0,
+    durationMs: 0,
+    failures: [],
+    tests: [
+      {
+        file: '/tmp/flows/skipped.ad',
+        status: 'skipped',
+        durationMs: 0,
+        reason: 'skipped-by-filter',
+        message: TRICKY_TITLE,
+      },
+    ],
+  };
+
+  const nodes = writeSuiteAndParse(suite);
+  const testcase = findChild(findChild(nodes[0]!, 'testsuite')!, 'testcase');
+  assert.ok(testcase);
+  const skipped = findChild(testcase, 'skipped');
+  assert.ok(skipped);
+  assert.equal(skipped.attributes.message, TRICKY_TITLE);
+});