Merge pull request #198 from BjoernSchotte/codex/parallel-browser-evidence

test(extension): parallel browser evidence lanes

Author: BjoernSchotte

.github/workflows/ci.yml

@@ -309,11 +309,11 @@ jobs:
       - name: Sink commit-protocol tests
         run: bun --conditions=development test apps/cli/src/commands/export-pdf-sink.test.ts
 
-  browser-export-harness:
+  browser-export-assets:
     needs: changes
     if: needs.changes.outputs.proofMode == 'required' && needs.changes.outputs.browserHarness == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v7
 
@@ -343,21 +343,12 @@ jobs:
             ${{ runner.os }}-turbo-browser-
             ${{ runner.os }}-turbo-
 
-      - name: Restore Playwright Chromium
-        uses: actions/cache@v6
-        with:
-          path: ~/.cache/ms-playwright
-          key: ${{ runner.os }}-playwright-chromium-1.55.0
-
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
       - name: Provision pinned PDF fonts
         run: bun run fonts:ensure
 
-      - name: Install Playwright Chromium
-        run: bunx playwright@1.55.0 install --with-deps chromium
-
       - name: Type check browser export harness
         run: bun run typecheck:browser-export-harness
 
@@ -370,29 +361,19 @@ jobs:
       - name: Assert conformance-case registry
         run: bun run assert:conformance-cases
 
-      - name: Run browser export harness E2E
-        run: bun run test:browser-export-harness
-
       - name: Build packed MV3 extension once
         run: bun run build:extension
 
-      - name: Verify PDF.js worker in the built MV3 extension
-        run: bun run --cwd apps/extension test:worker-extension-browser:prebuilt
-
-      - name: Verify durable export jobs in the built MV3 extension
-        run: bun run --cwd apps/extension test:jobs-extension-browser:prebuilt
-
-      - name: Verify Research in the built MV3 extension
-        run: bun run --cwd apps/extension test:research-extension-browser:prebuilt
-
-      - name: Verify Rovo visibility in the built MV3 extension
-        run: bun run --cwd apps/extension test:rovo-extension-browser:prebuilt
-
-      - name: Verify action palette in the built MV3 extension
-        run: bun run --cwd apps/extension test:palette-extension-browser:prebuilt
-
-      - name: Shape-parity gate
-        run: bun run check:parity
+      - name: Upload prebuilt browser assets
+        uses: actions/upload-artifact@v7
+        with:
+          name: browser-assets-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            apps/browser-export-harness/dist
+            apps/extension/.output/chrome-mv3
+          if-no-files-found: error
+          include-hidden-files: true
+          retention-days: 1
 
       - name: Sanitize Turbo run summaries
         if: always()
@@ -412,6 +393,98 @@ jobs:
           if-no-files-found: error
           retention-days: 14
 
+  browser-export-harness:
+    name: Browser evidence / ${{ matrix.lane }}
+    needs: [changes, browser-export-assets]
+    if: always() && needs.changes.outputs.proofMode == 'required' && needs.changes.outputs.browserHarness == 'true' && needs.browser-export-assets.result == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        lane: [neutral-palette, research-worker-rovo, jobs]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      ATLCLI_BROWSER_EVIDENCE_ROOT: ${{ github.workspace }}/.artifacts/browser-evidence
+      # Shared GitHub runners record timing but do not enforce local/homelab
+      # latency budgets. Functional, network, long-task, and size gates remain.
+      ATLCLI_BROWSER_ASSERT_TIMING: "0"
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Restore Bun package cache
+        uses: actions/cache@v6
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
+
+      - name: Restore pinned PDF fonts
+        uses: actions/cache@v6
+        with:
+          path: packages/pdf/.fonts
+          key: ${{ runner.os }}-pdf-fonts-v1-${{ hashFiles('packages/pdf/src/runtime-assets.ts', 'packages/pdf/scripts/ensure-fonts.ts') }}
+
+      - name: Restore Playwright Chromium
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-chromium-1.55.0
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Provision pinned PDF fonts
+        run: bun run fonts:ensure
+
+      - name: Install Playwright Chromium
+        run: bunx playwright@1.55.0 install --with-deps chromium
+
+      - name: Download exact prebuilt browser assets
+        uses: actions/download-artifact@v8
+        with:
+          name: browser-assets-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          # upload-artifact stores both inputs relative to their common apps/
+          # root, so restore that root explicitly in each isolated runner.
+          path: apps
+
+      - name: Verify prebuilt browser assets
+        run: |
+          test -d apps/browser-export-harness/dist
+          test -f apps/extension/.output/chrome-mv3/manifest.json
+
+      - name: Run isolated browser lane
+        run: bun scripts/ci/run-browser-lane.ts "${{ matrix.lane }}"
+
+      - name: Validate synthetic browser evidence
+        id: browser-evidence-validation
+        if: always()
+        run: bun scripts/ci/validate-browser-evidence.ts "$ATLCLI_BROWSER_EVIDENCE_ROOT"
+
+      - name: Upload browser evidence summaries
+        if: always() && steps.browser-evidence-validation.outcome == 'success'
+        uses: actions/upload-artifact@v7
+        with:
+          name: browser-evidence-summary-${{ matrix.lane }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .artifacts/browser-evidence/*/junit.xml
+            .artifacts/browser-evidence/*/summary.json
+            .artifacts/browser-evidence/*/manifest.json
+          if-no-files-found: error
+          retention-days: 14
+
+      - name: Upload browser failure evidence
+        if: failure() && steps.browser-evidence-validation.outcome == 'success'
+        uses: actions/upload-artifact@v7
+        with:
+          name: browser-evidence-failures-${{ matrix.lane }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: .artifacts/browser-evidence/*/failures
+          if-no-files-found: error
+          retention-days: 7
+
   # Compatibility signal only. GitHub runner Chrome is intentionally not a
   # substitute for Playwright-matched Chromium in the required MV3 proof.
   browser-system-chrome-canary:

apps/browser-export-harness/package.json

@@ -12,10 +12,12 @@
     "check:output": "bun --conditions=development scripts/check-output.ts",
     "assert:cases": "bun scripts/assert-case-manifest.ts",
     "precheck:parity": "bun run --cwd ../.. fonts:ensure && bun run --cwd ../.. vendor:typst && bun run --cwd ../.. vendor:pdfium",
-    "check:parity": "bun --conditions=development scripts/check-parity.ts",
+    "check:parity": "bun --conditions=development run check:parity:prebuilt",
+    "check:parity:prebuilt": "bun --conditions=development scripts/check-parity.ts",
     "test:unit": "bun --conditions=development test tests/boundaries.test.ts tests/output-scan.test.ts tests/pdf-worker-client.test.ts tests/pdf-job-parity.test.ts tests/docx-job-parity.test.ts scripts/check-parity.test.ts",
     "pretest:e2e": "bun run --cwd ../.. fonts:ensure && bun run --cwd ../.. vendor:typst && bun run --cwd ../.. vendor:pdfium",
-    "test:e2e": "playwright test"
+    "test:e2e": "bun --conditions=development run test:e2e:prebuilt",
+    "test:e2e:prebuilt": "playwright test"
   },
   "dependencies": {
     "@atlcli/code-highlight": "workspace:*",

apps/browser-export-harness/playwright.config.ts

@@ -1,3 +1,4 @@
+import { join, resolve } from "node:path";
 import { defineConfig, devices } from "@playwright/test";
 
 const port = Number(process.env.ATLCLI_HARNESS_PORT ?? "4179");
@@ -9,6 +10,10 @@ const browserChannel = process.env.ATLCLI_PLAYWRIGHT_CHANNEL as
   | "chrome"
   | "chromium"
   | undefined;
+const evidenceDir = resolve(
+  process.env.ATLCLI_BROWSER_EVIDENCE_DIR ??
+    join(import.meta.dirname, "../../.artifacts/browser-evidence/neutral"),
+);
 
 export default defineConfig({
   testDir: "./tests",
@@ -18,12 +23,23 @@ export default defineConfig({
   retries: 0,
   timeout: 120_000,
   expect: { timeout: 60_000 },
-  reporter: "line",
+  reporter: [
+    ["line"],
+    ["junit", { outputFile: join(evidenceDir, "junit.xml") }],
+  ],
+  outputDir: join(evidenceDir, ".playwright"),
   use: {
     ...devices["Desktop Chrome"],
     baseURL: mountUrl,
     headless: true,
-    trace: "retain-on-failure",
+    trace: {
+      mode: "retain-on-failure",
+      screenshots: true,
+      snapshots: true,
+      sources: false,
+    },
+    screenshot: "only-on-failure",
+    video: "retain-on-failure",
   },
   webServer: {
     command: "bun scripts/serve-dist.ts",

apps/extension/README.md

@@ -169,6 +169,43 @@ The repository-level `bun run check:browser-export-harness` and
 `bun run test:browser-export-harness` commands prove the reusable DOCX/PDF package contracts in
 an independent production Vite/Chromium host.
 
+### Parallel packed-browser checks
+
+Build the production artifacts once, then run either the complete local
+orchestrator or one fixed lane:
+
+```bash
+bun run build:browser-export-harness
+bun run build:extension
+
+# Performance-sensitive checks first; long jobs/research checks then overlap.
+ATLCLI_BROWSER_EVIDENCE_ROOT="$PWD/.artifacts/browser-evidence" \
+  bun scripts/ci/run-browser-lanes.ts
+
+# Useful when separate worktrees or homelab workers own separate lanes.
+ATLCLI_BROWSER_EVIDENCE_ROOT="$PWD/.artifacts/browser-evidence-jobs" \
+  bun scripts/ci/run-browser-lane.ts jobs
+```
+
+The fixed lanes are `neutral-palette`, `research-worker-rovo`, and `jobs`.
+Give every concurrently running worktree or worker its own evidence root; the
+default is already worktree-local, but an explicit path makes the ownership
+visible. Local and homelab runs enforce the palette latency budget by default.
+Set `ATLCLI_BROWSER_ASSERT_TIMING=0` only on an uncontrolled shared runner;
+functional, network-isolation, long-task, and bundle-size checks still run.
+
+Each suite writes JUnit, a summary, and a SHA/run/digest-bound manifest. Passed
+suites discard browser media. Failed suites retain an opaque test directory
+with a Playwright trace plus screenshots and videos. GitHub CI builds the
+browser artifacts once and fans the three lanes out to separate runners. It
+uploads evidence only after the synthetic-evidence validator has rejected live
+tenant data, credentials, private paths, unsafe archives, and unexpected files.
+
+Live Atlassian-account checks are intentionally outside GitHub CI and its
+artifact publication path. Run those only on an authorized local or homelab
+worker, keep profiles and captures outside Git, and clean up created tenant
+resources after the run.
+
 ## Design notes
 
 - The compact shell has a scalable product-area switcher. **Publishing** is active;

apps/extension/tests/jobs/packed/job-recovery.e2e.ts

@@ -6,6 +6,7 @@ import { join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { buildDocx, para } from "@atlcli/docx/fixtures";
 import { EXTENSION_ROOT, OUTPUT_DIR } from "../../build-helper.js";
+import { createPackedBrowserEvidence } from "../../support/packed-browser-evidence.js";
 
 const JOB_A = "123e4567-e89b-42d3-a456-426614174000";
 const JOB_B = "223e4567-e89b-42d3-a456-426614174000";
@@ -35,6 +36,7 @@ let baseExtensionDir: string;
 let userDataDir: string;
 let page: Page;
 let packedOffscreen: ChildTargetSession | undefined;
+const browserEvidence = createPackedBrowserEvidence("jobs");
 
 class ChildTargetSession {
   private id = 0;
@@ -355,11 +357,12 @@ async function createPackedPngBytes(): Promise<number[]> {
 }
 
 async function launchPackedBrowser(): Promise<void> {
-  context = await chromium.launchPersistentContext(userDataDir, {
+  context = await chromium.launchPersistentContext(userDataDir, browserEvidence.launchOptions({
     channel: "chromium",
     headless: true,
     args: [`--disable-extensions-except=${baseExtensionDir}`, `--load-extension=${baseExtensionDir}`],
-  });
+  }));
+  await browserEvidence.attachContext(context);
   let serviceWorker = context.serviceWorkers()[0];
   serviceWorker ??= await context.waitForEvent("serviceworker", { timeout: 30_000 });
   extensionId = new URL(serviceWorker.url()).host;
@@ -471,6 +474,10 @@ test.beforeAll(async () => {
   await launchPackedBrowser();
 });
 
+test.beforeEach(async ({}, testInfo) => {
+  await browserEvidence.startTest(testInfo);
+});
+
 test.beforeEach(async () => {
   await packedOffscreen?.close();
   packedOffscreen = undefined;
@@ -508,10 +515,18 @@ test.beforeEach(async () => {
   });
 });
 
+test.afterEach(async ({}, testInfo) => {
+  await browserEvidence.finishTest(testInfo);
+});
+
 test.afterAll(async () => {
-  await packedOffscreen?.close();
-  await context?.close();
-  rmSync(suiteRoot, { recursive: true, force: true });
+  try {
+    await packedOffscreen?.close();
+    if (context) await browserEvidence.closeContext(context);
+  } finally {
+    browserEvidence.finalize();
+    rmSync(suiteRoot, { recursive: true, force: true });
+  }
 });
 
 async function sendWake(ids?: string[]): Promise<{ kind: string; claimedJobId?: string }> {
@@ -2217,7 +2232,7 @@ test("a full persistent-browser restart automatically reclaims checkpointed work
   const originalExtensionId = extensionId;
   await packedOffscreen?.close();
   packedOffscreen = undefined;
-  await context.close();
+  await browserEvidence.closeContext(context);
   installBrowserRestartFetchStub();
   await launchPackedBrowser();
 

apps/extension/tests/jobs/packed/playwright.config.ts

@@ -1,6 +1,7 @@
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { defineConfig } from "@playwright/test";
+import { browserEvidenceSuiteDir } from "../../support/packed-browser-evidence.js";
 
 export default defineConfig({
   testDir: ".",
@@ -9,5 +10,8 @@ export default defineConfig({
   timeout: 180_000,
   workers: 1,
   retries: 0,
-  use: { trace: "retain-on-failure" },
+  reporter: [
+    ["line"],
+    ["junit", { outputFile: join(browserEvidenceSuiteDir("jobs"), "junit.xml") }],
+  ],
 });