Audit and harden the test suite

Scripts/test_ci_app_test_runner.py

@@ -97,6 +97,24 @@ def write_ledger(self, directory: Path, rows: list[dict[str, str]]) -> Path:
             handle.write("\t".join(columns) + "\n")
             for row in rows:
                 values = {column: "" for column in columns}
+                values.update(
+                    {
+                        "method_id": f"root/{row['suite']}/{row['method']}",
+                        "target": "root",
+                        "file": "Tests/Fake.swift",
+                        "domain": "Root",
+                        "primary_contract_id": "contract",
+                        "validation_class": "unit",
+                        "layer": "root_swiftpm",
+                        "execution_tier": "fast",
+                        "scenario_count": "1",
+                        "observable_oracle": "oracle",
+                        "failure_risk": "low",
+                        "lifecycle_owner": "owner",
+                        "current_disposition": "retain",
+                        "preserved_scenario_delta": "0",
+                    }
+                )
                 values.update(row)
                 handle.write("\t".join(values[column] for column in columns) + "\n")
         return path
@@ -1379,59 +1397,6 @@ def test_main_rejects_missing_explicit_bundle_name(self) -> None:
         self.assertIn("did not match any built XCTest bundle", output.getvalue())
         run_all_suites.assert_not_called()
 
-    def write_ledger(self, directory: Path, rows: list[dict[str, str]]) -> Path:
-        header = [
-            "method_id",
-            "target",
-            "file",
-            "suite",
-            "method",
-            "domain",
-            "primary_contract_id",
-            "secondary_contract_tags",
-            "validation_class",
-            "layer",
-            "execution_tier",
-            "scenario_count",
-            "fixture_ids",
-            "observable_oracle",
-            "failure_risk",
-            "runtime_seconds",
-            "resource_cost_tags",
-            "shared_state_tags",
-            "lifecycle_owner",
-            "current_disposition",
-            "replacement_method_id",
-            "preserved_scenario_delta",
-            "notes",
-        ]
-        path = directory / "ledger.tsv"
-        lines = ["\t".join(header)]
-        for row in rows:
-            complete = {key: "" for key in header}
-            complete.update(
-                {
-                    "method_id": f"root/{row['suite']}/{row['method']}",
-                    "target": "root",
-                    "file": "Tests/Fake.swift",
-                    "domain": "Root",
-                    "primary_contract_id": "contract",
-                    "validation_class": "unit",
-                    "layer": "root_swiftpm",
-                    "execution_tier": "fast",
-                    "scenario_count": "1",
-                    "observable_oracle": "oracle",
-                    "failure_risk": "low",
-                    "lifecycle_owner": "owner",
-                    "current_disposition": "retain",
-                    "preserved_scenario_delta": "0",
-                }
-            )
-            complete.update(row)
-            lines.append("\t".join(complete[key] for key in header))
-        path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-        return path
-
     def test_plan_selected_suites_uses_runtime_balanced_shard_and_slow_first(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             ledger = self.write_ledger(Path(tmp), [

Tests/RepoPromptTests/AI/CLIProcessRunnerLifecycleTests.swift

@@ -194,29 +194,35 @@ final class CLIProcessRunnerLifecycleTests: XCTestCase {
     }
 
     private static func waitForPIDFile(_ url: URL, timeout: TimeInterval = 3) async throws -> pid_t {
-        let deadline = Date().addingTimeInterval(timeout)
-        while Date() < deadline {
+        var processID: pid_t?
+        try await AsyncTestWait.waitUntil("CLI process PID file", timeout: timeout) {
             if let text = try? String(contentsOf: url, encoding: .utf8),
                let value = Int32(text.trimmingCharacters(in: .whitespacesAndNewlines))
             {
-                return value
+                processID = value
+                return true
             }
-            try? await Task.sleep(for: .milliseconds(20))
+            return false
         }
-        throw CLIProcessRunnerLifecycleTestError.pidFileTimedOut
+        guard let processID else { throw CLIProcessRunnerLifecycleTestError.pidFileTimedOut }
+        return processID
     }
 
     private static func waitUntilProcessGone(_ pid: pid_t, timeout: TimeInterval = 5) async -> Bool {
-        let deadline = Date().addingTimeInterval(timeout)
-        while Date() < deadline {
-            if !processExists(pid) { return true }
-            try? await Task.sleep(for: .milliseconds(20))
+        do {
+            try await AsyncTestWait.waitUntil("CLI process \(pid) to exit", timeout: timeout) {
+                !processExists(pid)
+            }
+            return true
+        } catch {
+            return false
         }
-        return !processExists(pid)
     }
 
     private static func processExists(_ pid: pid_t) -> Bool {
-        if Darwin.kill(pid, 0) == 0 { return true }
+        if Darwin.kill(pid, 0) == 0 {
+            return true
+        }
         return errno == EPERM
     }
 
@@ -276,27 +282,30 @@ private actor ProcessLifecycleRecorder {
     }
 
     func waitForStart() async -> Bool {
-        for _ in 0 ..< 100 {
-            if startedPID != nil { return true }
-            try? await Task.sleep(for: .milliseconds(10))
+        do {
+            try await AsyncTestWait.waitUntil("process lifecycle start", timeout: 1) { await self.startedPID != nil }
+            return true
+        } catch {
+            return false
         }
-        return false
     }
 
     func waitForReadiness() async -> Bool {
-        for _ in 0 ..< 100 {
-            if readinessObserved { return true }
-            try? await Task.sleep(for: .milliseconds(10))
+        do {
+            try await AsyncTestWait.waitUntil("process lifecycle readiness", timeout: 1) { await self.readinessObserved }
+            return true
+        } catch {
+            return false
         }
-        return false
     }
 
     func waitForTermination() async -> Bool {
-        for _ in 0 ..< 300 {
-            if terminatedPID != nil { return true }
-            try? await Task.sleep(for: .milliseconds(10))
+        do {
+            try await AsyncTestWait.waitUntil("process lifecycle termination", timeout: 3) { await self.terminatedPID != nil }
+            return true
+        } catch {
+            return false
         }
-        return false
     }
 
     func snapshot() -> (startedPID: pid_t?, terminatedPID: pid_t?) {

Tests/RepoPromptTests/AI/CodexModelPollingServiceTests.swift

@@ -36,18 +36,14 @@ final class CodexModelPollingServiceTests: XCTestCase {
     }
 
     private func waitUntil(
-        timeout: Duration = .seconds(2),
+        timeout: TimeInterval = 2,
         condition: @escaping @Sendable () async -> Bool
     ) async throws {
-        let clock = ContinuousClock()
-        let deadline = clock.now.advanced(by: timeout)
-        while await !condition() {
-            guard clock.now < deadline else {
-                XCTFail("Timed out waiting for condition")
-                return
-            }
-            try await Task.sleep(for: .milliseconds(10))
-        }
+        try await AsyncTestWait.waitUntil(
+            "Codex model polling condition",
+            timeout: timeout,
+            condition: condition
+        )
     }
 }
 

Tests/RepoPromptTests/AgentMode/AgentModeMCPWaitEpochTests.swift

@@ -687,13 +687,9 @@ final class AgentModeMCPWaitEpochTests: XCTestCase {
     }
 
     private func waitForWaiter(registration: AgentRunSessionStore.Registration) async throws {
-        for _ in 0 ..< 200 {
-            if await AgentRunSessionStore.shared.test_waiterCount(registration: registration) == 1 {
-                return
-            }
-            await Task.yield()
+        try await AsyncTestWait.waitUntil("Agent Run session waiter registration") {
+            await AgentRunSessionStore.shared.test_waiterCount(registration: registration) == 1
         }
-        XCTFail("Timed out waiting for waiter")
     }
 
     private func makeViewModel() -> AgentModeViewModel {
@@ -755,7 +751,9 @@ private actor EpochBeginGate {
     }
 
     func waitUntilPaused() async {
-        if isPaused { return }
+        if isPaused {
+            return
+        }
         await withCheckedContinuation { continuation in
             pauseWaiters.append(continuation)
         }

Tests/RepoPromptTests/AgentMode/AgentModeRunServiceLifecycleTests.swift

@@ -2030,11 +2030,9 @@ final class AgentModeRunServiceLifecycleTests: XCTestCase {
         _ message: String,
         condition: @escaping @MainActor () -> Bool
     ) async throws {
-        for _ in 0 ..< 500 {
-            if condition() { return }
-            try? await Task.sleep(nanoseconds: 1_000_000)
+        try await AsyncTestWait.waitUntil(message, timeout: 0.5) {
+            await MainActor.run { condition() }
         }
-        throw LifecycleTimeoutError(operation: message, timeoutSeconds: 0.5)
     }
 
     func assertOrderedEvents(

Tests/RepoPromptTests/AgentMode/AgentModeStopSubmitTargetTests.swift

@@ -317,12 +317,11 @@ final class AgentModeStopSubmitTargetTests: XCTestCase {
         timeout: TimeInterval = 2,
         _ predicate: @escaping () -> Bool
     ) async throws {
-        let deadline = Date().addingTimeInterval(timeout)
-        while Date() < deadline {
-            if predicate() { return }
-            try await Task.sleep(nanoseconds: 10_000_000)
-        }
-        XCTFail("Timed out waiting for asynchronous Codex submission")
+        try await AsyncTestWait.waitUntil(
+            "asynchronous Codex submission",
+            timeout: timeout,
+            condition: predicate
+        )
     }
 
     func testGuardedFirstSendRejectsReusedSourceTargetBeforeCreatingAnotherDestination() async throws {

Tests/RepoPromptTests/AgentMode/AgentModeViewModelInactiveRefreshTests.swift

@@ -993,7 +993,8 @@ final class AgentModeViewModelInactiveRefreshTests: XCTestCase {
         let owner = viewModel.test_receiveWorkspaceSwitchNotification(workspace)
 
         await viewModel.test_handleWorkspaceSwitch(workspace, owner: owner)
-        await harness.waitForRequestCount(1)
+        let firstRequestStarted = await harness.waitForRequestCount(1)
+        XCTAssertTrue(firstRequestStarted)
         let generationWhileBindingIsInstalled = try XCTUnwrap(viewModel.test_activeSessionIndexRefreshGeneration)
         let restoredSession = try XCTUnwrap(viewModel.session(for: rootTabID, createIfNeeded: true))
         XCTAssertEqual(restoredSession.activeAgentSessionID, rootSessionID)
@@ -1155,7 +1156,8 @@ final class AgentModeViewModelInactiveRefreshTests: XCTestCase {
         let owner = viewModel.test_receiveWorkspaceSwitchNotification(workspace)
 
         await viewModel.test_handleWorkspaceSwitch(workspace, owner: owner)
-        await harness.waitForRequestCount(1)
+        let firstRequestStarted = await harness.waitForRequestCount(1)
+        XCTAssertTrue(firstRequestStarted)
         let originalGeneration = try XCTUnwrap(viewModel.test_activeSessionIndexRefreshGeneration)
         try await waitUntil {
             viewModel.test_ownerValidatedSessionIndex[originalSessionID] != nil
@@ -1166,7 +1168,8 @@ final class AgentModeViewModelInactiveRefreshTests: XCTestCase {
         _ = viewModel.test_installPersistentSessionBinding(sessionID: replacementSessionID, on: session)
         let successorGeneration = try XCTUnwrap(viewModel.test_activeSessionIndexRefreshGeneration)
         XCTAssertGreaterThan(successorGeneration, originalGeneration)
-        await harness.waitForRequestCount(2)
+        let successorRequestStarted = await harness.waitForRequestCount(2)
+        XCTAssertTrue(successorRequestStarted)
         XCTAssertTrue(viewModel.test_ownerValidatedSessionIndex.isEmpty)
 
         await firstGate.release()
@@ -1227,7 +1230,8 @@ final class AgentModeViewModelInactiveRefreshTests: XCTestCase {
         XCTAssertEqual(Set(viewModel.test_ownerValidatedSessionIndex.keys), [rootSessionID, childSessionID])
 
         viewModel.test_refreshSessionListCache(for: workspace)
-        await harness.waitForRequestCount(2)
+        let secondRequestStarted = await harness.waitForRequestCount(2)
+        XCTAssertTrue(secondRequestStarted)
         try await waitUntil {
             viewModel.test_ownerValidatedSessionIndex[rootSessionID]?.savedAt == updatedRootEntry.savedAt
         }
@@ -1673,12 +1677,10 @@ final class AgentModeViewModelInactiveRefreshTests: XCTestCase {
         timeout: TimeInterval = 3,
         _ condition: @escaping @MainActor () -> Bool
     ) async throws {
-        let deadline = Date().addingTimeInterval(timeout)
-        while Date() < deadline {
-            if condition() { return }
-            try await Task.sleep(nanoseconds: 1_000_000)
-        }
-        XCTFail("Timed out waiting for condition")
+        try await AsyncTestWait.waitUntil(
+            "inactive Agent Mode refresh condition",
+            timeout: timeout
+        ) { await MainActor.run { condition() } }
     }
 
     private func makeViewModel() -> AgentModeViewModel {
@@ -1824,9 +1826,14 @@ private actor SidebarIndexStreamHarness {
         return boundSessionIDByTabIDByRequest[requestIndex][tabID]
     }
 
-    func waitForRequestCount(_ expectedCount: Int) async {
-        while requestCount < expectedCount {
-            try? await Task.sleep(nanoseconds: 1_000_000)
+    func waitForRequestCount(_ expectedCount: Int) async -> Bool {
+        do {
+            try await AsyncTestWait.waitUntil("inactive refresh request count \(expectedCount)") {
+                await self.requestCount >= expectedCount
+            }
+            return true
+        } catch {
+            return false
         }
     }
 }