mason: classify Dreamer provider outage completions

Recognize invalid near-zero stop completions with zero reasoning as transient provider failures before the manifest parser error is surfaced. Abort a verify run after two identical provider-failure batches while preserving already-banked broad-cycle progress for scheduler retries.

Author: ualtinok

packages/plugin/src/features/magic-context/dreamer/provider-output-failure.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from "bun:test";
+
+import {
+    DreamerProviderOutputFailureError,
+    providerOutputFailureFromInvalidManifest,
+} from "./provider-output-failure";
+
+function assistantCompletion(args: {
+    created: number;
+    output: number;
+    reasoning: number;
+    finish?: string;
+    error?: unknown;
+}) {
+    return {
+        info: {
+            role: "assistant",
+            time: { created: args.created },
+            finish: args.finish ?? "stop",
+            error: args.error ?? null,
+            tokens: { output: args.output, reasoning: args.reasoning },
+        },
+        parts: [{ type: "text", text: "provider text" }],
+    };
+}
+
+describe("providerOutputFailureFromInvalidManifest", () => {
+    it("classifies the latest near-zero no-reasoning completion as a transient provider failure", () => {
+        const messages = [
+            assistantCompletion({ created: 1, output: 8, reasoning: 0 }),
+            assistantCompletion({ created: 2, output: 8, reasoning: 0 }),
+            assistantCompletion({ created: 3, output: 8, reasoning: 0 }),
+        ];
+
+        const failure = providerOutputFailureFromInvalidManifest(
+            messages,
+            "All Antigravity endpoints failed",
+        );
+
+        expect(failure).toBeInstanceOf(DreamerProviderOutputFailureError);
+        expect(failure?.transient).toBe(true);
+        expect(failure?.outputTokens).toBe(8);
+        expect(failure?.reasoningTokens).toBe(0);
+        expect(failure?.message).toContain("provider-outage completion");
+        expect(failure?.message).not.toContain("manifest missing");
+    });
+
+    it("requires the complete outage token shape instead of matching response wording", () => {
+        const responseText = "All Antigravity endpoints failed";
+
+        expect(
+            providerOutputFailureFromInvalidManifest(
+                [assistantCompletion({ created: 1, output: 33, reasoning: 0 })],
+                responseText,
+            ),
+        ).toBeNull();
+        expect(
+            providerOutputFailureFromInvalidManifest(
+                [assistantCompletion({ created: 1, output: 8, reasoning: 1 })],
+                responseText,
+            ),
+        ).toBeNull();
+        expect(
+            providerOutputFailureFromInvalidManifest(
+                [assistantCompletion({ created: 1, output: 8, reasoning: 0, finish: "length" })],
+                responseText,
+            ),
+        ).toBeNull();
+        expect(
+            providerOutputFailureFromInvalidManifest(
+                [
+                    assistantCompletion({
+                        created: 1,
+                        output: 8,
+                        reasoning: 0,
+                        error: { name: "ProviderError" },
+                    }),
+                ],
+                responseText,
+            ),
+        ).toBeNull();
+    });
+});

packages/plugin/src/features/magic-context/dreamer/provider-output-failure.ts

@@ -0,0 +1,100 @@
+import { createHash } from "node:crypto";
+
+import { isRecord } from "../../../shared/record-type-guard";
+
+const MAX_NEAR_ZERO_OUTPUT_TOKENS = 32;
+
+interface AssistantCompletionShape {
+    createdAt: number;
+    finish: string | null;
+    error: unknown;
+    outputTokens: number | null;
+    reasoningTokens: number | null;
+}
+
+/** A provider transport failure that arrived as an ordinary assistant completion. */
+export class DreamerProviderOutputFailureError extends Error {
+    readonly transient = true;
+
+    constructor(
+        readonly fingerprint: string,
+        readonly outputTokens: number,
+        readonly reasoningTokens: number,
+        responseText: string,
+    ) {
+        const preview = responseText.trim().replace(/\s+/g, " ").slice(0, 160);
+        super(
+            `verify provider-outage completion (output_tokens=${outputTokens}, reasoning_tokens=${reasoningTokens}): ${JSON.stringify(preview)}`,
+        );
+        this.name = "DreamerProviderOutputFailureError";
+    }
+}
+
+function finiteTokenCount(value: unknown): number | null {
+    return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function completionShape(value: unknown): AssistantCompletionShape | null {
+    if (!isRecord(value)) return null;
+    const info = isRecord(value.info) ? value.info : value;
+    if (info.role !== "assistant") return null;
+
+    const time = isRecord(info.time) ? info.time : null;
+    const tokens = isRecord(info.tokens) ? info.tokens : null;
+    return {
+        createdAt: typeof time?.created === "number" ? time.created : 0,
+        finish:
+            typeof info.finish === "string"
+                ? info.finish
+                : typeof info.finish_reason === "string"
+                  ? info.finish_reason
+                  : typeof info.finishReason === "string"
+                    ? info.finishReason
+                    : null,
+        error: info.error,
+        outputTokens: finiteTokenCount(tokens?.output),
+        reasoningTokens: finiteTokenCount(tokens?.reasoning),
+    };
+}
+
+function latestAssistantCompletion(messages: unknown): AssistantCompletionShape | null {
+    if (!Array.isArray(messages)) return null;
+    let latest: AssistantCompletionShape | null = null;
+    for (const message of messages) {
+        const completion = completionShape(message);
+        if (completion && (!latest || completion.createdAt >= latest.createdAt))
+            latest = completion;
+    }
+    return latest;
+}
+
+/**
+ * OpenCode can serialize a provider outage as a successful `finish=stop` assistant
+ * message. Only classify that shape after manifest validation has already failed:
+ * a real manifest remains authoritative regardless of its token counts.
+ */
+export function providerOutputFailureFromInvalidManifest(
+    messages: unknown,
+    responseText: string,
+): DreamerProviderOutputFailureError | null {
+    const completion = latestAssistantCompletion(messages);
+    if (completion?.finish?.toLowerCase() !== "stop") return null;
+    if (
+        completion.error != null ||
+        completion.outputTokens === null ||
+        completion.outputTokens > MAX_NEAR_ZERO_OUTPUT_TOKENS ||
+        completion.reasoningTokens !== 0
+    ) {
+        return null;
+    }
+
+    const normalized = responseText.trim().replace(/\s+/g, " ").toLowerCase();
+    if (!normalized) return null;
+    const fingerprint = createHash("sha256").update(normalized).digest("hex").slice(0, 16);
+    return new DreamerProviderOutputFailureError(
+        fingerprint,
+        completion.outputTokens,
+        completion.reasoningTokens,
+        responseText,
+    );
+}

packages/plugin/src/features/magic-context/dreamer/task-executor.test.ts

@@ -46,6 +46,21 @@ function assistantMessages(text: string) {
     ];
 }
 
+function providerFailureMessages(text: string) {
+    return [
+        {
+            info: {
+                role: "assistant",
+                time: { created: Date.now() },
+                finish: "stop",
+                error: null,
+                tokens: { output: 8, reasoning: 0 },
+            },
+            parts: [{ type: "text", text }],
+        },
+    ];
+}
+
 describe("createDreamTaskExecutor — curate", () => {
     test("runs whole-pool curation without verification gate or watermark patch", async () => {
         db = freshDb();
@@ -234,6 +249,58 @@ describe("createDreamTaskExecutor — verify-broad disposition", () => {
         expect(task.backlog).toMatchObject({ pendingAtStart: 51, pendingAtEnd: 1, processed: 50 });
     });
 
+    test("surfaces provider-outage completions as transient task failures", async () => {
+        db = freshDb();
+        const project = "/repo/verify-broad-provider-outage";
+        seedTaskScheduleState(db, project, "verify-broad", null, null, "0 3 * * 0");
+        const memory = insertMemory(db, {
+            projectPath: project,
+            category: "ARCHITECTURE",
+            content: "Mapped fact blocked by a provider outage.",
+        });
+        recordMemoryVerifications(db, memory.id, ["src/fact.ts"], 1_000);
+        const client = {
+            session: {
+                list: mock(async () => ({ data: [] })),
+                create: mock(async () => ({ data: { id: "verify-provider-outage" } })),
+                prompt: mock(async () => ({})),
+                messages: mock(async () => ({
+                    data: providerFailureMessages("All Antigravity endpoints failed"),
+                })),
+                delete: mock(async () => ({})),
+            },
+        };
+        const executor = createDreamTaskExecutor({
+            client: client as never,
+            sessionDirectory: project,
+            openOpenCodeDb: () => null,
+        });
+        const leaseKey = leaseKeyFor("verify-broad", project);
+        expect(acquireLease(db, "holder-broad-provider-outage", leaseKey)).toBe(true);
+
+        const result = await executor(
+            { task: "verify-broad", schedule: "0 3 * * 0", timeoutMinutes: 20 },
+            {
+                db,
+                projectIdentity: project,
+                holderId: "holder-broad-provider-outage",
+                leaseKey,
+            },
+        );
+
+        expect(result.status).toBe("failed");
+        expect(result.transient).toBe(true);
+        expect(result.error).toContain("provider-outage completion");
+        expect(result.error).not.toContain("manifest missing");
+        expect(getTaskScheduleState(db, project, "verify-broad")?.lastBroadRunAt).toBeGreaterThan(
+            0,
+        );
+        const run = getDreamRuns(db, project)[0];
+        expect(run?.tasks_failed).toBe(1);
+        const task = JSON.parse(run?.tasks_json ?? "[]")[0] as { error?: string };
+        expect(task.error).toContain("provider-outage completion");
+    });
+
     test("keeps a zero-progress broad run failed", async () => {
         db = freshDb();
         const project = "/repo/verify-broad-zero";