feat: opt-in control-plane fallback on browser_gone 404

Rework the routing-layer control-plane fallback to a tight, opt-in design
that pairs with metro-api kernel#2317 (routed requests to a deleted/gone
browser return HTTP 404 with body {"code":"browser_gone"}).

Replaces the previous broad trigger (fall back on any 5xx for any
idempotent GET), which retried the dead VM then fell back, adding latency
on transient errors.

New semantics in routeRequest, applied only when the request was actually
routed to the VM (allowlisted subresource + cached route): fall back IFF
method is GET, the routed (subresource + suffix) path is in the
fallback-eligible registry, and the VM returns HTTP 404 whose JSON body has
code == "browser_gone". On fallback, evict the cached route and re-issue the
ORIGINAL request to the control plane exactly once (original URL, restore
Authorization, drop the jwt query param); return that response, never loop.
Success, transient 5xx, network errors, other 4xx, and non-browser_gone
404s propagate unchanged. The 404 body is read via response.clone() so a
non-fallback response is returned intact.

Per kernel#2317 there is no special response header, so the gone check keys
off the body code only (no content-type gate), matching the python SDK.

Adds an isFallbackEligible(subresource, suffix) predicate backed by a
registry that is default-OFF. Pre-registers only the prospective pull
endpoint GET /browsers/{id}/telemetry/events; adding future eligible
endpoints is a one-line registry edit.

Scoped to the fallback only: does not modify the default routing
subresource list (owned by the telemetry-default-routing PR).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: rgarcia

src/lib/browser-routing.ts

@@ -34,6 +34,56 @@ const BROWSER_POOL_ACQUIRE_PATH = /^\/(?:v\d+\/)?browser_pools\/[^/]+\/acquire\/
 const BROWSER_DELETE_BY_ID_PATH = /^\/(?:v\d+\/)?browsers\/([^/]+)\/?$/;
 const BROWSER_POOL_RELEASE_PATH = /^\/(?:v\d+\/)?browser_pools\/[^/]+\/release\/?$/;
 
+/**
+ * Registry of routed (subresource + suffix) paths that are eligible for
+ * control-plane fallback when the VM reports the browser is authoritatively
+ * gone (HTTP 404 with body code "browser_gone"). Everything not listed here is
+ * fallback-OFF by default.
+ *
+ * Adding a future eligible endpoint is intentionally a one-line edit: append
+ * another `${subresource} ${suffix}` entry below.
+ */
+const FALLBACK_ELIGIBLE_ROUTES = new Set<string>([
+  // PROSPECTIVE: GET /browsers/{id}/telemetry/events. This pull endpoint /
+  // method does not exist on the SDK yet; this entry pre-wires the opt-in so
+  // control-plane fallback works the moment the method ships. Remove this
+  // comment once the method lands.
+  fallbackRouteKey('telemetry', '/events'),
+]);
+
+const BROWSER_GONE_CODE = 'browser_gone';
+
+function fallbackRouteKey(subresource: string, suffix: string): string {
+  return `${subresource} ${suffix}`;
+}
+
+/**
+ * Whether a routed path (parsed subresource + suffix) is opted in to
+ * control-plane fallback. Suffix is the portion after the subresource (e.g.
+ * "/events"), or "" when the request targets the bare subresource.
+ */
+export function isFallbackEligible(subresource: string, suffix: string): boolean {
+  return FALLBACK_ELIGIBLE_ROUTES.has(fallbackRouteKey(subresource, suffix));
+}
+
+async function isBrowserGone404(response: Response): Promise<boolean> {
+  if (response.status !== 404) {
+    return false;
+  }
+  // Key off the body code only (per kernel#2317: there is NO special response
+  // header). We do not gate on content-type so behavior matches the spec and
+  // the python SDK, which simply attempts response.json(). A non-JSON body just
+  // fails to parse and returns false.
+  try {
+    const body = await response.clone().json();
+    return (
+      !!body && typeof body === 'object' && (body as Record<string, unknown>)['code'] === BROWSER_GONE_CODE
+    );
+  } catch {
+    return false;
+  }
+}
+
 export function browserRoutingSubresourcesFromEnv(): string[] {
   const raw = readBrowserRoutingSubresourcesEnv();
   if (raw === undefined) {
@@ -229,6 +279,7 @@ async function routeRequest(
 
   const sessionId = decodeURIComponent(match[1] ?? '');
   const subresource = match[2] ?? '';
+  const suffix = match[3] ?? '';
   if (!sessionId || !allowed.has(subresource)) {
     return innerFetch(input, init);
   }
@@ -237,7 +288,7 @@ async function routeRequest(
     return innerFetch(input, init);
   }
 
-  const target = new URL(joinURL(route.baseURL, `/${subresource}${match[3] ?? ''}`));
+  const target = new URL(joinURL(route.baseURL, `/${subresource}${suffix}`));
   url.searchParams.forEach((value, key) => {
     if (key !== 'jwt') {
       target.searchParams.append(key, value);
@@ -249,7 +300,32 @@ async function routeRequest(
 
   const headers = new Headers(request.headers);
   headers.delete('authorization');
-  return innerFetch(target.toString(), buildRoutedInit(request, init, headers));
+  const response = await innerFetch(target.toString(), buildRoutedInit(request, init, headers));
+
+  // Control-plane fallback: the request was actually routed to the VM, so this
+  // is the only place we attempt it. Fall back IFF the method is GET, the routed
+  // path is opted in, and the VM authoritatively reports the browser is gone
+  // (HTTP 404 with body code "browser_gone"). Everything else — success,
+  // transient 5xx, network errors, other 4xx, or a 404 without that code —
+  // propagates unchanged.
+  const method = request.method.toUpperCase();
+  if (method !== 'GET' || !isFallbackEligible(subresource, suffix)) {
+    return response;
+  }
+  if (!(await isBrowserGone404(response))) {
+    return response;
+  }
+
+  // The browser is authoritatively gone: evict the now-stale route and re-issue
+  // the ORIGINAL request to the control plane exactly once. Restore the original
+  // Authorization (still present on `request.headers`) and drop the jwt query
+  // param so we hit the CP, not the VM. Never loops back through routing.
+  cache.delete(sessionId);
+
+  const cpURL = new URL(request.url);
+  cpURL.searchParams.delete('jwt');
+  const cpHeaders = new Headers(request.headers);
+  return innerFetch(cpURL.toString(), buildRoutedInit(request, init, cpHeaders));
 }
 
 function buildRoutedInit(

tests/lib/browser-routing.test.ts

@@ -4,6 +4,7 @@ import {
   BrowserRouteCache,
   browserRoutingSubresourcesFromEnv,
   createRoutingFetch,
+  isFallbackEligible,
 } from '../../src/lib/browser-routing';
 
 describe('browser routing', () => {
@@ -425,3 +426,256 @@ describe('browser routing', () => {
     });
   });
 });
+
+describe('browser routing control-plane fallback', () => {
+  const ELIGIBLE_PATH = 'https://api.example/browsers/sess-1/telemetry/events';
+  const VM_BASE = 'http://browser-session.test/browser/kernel';
+
+  const browserGone404 = () =>
+    Response.json({ code: 'browser_gone', message: 'browser not found' }, { status: 404 });
+
+  const warmedCache = () => {
+    const cache = new BrowserRouteCache();
+    cache.set({ sessionId: 'sess-1', baseURL: VM_BASE, jwt: 'token-abc' });
+    return cache;
+  };
+
+  // The "telemetry" subresource must be routed; the SUT's default-subresources
+  // constant is out of scope for this PR, so tests enable it explicitly here.
+  const makeFetch = (
+    cache: BrowserRouteCache,
+    inner: (input: any, init?: RequestInit) => Promise<Response>,
+  ) =>
+    createRoutingFetch(inner as any, {
+      apiBaseURL: 'https://api.example/',
+      subresources: ['telemetry'],
+      cache,
+    });
+
+  const urlOf = (input: any) =>
+    typeof input === 'string' ? input
+    : input instanceof URL ? input.toString()
+    : input.url;
+
+  test('isFallbackEligible only matches the pre-registered telemetry/events route', () => {
+    expect(isFallbackEligible('telemetry', '/events')).toBe(true);
+    expect(isFallbackEligible('telemetry', '/stream')).toBe(false);
+    expect(isFallbackEligible('telemetry', '')).toBe(false);
+    expect(isFallbackEligible('curl', '/events')).toBe(false);
+  });
+
+  test('eligible GET + 404 browser_gone falls back to the control plane', async () => {
+    const cache = warmedCache();
+    const calls: Array<{ url: string; method: string; auth: string | null }> = [];
+    const wrapped = makeFetch(cache, async (input, init) => {
+      const url = urlOf(input);
+      const headers = new Headers(init?.headers);
+      calls.push({ url, method: (init?.method ?? 'GET').toUpperCase(), auth: headers.get('authorization') });
+      if (url.startsWith(VM_BASE)) {
+        return browserGone404();
+      }
+      return Response.json({ events: [] }, { status: 200 });
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, {
+      method: 'GET',
+      headers: { authorization: 'Bearer original-key' },
+    });
+
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual({ events: [] });
+    // Exactly one VM attempt + exactly one CP re-issue. No loop.
+    expect(calls).toHaveLength(2);
+    // VM attempt: routed, Authorization stripped, jwt added.
+    expect(calls[0]?.url).toBe(`${VM_BASE}/telemetry/events?jwt=token-abc`);
+    expect(calls[0]?.auth).toBeNull();
+    // CP re-issue: original URL, Authorization restored, jwt dropped.
+    expect(calls[1]?.url).toBe(ELIGIBLE_PATH);
+    expect(calls[1]?.method).toBe('GET');
+    expect(calls[1]?.auth).toBe('Bearer original-key');
+    // Route evicted as authoritatively gone.
+    expect(cache.get('sess-1')).toBeUndefined();
+  });
+
+  test('eligible GET + 404 browser_gone where CP also errors returns CP response, no loop', async () => {
+    const cache = warmedCache();
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      const url = urlOf(input);
+      calls.push(url);
+      if (url.startsWith(VM_BASE)) {
+        return browserGone404();
+      }
+      return new Response('boom', { status: 500, headers: { 'content-type': 'text/plain' } });
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } });
+
+    expect(res.status).toBe(500);
+    expect(await res.text()).toBe('boom');
+    // One VM attempt + one CP re-issue only; the CP 500 is returned as-is.
+    expect(calls).toHaveLength(2);
+    expect(calls[0]).toBe(`${VM_BASE}/telemetry/events?jwt=token-abc`);
+    expect(calls[1]).toBe(ELIGIBLE_PATH);
+  });
+
+  test('not-eligible routed path + 404 browser_gone does NOT fall back', async () => {
+    const cache = warmedCache();
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      calls.push(urlOf(input));
+      return browserGone404();
+    });
+
+    const res = await wrapped('https://api.example/browsers/sess-1/telemetry/stream', {
+      method: 'GET',
+      headers: { authorization: 'Bearer k' },
+    });
+
+    expect(res.status).toBe(404);
+    expect(await res.json()).toEqual({ code: 'browser_gone', message: 'browser not found' });
+    expect(calls).toEqual([`${VM_BASE}/telemetry/stream?jwt=token-abc`]);
+    expect(cache.get('sess-1')).toMatchObject({ sessionId: 'sess-1' });
+  });
+
+  test('eligible GET + transient 502 does NOT fall back (propagated)', async () => {
+    const cache = warmedCache();
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      calls.push(urlOf(input));
+      return new Response('bad gateway', { status: 502, headers: { 'content-type': 'text/plain' } });
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } });
+
+    expect(res.status).toBe(502);
+    expect(calls).toEqual([`${VM_BASE}/telemetry/events?jwt=token-abc`]);
+    expect(cache.get('sess-1')).toMatchObject({ sessionId: 'sess-1' });
+  });
+
+  test('eligible GET + connection error does NOT fall back (propagated)', async () => {
+    const cache = warmedCache();
+    let attempts = 0;
+    const wrapped = makeFetch(cache, async () => {
+      attempts++;
+      throw new TypeError('network down');
+    });
+
+    await expect(
+      wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } }),
+    ).rejects.toThrow(/network down/);
+    expect(attempts).toBe(1);
+    expect(cache.get('sess-1')).toMatchObject({ sessionId: 'sess-1' });
+  });
+
+  test('eligible GET + 200 does NOT fall back', async () => {
+    const cache = warmedCache();
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      calls.push(urlOf(input));
+      return Response.json({ events: [{ seq: 1 }] }, { status: 200 });
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } });
+
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual({ events: [{ seq: 1 }] });
+    expect(calls).toEqual([`${VM_BASE}/telemetry/events?jwt=token-abc`]);
+    expect(cache.get('sess-1')).toMatchObject({ sessionId: 'sess-1' });
+  });
+
+  test('eligible path but POST does NOT fall back even on 404 browser_gone', async () => {
+    const cache = warmedCache();
+    const calls: Array<{ url: string; method: string }> = [];
+    const wrapped = makeFetch(cache, async (input, init) => {
+      calls.push({ url: urlOf(input), method: (init?.method ?? 'GET').toUpperCase() });
+      return browserGone404();
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, {
+      method: 'POST',
+      headers: { authorization: 'Bearer k' },
+      body: '{}',
+    });
+
+    expect(res.status).toBe(404);
+    expect(calls).toHaveLength(1);
+    expect(calls[0]?.method).toBe('POST');
+    expect(cache.get('sess-1')).toMatchObject({ sessionId: 'sess-1' });
+  });
+
+  test('a 404 whose body code is not browser_gone does NOT fall back', async () => {
+    const cache = warmedCache();
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      calls.push(urlOf(input));
+      return Response.json({ code: 'not_found', message: 'no such event' }, { status: 404 });
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } });
+
+    expect(res.status).toBe(404);
+    expect(await res.json()).toEqual({ code: 'not_found', message: 'no such event' });
+    expect(calls).toEqual([`${VM_BASE}/telemetry/events?jwt=token-abc`]);
+    expect(cache.get('sess-1')).toMatchObject({ sessionId: 'sess-1' });
+  });
+
+  test('keys off the body code only: browser_gone 404 without a JSON content-type still falls back', async () => {
+    const cache = warmedCache();
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      const url = urlOf(input);
+      calls.push(url);
+      if (url.startsWith(VM_BASE)) {
+        // Body is JSON but the proxy omitted the application/json content-type.
+        // Per kernel#2317 we key off the body code only, so this must fall back.
+        return new Response(JSON.stringify({ code: 'browser_gone', message: 'gone' }), {
+          status: 404,
+          headers: { 'content-type': 'text/plain' },
+        });
+      }
+      return Response.json({ events: [] }, { status: 200 });
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } });
+
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual({ events: [] });
+    expect(calls).toEqual([`${VM_BASE}/telemetry/events?jwt=token-abc`, ELIGIBLE_PATH]);
+    expect(cache.get('sess-1')).toBeUndefined();
+  });
+
+  test('a non-JSON 404 body does NOT fall back', async () => {
+    const cache = warmedCache();
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      calls.push(urlOf(input));
+      return new Response('plain not found', {
+        status: 404,
+        headers: { 'content-type': 'text/plain' },
+      });
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } });
+
+    expect(res.status).toBe(404);
+    expect(await res.text()).toBe('plain not found');
+    expect(calls).toEqual([`${VM_BASE}/telemetry/events?jwt=token-abc`]);
+    expect(cache.get('sess-1')).toMatchObject({ sessionId: 'sess-1' });
+  });
+
+  test('non-routed request (cache miss) is untouched and never falls back', async () => {
+    const cache = new BrowserRouteCache(); // no route warmed -> not routed to VM
+    const calls: string[] = [];
+    const wrapped = makeFetch(cache, async (input) => {
+      calls.push(urlOf(input));
+      return browserGone404();
+    });
+
+    const res = await wrapped(ELIGIBLE_PATH, { method: 'GET', headers: { authorization: 'Bearer k' } });
+
+    expect(res.status).toBe(404);
+    // Hit the original CP URL directly (not routed to a VM), and no re-issue.
+    expect(calls).toEqual([ELIGIBLE_PATH]);
+  });
+});