feat: add control-plane fallback for direct-to-VM browser routing

When a GET/HEAD request is routed directly to a browser VM and that VM is
unreachable or gone, transparently re-issue the original request to the
control plane exactly once.

The direct-VM middleware now snapshots the original control-plane request
target (URL/Host/Authorization) before rewriting the request to the VM.
After the routed attempt, if the method is idempotent (GET/HEAD) and the
attempt failed as 'VM unreachable / session gone' -- a transport error,
HTTP 502/503/504, or the clean gone signal (404 + X-Kernel-Upstream: gone
+ JSON body code 'browser_gone') -- the request is restored to its
original target (Authorization re-added, jwt query param dropped) and sent
once more to the control plane. The cache sniff/evict in finalizeResponse
runs on the final response.

Never falls back for non-idempotent methods, non-routed requests, normal
success, or ordinary 4xx (incl. a 404 without the gone marker). At most
one fallback attempt -- no looping.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: rgarcia

lib/browserrouting/route_cache.go

@@ -86,6 +86,13 @@ func DirectVMRoutingMiddleware(cache *RouteCache, subresources []string) option.
 		if err != nil {
 			return nil, err
 		}
+
+		// snapshot holds the original control-plane request target so a routed
+		// attempt can be transparently re-issued to the control plane if the VM
+		// turns out to be unreachable/gone. It stays nil for requests that are
+		// not actually rewritten to the VM.
+		var snapshot *originalRequestTarget
+
 		sessionID, subresource, suffix, ok := parseDirectVMPath(req.URL.Path)
 		if ok {
 			if _, ok := allowed[subresource]; ok {
@@ -95,6 +102,11 @@ func DirectVMRoutingMiddleware(cache *RouteCache, subresources []string) option.
 					if err != nil {
 						return nil, err
 					}
+
+					// Capture the original target before mutating the request so
+					// we can restore it for a control-plane fallback.
+					snapshot = snapshotRequestTarget(req)
+
 					req.Header.Del("Authorization")
 					if route.JWT != "" {
 						q := req.URL.Query()
@@ -114,13 +126,157 @@ func DirectVMRoutingMiddleware(cache *RouteCache, subresources []string) option.
 		}
 
 		res, err := next(req)
+
+		// Control-plane fallback: only for requests that were actually routed to
+		// the VM. If the direct-VM attempt failed as "VM unreachable / session
+		// gone" and the method is idempotent (GET/HEAD), re-issue the ORIGINAL
+		// request to the control plane exactly once. We never fall back for
+		// non-idempotent methods (could double-execute side effects), and we
+		// never loop: at most one fallback attempt.
+		if snapshot != nil && shouldFallBackToControlPlane(req.Method, res, err) {
+			// Drain and close any partial VM response body before retrying so we
+			// don't leak the connection.
+			if res != nil && res.Body != nil {
+				_, _ = io.Copy(io.Discard, res.Body)
+				_ = res.Body.Close()
+			}
+
+			// Restore the original control-plane target: URL/Host/Authorization
+			// and drop the VM jwt query param. A body-less GET/HEAD is safe to
+			// resend.
+			snapshot.restore(req)
+
+			res, err = next(req)
+		}
+
 		if err != nil {
 			return res, err
 		}
+		// Cache sniff/evict always runs on the FINAL response (VM or fallback).
 		return finalizeResponse(res, cache, lifecycle)
 	}
 }
 
+// originalRequestTarget captures the parts of a request that the direct-VM
+// rewrite mutates, so the request can be reconstructed for a control-plane
+// fallback.
+type originalRequestTarget struct {
+	scheme        string
+	host          string // URL.Host
+	path          string
+	rawPath       string
+	rawQuery      string
+	reqHost       string // Request.Host
+	authorization []string
+	hadAuthHeader bool
+}
+
+func snapshotRequestTarget(req *http.Request) *originalRequestTarget {
+	snap := &originalRequestTarget{
+		scheme:   req.URL.Scheme,
+		host:     req.URL.Host,
+		path:     req.URL.Path,
+		rawPath:  req.URL.RawPath,
+		rawQuery: req.URL.RawQuery,
+		reqHost:  req.Host,
+	}
+	if values, ok := req.Header["Authorization"]; ok {
+		snap.hadAuthHeader = true
+		snap.authorization = append([]string(nil), values...)
+	}
+	return snap
+}
+
+// restore rewrites req back to the original control-plane target: it restores
+// the URL/Host and Authorization header and drops the jwt query param that the
+// VM rewrite added.
+func (s *originalRequestTarget) restore(req *http.Request) {
+	req.URL.Scheme = s.scheme
+	req.URL.Host = s.host
+	req.URL.Path = s.path
+	req.URL.RawPath = s.rawPath
+	// Restoring the original RawQuery inherently drops the jwt query param, which
+	// only existed in the VM-rewritten request.
+	req.URL.RawQuery = s.rawQuery
+	req.Host = s.reqHost
+
+	if s.hadAuthHeader {
+		req.Header["Authorization"] = append([]string(nil), s.authorization...)
+	} else {
+		req.Header.Del("Authorization")
+	}
+}
+
+// shouldFallBackToControlPlane reports whether a routed direct-VM attempt should
+// be retried against the control plane. It triggers only for idempotent methods
+// (GET/HEAD) when the VM attempt failed as "VM unreachable / session gone":
+//   - a transport/connection error, OR
+//   - HTTP 502/503/504, OR
+//   - the clean gone signal: 404 with header X-Kernel-Upstream: gone and JSON
+//     body {"code":"browser_gone",...}.
+//
+// It does NOT trigger on a normal success or an ordinary 4xx from a live VM
+// (e.g. 400/401/403, or a 404 that is not the gone marker).
+func shouldFallBackToControlPlane(method string, res *http.Response, err error) bool {
+	switch strings.ToUpper(method) {
+	case http.MethodGet, http.MethodHead:
+	default:
+		return false
+	}
+
+	// Transport/network/connection error reaching the VM.
+	if err != nil {
+		return true
+	}
+	if res == nil {
+		return false
+	}
+
+	switch res.StatusCode {
+	case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
+		// 502/503/504 -- today's dead-VM signal ("Upstream not available").
+		return true
+	case http.StatusNotFound:
+		// Future clean gone signal: 404 + X-Kernel-Upstream: gone + browser_gone.
+		return isBrowserGoneResponse(res)
+	default:
+		return false
+	}
+}
+
+// isBrowserGoneResponse detects the clean "browser gone" marker: an
+// X-Kernel-Upstream: gone response header plus a JSON body whose code field is
+// "browser_gone". It buffers and restores the response body so the check is
+// non-destructive for callers that still read it.
+func isBrowserGoneResponse(res *http.Response) bool {
+	if res == nil {
+		return false
+	}
+	if !strings.EqualFold(strings.TrimSpace(res.Header.Get("X-Kernel-Upstream")), "gone") {
+		return false
+	}
+	if res.Body == nil || !isJSONResponse(res.Header) {
+		return false
+	}
+
+	body, err := io.ReadAll(res.Body)
+	_ = res.Body.Close()
+	// Restore the body so the response remains readable regardless of outcome.
+	res.Body = io.NopCloser(bytes.NewReader(body))
+	res.ContentLength = int64(len(body))
+	if err != nil {
+		return false
+	}
+
+	var payload struct {
+		Code string `json:"code"`
+	}
+	if err := json.Unmarshal(body, &payload); err != nil {
+		return false
+	}
+	return payload.Code == "browser_gone"
+}
+
 func parseCacheLifecycle(req *http.Request) (cacheLifecycle, error) {
 	if req == nil || req.URL == nil {
 		return cacheLifecycle{}, nil