REMOTE-2931: carry HTTP status on public-API errors so deterministic 4xx stop retrying

error_from_response converted non-success public API responses into
ClientError/CloudAgentCapacityError/AIApiError, none of which carry an
HTTP status. is_transient_http_error defaults to transient when no
HttpStatusError is in the error chain, so checkpoint commit retried a
deterministic 400 three times before giving up.

Wrap every branch of error_from_response as
anyhow::Error::new(HttpStatusError { status, body }).context(...),
preserving the existing Display text and downcast_ref::<T>() behavior
(anyhow's context-chain downcasting matches on the context type
directly), while making the HttpStatusError reachable via e.chain()
for is_transient_http_error and is_auth_error.

commit_snapshot (checkpoint commit) is the only current with_bounded_retry
caller that goes through this path for POST/PUT/PATCH/DELETE public-API
calls, since GET already carried status correctly via
warp_server_client::public_api. No other retry behavior changes today.

Author: warp-agent-staging[bot]

app/src/server/server_api.rs

@@ -44,6 +44,7 @@ use warp_core::context_flag::ContextFlag;
 use warp_core::telemetry::TelemetryEvent;
 use warp_errors::{AnyhowErrorExt, ErrorExt, register_error, report_error};
 use warp_managed_secrets::client::ManagedSecretsClient;
+use warp_server_client::HttpStatusError;
 use warp_server_client::auth::{AuthClientImpl, AuthEvent, EXPERIMENT_ID_HEADER};
 use warp_server_client::base_client::{
     AmbientHeaderPolicy, AuthenticatedGraphqlConfig, BaseClient, GraphqlRoutingConfig,
@@ -676,7 +677,11 @@ impl ServerApi {
         }
     }
 
-    /// Converts a non-success public API response into the most specific client error available.
+    /// Converts a non-success public API response into the most specific client error
+    /// available. The returned error always carries an [`HttpStatusError`] in its chain
+    /// (via [`anyhow::Error::context`]) so callers retrying through
+    /// [`is_transient_http_error`](super::retry_strategies::is_transient_http_error) fail
+    /// fast on a deterministic 4xx instead of defaulting to a transient retry.
     async fn error_from_response(response: http_client::Response) -> anyhow::Error {
         let status = response.status();
         let is_at_capacity = response
@@ -692,28 +697,32 @@ impl ServerApi {
 
         // Get the response text first since we may need to try multiple deserializations.
         let response_text = response.text().await.unwrap_or_default();
+        let status_error = HttpStatusError {
+            status: status.as_u16(),
+            body: response_text.clone(),
+        };
 
         // Check for AT_CAPACITY error code header.
         if is_at_capacity
             && let Ok(capacity_error) =
                 serde_json::from_str::<CloudAgentCapacityError>(&response_text)
         {
-            return capacity_error.into();
+            return anyhow::Error::new(status_error).context(capacity_error);
         }
         if status == StatusCode::TOO_MANY_REQUESTS && is_out_of_credits {
             let user_display_message = serde_json::from_str::<OutOfCreditsResponse>(&response_text)
                 .ok()
                 .and_then(|r| r.user_display_message);
-            return AIApiError::QuotaLimit {
+            return anyhow::Error::new(status_error).context(AIApiError::QuotaLimit {
                 user_display_message,
-            }
-            .into();
+            });
         }
 
         // Try to deserialize error response as { "error": "message" }
         match serde_json::from_str::<ClientError>(&response_text) {
-            Ok(error_response) => error_response.into(),
-            Err(_) => anyhow!("API request failed with status {status}"),
+            Ok(error_response) => anyhow::Error::new(status_error).context(error_response),
+            Err(_) => anyhow::Error::new(status_error)
+                .context(format!("API request failed with status {status}")),
         }
     }
 
@@ -1416,3 +1425,7 @@ impl Entity for ServerApiProvider {
 }
 
 impl SingletonEntity for ServerApiProvider {}
+
+#[cfg(test)]
+#[path = "server_api_tests.rs"]
+mod tests;

app/src/server/server_api_tests.rs

@@ -0,0 +1,110 @@
+use futures::executor::block_on;
+use mockito::Server;
+
+use super::*;
+use crate::server::retry_strategies::is_transient_http_error;
+
+/// Sends a GET request to a mock endpoint returning `status`/`headers`/`body`, then feeds the
+/// resulting response through [`ServerApi::error_from_response`].
+fn error_from_mock_response(status: usize, headers: &[(&str, &str)], body: &str) -> anyhow::Error {
+    let mut server = Server::new();
+    let mut mock = server
+        .mock("GET", "/error")
+        .with_status(status)
+        .with_body(body);
+    for (name, value) in headers {
+        mock = mock.with_header(*name, value);
+    }
+    mock.create();
+
+    let url = format!("{}/error", server.url());
+    block_on(async move {
+        let response = http_client::Client::new_for_test()
+            .get(url)
+            .send()
+            .await
+            .unwrap();
+        ServerApi::error_from_response(response).await
+    })
+}
+
+/// The status carried by the [`HttpStatusError`] in `err`'s chain, if any.
+fn status_in_chain(err: &anyhow::Error) -> Option<u16> {
+    err.chain()
+        .find_map(|cause| cause.downcast_ref::<HttpStatusError>())
+        .map(|status_error| status_error.status)
+}
+
+#[test]
+fn permanent_4xx_client_error_carries_status_and_fails_fast() {
+    let err = error_from_mock_response(
+        403,
+        &[],
+        r#"{"error": "checkpoint generation is incomplete"}"#,
+    );
+
+    assert_eq!(status_in_chain(&err), Some(403));
+    assert!(!is_transient_http_error(&err));
+    assert_eq!(err.to_string(), "checkpoint generation is incomplete");
+    assert_eq!(
+        err.downcast_ref::<ClientError>().unwrap().error,
+        "checkpoint generation is incomplete"
+    );
+}
+
+#[test]
+fn permanent_4xx_without_parseable_body_still_carries_status() {
+    let err = error_from_mock_response(404, &[], "not found");
+
+    assert_eq!(status_in_chain(&err), Some(404));
+    assert!(!is_transient_http_error(&err));
+    assert_eq!(
+        err.to_string(),
+        "API request failed with status 404 Not Found"
+    );
+}
+
+#[test]
+fn transient_5xx_still_retries() {
+    let err = error_from_mock_response(503, &[], "unavailable");
+
+    assert_eq!(status_in_chain(&err), Some(503));
+    assert!(is_transient_http_error(&err));
+}
+
+#[test]
+fn at_capacity_header_wraps_capacity_error_and_still_carries_status() {
+    let err = error_from_mock_response(
+        403,
+        &[(WARP_ERROR_CODE_HEADER, WARP_ERROR_CODE_AT_CAPACITY)],
+        r#"{"error": "at capacity", "running_agents": 5}"#,
+    );
+
+    assert_eq!(status_in_chain(&err), Some(403));
+    assert!(!is_transient_http_error(&err));
+    assert_eq!(
+        err.downcast_ref::<CloudAgentCapacityError>()
+            .unwrap()
+            .running_agents,
+        5
+    );
+}
+
+#[test]
+fn out_of_credits_429_wraps_quota_limit_and_stays_transient() {
+    let err = error_from_mock_response(
+        429,
+        &[(WARP_ERROR_CODE_HEADER, WARP_ERROR_CODE_OUT_OF_CREDITS)],
+        r#"{"userDisplayMessage": "You're out of credits"}"#,
+    );
+
+    // 429 always retries regardless of error code, matching every other public-API caller.
+    assert_eq!(status_in_chain(&err), Some(429));
+    assert!(is_transient_http_error(&err));
+    assert!(matches!(
+        err.downcast_ref::<AIApiError>().unwrap(),
+        AIApiError::QuotaLimit {
+            user_display_message: Some(message)
+        } if message == "You're out of credits"
+    ));
+}