feat(ecs): track ECS Service deployment stability via Phase A/B Status polling

Previously the AWS::ECS::Service provisioner returned Success at
CCAPI-ack time, so downstream resources (ALB Listener URL consumers
such as Grafana) frequently saw 503s before tasks were actually
serving traffic. This change holds InProgress until the deployment is
operationally stable: rolloutState=COMPLETED, runningCount equals
desiredCount, and at least one healthy target exists per attached
target group.

Stability is tracked by a two-phase Status FSM. Phase A polls the
CCAPI request token until it resolves. Once Successful, Phase B polls
DescribeServices and DescribeTargetHealth until the operational
criteria are met. State is encoded in a composite RequestID of the
form formae-ecs/<op>/<unixStart>/<ccapiToken>, set once at Create or
Update return and preserved across polls via the SDK's existing
StatusCheck semantics. Non-default service shapes — CODE_DEPLOY and
EXTERNAL controllers, DAEMON scheduling strategy, classic-ELB
attachments without targetGroupArn, desiredCount=0, and missing
Cluster — fall through to safe defaults rather than entering Phase B.

Two new fixtures, ecs-service-with-lb.pkl and its -update.pkl
counterpart, exercise the full TG-health-gated lifecycle end-to-end
on real AWS. The debug-conformance workflow timeout is raised from
60m to 90m so ALB-based fixtures have enough wall-clock headroom.

Author: JeroenSoeters

.github/workflows/debug-conformance.yml

@@ -157,7 +157,10 @@ jobs:
           AWS_REGION: us-east-1
           FORMAE_TEST_RUN_ID: debug-${{ github.run_id }}-${{ github.run_attempt }}
           POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
-        run: make conformance-test-crud-run conformance-test-discovery-run TEST=${{ matrix.test-case }} PARALLEL=1 TIMEOUT=60
+        # 90-minute timeout accommodates LB-attached fixtures (ALB provisioning
+        # + ECS stabilization + destroy + reapply takes 65-80m). Smaller fixtures
+        # are unaffected — they finish well under the bound.
+        run: make conformance-test-crud-run conformance-test-discovery-run TEST=${{ matrix.test-case }} PARALLEL=1 TIMEOUT=90
 
       - name: Dump formae client log on failure
         if: failure()

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.299.0
 	github.com/aws/aws-sdk-go-v2/service/ecs v1.77.0
 	github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk v1.33.20
+	github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.12
 	github.com/aws/aws-sdk-go-v2/service/iam v1.53.8
 	github.com/aws/aws-sdk-go-v2/service/lambda v1.90.0
 	github.com/aws/aws-sdk-go-v2/service/route53 v1.62.6

go.sum

@@ -38,6 +38,8 @@ github.com/aws/aws-sdk-go-v2/service/ecs v1.77.0 h1:g3RYQmK6uRU5kOuwDthemuiiTbmw
 github.com/aws/aws-sdk-go-v2/service/ecs v1.77.0/go.mod h1:QkWmubOYmjj3cHn7A4CoUU7BKJhVeo39Gp6NH7IyhZw=
 github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk v1.33.20 h1:0jvM1QtdJPeox3KX7zLV0XWb1rJpvHmVkoAPY3P3hz0=
 github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk v1.33.20/go.mod h1:73bxyBZbIDX0ulG9AgJsGgodZSiqwMoWDyiDY8Rj6Zk=
+github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.12 h1:TJXv7kZjdXA2maPDaJFFEQPBrPmvPtMybN3qYDOpJ4Y=
+github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.12/go.mod h1:lwjtb9DHOAmNt7EUW68Zd1Qd+cPyFxacXHN5c9JZ2VY=
 github.com/aws/aws-sdk-go-v2/service/iam v1.53.8 h1:p0oB4eZfBfBAOasnKvHJOlNcuHVE/ieuWs7uIZgQlyQ=
 github.com/aws/aws-sdk-go-v2/service/iam v1.53.8/go.mod h1:epCaPnGVdiX5ra1lHPfRkVuiQGxrdY8bRI2FBJU+6ok=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 h1:HtOTYcbVcGABLOVuPYaIihj6IlkqubBwFj10K5fxRek=

pkg/cfres/ecs/classify.go

@@ -0,0 +1,129 @@
+// © 2025 Platform Engineering Labs Inc.
+//
+// SPDX-License-Identifier: FSL-1.1-ALv2
+
+package ecs
+
+import (
+	"errors"
+	"net"
+
+	"github.com/aws/smithy-go"
+
+	"github.com/platform-engineering-labs/formae/pkg/plugin/resource"
+)
+
+// classifyAWSError maps an error from an AWS SDK call (or our own ccx layer) to
+// a (errorCode, retryable) verdict. See design Q5 for the full mapping.
+//
+// Discipline: unknown errors default to TERMINAL (GeneralServiceException) — fail
+// loudly rather than poll forever on something we don't recognize. The 20-minute
+// operation timeout (via inProgressOrTimeout) means even retryable verdicts can
+// escalate to terminal Failure if they persist.
+func classifyAWSError(err error) (resource.OperationErrorCode, bool) {
+	if err == nil {
+		return "", false
+	}
+	var ae smithy.APIError
+	if errors.As(err, &ae) {
+		switch ae.ErrorCode() {
+		// Retryable
+		case "Throttling", "ThrottlingException", "RequestLimitExceeded",
+			"TooManyRequestsException":
+			return resource.OperationErrorCodeThrottling, true
+		case "ServiceUnavailable", "ServiceUnavailableException":
+			return resource.OperationErrorCodeServiceInternalError, true
+		case "InternalFailure", "InternalServerError":
+			return resource.OperationErrorCodeServiceInternalError, true
+		// Terminal — auth/permissions
+		case "AccessDenied", "AccessDeniedException", "UnauthorizedOperation":
+			return resource.OperationErrorCodeAccessDenied, false
+		case "ExpiredToken", "InvalidClientTokenId",
+			"InvalidSignatureException", "SignatureDoesNotMatch":
+			return resource.OperationErrorCodeInvalidCredentials, false
+		// Terminal — validation
+		case "ValidationException", "InvalidParameterException",
+			"InvalidParameterValueException", "InvalidInputException":
+			return resource.OperationErrorCodeInvalidRequest, false
+		}
+	}
+	// Network errors with .Timeout() == true → retryable
+	var netErr net.Error
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return resource.OperationErrorCodeNetworkFailure, true
+	}
+	return resource.OperationErrorCodeGeneralServiceException, false
+}
+
+// classifyForEntry is used by Create/Update entry. Retryable AWS errors return
+// a recoverable ErrorCode so the operator's handlePluginResult schedules a CRUD
+// retry of the entire operation. Terminal errors get a non-recoverable code so
+// the operator surfaces the failure immediately.
+func classifyForEntry(err error, op resource.Operation, nativeID, contextMsg string) *resource.ProgressResult {
+	code, retryable := classifyAWSError(err)
+	if retryable {
+		// Map our verdict to a recoverable code the SDK's recoverableErrorCodes
+		// table actually recognises. (Throttling, NetworkFailure, ServiceInternalError
+		// are all in the table — see pkg/plugin/resource/resource.go:172-181.)
+		switch code {
+		case resource.OperationErrorCodeThrottling,
+			resource.OperationErrorCodeNetworkFailure,
+			resource.OperationErrorCodeServiceInternalError:
+			// already recoverable
+		default:
+			code = resource.OperationErrorCodeThrottling
+		}
+	}
+	return terminalFailurePR(op, nativeID, "", code, contextMsg+": "+err.Error())
+}
+
+// terminalFailurePR builds a populated Failure ProgressResult.
+func terminalFailurePR(op resource.Operation, nativeID, requestID string,
+	code resource.OperationErrorCode, msg string) *resource.ProgressResult {
+	return &resource.ProgressResult{
+		Operation:       op,
+		OperationStatus: resource.OperationStatusFailure,
+		NativeID:        nativeID,
+		RequestID:       requestID,
+		ErrorCode:       code,
+		StatusMessage:   msg,
+	}
+}
+
+// classifyReadResultForFinal classifies a post-stability Read outcome. Handles
+// both Go errors and ReadResult.ErrorCode (ccx.ReadResource maps CCAPI errors
+// into ErrorCode without returning a Go error — see pkg/ccx/client.go:294-303).
+//
+// Returns:
+//   - ok=true:                 Read returned non-empty Properties → caller emits Success
+//   - ok=false, retryable=true: route through inProgressOrFinalReadTimeout (grace-bounded)
+//   - ok=false, retryable=false: terminal Failure with `code`
+func classifyReadResultForFinal(rr *resource.ReadResult, readErr error) (resource.OperationErrorCode, bool, bool) {
+	if readErr != nil {
+		code, retryable := classifyAWSError(readErr)
+		return code, retryable, false
+	}
+	if rr == nil {
+		return resource.OperationErrorCodeGeneralServiceException, false, false
+	}
+	switch rr.ErrorCode {
+	case "":
+		if rr.Properties == "" {
+			return "", true, false // retryable: empty body without error
+		}
+		return "", false, true // success
+	case resource.OperationErrorCodeNotFound,
+		resource.OperationErrorCodeThrottling,
+		resource.OperationErrorCodeServiceInternalError,
+		resource.OperationErrorCodeServiceTimeout,
+		resource.OperationErrorCodeNetworkFailure,
+		resource.OperationErrorCodeInternalFailure:
+		return rr.ErrorCode, true, false
+	case resource.OperationErrorCodeAccessDenied,
+		resource.OperationErrorCodeInvalidCredentials,
+		resource.OperationErrorCodeInvalidRequest:
+		return rr.ErrorCode, false, false
+	default:
+		return resource.OperationErrorCodeGeneralServiceException, false, false
+	}
+}

pkg/cfres/ecs/classify_test.go

@@ -0,0 +1,122 @@
+// © 2025 Platform Engineering Labs Inc.
+//
+// SPDX-License-Identifier: FSL-1.1-ALv2
+
+//go:build unit
+
+package ecs
+
+import (
+	"errors"
+	"net"
+	"testing"
+
+	"github.com/aws/smithy-go"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/platform-engineering-labs/formae/pkg/plugin/resource"
+)
+
+type fakeAPIError struct{ code, msg string }
+
+func (e *fakeAPIError) Error() string                 { return e.code + ": " + e.msg }
+func (e *fakeAPIError) ErrorCode() string             { return e.code }
+func (e *fakeAPIError) ErrorMessage() string          { return e.msg }
+func (e *fakeAPIError) ErrorFault() smithy.ErrorFault { return smithy.FaultServer }
+
+func TestClassifyAWSError_Throttling(t *testing.T) {
+	code, retryable := classifyAWSError(&fakeAPIError{code: "Throttling", msg: "Rate exceeded"})
+	assert.True(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeThrottling, code)
+}
+
+func TestClassifyAWSError_AccessDenied(t *testing.T) {
+	code, retryable := classifyAWSError(&fakeAPIError{code: "AccessDenied", msg: "denied"})
+	assert.False(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeAccessDenied, code)
+}
+
+func TestClassifyAWSError_ExpiredToken(t *testing.T) {
+	code, retryable := classifyAWSError(&fakeAPIError{code: "ExpiredToken", msg: "expired"})
+	assert.False(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeInvalidCredentials, code)
+}
+
+func TestClassifyAWSError_ValidationException(t *testing.T) {
+	code, retryable := classifyAWSError(&fakeAPIError{code: "ValidationException", msg: "bad input"})
+	assert.False(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeInvalidRequest, code)
+}
+
+func TestClassifyAWSError_UnknownCode_TerminalGeneralServiceException(t *testing.T) {
+	code, retryable := classifyAWSError(&fakeAPIError{code: "SomethingBrandNew", msg: "?"})
+	assert.False(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeGeneralServiceException, code)
+}
+
+type timeoutErr struct{}
+
+func (timeoutErr) Error() string   { return "i/o timeout" }
+func (timeoutErr) Timeout() bool   { return true }
+func (timeoutErr) Temporary() bool { return true }
+
+var _ net.Error = timeoutErr{}
+
+func TestClassifyAWSError_NetworkTimeout_Retryable(t *testing.T) {
+	code, retryable := classifyAWSError(timeoutErr{})
+	assert.True(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeNetworkFailure, code)
+}
+
+func TestClassifyAWSError_PlainError_TerminalGeneralServiceException(t *testing.T) {
+	code, retryable := classifyAWSError(errors.New("some random non-AWS error"))
+	assert.False(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeGeneralServiceException, code)
+}
+
+func TestTerminalFailurePR(t *testing.T) {
+	pr := terminalFailurePR(resource.OperationCreate, "nid", "rid",
+		resource.OperationErrorCodeAccessDenied, "denied here")
+	assert.Equal(t, resource.OperationCreate, pr.Operation)
+	assert.Equal(t, resource.OperationStatusFailure, pr.OperationStatus)
+	assert.Equal(t, "nid", pr.NativeID)
+	assert.Equal(t, "rid", pr.RequestID)
+	assert.Equal(t, resource.OperationErrorCodeAccessDenied, pr.ErrorCode)
+	assert.Contains(t, pr.StatusMessage, "denied here")
+}
+
+func TestClassifyReadResultForFinal_Success(t *testing.T) {
+	rr := &resource.ReadResult{Properties: `{"k":"v"}`}
+	code, retryable, ok := classifyReadResultForFinal(rr, nil)
+	assert.True(t, ok)
+	assert.False(t, retryable)
+	assert.Equal(t, resource.OperationErrorCode(""), code)
+}
+
+func TestClassifyReadResultForFinal_NotFound_Retryable(t *testing.T) {
+	rr := &resource.ReadResult{ErrorCode: resource.OperationErrorCodeNotFound}
+	_, retryable, ok := classifyReadResultForFinal(rr, nil)
+	assert.False(t, ok)
+	assert.True(t, retryable)
+}
+
+func TestClassifyReadResultForFinal_AccessDenied_Terminal(t *testing.T) {
+	rr := &resource.ReadResult{ErrorCode: resource.OperationErrorCodeAccessDenied}
+	code, retryable, ok := classifyReadResultForFinal(rr, nil)
+	assert.False(t, ok)
+	assert.False(t, retryable)
+	assert.Equal(t, resource.OperationErrorCodeAccessDenied, code)
+}
+
+func TestClassifyReadResultForFinal_EmptyProperties_Retryable(t *testing.T) {
+	rr := &resource.ReadResult{ErrorCode: "", Properties: ""}
+	_, retryable, ok := classifyReadResultForFinal(rr, nil)
+	assert.False(t, ok)
+	assert.True(t, retryable)
+}
+
+func TestClassifyReadResultForFinal_GoError_GoesThroughAWSClassifier(t *testing.T) {
+	_, retryable, ok := classifyReadResultForFinal(nil, &fakeAPIError{code: "Throttling"})
+	assert.False(t, ok)
+	assert.True(t, retryable)
+}

pkg/cfres/ecs/clients.go

@@ -0,0 +1,56 @@
+// © 2025 Platform Engineering Labs Inc.
+//
+// SPDX-License-Identifier: FSL-1.1-ALv2
+
+package ecs
+
+import (
+	"context"
+	"fmt"
+
+	awsecs "github.com/aws/aws-sdk-go-v2/service/ecs"
+	awselbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
+	"github.com/platform-engineering-labs/formae/pkg/plugin/resource"
+
+	"github.com/platform-engineering-labs/formae-plugin-aws/pkg/ccx"
+	"github.com/platform-engineering-labs/formae-plugin-aws/pkg/config"
+)
+
+// ccxClient is the surface of pkg/ccx the ECS provisioner depends on. Defined
+// here so unit tests can mock just the methods we actually call.
+type ccxClient interface {
+	CreateResource(ctx context.Context, req *resource.CreateRequest) (*resource.CreateResult, error)
+	UpdateResource(ctx context.Context, req *resource.UpdateRequest) (*resource.UpdateResult, error)
+	StatusResource(ctx context.Context, req *resource.StatusRequest, readFunc func(context.Context, *resource.ReadRequest) (*resource.ReadResult, error)) (*resource.StatusResult, error)
+	ReadResource(ctx context.Context, req *resource.ReadRequest) (*resource.ReadResult, error)
+}
+
+// ecsClient is the surface of the AWS ECS SDK used by the Service provisioner for stability polling.
+type ecsClient interface {
+	DescribeServices(ctx context.Context, params *awsecs.DescribeServicesInput, optFns ...func(*awsecs.Options)) (*awsecs.DescribeServicesOutput, error)
+}
+
+// elbv2Client is the surface of the AWS ELBv2 SDK used by the Service provisioner for target-health checks.
+type elbv2Client interface {
+	DescribeTargetHealth(ctx context.Context, params *awselbv2.DescribeTargetHealthInput, optFns ...func(*awselbv2.Options)) (*awselbv2.DescribeTargetHealthOutput, error)
+}
+
+func defaultCCXClientFactory(cfg *config.Config) (ccxClient, error) {
+	return ccx.NewClient(cfg)
+}
+
+func defaultECSClientFactory(cfg *config.Config) (ecsClient, error) {
+	awsCfg, err := cfg.ToAwsConfig(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("ecs: build AWS config: %w", err)
+	}
+	return awsecs.NewFromConfig(awsCfg), nil
+}
+
+func defaultELBv2ClientFactory(cfg *config.Config) (elbv2Client, error) {
+	awsCfg, err := cfg.ToAwsConfig(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("elbv2: build AWS config: %w", err)
+	}
+	return awselbv2.NewFromConfig(awsCfg), nil
+}