cloudposse
diff --git a/‎NOTICE‎
Lines changed: 1 addition & 1 deletion b/‎NOTICE‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎errors/errors.go‎
Lines changed: 2 additions & 0 deletions b/‎errors/errors.go‎
Lines changed: 2 additions & 0 deletions
diff --git a/‎go.mod‎
Lines changed: 1 addition & 1 deletion b/‎go.mod‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎go.sum‎
Lines changed: 2 additions & 2 deletions b/‎go.sum‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎internal/exec/describe_affected_test.go‎
Lines changed: 62 additions & 6 deletions b/‎internal/exec/describe_affected_test.go‎
Lines changed: 62 additions & 6 deletions
diff --git a/‎internal/exec/describe_component_test.go‎
Lines changed: 8 additions & 0 deletions b/‎internal/exec/describe_component_test.go‎
Lines changed: 8 additions & 0 deletions
diff --git a/‎internal/exec/helmfile.go‎
Lines changed: 36 additions & 10 deletions b/‎internal/exec/helmfile.go‎
Lines changed: 36 additions & 10 deletions
diff --git a/‎internal/exec/helmfile_test.go‎
Lines changed: 112 additions & 0 deletions b/‎internal/exec/helmfile_test.go‎
Lines changed: 112 additions & 0 deletions
diff --git a/‎internal/exec/packer.go‎
Lines changed: 35 additions & 7 deletions b/‎internal/exec/packer.go‎
Lines changed: 35 additions & 7 deletions
@@ -719,7 +719,7 @@ APACHE 2.0 LICENSED DEPENDENCIES
 
   - google.golang.org/grpc
     License: Apache-2.0
-    URL: https://github.com/grpc/grpc-go/blob/v1.83.1/LICENSE
+    URL: https://github.com/grpc/grpc-go/blob/v1.83.2/LICENSE
 
   - gopkg.in/ini.v1
     License: Apache-2.0
 
@@ -524,6 +524,7 @@ var (
 	ErrInvalidComponentOverridesRequiredVersion   = errors.New("invalid component overrides required_version attribute")
 	ErrInvalidComponentOverridesHooks             = errors.New("invalid component overrides hooks section")
 	ErrInvalidComponentOverridesGenerate          = errors.New("invalid component overrides generate section")
+	ErrInvalidComponentOverridesProvision         = errors.New("invalid component overrides provision section")
 	ErrInvalidComponentOverridesFlags             = errors.New("invalid component overrides flags section")
 	ErrInvalidComponentAttribute                  = errors.New("invalid component attribute")
 	ErrInvalidComponentMetadataComponent          = errors.New("invalid component metadata.component attribute")
@@ -655,6 +656,7 @@ var (
 	ErrInvalidAuthSection               = errors.New("invalid auth section")
 	ErrInvalidGlobalMetadataSection     = errors.New("invalid metadata section")
 	ErrGlobalMetadataFieldNotAllowed    = errors.New("metadata field is not allowed at global (stack-wide) scope")
+	ErrInvalidGlobalRetrySection        = errors.New("invalid retry section")
 	ErrInvalidImportSection             = errors.New("invalid import section")
 	ErrInvalidImport                    = errors.New("invalid import")
 	ErrInvalidRemoteImport              = errors.New("invalid remote import")
 
@@ -2,6 +2,8 @@ package exec
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -291,7 +293,7 @@ func shouldSkipRepoCopyPath(src string) bool {
 //     housekeeping (e.g. an automatic repack writes tmp_pack_*/tmp_idx_*/tmp_rev_* files and
 //     renames or removes them within milliseconds) -- the otiai10/copy directory walk stats every
 //     entry it lists, so it can observe one of these files mid-flight and fail with
-//     "no such file or directory" (os.IsNotExist).
+//     "no such file or directory" (os.ErrNotExist, possibly wrapped).
 //   - fixture files locked by another concurrently running test's terraform process (e.g. a
 //     terraform.tfstate under tests/fixtures/scenarios/plan-diff held open mid-plan/apply), which
 //     on Windows surfaces as a sharing/lock violation rather than IsNotExist.
@@ -319,12 +321,17 @@ func copyRepoWithRetry(t *testing.T, src, dest string, opts *cp.Options) error {
 }
 
 // isTransientRepoCopyError reports whether err is expected to resolve on its own shortly, and so
-// is worth retrying rather than failing the test outright. Windows reports a locked file via its
-// error message text rather than a portable sentinel/errno, so this matches on that text the same
-// way pkg/git/worktree.go's isTransientWorktreeRemoveError does for transient worktree-removal
-// errors.
+// is worth retrying rather than failing the test outright. See copyRepoWithRetry's doc comment
+// for the two known causes. Windows reports a locked file via its error message text rather than
+// a portable sentinel/errno, so this matches on that text the same way pkg/git/worktree.go's
+// isTransientWorktreeRemoveError does for transient worktree-removal errors.
 func isTransientRepoCopyError(err error) bool {
-	if os.IsNotExist(err) {
+	if err == nil {
+		return false
+	}
+	// errors.Is (not os.IsNotExist) so a wrapped os.ErrNotExist is still recognized --
+	// os.IsNotExist does not reliably unwrap.
+	if errors.Is(err, os.ErrNotExist) {
 		return true
 	}
 	lower := strings.ToLower(err.Error())
@@ -340,6 +347,55 @@ func isTransientRepoCopyError(err error) bool {
 	return false
 }
 
+func TestIsTransientRepoCopyError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{
+			name: "nil error is not transient",
+			err:  nil,
+			want: false,
+		},
+		{
+			name: "not-exist error is transient",
+			err:  os.ErrNotExist,
+			want: true,
+		},
+		{
+			name: "wrapped not-exist error is transient",
+			// os.IsNotExist does not reliably unwrap; errors.Is does. This case
+			// guards against a regression back to os.IsNotExist.
+			err:  fmt.Errorf("copy failed: %w", os.ErrNotExist),
+			want: true,
+		},
+		{
+			name: "windows lock violation is transient",
+			// Exact error text observed on Windows CI (ERROR_LOCK_VIOLATION).
+			err:  errors.New(`read ..\..\tests\fixtures\scenarios\hooks-test\components\terraform\hook-and-store\terraform.tfstate.d\test-component2\terraform.tfstate: The process cannot access the file because another process has locked a portion of the file.`),
+			want: true,
+		},
+		{
+			name: "windows sharing violation is transient",
+			// ERROR_SHARING_VIOLATION wording, distinct from the lock-violation text above.
+			err:  errors.New(`open foo.txt: The process cannot access the file because it is being used by another process.`),
+			want: true,
+		},
+		{
+			name: "unrelated error is not transient",
+			err:  errors.New("permission denied"),
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, isTransientRepoCopyError(tt.err))
+		})
+	}
+}
+
 // setupDescribeAffectedTest sets up the test environment for describe affected tests.
 func setupDescribeAffectedTest(t *testing.T) (atmosConfig schema.AtmosConfiguration, repoPath, componentPath string) {
 	t.Helper()
 
@@ -469,6 +469,9 @@ func TestDescribeComponentWithOverridesSection(t *testing.T) {
 	assert.Contains(t, y, "b: b")
 	assert.Contains(t, y, "c: c")
 	assert.Contains(t, y, "d: d")
+	// `catalog/overrides` (with its `retry` override) is imported after `catalog/c1`,
+	// so it must not apply — same file-scoping rule as the `vars` override above.
+	assert.NotContains(t, y, "retry:")
 
 	// `test3`
 	res, err = ExecuteDescribeComponent(&ExecuteDescribeComponentParams{
@@ -487,6 +490,11 @@ func TestDescribeComponentWithOverridesSection(t *testing.T) {
 	assert.Contains(t, y, "b: b-overridden")
 	assert.Contains(t, y, "c: c")
 	assert.Contains(t, y, "d: d")
+	// `catalog/overrides` (with its `retry` override) is imported before `catalog/c1`,
+	// so the retry policy from the mixin-style overrides file must apply here — the
+	// same mechanism documented for sharing a `retry` policy via a mixin.
+	assert.Contains(t, y, "max_attempts: 5")
+	assert.Contains(t, y, "- /GOAWAY/")
 }
 
 func TestDescribeComponent_Packer(t *testing.T) {
 
@@ -454,16 +454,11 @@ func ExecuteHelmfile(info schema.ConfigAndStacksInfo) error {
 	// toolchain-installed helmfile (under the install path, not the system PATH)
 	// is found — mirroring the `version` subcommand above. Falls back to the bare
 	// command name when no toolchain dependency provides it.
-	err = ExecuteShellCommand(
-		atmosConfig,
-		tenv.Resolve(info.Command),
-		allArgsAndFlags,
-		componentPath,
-		envVars,
-		info.DryRun,
-		info.RedirectStdErr,
-		shellOpts...,
-	)
+	err = executeHelmfileCommandWithRetry(&atmosConfig, &info, tenv, retryExecParams{
+		allArgsAndFlags: allArgsAndFlags,
+		componentPath:   componentPath,
+		envVars:         envVars,
+	}, shellOpts...)
 	if info.NodeHooks != nil {
 		if afterErr := info.NodeHooks.After(context.Background(), &info, stdoutBuf.String()+stderrBuf.String(), err); afterErr != nil && err == nil {
 			err = afterErr
@@ -482,6 +477,37 @@ func ExecuteHelmfile(info schema.ConfigAndStacksInfo) error {
 	return nil
 }
 
+// executeHelmfileCommandWithRetry runs the resolved helmfile subcommand through
+// ExecuteShellCommandWithRetry. Extracted from ExecuteHelmfile so the retry wiring can
+// be unit-tested directly with a fake invoke, without standing up ExecuteHelmfile's full
+// stack-processing/auth/toolchain preamble or requiring a real helmfile binary.
+func executeHelmfileCommandWithRetry(
+	atmosConfig *schema.AtmosConfiguration,
+	info *schema.ConfigAndStacksInfo,
+	tenv *dependencies.ToolchainEnvironment,
+	params retryExecParams,
+	shellOpts ...ShellCommandOption,
+) error {
+	return ExecuteShellCommandWithRetry(
+		atmosConfig,
+		info,
+		info.SubCommand,
+		func(o ...ShellCommandOption) error {
+			return ExecuteShellCommand(
+				*atmosConfig,
+				tenv.Resolve(info.Command),
+				params.allArgsAndFlags,
+				params.componentPath,
+				params.envVars,
+				info.DryRun,
+				info.RedirectStdErr,
+				o...,
+			)
+		},
+		shellOpts...,
+	)
+}
+
 // renderAndDeliver is a seam over helmfile.RenderAndDeliver so the inline
 // call-site can be unit-tested without invoking the helmfile binary.
 var renderAndDeliver = helmfile.RenderAndDeliver
 
@@ -1,6 +1,7 @@
 package exec
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"os"
@@ -325,3 +326,114 @@ func TestExecuteHelmfileNodeHooks_AfterErrorDroppedWhenExecAlreadyFailed(t *test
 	assert.NotErrorIs(t, err, afterErr, "the after-hook error must be dropped when the exec error already won")
 	assert.Error(t, nodeHooks.afterExecErr, "After must have been called with the real (non-nil) exec error")
 }
+
+// TestExecuteHelmfileCommandWithRetry_MatchingError_Retries proves the retry wiring added
+// to ExecuteHelmfile actually triggers through the real call chain
+// (executeHelmfileCommandWithRetry -> ExecuteShellCommandWithRetry -> ExecuteShellCommand),
+// not just that the shared helper works in isolation. Uses the test binary itself as the
+// "helmfile" command (cross-platform, no real helmfile install needed) via the
+// _ATMOS_TEST_EXIT_ONE/_ATMOS_TEST_STDERR TestMain gate.
+func TestExecuteHelmfileCommandWithRetry_MatchingError_Retries(t *testing.T) {
+	exePath, err := os.Executable()
+	require.NoError(t, err)
+
+	counterFile := filepath.Join(t.TempDir(), "counter")
+
+	info := &schema.ConfigAndStacksInfo{
+		Command:    exePath,
+		SubCommand: "sync",
+		ComponentRetrySection: &schema.RetryConfig{
+			MaxAttempts: intPtr(3),
+			Conditions:  []string{"/Bad Gateway/"},
+		},
+	}
+	envVars := []string{
+		"_ATMOS_TEST_COUNTER_FILE=" + counterFile,
+		"_ATMOS_TEST_EXIT_ONE=1",
+		"_ATMOS_TEST_STDERR=Error: 502 Bad Gateway returned",
+	}
+
+	atmosConfig := schema.AtmosConfiguration{}
+	err = executeHelmfileCommandWithRetry(&atmosConfig, info, nil, retryExecParams{
+		allArgsAndFlags: []string{"sync"},
+		componentPath:   t.TempDir(),
+		envVars:         envVars,
+	})
+	require.Error(t, err, "all 3 attempts fail in this fixture, so the final error must propagate")
+
+	counterBytes, readErr := os.ReadFile(counterFile)
+	require.NoError(t, readErr)
+	assert.Len(t, counterBytes, 3, "all 3 configured attempts must execute before the final error propagates")
+}
+
+// TestExecuteHelmfileCommandWithRetry_NonMatchingError_FailsFast proves a real helmfile
+// failure whose output does not match `conditions` is NOT retried through the real call
+// chain -- the counter file lets us assert exactly one subprocess invocation happened.
+func TestExecuteHelmfileCommandWithRetry_NonMatchingError_FailsFast(t *testing.T) {
+	exePath, err := os.Executable()
+	require.NoError(t, err)
+
+	counterFile := filepath.Join(t.TempDir(), "counter")
+
+	info := &schema.ConfigAndStacksInfo{
+		Command:    exePath,
+		SubCommand: "sync",
+		ComponentRetrySection: &schema.RetryConfig{
+			MaxAttempts: intPtr(3),
+			Conditions:  []string{"/Bad Gateway/"},
+		},
+	}
+	envVars := []string{
+		"_ATMOS_TEST_COUNTER_FILE=" + counterFile,
+		"_ATMOS_TEST_EXIT_ONE=1",
+		"_ATMOS_TEST_STDERR=permission denied",
+	}
+
+	atmosConfig := schema.AtmosConfiguration{}
+	err = executeHelmfileCommandWithRetry(&atmosConfig, info, nil, retryExecParams{
+		allArgsAndFlags: []string{"sync"},
+		componentPath:   t.TempDir(),
+		envVars:         envVars,
+	})
+	require.Error(t, err)
+
+	counterBytes, readErr := os.ReadFile(counterFile)
+	require.NoError(t, readErr)
+	assert.Len(t, counterBytes, 1, "non-matching error must fail fast on the first attempt")
+}
+
+// TestExecuteHelmfileCommandWithRetry_ComposesWithNodeHooksCapture proves that
+// executeHelmfileCommandWithRetry's caller-supplied NodeHooks capture buffers (shellOpts)
+// still receive output when retry is also configured -- guards the
+// ExecuteShellCommandWithRetry MultiWriter composition fix at the actual helmfile call site,
+// not just in the shared helper's own unit tests.
+func TestExecuteHelmfileCommandWithRetry_ComposesWithNodeHooksCapture(t *testing.T) {
+	exePath, err := os.Executable()
+	require.NoError(t, err)
+
+	info := &schema.ConfigAndStacksInfo{
+		Command:    exePath,
+		SubCommand: "sync",
+		ComponentRetrySection: &schema.RetryConfig{
+			MaxAttempts: intPtr(2),
+			Conditions:  []string{"/Bad Gateway/"},
+		},
+	}
+	envVars := []string{
+		"_ATMOS_TEST_EXIT_ONE=1",
+		"_ATMOS_TEST_STDERR=502 Bad Gateway",
+	}
+
+	var nodeHooksStderr bytes.Buffer
+	shellOpts := []ShellCommandOption{WithStderrCapture(&nodeHooksStderr)}
+
+	atmosConfig := schema.AtmosConfiguration{}
+	err = executeHelmfileCommandWithRetry(&atmosConfig, info, nil, retryExecParams{
+		allArgsAndFlags: []string{"sync"},
+		componentPath:   t.TempDir(),
+		envVars:         envVars,
+	}, shellOpts...)
+	require.Error(t, err)
+	assert.Contains(t, nodeHooksStderr.String(), "502 Bad Gateway",
+		"the NodeHooks-style caller capture buffer must still receive output when retry is also active")
+}
@@ -301,14 +301,42 @@ func ExecutePacker(
 	// "executable file not found in $PATH", because exec.Command resolves the
 	// binary via the process's real PATH at call time, not via the PATH=...
 	// entry later added to envVars.
-	return executePackerShellCommand(
+	return executePackerCommandWithRetry(&atmosConfig, info, tenv, retryExecParams{
+		allArgsAndFlags: allArgsAndFlags,
+		componentPath:   componentPath,
+		envVars:         envVars,
+	})
+}
+
+// executePackerCommandWithRetry runs the resolved packer subcommand through
+// ExecuteShellCommandWithRetry. Extracted from ExecutePacker so the retry wiring can
+// be unit-tested directly with a fake invoke, without standing up ExecutePacker's full
+// stack-processing/toolchain preamble or requiring a real packer binary. The inner call
+// goes through the executePackerShellCommand seam (not ExecuteShellCommand directly) so
+// auth-credential-injection tests that swap that seam still intercept the real subprocess
+// invocation, retry or not.
+func executePackerCommandWithRetry(
+	atmosConfig *schema.AtmosConfiguration,
+	info *schema.ConfigAndStacksInfo,
+	tenv *dependencies.ToolchainEnvironment,
+	params retryExecParams,
+) error {
+	return ExecuteShellCommandWithRetry(
 		atmosConfig,
-		tenv.Resolve(info.Command),
-		allArgsAndFlags,
-		componentPath,
-		envVars,
-		info.DryRun,
-		info.RedirectStdErr,
+		info,
+		info.SubCommand,
+		func(o ...ShellCommandOption) error {
+			return executePackerShellCommand(
+				*atmosConfig,
+				tenv.Resolve(info.Command),
+				params.allArgsAndFlags,
+				params.componentPath,
+				params.envVars,
+				info.DryRun,
+				info.RedirectStdErr,
+				o...,
+			)
+		},
 		WithEnvironment(info.SanitizedEnv),
 	)
 }