Avoid counting snapshot failures twice in SLM

docs/changelog/136759.yaml

@@ -0,0 +1,5 @@
+pr: 136759
+summary: Avoid counting snapshot failures twice in SLM
+area: ILM+SLM
+type: bug
+issues: []

x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java

@@ -496,7 +496,8 @@ public ClusterState execute(ClusterState currentState) throws Exception {
             final SnapshotLifecyclePolicyMetadata.Builder newPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder(policyMetadata);
             SnapshotLifecycleStats newStats = snapMeta.getStats();
 
-            if (registeredSnapshots.contains(snapshotId) == false) {
+            final boolean snapshotIsRegistered = registeredSnapshots.contains(snapshotId);
+            if (snapshotIsRegistered == false) {
                 logger.warn(
                     "Snapshot [{}] not found in registered set after snapshot completion. This means snapshot was"
                         + " recorded as a failure by another snapshot's cleanup run.",
@@ -563,7 +564,11 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         exception.map(SnapshotLifecycleTask::exceptionToString).orElse(null)
                     )
                 );
-                newPolicyMetadata.incrementInvocationsSinceLastSuccess();
+                // If the snapshot was not registered, it means it was already counted as a failure by another snapshot's cleanup run
+                // so we should not increment the invocationsSinceLastSuccess again.
+                if (snapshotIsRegistered) {
+                    newPolicyMetadata.incrementInvocationsSinceLastSuccess();
+                }
             } else {
                 newStats = newStats.withTakenIncremented(policyName);
                 newPolicyMetadata.setLastSuccess(

x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java

@@ -528,7 +528,8 @@ public void testCleanUpRegisteredInitiatedByFailure() throws Exception {
                 inferredFailureSnapshot,
                 snapshotInfoSuccess.snapshotId(),
                 snapshotInfoFailure1.snapshotId(),
-                snapshotInfoFailure2.snapshotId()
+                snapshotInfoFailure2.snapshotId(),
+                initiatingSnapshot
             )
         );
         var inProgress = Map.of(policyId, List.of(stillRunning));