Fix ML tests failing with "no shards available"

muted-tests.yml

@@ -137,18 +137,12 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/10_basic/Create a source only snapshot and then restore it}
   issue: https://github.com/elastic/elasticsearch/issues/122755
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/data_frame_analytics_crud/Test get stats given multiple analytics}
-  issue: https://github.com/elastic/elasticsearch/issues/123034
 - class: org.elasticsearch.indices.recovery.IndexRecoveryIT
   method: testSourceThrottling
   issue: https://github.com/elastic/elasticsearch/issues/123680
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/3rd_party_deployment/Test start deployment fails while model download in progress}
   issue: https://github.com/elastic/elasticsearch/issues/120814
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable is missing}
-  issue: https://github.com/elastic/elasticsearch/issues/124168
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments}
   issue: https://github.com/elastic/elasticsearch/issues/124315
@@ -161,15 +155,6 @@ tests:
 - class: org.elasticsearch.packaging.test.BootstrapCheckTests
   method: test10Install
   issue: https://github.com/elastic/elasticsearch/issues/124957
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/data_frame_analytics_crud/Test get stats on newly created config}
-  issue: https://github.com/elastic/elasticsearch/issues/121726
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection}
-  issue: https://github.com/elastic/elasticsearch/issues/125641
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header}
-  issue: https://github.com/elastic/elasticsearch/issues/125642
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test schedule_now on an already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/120720
@@ -179,9 +164,6 @@ tests:
 - class: org.elasticsearch.xpack.core.common.notifications.AbstractAuditorTests
   method: testRecreateTemplateWhenDeleted
   issue: https://github.com/elastic/elasticsearch/issues/123232
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/start_data_frame_analytics/Test start given dest index is not empty}
-  issue: https://github.com/elastic/elasticsearch/issues/125909
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_stats/Test get transform stats with timeout}
   issue: https://github.com/elastic/elasticsearch/issues/125975
@@ -197,15 +179,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_stats/Test get transform stats}
   issue: https://github.com/elastic/elasticsearch/issues/126270
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable cardinality is too low}
-  issue: https://github.com/elastic/elasticsearch/issues/126299
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable cardinality is too low}
-  issue: https://github.com/elastic/elasticsearch/issues/123200
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/trained_model_cat_apis/Test cat trained models}
-  issue: https://github.com/elastic/elasticsearch/issues/125750
 - class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT
   method: testEnterpriseDownloaderTask
   issue: https://github.com/elastic/elasticsearch/issues/126124
@@ -245,9 +218,6 @@ tests:
 - class: org.elasticsearch.cli.keystore.AddStringKeyStoreCommandTests
   method: testStdinWithMultipleValues
   issue: https://github.com/elastic/elasticsearch/issues/126882
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header}
-  issue: https://github.com/elastic/elasticsearch/issues/127625
 - class: org.elasticsearch.xpack.ccr.action.ShardFollowTaskReplicationTests
   method: testChangeFollowerHistoryUUID
   issue: https://github.com/elastic/elasticsearch/issues/127680
@@ -333,9 +303,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test171AdditionalCliOptionsAreForwarded
   issue: https://github.com/elastic/elasticsearch/issues/120925
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/delete_expired_data/Test delete expired data with body parameters}
-  issue: https://github.com/elastic/elasticsearch/issues/131364
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test070BindMountCustomPathConfAndJvmOptions
   issue: https://github.com/elastic/elasticsearch/issues/131366

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ResetAuditorAction.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ResetMlComponentsAction.java

@@ -22,12 +22,12 @@
 import java.util.List;
 import java.util.Objects;
 
-public class ResetAuditorAction extends ActionType<ResetAuditorAction.Response> {
+public class ResetMlComponentsAction extends ActionType<ResetMlComponentsAction.Response> {
 
-    public static final ResetAuditorAction INSTANCE = new ResetAuditorAction();
+    public static final ResetMlComponentsAction INSTANCE = new ResetMlComponentsAction();
     public static final String NAME = "cluster:internal/xpack/ml/auditor/reset";
 
-    private ResetAuditorAction() {
+    private ResetMlComponentsAction() {
         super(NAME);
     }
 

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java

@@ -105,7 +105,6 @@ public void testDeleteExpiredData_GivenNothingToDelete() throws Exception {
         client().execute(DeleteExpiredDataAction.INSTANCE, new DeleteExpiredDataAction.Request()).get();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/62699")
     public void testDeleteExpiredDataNoThrottle() throws Exception {
         testExpiredDeletion(null, 10010);
     }
@@ -152,7 +151,6 @@ public void testDeleteExpiredDataActionDeletesEmptyStateIndices() throws Excepti
         );
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/62699")
     public void testDeleteExpiredDataWithStandardThrottle() throws Exception {
         testExpiredDeletion(-1.0f, 100);
     }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -160,8 +160,8 @@
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAliasAction;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelDefinitionPartAction;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelVocabularyAction;
-import org.elasticsearch.xpack.core.ml.action.ResetAuditorAction;
 import org.elasticsearch.xpack.core.ml.action.ResetJobAction;
+import org.elasticsearch.xpack.core.ml.action.ResetMlComponentsAction;
 import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.action.SetResetModeAction;
 import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction;
@@ -271,8 +271,8 @@
 import org.elasticsearch.xpack.ml.action.TransportPutTrainedModelAliasAction;
 import org.elasticsearch.xpack.ml.action.TransportPutTrainedModelDefinitionPartAction;
 import org.elasticsearch.xpack.ml.action.TransportPutTrainedModelVocabularyAction;
-import org.elasticsearch.xpack.ml.action.TransportResetAuditorAction;
 import org.elasticsearch.xpack.ml.action.TransportResetJobAction;
+import org.elasticsearch.xpack.ml.action.TransportResetMlComponentsAction;
 import org.elasticsearch.xpack.ml.action.TransportRevertModelSnapshotAction;
 import org.elasticsearch.xpack.ml.action.TransportSetResetModeAction;
 import org.elasticsearch.xpack.ml.action.TransportSetUpgradeModeAction;
@@ -805,7 +805,7 @@ public void loadExtensions(ExtensionLoader loader) {
     private final SetOnce<LearningToRankService> learningToRankService = new SetOnce<>();
     private final SetOnce<MlAutoscalingDeciderService> mlAutoscalingDeciderService = new SetOnce<>();
     private final SetOnce<DeploymentManager> deploymentManager = new SetOnce<>();
-    private final SetOnce<TrainedModelAssignmentClusterService> trainedModelAllocationClusterServiceSetOnce = new SetOnce<>();
+    private final SetOnce<TrainedModelAssignmentClusterService> trainedModelAllocationClusterService = new SetOnce<>();
 
     private final SetOnce<MachineLearningExtension> machineLearningExtension = new SetOnce<>();
 
@@ -1315,7 +1315,7 @@ public Collection<?> createComponents(PluginServices services) {
             clusterService,
             threadPool
         );
-        trainedModelAllocationClusterServiceSetOnce.set(
+        trainedModelAllocationClusterService.set(
             new TrainedModelAssignmentClusterService(
                 settings,
                 clusterService,
@@ -1391,7 +1391,8 @@ public Collection<?> createComponents(PluginServices services) {
             trainedModelCacheMetadataService,
             trainedModelProvider,
             trainedModelAssignmentService,
-            trainedModelAllocationClusterServiceSetOnce.get(),
+            trainedModelAllocationClusterService.get(),
+            trainedModelStatsService,
             deploymentManager.get(),
             nodeAvailabilityZoneMapper,
             new MachineLearningExtensionHolder(machineLearningExtension.get()),
@@ -1564,7 +1565,7 @@ public List<ActionHandler> getActions() {
         actionHandlers.add(new ActionHandler(MlMemoryAction.INSTANCE, TransportMlMemoryAction.class));
         actionHandlers.add(new ActionHandler(SetUpgradeModeAction.INSTANCE, TransportSetUpgradeModeAction.class));
         actionHandlers.add(new ActionHandler(SetResetModeAction.INSTANCE, TransportSetResetModeAction.class));
-        actionHandlers.add(new ActionHandler(ResetAuditorAction.INSTANCE, TransportResetAuditorAction.class));
+        actionHandlers.add(new ActionHandler(ResetMlComponentsAction.INSTANCE, TransportResetMlComponentsAction.class));
         // Included in this section as it's used by MlMemoryAction
         actionHandlers.add(new ActionHandler(TrainedModelCacheInfoAction.INSTANCE, TransportTrainedModelCacheInfoAction.class));
         actionHandlers.add(new ActionHandler(GetMlAutoscalingStats.INSTANCE, TransportGetMlAutoscalingStats.class));
@@ -2177,17 +2178,17 @@ public void cleanUpFeature(
         });
 
         ActionListener<ResetFeatureStateResponse.ResetFeatureStateStatus> resetAuditors = ActionListener.wrap(success -> {
-            // reset the auditors as aliases used may be removed
+            // reset components, such as the auditors the trained model stats queue
             client.execute(
-                ResetAuditorAction.INSTANCE,
-                ResetAuditorAction.Request.RESET_AUDITOR_REQUEST,
+                ResetMlComponentsAction.INSTANCE,
+                ResetMlComponentsAction.Request.RESET_AUDITOR_REQUEST,
                 ActionListener.wrap(ignored -> unsetResetModeListener.onResponse(success), unsetResetModeListener::onFailure)
             );
         }, failure -> {
             logger.error("failed to reset machine learning", failure);
             client.execute(
-                ResetAuditorAction.INSTANCE,
-                ResetAuditorAction.Request.RESET_AUDITOR_REQUEST,
+                ResetMlComponentsAction.INSTANCE,
+                ResetMlComponentsAction.Request.RESET_AUDITOR_REQUEST,
                 ActionListener.wrap(ignored -> unsetResetModeListener.onFailure(failure), unsetResetModeListener::onFailure)
             );
         });
@@ -2329,11 +2330,11 @@ public void cleanUpFeature(
             );
             client.execute(CancelJobModelSnapshotUpgradeAction.INSTANCE, cancelSnapshotUpgradesReq, delegate);
         }).delegateFailureAndWrap((delegate, acknowledgedResponse) -> {
-            if (trainedModelAllocationClusterServiceSetOnce.get() == null || machineLearningExtension.get().isNlpEnabled() == false) {
+            if (trainedModelAllocationClusterService.get() == null || machineLearningExtension.get().isNlpEnabled() == false) {
                 delegate.onResponse(AcknowledgedResponse.TRUE);
                 return;
             }
-            trainedModelAllocationClusterServiceSetOnce.get().removeAllModelAssignments(delegate);
+            trainedModelAllocationClusterService.get().removeAllModelAssignments(delegate);
         });
 
         // validate no pipelines are using machine learning models

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java

@@ -61,6 +61,7 @@
 import org.elasticsearch.xpack.ml.utils.persistence.MlParserUtils;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -278,7 +279,7 @@ private void searchStats(DataFrameAnalyticsConfig config, TaskId parentTaskId, A
                             () -> format(
                                 "[%s] Item failure encountered during multi search for request [indices=%s, source=%s]: %s",
                                 config.getId(),
-                                itemRequest.indices(),
+                                Arrays.toString(itemRequest.indices()),
                                 itemRequest.source(),
                                 itemResponse.getFailureMessage()
                             ),

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetAuditorAction.java → x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetMlComponentsAction.java

@@ -17,72 +17,77 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.action.ResetAuditorAction;
+import org.elasticsearch.xpack.core.ml.action.ResetMlComponentsAction;
+import org.elasticsearch.xpack.ml.inference.TrainedModelStatsService;
 import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
 
 import java.io.IOException;
 import java.util.List;
 
-public class TransportResetAuditorAction extends TransportNodesAction<
-    ResetAuditorAction.Request,
-    ResetAuditorAction.Response,
-    ResetAuditorAction.NodeRequest,
-    ResetAuditorAction.Response.ResetResponse,
+public class TransportResetMlComponentsAction extends TransportNodesAction<
+    ResetMlComponentsAction.Request,
+    ResetMlComponentsAction.Response,
+    ResetMlComponentsAction.NodeRequest,
+    ResetMlComponentsAction.Response.ResetResponse,
     Void> {
 
     private final AnomalyDetectionAuditor anomalyDetectionAuditor;
     private final DataFrameAnalyticsAuditor dfaAuditor;
     private final InferenceAuditor inferenceAuditor;
+    private final TrainedModelStatsService trainedModelStatsService;
 
     @Inject
-    public TransportResetAuditorAction(
+    public TransportResetMlComponentsAction(
         ThreadPool threadPool,
         ClusterService clusterService,
         TransportService transportService,
         ActionFilters actionFilters,
         AnomalyDetectionAuditor anomalyDetectionAuditor,
         DataFrameAnalyticsAuditor dfaAuditor,
-        InferenceAuditor inferenceAuditor
+        InferenceAuditor inferenceAuditor,
+        TrainedModelStatsService trainedModelStatsService
     ) {
         super(
-            ResetAuditorAction.NAME,
+            ResetMlComponentsAction.NAME,
             clusterService,
             transportService,
             actionFilters,
-            ResetAuditorAction.NodeRequest::new,
+            ResetMlComponentsAction.NodeRequest::new,
             threadPool.executor(ThreadPool.Names.MANAGEMENT)
         );
         this.anomalyDetectionAuditor = anomalyDetectionAuditor;
         this.dfaAuditor = dfaAuditor;
         this.inferenceAuditor = inferenceAuditor;
+        this.trainedModelStatsService = trainedModelStatsService;
     }
 
     @Override
-    protected ResetAuditorAction.Response newResponse(
-        ResetAuditorAction.Request request,
-        List<ResetAuditorAction.Response.ResetResponse> resetResponses,
+    protected ResetMlComponentsAction.Response newResponse(
+        ResetMlComponentsAction.Request request,
+        List<ResetMlComponentsAction.Response.ResetResponse> resetResponses,
         List<FailedNodeException> failures
     ) {
-        return new ResetAuditorAction.Response(clusterService.getClusterName(), resetResponses, failures);
+        return new ResetMlComponentsAction.Response(clusterService.getClusterName(), resetResponses, failures);
     }
 
     @Override
-    protected ResetAuditorAction.NodeRequest newNodeRequest(ResetAuditorAction.Request request) {
-        return new ResetAuditorAction.NodeRequest();
+    protected ResetMlComponentsAction.NodeRequest newNodeRequest(ResetMlComponentsAction.Request request) {
+        return new ResetMlComponentsAction.NodeRequest();
     }
 
     @Override
-    protected ResetAuditorAction.Response.ResetResponse newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException {
-        return new ResetAuditorAction.Response.ResetResponse(in);
+    protected ResetMlComponentsAction.Response.ResetResponse newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException {
+        return new ResetMlComponentsAction.Response.ResetResponse(in);
     }
 
     @Override
-    protected ResetAuditorAction.Response.ResetResponse nodeOperation(ResetAuditorAction.NodeRequest request, Task task) {
+    protected ResetMlComponentsAction.Response.ResetResponse nodeOperation(ResetMlComponentsAction.NodeRequest request, Task task) {
         anomalyDetectionAuditor.reset();
         dfaAuditor.reset();
         inferenceAuditor.reset();
-        return new ResetAuditorAction.Response.ResetResponse(clusterService.localNode(), true);
+        trainedModelStatsService.clearQueue();
+        return new ResetMlComponentsAction.Response.ResetResponse(clusterService.localNode(), true);
     }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java

@@ -295,4 +295,8 @@ static UpdateRequest buildUpdateRequest(InferenceStats stats) {
         }
         return null;
     }
+
+    public void clearQueue() {
+        statsQueue.clear();
+    }
 }