Issue 5688 - Parallelise nightly system tests

docs/development/system-test-suites.md

@@ -0,0 +1,19 @@
+# Current Slow and Expensive test suites
+### This looks imbalanced but EKSBulkImportST is a lot slower than others
+
+These tables show which system tests run in which suite. Each suite runs in parallel to the others to speed up the time
+it takes to complete the nightly system tests.
+
+| Slow1                      | Slow2                      | Slow3                           |
+| -------------------------- | -------------------------- | ------------------------------- |
+| AutoStopEcsTaskST          | EksBulkImportST            | CompactionCreationST            |
+| AutoDeleteS3ObjectsST      | CompactionOnEC2ST          | MultipleTablesST                |
+| RedeployOptionalStacksST   |                            | StateStoreCommitterThroughputST |
+| EmrPersistentBulkImportST  |
+| OptionalFeaturesDisabledST |
+
+
+| Expensive1                        | Expensive2                 | Expensive3            |
+| --------------------------------- | -------------------------- | --------------------- |
+| CompactionDataFusionPerformanceST | CompactionPerformanceST    | IngestPerformanceST   |
+| CompactionVeryLargeST             | EmrBulkImportPerformanceST | ParallelCompactionsST |

docs/development/system-tests.md

@@ -76,13 +76,27 @@ of that class, and look at the tests in that module for examples.
 
 This test module contains several JUnit test suites:
 - QuickSystemTestSuite (the default)
-- NightlyFunctionalSystemTestSuite
-- NightlyPerformanceSystemTestSuite
+- SlowSystemTestSuite1-n
+- ExpensiveSystemTestSuite1-n
 
-Tests that are tagged as Slow or Expensive will not be included in the quick suite. The quick suite is intended to run
-in around 40 minutes. The nightly functional suite includes tests tagged as Slow, and will take a bit longer. The
-nightly performance suite includes all tests, including ones tagged as Expensive. The performance tests work with a
+The nightly test runs have been broken down into a number of test suites that each run in parallel.
+This is to speed up the time it takes for the nightly system tests to complete. Each suite (Quick, Slow1-n, Expensive1-n)
+begins by creating a copy of the Sleeper project and then running specific system tests inside that copy. Results are then
+collated once all suites have finished, then uploaded to an s3 bucket. The suite's Sleeper instances are isolated from one
+another so that they can run in parallel.
+
+For more information about the nightly system tests see [here](../../scripts/test/nightly/README.md).
+
+Tests that are tagged as Slow(1-n) or Expensive(1-n) will not be included in the quick suite. The quick suite is intended
+to run in around 40 minutes. The nightly functional run includes tests tagged as Slow, and will take a bit longer. The
+nightly performance run includes all tests, including ones tagged as Expensive. The performance tests work with a
 larger bulk of data. They take time to run and can be costly to run frequently.
+When adding a new Slow or Expensive System test add either the Slow1-n or Expensive1-n tag.
+
+### Current Slow and Expensive test suites
+The current system tests running in each suite can be seen in [system-test-suites](system-test-suites.md).
+
+When adding new slow or expensive system tests ensure this documentation is updated.
 
 ### Running tests
 

java/cdk-environment/src/main/java/sleeper/environment/cdk/buildec2/BuildEC2Deployment.java

@@ -58,7 +58,7 @@ public BuildEC2Deployment(Construct scope, IVpc vpc, NightlyTestDeployment night
                 .vpc(vpc)
                 .securityGroup(createSecurityGroup(scope))
                 .machineImage(image.machineImage())
-                .instanceType(InstanceType.of(InstanceClass.T3, InstanceSize.LARGE))
+                .instanceType(InstanceType.of(InstanceClass.T3, InstanceSize.XLARGE2))
                 .vpcSubnets(SubnetSelection.builder().subnetType(SubnetType.PRIVATE_WITH_EGRESS).build())
                 .userData(UserData.custom(LoadUserDataUtil.userData(params)))
                 .userDataCausesReplacement(true)

java/cdk-environment/src/main/java/sleeper/environment/cdk/config/AppParameters.java

@@ -32,7 +32,7 @@ private AppParameters() {
     public static final StringParameter BUILD_IMAGE_OWNER = StringParameter.keyAndDefault("buildImageOwner", "099720109477");
     public static final StringParameter BUILD_IMAGE_LOGIN_USER = StringParameter.keyAndDefault("buildImageLoginUser", "ubuntu");
     public static final StringParameter BUILD_IMAGE_ROOT_DEVICE_NAME = StringParameter.keyAndDefault("buildImageRootDeviceName", "/dev/sda1");
-    public static final IntParameter BUILD_ROOT_VOLUME_SIZE_GIB = IntParameter.keyAndDefault("buildRootVolumeSizeGiB", 200);
+    public static final IntParameter BUILD_ROOT_VOLUME_SIZE_GIB = IntParameter.keyAndDefault("buildRootVolumeSizeGiB", 350);
 
     public static final OptionalStringParameter LOG_RETENTION_DAYS = OptionalStringParameter.key("logRetentionDays");
     public static final OptionalStringParameter BUILD_UPTIME_LAMBDA_JAR = OptionalStringParameter.key("buildUptimeLambdaJar");

java/cdk-environment/src/test/java/sleeper/environment/cdk/config/IntParameterTest.java

@@ -26,7 +26,7 @@ public class IntParameterTest {
     @Test
     public void useDefaultValueWhenUnset() {
         AppContext context = AppContext.empty();
-        assertThat(context.get(BUILD_ROOT_VOLUME_SIZE_GIB)).isEqualTo(200);
+        assertThat(context.get(BUILD_ROOT_VOLUME_SIZE_GIB)).isEqualTo(350);
     }
 
     @Test

java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java

@@ -53,6 +53,7 @@ public class DeployExistingInstance {
     private final EcrClient ecr;
     private final boolean deployPaused;
     private final CommandPipelineRunner runCommand;
+    private final boolean createMultiPlatformBuilder;
 
     private DeployExistingInstance(Builder builder) {
         scriptsDirectory = builder.scriptsDirectory;
@@ -62,6 +63,7 @@ private DeployExistingInstance(Builder builder) {
         ecr = builder.ecr;
         deployPaused = builder.deployPaused;
         runCommand = builder.runCommand;
+        createMultiPlatformBuilder = builder.createMultiPlatformBuilder;
     }
 
     public static Builder builder() {
@@ -97,6 +99,7 @@ public void update() throws IOException, InterruptedException {
                                 .scriptsDirectory(scriptsDirectory)
                                 .deployConfig(DeployConfiguration.fromScriptsDirectory(scriptsDirectory))
                                 .commandRunner(runCommand)
+                                .createMultiplatformBuilder(createMultiPlatformBuilder)
                                 .build(),
                         CheckVersionExistsInEcr.withEcrClient(ecr)),
                 DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory),
@@ -120,6 +123,7 @@ public static final class Builder {
         private EcrClient ecr;
         private boolean deployPaused;
         private CommandPipelineRunner runCommand = CommandUtils::runCommandInheritIO;
+        private boolean createMultiPlatformBuilder = true;
 
         private Builder() {
         }
@@ -164,6 +168,11 @@ public Builder runCommand(CommandPipelineRunner runCommand) {
             return this;
         }
 
+        public Builder createMultiPlatformBuilder(boolean createMultiPlatformBuilder) {
+            this.createMultiPlatformBuilder = createMultiPlatformBuilder;
+            return this;
+        }
+
         public Builder loadPropertiesFromS3(S3Client s3Client, DynamoDbClient dynamoCient) {
             properties = S3InstanceProperties.loadGivenInstanceId(s3Client, instanceId);
             tablePropertiesList = S3TableProperties.createStore(properties, s3Client, dynamoCient)

java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java

@@ -61,6 +61,7 @@ public class DeployNewInstance {
     private final InvokeCdk.Type instanceType;
     private final CommandPipelineRunner runCommand;
     private final boolean deployPaused;
+    private final boolean createMultiPlatformBuilder;
 
     private DeployNewInstance(Builder builder) {
         s3Client = builder.s3Client;
@@ -72,6 +73,7 @@ private DeployNewInstance(Builder builder) {
         instanceType = builder.instanceType;
         runCommand = builder.runCommand;
         deployPaused = builder.deployPaused;
+        createMultiPlatformBuilder = builder.createMultiPlatformBuilder;
     }
 
     public static Builder builder() {
@@ -108,14 +110,14 @@ public void deploy() throws IOException, InterruptedException {
         LOGGER.info("Running Deployment");
         LOGGER.info("-------------------------------------------------------");
         deployInstanceConfiguration.validate();
-
         DeployInstance deployInstance = new DeployInstance(
                 SyncJars.fromScriptsDirectory(s3Client, scriptsDirectory),
                 new UploadDockerImagesToEcr(
                         UploadDockerImages.builder()
                                 .scriptsDirectory(scriptsDirectory)
                                 .deployConfig(DeployConfiguration.fromScriptsDirectory(scriptsDirectory))
                                 .commandRunner(runCommand)
+                                .createMultiplatformBuilder(createMultiPlatformBuilder)
                                 .build(),
                         CheckVersionExistsInEcr.withEcrClient(ecrClient)),
                 DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory),
@@ -149,6 +151,7 @@ public static final class Builder {
         private InvokeCdk.Type instanceType;
         private CommandPipelineRunner runCommand = CommandUtils::runCommandInheritIO;
         private boolean deployPaused;
+        private boolean createMultiPlatformBuilder = true;
 
         private Builder() {
         }
@@ -198,6 +201,11 @@ public Builder deployPaused(boolean deployPaused) {
             return this;
         }
 
+        public Builder createMultiPlatformBuilder(boolean createMultiPlatformBuilder) {
+            this.createMultiPlatformBuilder = createMultiPlatformBuilder;
+            return this;
+        }
+
         public DeployNewInstance build() {
             return new DeployNewInstance(this);
         }

java/pom.xml

@@ -241,6 +241,7 @@
         <sleeper.system.test.force.redeploy>false</sleeper.system.test.force.redeploy>
         <sleeper.system.test.instances.force.redeploy>false</sleeper.system.test.instances.force.redeploy>
         <sleeper.system.test.force.statestore.classname />
+        <sleeper.system.test.create.multi.platform.builder>true</sleeper.system.test.create.multi.platform.builder>
         <sleeper.system.test.standalone.properties.template />
         <sleeper.system.test.instance.properties.overrides />
         <!-- Maven plugins -->
@@ -1944,6 +1945,10 @@
                                 <name>sleeper.system.test.force.statestore.classname</name>
                                 <value>${sleeper.system.test.force.statestore.classname}</value>
                             </property>
+                            <property>
+                                <name>sleeper.system.test.create.multi.platform.builder</name>
+                                <value>${sleeper.system.test.create.multi.platform.builder}</value>
+                            </property>
                             <property>
                                 <name>sleeper.system.test.standalone.properties.template</name>
                                 <value>${sleeper.system.test.standalone.properties.template}</value>

java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java

@@ -94,6 +94,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf
                     .deployInstanceConfiguration(deployConfig)
                     .instanceType(InvokeCdk.Type.STANDARD)
                     .runCommand(CommandUtils::runCommandLogOutput)
+                    .createMultiPlatformBuilder(parameters.isCreateMultiPlatformBuilder())
                     .deployWithClients(s3, dynamoDB, ecr);
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
@@ -128,6 +129,7 @@ public void redeploy(InstanceProperties instanceProperties, List<TableProperties
                     .tablePropertiesList(tableProperties)
                     .scriptsDirectory(parameters.getScriptsDirectory())
                     .runCommand(CommandUtils::runCommandLogOutput)
+                    .createMultiPlatformBuilder(parameters.isCreateMultiPlatformBuilder())
                     .build().update();
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();

java/system-test/system-test-dsl/src/main/java/sleeper/systemtest/dsl/instance/SystemTestParameters.java

@@ -63,6 +63,7 @@ public class SystemTestParameters {
     private final boolean forceRedeploySystemTest;
     private final boolean forceRedeployInstances;
     private final String forceStateStoreClassname;
+    private final boolean createMultiPlatformBuilder;
     private final SystemTestStandaloneProperties standalonePropertiesTemplate;
     private final InstanceProperties instancePropertiesOverrides;
 
@@ -79,6 +80,7 @@ private SystemTestParameters(Builder builder) {
         forceRedeploySystemTest = builder.forceRedeploySystemTest;
         forceRedeployInstances = builder.forceRedeployInstances;
         forceStateStoreClassname = builder.forceStateStoreClassname;
+        createMultiPlatformBuilder = builder.createMultiPlatformBuilder;
         standalonePropertiesTemplate = Objects.requireNonNull(builder.standalonePropertiesTemplate, "standalonePropertiesTemplate must not be null");
         instancePropertiesOverrides = Objects.requireNonNull(builder.instancePropertiesOverrides, "instancePropertiesOverrides must not be null");
         // Combines with SystemTestInstanceConfiguration.shortName and a hyphen to create an instance ID within maximum length
@@ -155,6 +157,10 @@ public boolean isForceRedeployInstances() {
         return forceRedeployInstances;
     }
 
+    public boolean isCreateMultiPlatformBuilder() {
+        return createMultiPlatformBuilder;
+    }
+
     public TableProperties createTableProperties(InstanceProperties instanceProperties, Schema schema) {
         TableProperties tableProperties = new TableProperties(instanceProperties);
         tableProperties.setSchema(schema);
@@ -244,6 +250,7 @@ public static final class Builder {
         private boolean forceRedeploySystemTest;
         private boolean forceRedeployInstances;
         private String forceStateStoreClassname;
+        private boolean createMultiPlatformBuilder = true;
         private SystemTestStandaloneProperties standalonePropertiesTemplate;
         private InstanceProperties instancePropertiesOverrides;
 
@@ -310,6 +317,11 @@ public Builder forceStateStoreClassname(String forceStateStoreClassname) {
             return this;
         }
 
+        public Builder createMultiPlatformBuilder(boolean createMultiPlatformBuilder) {
+            this.createMultiPlatformBuilder = createMultiPlatformBuilder;
+            return this;
+        }
+
         public Builder systemTestStandalonePropertiesTemplate(SystemTestStandaloneProperties standalonePropertiesTemplate) {
             this.standalonePropertiesTemplate = standalonePropertiesTemplate;
             return this;
@@ -332,6 +344,7 @@ public Builder loadFromSystemProperties() {
                     .forceRedeploySystemTest(getBooleanProperty("sleeper.system.test.force.redeploy", false))
                     .forceRedeployInstances(getBooleanProperty("sleeper.system.test.instances.force.redeploy", false))
                     .forceStateStoreClassname(getOptionalProperty("sleeper.system.test.force.statestore.classname").orElse(null))
+                    .createMultiPlatformBuilder(getBooleanProperty("sleeper.system.test.create.multi.platform.builder", true))
                     .systemTestStandalonePropertiesTemplate(getOptionalProperty("sleeper.system.test.standalone.properties.template")
                             .map(Paths::get)
                             .map(SystemTestStandaloneProperties::fromFile)

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/AutoDeleteS3ObjectsST.java

@@ -25,8 +25,8 @@
 import sleeper.query.core.output.ResultsOutput;
 import sleeper.query.runner.output.S3ResultsOutput;
 import sleeper.systemtest.dsl.SleeperDsl;
-import sleeper.systemtest.suite.testutil.Slow;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.parallel.Slow1;
 
 import java.nio.file.Path;
 import java.util.List;
@@ -38,7 +38,7 @@
 
 @SystemTest
 // Slow because it needs to do multiple CDK deployments
-@Slow
+@Slow1
 public class AutoDeleteS3ObjectsST {
 
     @TempDir

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/AutoStopEcsTaskST.java

@@ -23,8 +23,8 @@
 import sleeper.core.statestore.FileReference;
 import sleeper.core.util.PollWithRetries;
 import sleeper.systemtest.dsl.SleeperDsl;
-import sleeper.systemtest.suite.testutil.Slow;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.parallel.Slow1;
 
 import java.time.Duration;
 import java.util.List;
@@ -36,7 +36,7 @@
 
 @SystemTest
 // Slow because it needs to do multiple CDK deployments
-@Slow
+@Slow1
 public class AutoStopEcsTaskST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionCreationST.java

@@ -25,8 +25,8 @@
 import sleeper.core.util.PollWithRetries;
 import sleeper.systemtest.dsl.SleeperDsl;
 import sleeper.systemtest.dsl.compaction.FoundCompactionJobs;
-import sleeper.systemtest.suite.testutil.Slow;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.parallel.Slow1;
 
 import java.time.Duration;
 import java.util.List;
@@ -45,7 +45,8 @@
 import static sleeper.systemtest.suite.testutil.FileReferenceSystemTestHelper.fileFactory;
 
 @SystemTest
-@Slow // Slow because it deploys its own instance so it can drain the whole compaction jobs queue
+// Slow because it deploys its own instance so it can drain the whole compaction jobs queue
+@Slow1
 @Execution(SAME_THREAD)
 public class CompactionCreationST {
 

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionDataFusionPerformanceST.java

@@ -26,8 +26,8 @@
 import sleeper.systemtest.dsl.SleeperDsl;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.parallel.Expensive1;
 
 import java.time.Duration;
 import java.util.Map;
@@ -41,7 +41,8 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.COMPACTION_PERFORMANCE_DATAFUSION;
 
 @SystemTest
-@Expensive // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
+// Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
+@Expensive1
 public class CompactionDataFusionPerformanceST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionOnEC2ST.java

@@ -25,8 +25,8 @@
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
 import sleeper.systemtest.dsl.sourcedata.RowNumbers;
-import sleeper.systemtest.suite.testutil.Slow;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.parallel.Slow2;
 
 import java.nio.file.Path;
 import java.util.Map;
@@ -41,7 +41,7 @@
 import static sleeper.systemtest.suite.testutil.TestResources.exampleString;
 
 @SystemTest
-@Slow
+@Slow2
 public class CompactionOnEC2ST {
     @TempDir
     private Path tempDir;

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionPerformanceST.java

@@ -26,8 +26,8 @@
 import sleeper.systemtest.dsl.SleeperDsl;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.parallel.Expensive2;
 
 import java.time.Duration;
 import java.util.Map;
@@ -40,7 +40,8 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.COMPACTION_PERFORMANCE;
 
 @SystemTest
-@Expensive // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
+// Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
+@Expensive2
 public class CompactionPerformanceST {
 
     @BeforeEach