diff --git a/docs/development/system-test-suites.md b/docs/development/system-test-suites.md new file mode 100644 index 00000000000..52b5984807c --- /dev/null +++ b/docs/development/system-test-suites.md @@ -0,0 +1,19 @@ +# Current Slow and Expensive test suites +### This looks imbalanced but EKSBulkImportST is a lot slower than others + +These tables show which system tests run in which suite. Each suite runs in parallel to the others to speed up the time +it takes to complete the nightly system tests. + +| Slow1 | Slow2 | Slow3 | +| -------------------------- | -------------------------- | ------------------------------- | +| AutoStopEcsTaskST | EksBulkImportST | CompactionCreationST | +| AutoDeleteS3ObjectsST | CompactionOnEC2ST | MultipleTablesST | +| RedeployOptionalStacksST | | StateStoreCommitterThroughputST | +| EmrPersistentBulkImportST | +| OptionalFeaturesDisabledST | + + +| Expensive1 | Expensive2 | Expensive3 | +| --------------------------------- | -------------------------- | --------------------- | +| CompactionDataFusionPerformanceST | CompactionPerformanceST | IngestPerformanceST | +| CompactionVeryLargeST | EmrBulkImportPerformanceST | ParallelCompactionsST | \ No newline at end of file diff --git a/docs/development/system-tests.md b/docs/development/system-tests.md index 1208074d206..01b8f545932 100644 --- a/docs/development/system-tests.md +++ b/docs/development/system-tests.md @@ -76,13 +76,27 @@ of that class, and look at the tests in that module for examples. This test module contains several JUnit test suites: - QuickSystemTestSuite (the default) -- NightlyFunctionalSystemTestSuite -- NightlyPerformanceSystemTestSuite +- SlowSystemTestSuite1-n +- ExpensiveSystemTestSuite1-n -Tests that are tagged as Slow or Expensive will not be included in the quick suite. The quick suite is intended to run -in around 40 minutes. The nightly functional suite includes tests tagged as Slow, and will take a bit longer. The -nightly performance suite includes all tests, including ones tagged as Expensive. The performance tests work with a +The nightly test runs have been broken down into a number of test suites that each run in parallel. +This is to speed up the time it takes for the nightly system tests to complete. Each suite (Quick, Slow1-n, Expensive1-n) +begins by creating a copy of the Sleeper project and then running specific system tests inside that copy. Results are then +collated once all suites have finished, then uploaded to an s3 bucket. The suite's Sleeper instances are isolated from one +another so that they can run in parallel. + +For more information about the nightly system tests see [here](../../scripts/test/nightly/README.md). + +Tests that are tagged as Slow(1-n) or Expensive(1-n) will not be included in the quick suite. The quick suite is intended +to run in around 40 minutes. The nightly functional run includes tests tagged as Slow, and will take a bit longer. The +nightly performance run includes all tests, including ones tagged as Expensive. The performance tests work with a larger bulk of data. They take time to run and can be costly to run frequently. +When adding a new Slow or Expensive System test add either the Slow1-n or Expensive1-n tag. + +### Current Slow and Expensive test suites +The current system tests running in each suite can be seen in [system-test-suites](system-test-suites.md). + +When adding new slow or expensive system tests ensure this documentation is updated. ### Running tests diff --git a/java/cdk-environment/src/main/java/sleeper/environment/cdk/buildec2/BuildEC2Deployment.java b/java/cdk-environment/src/main/java/sleeper/environment/cdk/buildec2/BuildEC2Deployment.java index 6d6c3f9d700..c8c347d51c3 100644 --- a/java/cdk-environment/src/main/java/sleeper/environment/cdk/buildec2/BuildEC2Deployment.java +++ b/java/cdk-environment/src/main/java/sleeper/environment/cdk/buildec2/BuildEC2Deployment.java @@ -58,7 +58,7 @@ public BuildEC2Deployment(Construct scope, IVpc vpc, NightlyTestDeployment night .vpc(vpc) .securityGroup(createSecurityGroup(scope)) .machineImage(image.machineImage()) - .instanceType(InstanceType.of(InstanceClass.T3, InstanceSize.LARGE)) + .instanceType(InstanceType.of(InstanceClass.T3, InstanceSize.XLARGE2)) .vpcSubnets(SubnetSelection.builder().subnetType(SubnetType.PRIVATE_WITH_EGRESS).build()) .userData(UserData.custom(LoadUserDataUtil.userData(params))) .userDataCausesReplacement(true) diff --git a/java/cdk-environment/src/main/java/sleeper/environment/cdk/config/AppParameters.java b/java/cdk-environment/src/main/java/sleeper/environment/cdk/config/AppParameters.java index 519146023d1..eeae6e1e388 100644 --- a/java/cdk-environment/src/main/java/sleeper/environment/cdk/config/AppParameters.java +++ b/java/cdk-environment/src/main/java/sleeper/environment/cdk/config/AppParameters.java @@ -32,7 +32,7 @@ private AppParameters() { public static final StringParameter BUILD_IMAGE_OWNER = StringParameter.keyAndDefault("buildImageOwner", "099720109477"); public static final StringParameter BUILD_IMAGE_LOGIN_USER = StringParameter.keyAndDefault("buildImageLoginUser", "ubuntu"); public static final StringParameter BUILD_IMAGE_ROOT_DEVICE_NAME = StringParameter.keyAndDefault("buildImageRootDeviceName", "/dev/sda1"); - public static final IntParameter BUILD_ROOT_VOLUME_SIZE_GIB = IntParameter.keyAndDefault("buildRootVolumeSizeGiB", 200); + public static final IntParameter BUILD_ROOT_VOLUME_SIZE_GIB = IntParameter.keyAndDefault("buildRootVolumeSizeGiB", 350); public static final OptionalStringParameter LOG_RETENTION_DAYS = OptionalStringParameter.key("logRetentionDays"); public static final OptionalStringParameter BUILD_UPTIME_LAMBDA_JAR = OptionalStringParameter.key("buildUptimeLambdaJar"); diff --git a/java/cdk-environment/src/test/java/sleeper/environment/cdk/config/IntParameterTest.java b/java/cdk-environment/src/test/java/sleeper/environment/cdk/config/IntParameterTest.java index b74003074d2..a7c963d6065 100644 --- a/java/cdk-environment/src/test/java/sleeper/environment/cdk/config/IntParameterTest.java +++ b/java/cdk-environment/src/test/java/sleeper/environment/cdk/config/IntParameterTest.java @@ -26,7 +26,7 @@ public class IntParameterTest { @Test public void useDefaultValueWhenUnset() { AppContext context = AppContext.empty(); - assertThat(context.get(BUILD_ROOT_VOLUME_SIZE_GIB)).isEqualTo(200); + assertThat(context.get(BUILD_ROOT_VOLUME_SIZE_GIB)).isEqualTo(350); } @Test diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index 435eab2062e..f19bb2d4884 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -53,6 +53,7 @@ public class DeployExistingInstance { private final EcrClient ecr; private final boolean deployPaused; private final CommandPipelineRunner runCommand; + private final boolean createMultiPlatformBuilder; private DeployExistingInstance(Builder builder) { scriptsDirectory = builder.scriptsDirectory; @@ -62,6 +63,7 @@ private DeployExistingInstance(Builder builder) { ecr = builder.ecr; deployPaused = builder.deployPaused; runCommand = builder.runCommand; + createMultiPlatformBuilder = builder.createMultiPlatformBuilder; } public static Builder builder() { @@ -97,6 +99,7 @@ public void update() throws IOException, InterruptedException { .scriptsDirectory(scriptsDirectory) .deployConfig(DeployConfiguration.fromScriptsDirectory(scriptsDirectory)) .commandRunner(runCommand) + .createMultiplatformBuilder(createMultiPlatformBuilder) .build(), CheckVersionExistsInEcr.withEcrClient(ecr)), DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), @@ -120,6 +123,7 @@ public static final class Builder { private EcrClient ecr; private boolean deployPaused; private CommandPipelineRunner runCommand = CommandUtils::runCommandInheritIO; + private boolean createMultiPlatformBuilder = true; private Builder() { } @@ -164,6 +168,11 @@ public Builder runCommand(CommandPipelineRunner runCommand) { return this; } + public Builder createMultiPlatformBuilder(boolean createMultiPlatformBuilder) { + this.createMultiPlatformBuilder = createMultiPlatformBuilder; + return this; + } + public Builder loadPropertiesFromS3(S3Client s3Client, DynamoDbClient dynamoCient) { properties = S3InstanceProperties.loadGivenInstanceId(s3Client, instanceId); tablePropertiesList = S3TableProperties.createStore(properties, s3Client, dynamoCient) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index b664ca8f226..e435c377a39 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -61,6 +61,7 @@ public class DeployNewInstance { private final InvokeCdk.Type instanceType; private final CommandPipelineRunner runCommand; private final boolean deployPaused; + private final boolean createMultiPlatformBuilder; private DeployNewInstance(Builder builder) { s3Client = builder.s3Client; @@ -72,6 +73,7 @@ private DeployNewInstance(Builder builder) { instanceType = builder.instanceType; runCommand = builder.runCommand; deployPaused = builder.deployPaused; + createMultiPlatformBuilder = builder.createMultiPlatformBuilder; } public static Builder builder() { @@ -108,7 +110,6 @@ public void deploy() throws IOException, InterruptedException { LOGGER.info("Running Deployment"); LOGGER.info("-------------------------------------------------------"); deployInstanceConfiguration.validate(); - DeployInstance deployInstance = new DeployInstance( SyncJars.fromScriptsDirectory(s3Client, scriptsDirectory), new UploadDockerImagesToEcr( @@ -116,6 +117,7 @@ public void deploy() throws IOException, InterruptedException { .scriptsDirectory(scriptsDirectory) .deployConfig(DeployConfiguration.fromScriptsDirectory(scriptsDirectory)) .commandRunner(runCommand) + .createMultiplatformBuilder(createMultiPlatformBuilder) .build(), CheckVersionExistsInEcr.withEcrClient(ecrClient)), DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), @@ -149,6 +151,7 @@ public static final class Builder { private InvokeCdk.Type instanceType; private CommandPipelineRunner runCommand = CommandUtils::runCommandInheritIO; private boolean deployPaused; + private boolean createMultiPlatformBuilder = true; private Builder() { } @@ -198,6 +201,11 @@ public Builder deployPaused(boolean deployPaused) { return this; } + public Builder createMultiPlatformBuilder(boolean createMultiPlatformBuilder) { + this.createMultiPlatformBuilder = createMultiPlatformBuilder; + return this; + } + public DeployNewInstance build() { return new DeployNewInstance(this); } diff --git a/java/pom.xml b/java/pom.xml index ffe857a3a19..739f18062a1 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -241,6 +241,7 @@ false false + true @@ -1944,6 +1945,10 @@ sleeper.system.test.force.statestore.classname ${sleeper.system.test.force.statestore.classname} + + sleeper.system.test.create.multi.platform.builder + ${sleeper.system.test.create.multi.platform.builder} + sleeper.system.test.standalone.properties.template ${sleeper.system.test.standalone.properties.template} diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 525876ab3cf..78aa4a3fedf 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -94,6 +94,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf .deployInstanceConfiguration(deployConfig) .instanceType(InvokeCdk.Type.STANDARD) .runCommand(CommandUtils::runCommandLogOutput) + .createMultiPlatformBuilder(parameters.isCreateMultiPlatformBuilder()) .deployWithClients(s3, dynamoDB, ecr); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -128,6 +129,7 @@ public void redeploy(InstanceProperties instanceProperties, List REDEPLOYABLE_STACKS = new LinkedHashSet<>(OptionalStack.all()); diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterThroughputST.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterThroughputST.java index f17b9758bd4..d79e64acf9e 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterThroughputST.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterThroughputST.java @@ -23,8 +23,8 @@ import sleeper.core.statestore.FileReferenceFactory; import sleeper.systemtest.dsl.SleeperDsl; import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage; -import sleeper.systemtest.suite.testutil.Slow; import sleeper.systemtest.suite.testutil.SystemTest; +import sleeper.systemtest.suite.testutil.parallel.Slow3; import java.time.Duration; import java.time.Instant; @@ -49,7 +49,7 @@ import static sleeper.systemtest.suite.fixtures.SystemTestInstance.COMMITTER_THROUGHPUT; @SystemTest -@Slow +@Slow3 @Execution(SAME_THREAD) public class StateStoreCommitterThroughputST { diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/NightlyFunctionalSystemTestSuite.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite1.java similarity index 85% rename from java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/NightlyFunctionalSystemTestSuite.java rename to java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite1.java index 48604c2563c..743ac14f7f0 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/NightlyFunctionalSystemTestSuite.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite1.java @@ -16,11 +16,11 @@ package sleeper.systemtest.suite.suites; -import org.junit.platform.suite.api.ExcludeTags; +import org.junit.platform.suite.api.IncludeTags; import sleeper.systemtest.suite.testutil.SystemTestSuite; @SystemTestSuite -@ExcludeTags("expensive") -public class NightlyFunctionalSystemTestSuite { +@IncludeTags("expensive1") +public class ExpensiveSystemTestSuite1 { } diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite2.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite2.java new file mode 100644 index 00000000000..97a78f851a7 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite2.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.suites; + +import org.junit.platform.suite.api.IncludeTags; + +import sleeper.systemtest.suite.testutil.SystemTestSuite; + +@SystemTestSuite +@IncludeTags("expensive2") +public class ExpensiveSystemTestSuite2 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite3.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite3.java new file mode 100644 index 00000000000..08e40c1445f --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/ExpensiveSystemTestSuite3.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.suites; + +import org.junit.platform.suite.api.IncludeTags; + +import sleeper.systemtest.suite.testutil.SystemTestSuite; + +@SystemTestSuite +@IncludeTags("expensive3") +public class ExpensiveSystemTestSuite3 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/NightlyPerformanceSystemTestSuite.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite1.java similarity index 87% rename from java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/NightlyPerformanceSystemTestSuite.java rename to java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite1.java index 9cf597a7676..a5a899790e4 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/NightlyPerformanceSystemTestSuite.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite1.java @@ -16,8 +16,11 @@ package sleeper.systemtest.suite.suites; +import org.junit.platform.suite.api.IncludeTags; + import sleeper.systemtest.suite.testutil.SystemTestSuite; @SystemTestSuite -public class NightlyPerformanceSystemTestSuite { +@IncludeTags("slow1") +public class SlowSystemTestSuite1 { } diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite2.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite2.java new file mode 100644 index 00000000000..674b6cf12e8 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite2.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.suites; + +import org.junit.platform.suite.api.IncludeTags; + +import sleeper.systemtest.suite.testutil.SystemTestSuite; + +@SystemTestSuite +@IncludeTags("slow2") +public class SlowSystemTestSuite2 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite3.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite3.java new file mode 100644 index 00000000000..1cba5df5fad --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/SlowSystemTestSuite3.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.suites; + +import org.junit.platform.suite.api.IncludeTags; + +import sleeper.systemtest.suite.testutil.SystemTestSuite; + +@SystemTestSuite +@IncludeTags("slow3") +public class SlowSystemTestSuite3 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/Expensive.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive.java similarity index 81% rename from java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/Expensive.java rename to java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive.java index eba272cae9a..e130a961860 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/Expensive.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package sleeper.systemtest.suite.testutil; +package sleeper.systemtest.suite.testutil.parallel; import org.junit.jupiter.api.Tag; @@ -23,6 +23,10 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +/** + * This tag should not be added directly to a test. + * It is used within Expensive1, Expensive2 & Expensive3 to declare a test as Expensive. + */ @Tag("expensive") @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive1.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive1.java new file mode 100644 index 00000000000..a29494d2dc7 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive1.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.testutil.parallel; + +import org.junit.jupiter.api.Tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Tag("expensive1") +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Expensive +public @interface Expensive1 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive2.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive2.java new file mode 100644 index 00000000000..c0a040cb220 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive2.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.testutil.parallel; + +import org.junit.jupiter.api.Tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Tag("expensive2") +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Expensive +public @interface Expensive2 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive3.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive3.java new file mode 100644 index 00000000000..cf27909e9b3 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Expensive3.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.testutil.parallel; + +import org.junit.jupiter.api.Tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Tag("expensive3") +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Expensive +public @interface Expensive3 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/Slow.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow.java similarity index 83% rename from java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/Slow.java rename to java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow.java index 96b95c731a4..817309d286b 100644 --- a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/Slow.java +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package sleeper.systemtest.suite.testutil; +package sleeper.systemtest.suite.testutil.parallel; import org.junit.jupiter.api.Tag; @@ -23,6 +23,10 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +/** + * This tag should not be added directly to a test. + * It is used within Slow1, Slow2 & Slow3 to declare a test as Slow. + */ @Tag("slow") @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow1.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow1.java new file mode 100644 index 00000000000..90a08a52ef4 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow1.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.testutil.parallel; + +import org.junit.jupiter.api.Tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Tag("slow1") +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Slow +public @interface Slow1 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow2.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow2.java new file mode 100644 index 00000000000..8d89e8b41b8 --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow2.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.testutil.parallel; + +import org.junit.jupiter.api.Tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Tag("slow2") +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Slow +public @interface Slow2 { +} diff --git a/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow3.java b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow3.java new file mode 100644 index 00000000000..6bedc39493d --- /dev/null +++ b/java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/testutil/parallel/Slow3.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.systemtest.suite.testutil.parallel; + +import org.junit.jupiter.api.Tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Tag("slow3") +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Slow +public @interface Slow3 { +} diff --git a/scripts/cli/builder/Dockerfile b/scripts/cli/builder/Dockerfile index 86307a6faa2..1aafe2091c4 100644 --- a/scripts/cli/builder/Dockerfile +++ b/scripts/cli/builder/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y \ jq \ uuid-runtime \ sudo \ + rsync \ # Rust build time dependencies build-essential pkg-config libssl-dev \ # Clang installation dependencies diff --git a/scripts/test/nightly/README.md b/scripts/test/nightly/README.md index ef13d75e7cf..057ea524637 100644 --- a/scripts/test/nightly/README.md +++ b/scripts/test/nightly/README.md @@ -26,6 +26,9 @@ git clone https://github.com/gchq/sleeper.git cd sleeper ``` +When running the performance build, which runs every test suite in parallel, it is recommended to have at least 350GiB +volume size on the running EC2 with an instance type that has at least 4vCPU's and 32GiB Memory. + ### Configuration You can configure the nightly tests inside the builder. Note that any files you put in the builder under the @@ -68,7 +71,12 @@ To run the tests as they would be run by the cron job, use this command from the sleeper builder ./sleeper/scripts/test/nightly/updateAndRunTests.sh /sleeper-builder/nightlyTestSettings.json &> /tmp/sleeperTests.log ``` -With the performance test suite, this will take 6 hours or so. +Running the nightly system tests should take around 2.5 hours. + +Due to the time taken to run all the system tests for either a functional or performance run, multiple test suites +now run in parallel. This is achieved by copying the Sleeper folders into a +temporary folder per batch (quick, slow1-n, expensive1-n). The nightly run script then runs each of these in parallel +with a short 60 seconds delay between each one starting up. Once all have completed it then uploads the results to S3. ### Output diff --git a/scripts/test/nightly/runTests.sh b/scripts/test/nightly/runTests.sh index 4ce4da3c6b4..8a72a7d32fd 100755 --- a/scripts/test/nightly/runTests.sh +++ b/scripts/test/nightly/runTests.sh @@ -34,25 +34,23 @@ VPC=$2 SUBNETS=$3 RESULTS_BUCKET=$4 MAIN_SUITE_NAME=$5 +SUITE_PARAMS=("-Dsleeper.system.test.cluster.enabled=true" "-DskipRust" "-Dsleeper.system.test.create.multi.platform.builder=false") + shift 4 -if [ "$MAIN_SUITE_NAME" == "performance" ]; then - shift - MAIN_SUITE_PARAMS=(-Dsleeper.system.test.cluster.enabled=true -DrunIT=NightlyPerformanceSystemTestSuite "$@") -elif [ "$MAIN_SUITE_NAME" == "functional" ]; then +if [ "$MAIN_SUITE_NAME" == "performance" ] || [ "$MAIN_SUITE_NAME" == "functional" ]; then shift - MAIN_SUITE_PARAMS=(-Dsleeper.system.test.cluster.enabled=true -DrunIT=NightlyFunctionalSystemTestSuite "$@") elif [ "$1" == "--main" ]; then MAIN_SUITE_NAME=custom - MAIN_SUITE_PARAMS=("$2") + SUITE_PARAMS=("$2") shift 2 else MAIN_SUITE_NAME=custom - MAIN_SUITE_PARAMS=("$@") + SUITE_PARAMS=("$@") fi echo "DEPLOY_ID=$DEPLOY_ID" echo "MAIN_SUITE_NAME=$MAIN_SUITE_NAME" -echo "MAIN_SUITE_PARAMS=(${MAIN_SUITE_PARAMS[*]})" +echo "SUITE_PARAMS=(${SUITE_PARAMS[*]})" source "$SCRIPTS_DIR/functions/timeUtils.sh" source "$SCRIPTS_DIR/functions/systemTestUtils.sh" @@ -65,11 +63,35 @@ mkdir -p "$OUTPUT_DIR" ../build/buildForTest.sh VERSION=$(cat "$SCRIPTS_DIR/templates/version.txt") SYSTEM_TEST_JAR="$SCRIPTS_DIR/jars/system-test-${VERSION}-utility.jar" + set +e END_EXIT_CODE=0 +docker buildx rm sleeper +set -e +docker buildx create --name sleeper --use +set +e + +copyFolderForParallelRun() { + echo "Making folder $1 for parallel build" + pushd $REPO_PARENT_DIR + sudo rm -rf $1 + mkdir $1 + sudo rsync -a --exclude=".*" sleeper/ $1 + popd +} + +removeFolderAfterParallelRun() { + echo "Removing folder $1" + pushd $REPO_PARENT_DIR + sudo rm -rf $1 + popd +} + runMavenSystemTests() { + # Setup + NEW_MAVEN_DIR=$(cd ../../java && pwd) SHORT_ID=$1 TEST_NAME=$2 shift 2 @@ -77,19 +99,23 @@ runMavenSystemTests() { EXTRA_MAVEN_PARAMS=("$@") TEST_OUTPUT_DIR="$OUTPUT_DIR/$TEST_NAME" mkdir "$TEST_OUTPUT_DIR" - pushd "$MAVEN_DIR" - mvn clean - popd + echo "Made output directory: $TEST_OUTPUT_DIR for SHORT_ID: $SHORT_ID" + + # Run tests ./maven/deployTest.sh "$SHORT_ID" "$VPC" "$SUBNETS" \ -Dsleeper.system.test.output.dir="$TEST_OUTPUT_DIR" \ "${EXTRA_MAVEN_PARAMS[@]}" \ &> "$OUTPUT_DIR/$TEST_NAME.log" RUN_TESTS_EXIT_CODE=$? + echo "Exit code for $SHORT_ID is $RUN_TESTS_EXIT_CODE" if [ $RUN_TESTS_EXIT_CODE -ne 0 ]; then END_EXIT_CODE=$RUN_TESTS_EXIT_CODE TEST_EXIT_CODE=$RUN_TESTS_EXIT_CODE fi - pushd "$MAVEN_DIR" + + # Generate site HTML + pushd "$NEW_MAVEN_DIR" + echo "Generating site HTML for $SHORT_ID" mvn --batch-mode site site:stage -pl system-test/system-test-suite \ -DskipTests=true \ -DstagingDirectory="$TEST_OUTPUT_DIR/site" @@ -98,6 +124,8 @@ runMavenSystemTests() { zip -r "$OUTPUT_DIR/$TEST_NAME-site.zip" "." popd rm -rf "$TEST_OUTPUT_DIR/site" + + # Tear down instances used for tests SHORT_INSTANCE_NAMES=$(read_short_instance_names_from_instance_ids "$SHORT_ID" "$TEST_OUTPUT_DIR/instanceIds.txt") ./maven/tearDown.sh "$SHORT_ID" "$SHORT_INSTANCE_NAMES" &> "$OUTPUT_DIR/$TEST_NAME.tearDown.log" TEARDOWN_EXIT_CODE=$? @@ -108,7 +136,40 @@ runMavenSystemTests() { echo -n "$TEST_EXIT_CODE $SHORT_ID" > "$OUTPUT_DIR/$TEST_NAME.status" } -runMavenSystemTests "${DEPLOY_ID}mvn${START_TIME_SHORT}" $MAIN_SUITE_NAME "${MAIN_SUITE_PARAMS[@]}" +runTestSuite(){ + SUITE=$3 + copyFolderForParallelRun "$SUITE" + # Wait short time to not have clashes with other process deploying + sleep $1 + shift 1 + echo "[$(time_str)] Starting test suite: $SUITE" + pushd "$REPO_PARENT_DIR/$SUITE/scripts/test" #Move into isolated repo copy + runMavenSystemTests "$@" + popd + echo "[$(time_str)] Finished test suite: $SUITE" + removeFolderAfterParallelRun "$SUITE" +} + +if [ "$MAIN_SUITE_NAME" == "performance" ]; then + echo "Running performance tests in parallel. Start time: [$(time_str)]" + runTestSuite 0 "${DEPLOY_ID}${START_TIME_SHORT}s1" "slow1" "${SUITE_PARAMS[@]}" "-DrunIT=SlowSystemTestSuite1" "$@" &> "$OUTPUT_DIR/slow1.suite.log" & + runTestSuite 60 "${DEPLOY_ID}${START_TIME_SHORT}s2" "slow2" "${SUITE_PARAMS[@]}" "-DrunIT=SlowSystemTestSuite2" "$@" &> "$OUTPUT_DIR/slow2.suite.log" & + runTestSuite 120 "${DEPLOY_ID}${START_TIME_SHORT}s3" "slow3" "${SUITE_PARAMS[@]}" "-DrunIT=SlowSystemTestSuite3" "$@" &> "$OUTPUT_DIR/slow3.suite.log" & + runTestSuite 180 "${DEPLOY_ID}${START_TIME_SHORT}e1" "expensive1" "${SUITE_PARAMS[@]}" "-DrunIT=ExpensiveSystemTestSuite1" "$@" &> "$OUTPUT_DIR/expensive1.suite.log" & + runTestSuite 240 "${DEPLOY_ID}${START_TIME_SHORT}e2" "expensive2" "${SUITE_PARAMS[@]}" "-DrunIT=ExpensiveSystemTestSuite2" "$@" &> "$OUTPUT_DIR/expensive2.suite.log" & + runTestSuite 300 "${DEPLOY_ID}${START_TIME_SHORT}e3" "expensive3" "${SUITE_PARAMS[@]}" "-DrunIT=ExpensiveSystemTestSuite3" "$@" &> "$OUTPUT_DIR/expensive3.suite.log" & + runTestSuite 360 "${DEPLOY_ID}${START_TIME_SHORT}q1" "quick" "${SUITE_PARAMS[@]}" "-DrunIT=QuickSystemTestSuite" "$@" &> "$OUTPUT_DIR/quick.suite.log" & + wait +elif [ "$MAIN_SUITE_NAME" == "functional" ]; then + echo "Running slow tests in parallel. Start time: [$(time_str)]" + runTestSuite 0 "${DEPLOY_ID}${START_TIME_SHORT}s1" "slow1" "${SUITE_PARAMS[@]}" "-DrunIT=SlowSystemTestSuite1" "$@" &> "$OUTPUT_DIR/slow1.suite.log" & + runTestSuite 60 "${DEPLOY_ID}${START_TIME_SHORT}s2" "slow2" "${SUITE_PARAMS[@]}" "-DrunIT=SlowSystemTestSuite2" "$@" &> "$OUTPUT_DIR/slow2.suite.log" & + runTestSuite 120 "${DEPLOY_ID}${START_TIME_SHORT}s3" "slow3" "${SUITE_PARAMS[@]}" "-DrunIT=SlowSystemTestSuite3" "$@" &> "$OUTPUT_DIR/slow3.suite.log" & + runTestSuite 0 "${DEPLOY_ID}${START_TIME_SHORT}q1" "quick" "${SUITE_PARAMS[@]}" "-DrunIT=QuickSystemTestSuite" "$@" &> "$OUTPUT_DIR/quick.suite.log" & + wait +else + runMavenSystemTests "${DEPLOY_ID}mvn${START_TIME_SHORT}" $MAIN_SUITE_NAME "${SUITE_PARAMS[@]}" +fi echo "[$(time_str)] Uploading test output" java -cp "${SYSTEM_TEST_JAR}" \