diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index fea5bdc046c..332992d53e3 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -171,6 +171,9 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexi public final class aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric : java/lang/Enum, aws/smithy/kotlin/runtime/businessmetrics/BusinessMetric { public static final field DDB_MAPPER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static final field S3_EXPRESS_BUCKET Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_DOWNLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_UPLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static fun getEntries ()Lkotlin/enums/EnumEntries; public fun getIdentifier ()Ljava/lang/String; public fun toString ()Ljava/lang/String; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt index 239686ee3df..bcee46a0020 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt @@ -62,6 +62,9 @@ internal fun formatMetrics(metrics: MutableSet, logger: Logger): public enum class AwsBusinessMetric(public override val identifier: String) : BusinessMetric { S3_EXPRESS_BUCKET("J"), DDB_MAPPER("d"), + S3_TRANSFER("G"), + S3_TRANSFER_UPLOAD_DIRECTORY("9"), + S3_TRANSFER_DOWNLOAD_DIRECTORY("+"), ; @InternalApi diff --git a/hll/build.gradle.kts b/hll/build.gradle.kts index e6c5661b02a..d44c4c45162 100644 --- a/hll/build.gradle.kts +++ b/hll/build.gradle.kts @@ -112,6 +112,8 @@ val projectsToIgnore = listOf( "dynamodb-mapper-ops-codegen", "dynamodb-mapper-schema-codegen", "dynamodb-mapper-schema-generator-plugin-test", + + "s3-transfer-manager-codegen", ).filter { it in subprojects.map { it.name }.toSet() } // Some projects may not be in the build depending on bootstrapping apiValidation { diff --git a/hll/hll-codegen/build.gradle.kts b/hll/hll-codegen/build.gradle.kts index 19c65cade4b..d01abfa1d68 100644 --- a/hll/hll-codegen/build.gradle.kts +++ b/hll/hll-codegen/build.gradle.kts @@ -1,3 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + /* * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 @@ -48,3 +57,14 @@ publishing { } } } + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + } +} diff --git a/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt b/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt index 54d62727b03..bbde36912be 100644 --- a/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt +++ b/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt @@ -23,6 +23,7 @@ public data class Member( val type: Type, val mutable: Boolean = false, val attributes: Attributes = emptyAttributes(), + val kDocs: String? = null, ) { @InternalSdkApi public companion object { @@ -34,6 +35,7 @@ public data class Member( name = prop.simpleName.getShortName(), type = Type.from(prop.type), mutable = prop.isMutable, + kDocs = prop.docString, ) return ModelParsingPlugin.transform(member, ModelParsingPlugin::postProcessMember) diff --git a/hll/s3-transfer-manager-codegen/build.gradle.kts b/hll/s3-transfer-manager-codegen/build.gradle.kts new file mode 100644 index 00000000000..174b099cf0d --- /dev/null +++ b/hll/s3-transfer-manager-codegen/build.gradle.kts @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +val sdkVersion: String by project +version = sdkVersion + +description = "S3 Transfer Manager Code Generation" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager Codegen" +extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager.codegen" + +plugins { + id(libs.plugins.kotlin.jvm.get().pluginId) +} + +dependencies { + implementation(libs.ksp.api) + implementation(project(":hll:hll-codegen")) + implementation(project(":services:s3")) +} + +kotlin { + explicitApi() + sourceSets.all { + listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + "kotlin.RequiresOptIn", + ).forEach(languageSettings::optIn) + } +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + freeCompilerArgs.add("-Xjdk-release=1.8") + freeCompilerArgs.add("-opt-in=kotlin.RequiresOptIn") + } +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt new file mode 100644 index 00000000000..68291cbea87 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen + +import aws.sdk.kotlin.hll.codegen.core.CodeGeneratorFactory +import aws.sdk.kotlin.hll.codegen.ksp.processors.HllKspProcessor +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.conversionMappings +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ioMappings +import aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers.ConversionRenderer +import aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers.IoRenderer +import com.google.devtools.ksp.processing.Resolver +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.symbol.KSAnnotated + +internal class S3TransferManagerSymbolProcessor(environment: SymbolProcessorEnvironment) : HllKspProcessor(environment) { + val rendererName = "s3-transfer-manager-code-generator" + val codeGenerator = environment.codeGenerator + val logger = environment.logger + + override fun processImpl(resolver: Resolver): List { + val ioMappingsContext = + RenderContext( + logger, + CodeGeneratorFactory(codeGenerator, logger), + "aws.sdk.kotlin.hll.s3transfermanager.model", + rendererName, + ) + + ioMappings.forEach { mapping -> + IoRenderer( + ioMappingsContext, + mapping.className, + mapping, + resolver, + ).render() + } + + val conversionMappingsContext = + RenderContext( + logger, + CodeGeneratorFactory(codeGenerator, logger), + "aws.sdk.kotlin.hll.s3transfermanager.model.utils", + rendererName, + ) + + ConversionRenderer( + conversionMappingsContext, + "Converters", + conversionMappings, + resolver, + ).render() + + return listOf() + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt new file mode 100644 index 00000000000..3deb764223a --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen + +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.processing.SymbolProcessorProvider + +public class S3TransferManagerSymbolProcessorProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor = + S3TransferManagerSymbolProcessor(environment) +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt new file mode 100644 index 00000000000..1458ce99677 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt @@ -0,0 +1,42 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings + +import aws.sdk.kotlin.hll.codegen.model.TypeRef + +/** + * Converts one type to another + */ +internal data class ConversionMapping( + val source: TypeRef, + val destination: TypeRef, + val members: Set, + val additionalImports: List = emptyList(), + val additionalParameters: List = emptyList(), + val additionalLogic: String = "", +) + +/** + * High level S3 TM request/response from low level S3 operation members + */ +internal data class IoMapping( + val type: MappingType, + val className: String, + val sourceOperation: String, + val members: Set, +) + +internal enum class MappingType { + /** + * Maps high level operation request members to low level request members + */ + REQUEST, + + /** + * Maps high level operation response members to low level response members + */ + RESPONSE, +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt new file mode 100644 index 00000000000..8acc5b25837 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt @@ -0,0 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings + +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadObject.uploadObjectConversions +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadObject.uploadObjectIoMappings + +internal val ioMappings = uploadObjectIoMappings +internal val conversionMappings = uploadObjectConversions diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadObject/Converters.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadObject/Converters.kt new file mode 100644 index 00000000000..c33cfe4dada --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadObject/Converters.kt @@ -0,0 +1,247 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadObject + +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ConversionMapping + +internal val uploadObjectConversions = listOf( + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "PutObjectResponse", + ), + destination = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadObjectResponse", + ), + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "sseCustomerAlgorithm", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompleteMultipartUploadResponse", + ), + destination = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadObjectResponse", + ), + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadObjectRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "PutObjectRequest", + ), + setOf( + "acl", + "body", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentLength", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "ifMatch", + "ifNoneMatch", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadObjectRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CreateMultipartUploadRequest", + ), + setOf( + "acl", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadObjectRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "UploadPartRequest", + ), + setOf( + "bucket", + "checksumAlgorithm", + "expectedBucketOwner", + "key", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + ), + listOf( + TypeRef( + "aws.smithy.kotlin.runtime.io", + "SdkBuffer", + ), + TypeRef( + "aws.smithy.kotlin.runtime.io", + "SdkSource", + ), + TypeRef( + "aws.smithy.kotlin.runtime.content", + "ByteStream", + ), + ), + listOf( + "currentPart: SdkBuffer", + "currentPartNumber: Int", + "mpuUploadId: String", + ), + """ + uploadId = mpuUploadId + body = object : ByteStream.SourceStream() { + override fun readFrom(): SdkSource = currentPart + override val contentLength: Long = currentPart.size + } + partNumber = currentPartNumber + """.trimIndent(), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadObjectRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompleteMultipartUploadRequest", + ), + setOf( + "bucket", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "expectedBucketOwner", + "ifMatch", + "ifNoneMatch", + "key", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + ), + listOf( + TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompletedPart", + ), + ), + listOf( + "mpuUploadId: String", + "uploadedParts: List", + ), + """ + uploadId = mpuUploadId + multipartUpload { + parts = uploadedParts + } + """.trimIndent(), + ), +) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadObject/IO.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadObject/IO.kt new file mode 100644 index 00000000000..44220488846 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadObject/IO.kt @@ -0,0 +1,81 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadObject + +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.IoMapping +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.MappingType + +internal val uploadObjectIoMappings = listOf( + IoMapping( + MappingType.REQUEST, + "UploadObjectRequest", + "putObject", + setOf( + "acl", + "body", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentLength", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "ifMatch", + "ifNoneMatch", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + ), + IoMapping( + MappingType.RESPONSE, + "UploadObjectResponse", + "putObject", + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "sseCustomerAlgorithm", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), +) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt new file mode 100644 index 00000000000..9259f0e4a33 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt @@ -0,0 +1,48 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers + +import aws.sdk.kotlin.hll.codegen.core.ImportDirective +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.codegen.rendering.RendererBase +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ConversionMapping +import com.google.devtools.ksp.processing.Resolver + +internal class ConversionRenderer( + ctx: RenderContext, + fileName: String, + val conversions: List, + val resolver: Resolver, +) : RendererBase(ctx, fileName) { + override fun generate() { + conversions.forEach { conversion -> + val functionName = "to${conversion.destination.shortName}" + + conversion.additionalImports.forEach { + imports += ImportDirective(it) + } + + withBlock( + "internal fun #1T.#2L(#3L): #4T = #4T {", + "}", + conversion.source, + functionName, + conversion.additionalParameters.joinToString(", "), + conversion.destination, + ) { + conversion.members.forEach { member -> + write( + "#1L = this@#2L.#1L", + member, + functionName, + ) + } + write(conversion.additionalLogic) + } + blankLine() + } + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IoRenderer.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IoRenderer.kt new file mode 100644 index 00000000000..4b35c9778ca --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IoRenderer.kt @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers + +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.codegen.rendering.RendererBase +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.IoMapping +import aws.sdk.kotlin.hll.s3transfermanager.codegen.utils.operationMembers +import com.google.devtools.ksp.processing.Resolver + +/** + * Renders request and response types + */ +internal class IoRenderer( + ctx: RenderContext, + val className: String, + val mapping: IoMapping, + val resolver: Resolver, +) : RendererBase(ctx, className) { + override fun generate() { + val members = resolver + .operationMembers( + mapping.sourceOperation, + mapping.type, + mapping.members, + ) + + withBlock( + "public class #L private constructor(builder: Builder) {", + "}", + className, + ) { + members.forEach { member -> + member.kDocs?.let { write(it) } // FIXME: KSP isn't detecting KDocs + write( + "public val #1L: #2T = builder.#1L", + member.name, + member.type, + ) + } + blankLine() + + withBlock( + "public companion object {", + "}", + ) { + write( + "public operator fun invoke(block: Builder.() -> Unit): #L = Builder().apply(block).build()", + className, + ) + } + blankLine() + + withBlock( + "public class Builder {", + "}", + ) { + members.forEach { member -> + write( + "public var #L: #T = null", + member.name, + member.type, + ) + } + blankLine() + + write("@PublishedApi") + write( + "internal fun build(): #1L = #1L(this)", + className, + ) + } + } + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt new file mode 100644 index 00000000000..a4d46df39b9 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.utils + +/** + * Exception thrown when an error occurs during S3 transfer codegen. + * + * @param message Description of the error. + * @param cause The underlying cause of the exception, if any. + */ +internal class S3TransferManagerCodegenException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt new file mode 100644 index 00000000000..aa574a93a37 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt @@ -0,0 +1,55 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.utils + +import aws.sdk.kotlin.hll.codegen.model.Member +import aws.sdk.kotlin.hll.codegen.model.Operation +import aws.sdk.kotlin.hll.codegen.model.Type +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.MappingType +import aws.sdk.kotlin.services.s3.S3Client +import com.google.devtools.ksp.getClassDeclarationByName +import com.google.devtools.ksp.getDeclaredFunctions +import com.google.devtools.ksp.processing.Resolver + +internal fun Resolver.operationMembers( + operationName: String, + type: MappingType, + relevantMembers: Set, +): List = + Operation.from( + this + .getClassDeclarationByName()!! + .getDeclaredFunctions() + .find { it.simpleName.getShortName().equals(operationName, ignoreCase = true) } + ?: throw S3TransferManagerCodegenException("Operation $operationName not found"), + ) + .let { + if (type == MappingType.REQUEST) { + it.request + } else { + it.response + } + } + .members + .filter { member -> + relevantMembers.any { it.equals(member.name, ignoreCase = true) } + } + +internal fun Type.renderMember(): String { + val code = StringBuilder() + code.append(this.shortName) // Map + + (this as TypeRef).genericArgs.let { args -> + if (args.isNotEmpty()) { + code.append( + args.joinToString(", ", "<", ">") { it.shortName }, // Map + ) + } + } + + return code.toString() +} diff --git a/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider new file mode 100644 index 00000000000..6526cc79fb7 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -0,0 +1 @@ +aws.sdk.kotlin.hll.s3transfermanager.codegen.S3TransferManagerSymbolProcessorProvider diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api new file mode 100644 index 00000000000..48d2efda2e9 --- /dev/null +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -0,0 +1,349 @@ +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion; + public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getInterceptors ()Ljava/util/List; + public final fun getMaxConcurrentPartUploads ()I + public final fun getMaxInMemoryParts ()I + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getS3Client ()Laws/sdk/kotlin/services/s3/S3Client; + public final fun getTargetPartSizeBytes ()J + public final fun uploadObject (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public final fun uploadObject (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder { + public fun ()V + public final fun getInterceptors ()Ljava/util/List; + public final fun getMaxConcurrentPartUploads ()I + public final fun getMaxInMemoryParts ()I + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getTargetPartSizeBytes ()J + public final fun setInterceptors (Ljava/util/List;)V + public final fun setMaxConcurrentPartUploads (I)V + public final fun setMaxInMemoryParts (I)V + public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;)V + public final fun setMultipartUploadThresholdBytes (J)V + public final fun setTargetPartSizeBytes (J)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion { + public final fun invoke (Laws/sdk/kotlin/services/s3/S3Client;Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext : aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptorContext { + public fun ()V + public fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;)V + public synthetic fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Ljava/lang/Object; + public final fun component2 ()Ljava/lang/Object; + public final fun component3 ()Ljava/lang/Object; + public final fun component4 ()Ljava/lang/Object; + public final fun component5 ()Ljava/lang/Long; + public final fun component6 ()Ljava/lang/Long; + public final fun component7 ()Ljava/lang/Long; + public final fun component8 ()Ljava/lang/Long; + public final fun copy (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;)Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext; + public static synthetic fun copy$default (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;ILjava/lang/Object;)Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext; + public fun equals (Ljava/lang/Object;)Z + public fun getS3Request ()Ljava/lang/Object; + public fun getS3Response ()Ljava/lang/Object; + public fun getTmRequest ()Ljava/lang/Object; + public fun getTmResponse ()Ljava/lang/Object; + public fun getTransferableBytes ()Ljava/lang/Long; + public fun getTransferableObjects ()Ljava/lang/Long; + public fun getTransferredBytes ()Ljava/lang/Long; + public fun getTransferredObjects ()Ljava/lang/Long; + public fun hashCode ()I + public fun setS3Request (Ljava/lang/Object;)V + public fun setS3Response (Ljava/lang/Object;)V + public fun setTmRequest (Ljava/lang/Object;)V + public fun setTmResponse (Ljava/lang/Object;)V + public fun setTransferableBytes (Ljava/lang/Long;)V + public fun setTransferableObjects (Ljava/lang/Long;)V + public fun setTransferredBytes (Ljava/lang/Long;)V + public fun setTransferredObjects (Ljava/lang/Long;)V + public fun toString ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext : aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptorContext { + public fun ()V + public fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;)V + public synthetic fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Ljava/lang/Object; + public final fun component2 ()Ljava/lang/Object; + public final fun component3 ()Ljava/lang/Object; + public final fun component4 ()Ljava/lang/Object; + public final fun component5 ()Ljava/lang/Long; + public final fun component6 ()Ljava/lang/Long; + public final fun component7 ()Ljava/lang/Long; + public final fun component8 ()Ljava/lang/Long; + public final fun copy (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;)Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext; + public static synthetic fun copy$default (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;ILjava/lang/Object;)Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext; + public fun equals (Ljava/lang/Object;)Z + public fun getS3Request ()Ljava/lang/Object; + public fun getS3Response ()Ljava/lang/Object; + public fun getTmRequest ()Ljava/lang/Object; + public fun getTmResponse ()Ljava/lang/Object; + public fun getTransferableBytes ()Ljava/lang/Long; + public fun getTransferableObjects ()Ljava/lang/Long; + public fun getTransferredBytes ()Ljava/lang/Long; + public fun getTransferredObjects ()Ljava/lang/Long; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor { + public fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun modifyAfterObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun modifyBeforeObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public fun readAfterObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public fun readBeforeObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor$DefaultImpls { + public static fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun modifyAfterObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun modifyBeforeObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/MutableTransferContext;)V + public static fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public static fun readAfterObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public static fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public static fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public static fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public static fun readBeforeObjectTransferred (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public static fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V + public static fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext;)V +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptorContext { + public abstract fun getS3Request ()Ljava/lang/Object; + public abstract fun getS3Response ()Ljava/lang/Object; + public abstract fun getTmRequest ()Ljava/lang/Object; + public abstract fun getTmResponse ()Ljava/lang/Object; + public abstract fun getTransferableBytes ()Ljava/lang/Long; + public abstract fun getTransferableObjects ()Ljava/lang/Long; + public abstract fun getTransferredBytes ()Ljava/lang/Long; + public abstract fun getTransferredObjects ()Ljava/lang/Long; +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType$Part : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType$Part; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType$Range : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType$Range; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest$Companion; + public synthetic fun (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentLength ()Ljava/lang/Long; + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest$Builder { + public fun ()V + public final fun build ()Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest; + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentLength ()Ljava/lang/Long; + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; + public final fun setAcl (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;)V + public final fun setBody (Laws/smithy/kotlin/runtime/content/ByteStream;)V + public final fun setBucket (Ljava/lang/String;)V + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setCacheControl (Ljava/lang/String;)V + public final fun setChecksumAlgorithm (Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setContentDisposition (Ljava/lang/String;)V + public final fun setContentEncoding (Ljava/lang/String;)V + public final fun setContentLanguage (Ljava/lang/String;)V + public final fun setContentLength (Ljava/lang/Long;)V + public final fun setContentType (Ljava/lang/String;)V + public final fun setExpectedBucketOwner (Ljava/lang/String;)V + public final fun setExpires (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setGrantFullControl (Ljava/lang/String;)V + public final fun setGrantRead (Ljava/lang/String;)V + public final fun setGrantReadAcp (Ljava/lang/String;)V + public final fun setGrantWriteAcp (Ljava/lang/String;)V + public final fun setIfMatch (Ljava/lang/String;)V + public final fun setIfNoneMatch (Ljava/lang/String;)V + public final fun setKey (Ljava/lang/String;)V + public final fun setMetadata (Ljava/util/Map;)V + public final fun setObjectLockLegalHoldStatus (Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;)V + public final fun setObjectLockMode (Laws/sdk/kotlin/services/s3/model/ObjectLockMode;)V + public final fun setObjectLockRetainUntilDate (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setRequestPayer (Laws/sdk/kotlin/services/s3/model/RequestPayer;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKey (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setStorageClass (Laws/sdk/kotlin/services/s3/model/StorageClass;)V + public final fun setTagging (Ljava/lang/String;)V + public final fun setWebsiteRedirectLocation (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectRequest; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectResponse { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectResponse$Companion; + public synthetic fun (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectResponse$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectResponse$Builder { + public fun ()V + public final fun build ()Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectResponse; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setChecksumType (Laws/sdk/kotlin/services/s3/model/ChecksumType;)V + public final fun setETag (Ljava/lang/String;)V + public final fun setExpiration (Ljava/lang/String;)V + public final fun setRequestCharged (Laws/sdk/kotlin/services/s3/model/RequestCharged;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setVersionId (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectResponse$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadObjectResponse; +} + diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts new file mode 100644 index 00000000000..ad4c9be284d --- /dev/null +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -0,0 +1,107 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import aws.sdk.kotlin.gradle.kmp.NATIVE_ENABLED +import com.google.devtools.ksp.gradle.KspTaskJvm +import com.google.devtools.ksp.gradle.KspTaskMetadata +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.project +import org.gradle.kotlin.dsl.sourceSets +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.tasks.KotlinCompilationTask +import java.nio.file.Files +import java.nio.file.StandardCopyOption + +val sdkVersion: String by project +version = sdkVersion + +description = "S3 Transfer Manager for the AWS SDK for Kotlin" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager" +extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager" + +plugins { + alias(libs.plugins.ksp) +} + +kotlin { + sourceSets { + commonMain { + dependencies { + implementation(project(":aws-runtime:aws-http")) + implementation(project(":services:s3")) + } + } + commonTest { + dependencies { + implementation(libs.smithy.kotlin.http.test) + } + } + } +} + +ksp { + dependencies { + ksp(project(":hll:s3-transfer-manager-codegen")) + } +} + +// This is copied from :hll:dynamodb-mapper:dynamodb-mapper. TODO: Commonize +if (project.NATIVE_ENABLED) { + // Configure KSP for multiplatform: https://kotlinlang.org/docs/ksp-multiplatform.html + // https://github.com/google/ksp/issues/963#issuecomment-1894144639 + // https://github.com/google/ksp/issues/965 + kotlin.sourceSets.commonMain { + tasks.withType { + // Wire up the generated source to the commonMain source set + kotlin.srcDir(destinationDirectory) + } + } +} else { + // FIXME This is a dirty hack for JVM-only builds which KSP doesn't consider to be "multiplatform". Explanation of + // hack follows in narrative, minimally-opinionated comments. + + // Then we need to move the generated source from jvm to common + val moveGenSrc by tasks.registering { + // Can't move src until the src is generated + dependsOn(tasks.named("kspKotlinJvm")) + + // Detecting these paths programmatically is complex; just hardcode them + val srcDir = file("build/generated/ksp/jvm/jvmMain") + val destDir = file("build/generated/ksp/common/commonMain") + + inputs.dir(srcDir) + outputs.dirs(srcDir, destDir) + + doLast { + if (destDir.exists()) { + // Clean out the existing destination, otherwise move fails + require(destDir.deleteRecursively()) { "Failed to delete $destDir before moving from $srcDir" } + } else { + // Create the destination directories, otherwise move fails + require(destDir.mkdirs()) { "Failed to create path $destDir" } + } + + Files.move(srcDir.toPath(), destDir.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + } + + listOf("jvmSourcesJar", "metadataSourcesJar", "jvmProcessResources").forEach { + tasks.named(it) { + dependsOn(moveGenSrc) + } + } + + tasks.withType> { + if (this !is KspTaskJvm) { + // Ensure that any **non-KSP** compile tasks depend on the generated src move + dependsOn(moveGenSrc) + } + } + + // Finally, wire up the generated source to the commonMain source set + kotlin.sourceSets.commonMain { + kotlin.srcDir("build/generated/ksp/common/commonMain/kotlin") + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt new file mode 100644 index 00000000000..7858dbb9442 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -0,0 +1,154 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.MultipartDownloadType +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadObjectResponse +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.uploadObjectImplementation +import aws.sdk.kotlin.services.s3.S3Client +import kotlinx.coroutines.sync.Semaphore + +/** + * High level utility for managing transfers to Amazon S3. + */ +public class S3TransferManager private constructor(public val s3Client: S3Client, builder: Builder) { + + /** + * Preferred part size for multipart uploads. + * If using this size would require more than 10,000 parts (the S3 limit), + * the smallest possible part size that results in 10,000 parts is used instead. + * + * Defaults to 8,000,000 bytes. + */ + public val targetPartSizeBytes: Long = builder.targetPartSizeBytes + + /** + * Threshold size at which an object upload uses multipart upload + * instead of a single [S3Client.putObject] request. + * + * Defaults to 16,000,000 bytes. + */ + public val multipartUploadThresholdBytes: Long = builder.multipartUploadThresholdBytes + + /** + * Strategy for multipart downloads, defined by [MultipartDownloadType]. + * Downloads can be performed either by specifying byte ranges or by requesting individual parts. + * + * Defaults to [MultipartDownloadType.Part]. + */ + public val multipartDownloadType: MultipartDownloadType = builder.multipartDownloadType + + /** + * Mutable list of [TransferInterceptor] instances, typically used to track transfers + * or inspect/modify low-level S3 requests. + */ + public val interceptors: MutableList = builder.interceptors + + /** + * The maximum amount of parts to buffer in memory while waiting for uploads to complete. + * The actual number of parts buffered at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public val maxInMemoryParts: Int = builder.maxInMemoryParts + + /** + * Maximum number of concurrent part uploads for an object. + * The actual number of uploads at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public val maxConcurrentPartUploads: Int = builder.maxConcurrentPartUploads + + public companion object { + public operator fun invoke(client: S3Client, block: Builder.() -> Unit): S3TransferManager = + Builder().apply(block).build(client) + } + + public class Builder { + /** + * Preferred part size for multipart uploads. + * If using this size would require more than 10,000 parts (the S3 limit), + * the smallest possible part size that results in 10,000 parts is used instead. + * + * Defaults to 8,000,000 bytes. + */ + public var targetPartSizeBytes: Long = 8_000_000 + + /** + * Threshold size at which an object upload uses multipart upload + * instead of a single [S3Client.putObject] request. + * + * Defaults to 16,000,000 bytes. + */ + public var multipartUploadThresholdBytes: Long = 16_000_000L + + /** + * Strategy for multipart downloads, defined by [MultipartDownloadType]. + * Downloads can be performed either by specifying byte ranges or by requesting individual parts. + * + * Defaults to [MultipartDownloadType.Part]. + */ + public var multipartDownloadType: MultipartDownloadType = MultipartDownloadType.Part + + /** + * Mutable list of [TransferInterceptor] instances, typically used to track transfers + * or inspect/modify low-level S3 requests. + */ + public var interceptors: MutableList = mutableListOf() + + /** + * The maximum amount of parts to buffer in memory while waiting for uploads to complete. + * The actual number of parts buffered at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public var maxInMemoryParts: Int = 5 + + /** + * Maximum number of concurrent part uploads for an object. + * The actual number of uploads at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public var maxConcurrentPartUploads: Int = 5 + + internal fun build(client: S3Client): S3TransferManager = + S3TransferManager(client, this) + } + + // Keeps track of how many parts are in memory for this S3 TM via permits + internal val bufferSemaphore = Semaphore(maxInMemoryParts) + + /** + * Uploads an object to S3 via [aws.smithy.kotlin.runtime.content.ByteStream]. + * Uses multipart uploads with concurrent uploads if the object size is more than the configured [multipartUploadThresholdBytes]. + */ + public suspend fun uploadObject( + uploadObjectRequest: UploadObjectRequest, + ): UploadObjectResponse = + uploadObjectImplementation( + uploadObjectRequest, + s3Client, + multipartUploadThresholdBytes, + targetPartSizeBytes, + interceptors, + maxInMemoryParts, + maxConcurrentPartUploads, + bufferSemaphore, + ) + + /** + * Uploads an object to S3 via [aws.smithy.kotlin.runtime.content.ByteStream]. + * Uses multipart uploads with concurrent uploads if the object size is more than the configured [multipartUploadThresholdBytes]. + */ + public suspend inline fun uploadObject( + crossinline block: UploadObjectRequest.Builder.() -> Unit, + ): UploadObjectResponse = + uploadObject(UploadObjectRequest.Builder().apply(block).build()) +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext.kt new file mode 100644 index 00000000000..11ee32fdaa1 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferContext.kt @@ -0,0 +1,95 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.interceptors + +// TODO: Create a sealed classes to hold possible s3 and tm request and response types. Should eliminate use of casting. +/** + * The context around an [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager] transfer. + * Used to track transfer progress or to modify in progress transfers. + */ +public interface TransferInterceptorContext { + /** + * Current low level S3 request + */ + public val s3Request: Any? + + /** + * Current low level S3 response + */ + public val s3Response: Any? + + /** + * Current high level transfer manager request + */ + public val tmRequest: Any? + + /** + * Current high level transfer manager response + */ + public val tmResponse: Any? + + /** + * The amount of transferable bytes for an object + */ + public val transferableBytes: Long? + + /** + * The amount of transferred bytes for an object + */ + public val transferredBytes: Long? + + /** + * The amount of transferable objects for a directory + */ + public val transferableObjects: Long? + + /** + * The amount of transferred objects for a directory + */ + public val transferredObjects: Long? +} + +/** + * The context around an [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager] transfer. + * Used to track transfer progress or to modify in progress transfers. + */ +public data class TransferContext( + override val s3Request: Any? = null, + override val s3Response: Any? = null, + override val tmRequest: Any? = null, + override val tmResponse: Any? = null, + override val transferableBytes: Long? = null, + override val transferredBytes: Long? = null, + override val transferableObjects: Long? = null, + override val transferredObjects: Long? = null, +) : TransferInterceptorContext + +/** + * The context around an [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager] transfer. + * Used to track transfer progress or to modify in progress transfers. + */ +public data class MutableTransferContext( + override var s3Request: Any? = null, + override var s3Response: Any? = null, + override var tmRequest: Any? = null, + override var tmResponse: Any? = null, + override var transferableBytes: Long? = null, + override var transferredBytes: Long? = null, + override var transferableObjects: Long? = null, + override var transferredObjects: Long? = null, +) : TransferInterceptorContext { + internal fun immutableCopy() = + TransferContext( + s3Request, + s3Response, + tmRequest, + tmResponse, + transferableBytes, + transferredBytes, + transferableObjects, + transferredObjects, + ) +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor.kt new file mode 100644 index 00000000000..d574a44991f --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferInterceptor.kt @@ -0,0 +1,42 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.interceptors + +/** + * A transfer interceptor allows peeking into the progress + * and context of an [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager] transfer at a certain phase using hooks. + * Also allows modifying a transfer in progress using [TransferInterceptorContext] parameters such as [TransferInterceptorContext.s3Response]. + * + * Terminology: + * Phase - A specific execution point for a transfer. + * Hook - Methods that allows interceptors to read/modify a transfer before and after a phase. + * Transfer context - See: [TransferInterceptorContext] + * Transfer initiated - The point in time a transfer is initiated. For example, in multipart uploads this is when a [aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest] is sent to S3. + * Bytes transferred - Any time bytes are transferred to S3 for either an upload or download + * Object transferred - Any time objects are transferred to S3 for either an upload or download + * Transfer completed - The point in time a transfer is completed. For example in multipart uploads this is when a [aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest] is sent to S3. + */ +public interface TransferInterceptor { + public fun readBeforeTransferInitiated(context: TransferContext) {} + public fun modifyBeforeTransferInitiated(context: MutableTransferContext) {} + public fun readAfterTransferInitiated(context: TransferContext) {} + public fun modifyAfterTransferInitiated(context: MutableTransferContext) {} + + public fun readBeforeBytesTransferred(context: TransferContext) {} + public fun modifyBeforeBytesTransferred(context: MutableTransferContext) {} + public fun readAfterBytesTransferred(context: TransferContext) {} + public fun modifyAfterBytesTransferred(context: MutableTransferContext) {} + + public fun readBeforeObjectTransferred(context: TransferContext) {} + public fun modifyBeforeObjectTransferred(context: MutableTransferContext) {} + public fun readAfterObjectTransferred(context: TransferContext) {} + public fun modifyAfterObjectTransferred(context: MutableTransferContext) {} + + public fun readBeforeTransferCompleted(context: TransferContext) {} + public fun modifyBeforeTransferCompleted(context: MutableTransferContext) {} + public fun readAfterTransferCompleted(context: TransferContext) {} + public fun modifyAfterTransferCompleted(context: MutableTransferContext) {} +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferPhase.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferPhase.kt new file mode 100644 index 00000000000..381bb8e7985 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/interceptors/TransferPhase.kt @@ -0,0 +1,94 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.interceptors + +/** + * Describes a type of phase that is executed during an [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager] transfer + */ +internal sealed interface TransferPhase { + object TransferInitiated : TransferPhase + object BytesTransferred : TransferPhase + object ObjectTransferred : TransferPhase + object TransferCompleted : TransferPhase +} + +/** + * Executes a sequence of hooks around a phase of an operation. + * + * The execution flow is as follows: + * 1. Runs all hooks scheduled to execute **before** the phase. + * 2. Executes the phase logic. + * 3. Runs all hooks scheduled to execute **after** the phase. + */ +internal suspend fun executePhase( + phase: TransferPhase, + context: MutableTransferContext, + interceptors: List, + block: suspend () -> Unit, +) { + when (phase) { + is TransferPhase.TransferInitiated -> { + var immutableContext = context.immutableCopy() + interceptors.forEachCatching { readBeforeTransferInitiated(immutableContext) } + interceptors.forEach { it.modifyBeforeTransferInitiated(context) } + block.invoke() + immutableContext = context.immutableCopy() + interceptors.forEachCatching { readAfterTransferInitiated(immutableContext) } + interceptors.forEach { it.modifyAfterTransferInitiated(context) } + } + is TransferPhase.BytesTransferred -> { + var immutableContext = context.immutableCopy() + interceptors.forEachCatching { readBeforeBytesTransferred(immutableContext) } + interceptors.forEach { it.modifyBeforeBytesTransferred(context) } + block.invoke() + immutableContext = context.immutableCopy() + interceptors.forEachCatching { readAfterBytesTransferred(immutableContext) } + interceptors.forEach { it.modifyAfterBytesTransferred(context) } + } + is TransferPhase.ObjectTransferred -> { + var immutableContext = context.immutableCopy() + interceptors.forEachCatching { readBeforeObjectTransferred(immutableContext) } + interceptors.forEach { it.modifyBeforeObjectTransferred(context) } + block.invoke() + immutableContext = context.immutableCopy() + interceptors.forEachCatching { readAfterObjectTransferred(immutableContext) } + interceptors.forEach { it.modifyAfterObjectTransferred(context) } + } + is TransferPhase.TransferCompleted -> { + var immutableContext = context.immutableCopy() + interceptors.forEachCatching { readBeforeTransferCompleted(immutableContext) } + interceptors.forEach { it.modifyBeforeTransferCompleted(context) } + block.invoke() + immutableContext = context.immutableCopy() + interceptors.forEachCatching { readAfterTransferCompleted(immutableContext) } + interceptors.forEach { it.modifyAfterTransferCompleted(context) } + } + } +} + +/** + * Executes an action for each [TransferInterceptor]. + * Collects all exceptions, if any, and finally throws the first one with the others suppressed. + */ +private fun List.forEachCatching( + action: TransferInterceptor.() -> Unit, +) { + var exception: Exception? = null + + this.forEach { + try { + it.action() + } catch (e: Exception) { + if (exception == null) { + exception = e + } else { + exception.addSuppressed(e) + } + } + } + + exception?.let { throw it } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt new file mode 100644 index 00000000000..c2788f69590 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt @@ -0,0 +1,23 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +/** + * Defines the strategy used for multipart downloads in [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager]. + * + * A multipart download can either be performed by specifying byte ranges or by requesting individual parts. + */ +public sealed interface MultipartDownloadType { + /** + * Download specific byte ranges from an object. + */ + public object Range : MultipartDownloadType + + /** + * Download individual parts of an object as defined by the multipart upload structure. + */ + public object Part : MultipartDownloadType +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/Part.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/Part.kt new file mode 100644 index 00000000000..7260d89ed8c --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/Part.kt @@ -0,0 +1,19 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +import aws.smithy.kotlin.runtime.io.SdkBuffer + +/** + * Represents a part in a multipart upload. + * + * @param number The part number. + * @param bytes The bytes of the part. + */ +internal data class Part( + val number: Int, + val bytes: SdkBuffer, +) diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/HelperFunctions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/HelperFunctions.kt new file mode 100644 index 00000000000..5d5d6b43e51 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/HelperFunctions.kt @@ -0,0 +1,127 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject + +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.io.SdkByteReadChannel +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.readFully +import aws.smithy.kotlin.runtime.io.readRemaining +import aws.smithy.kotlin.runtime.telemetry.logging.Logger + +// S3 imposed limit for parts in a multipart upload +private const val MAX_NUMBER_PARTS = 10_000L + +/** + * Determines the actual part size to use for a multipart S3 upload. + * + * This function calculates the part size based on the total size + * of the object and the requested part size. If the requested part size is + * too small to allow the upload to fit within S3's 10,000-part limit, the + * part size will be automatically increased so that exactly 10,000 parts + * are uploaded. + */ +internal fun resolvePartSize(contentLength: Long, targetPartSize: Long, objectName: String?, logger: Logger): Long { + val targetNumberOfParts = ceilDiv(contentLength, targetPartSize) + return if (targetNumberOfParts > MAX_NUMBER_PARTS) { + ceilDiv(contentLength, MAX_NUMBER_PARTS).also { + logger.info { + buildString { + append("The target part size of $targetPartSize bytes is too small to upload $objectName in $MAX_NUMBER_PARTS parts ") + append("(the maximum allowed by S3). ") + append("The object will be uploaded in parts of $it bytes instead.") + } + } + } + } else { + targetPartSize + } +} + +/** + * Determines what part source an S3 body will have: + * [ByteStream.Buffer] + * [ByteStream.ChannelStream] + * [ByteStream.SourceStream] + */ +internal fun resolveSource(body: ByteStream): Any = + when (body) { + is ByteStream.Buffer -> body.bytes() + is ByteStream.ChannelStream -> body.readFrom() + is ByteStream.SourceStream -> body.readFrom() + else -> + throw S3TransferManagerException( + "Unhandled body type: ${body::class.simpleName }", + ) + } + +/** + * Retrieves the bytes for the next part of a multipart upload from the given part source into a [SdkBuffer] + */ +internal suspend fun nextPartBytes( + partSource: Any, + partSize: Long, + lastPart: Boolean, + readBytes: Long, + readableBytes: Long, +): SdkBuffer { + val buffer = SdkBuffer() + + when (partSource) { + is ByteArray -> { + // Long to Int is safe here because the ByteArray max size is Int.MAX_VALUE, it's size is managed as an Int. + val readBytes = readBytes.toInt() + val readableBytes = readableBytes.toInt() + val partSize = partSize.toInt() + + if (lastPart) { + buffer.write( + partSource.sliceArray(readBytes.. { + if (lastPart) { + partSource.readRemaining(buffer) + } else { + partSource.readFully(buffer, partSize) + } + } + is SdkSource -> { + if (lastPart) { + partSource.readRemaining(buffer) + } else { + partSource.readFully(buffer, partSize) + } + } + } + + return buffer +} + +/** + * Returns the ceiling of the division + * + * This means the result is rounded up to the nearest integer if the dividend is not + * evenly divisible by the divisor + */ +internal fun ceilDiv(dividend: Long, divisor: Long): Long { + val div = dividend / divisor + val remainder = dividend % divisor + return if (remainder != 0L) { + div + 1 + } else { + div + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/UploadObject.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/UploadObject.kt new file mode 100644 index 00000000000..4592f3186ff --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/UploadObject.kt @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.MutableTransferContext +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadObjectResponse +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.phases.completeTransfer +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.phases.initiateTransfer +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.phases.transferBytes +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext + +// TODO: Create abstraction class to use on each S3 TM operation. It should cut down on parameters between functions. +internal suspend fun uploadObjectImplementation( + uploadObjectRequest: UploadObjectRequest, + client: S3Client, + multipartUploadThresholdBytes: Long, + partSizeBytes: Long, + interceptors: List, + maxInMemoryParts: Int, + maxConcurrentPartUploads: Int, + bufferSemaphore: Semaphore, +): UploadObjectResponse = withContext(currentCoroutineContext() + TelemetryProviderContext(client.config.telemetryProvider)) { + val contentLength = uploadObjectRequest.contentLength ?: uploadObjectRequest.body?.contentLength ?: throw S3TransferManagerException("Content length must be known. Please set it in the request parameters.") + val multipartUpload = contentLength >= multipartUploadThresholdBytes + val logger = coroutineContext.logger() + var transferContext = MutableTransferContext( + tmRequest = uploadObjectRequest, + ) + + val mpuUploadId = initiateTransfer( + multipartUpload, + transferContext, + contentLength, + uploadObjectRequest, + interceptors, + client, + ) + + val uploadedParts = transferBytes( + multipartUpload, + contentLength, + partSizeBytes, + logger, + uploadObjectRequest, + transferContext, + mpuUploadId, + interceptors, + client, + maxInMemoryParts, + maxConcurrentPartUploads, + bufferSemaphore, + ) + + completeTransfer( + multipartUpload, + transferContext, + uploadObjectRequest, + mpuUploadId, + uploadedParts, + interceptors, + client, + ) + + return@withContext transferContext.tmResponse as UploadObjectResponse +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/CompleteTransfer.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/CompleteTransfer.kt new file mode 100644 index 00000000000..007db410868 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/CompleteTransfer.kt @@ -0,0 +1,65 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.phases + +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.MutableTransferContext +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferPhase +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.executePhase +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toCompleteMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toUploadObjectResponse +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.hll.s3transfermanager.utils.withTmBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.PutObjectResponse + +internal suspend fun completeTransfer( + multipartUpload: Boolean, + context: MutableTransferContext, + uploadObjectRequest: UploadObjectRequest, + mpuUploadId: String?, + uploadedParts: List, + interceptors: List, + client: S3Client, +) { + if (multipartUpload) { + context.s3Request = + uploadObjectRequest.toCompleteMultipartUploadRequest( + mpuUploadId!!, + uploadedParts, + ) + } + + executePhase( + TransferPhase.TransferCompleted, + context, + interceptors, + ) { + if (multipartUpload) { + try { + context.s3Response = client.withTmBusinessMetric { + it.completeMultipartUpload(context.s3Request as CompleteMultipartUploadRequest) + } + } catch (e: Exception) { + throw S3TransferManagerException("Unable to complete multipart upload with ID: $mpuUploadId", e) + } + } + + when (context.s3Response) { + is PutObjectResponse -> (context.s3Response as PutObjectResponse).toUploadObjectResponse().also { + context.tmResponse = it + } + is CompleteMultipartUploadResponse -> (context.s3Response as CompleteMultipartUploadResponse).toUploadObjectResponse().also { + context.tmResponse = it + } + else -> throw S3TransferManagerException("Unexpected response type: ${context.s3Response}") + } + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/InitiateTransfer.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/InitiateTransfer.kt new file mode 100644 index 00000000000..b8a54dbc0fd --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/InitiateTransfer.kt @@ -0,0 +1,50 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.phases + +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.MutableTransferContext +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferPhase +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.executePhase +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toCreateMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toPutObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.withTmBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse + +internal suspend fun initiateTransfer( + multipartUpload: Boolean, + context: MutableTransferContext, + contentLength: Long, + uploadObjectRequest: UploadObjectRequest, + interceptors: List, + client: S3Client, +): String? { + context.transferredBytes = 0L + context.transferableBytes = contentLength + context.s3Request = if (multipartUpload) { + uploadObjectRequest.toCreateMultipartUploadRequest() + } else { + uploadObjectRequest.toPutObjectRequest() + } + + var mpuUploadId: String? = null + executePhase( + TransferPhase.TransferInitiated, + context, + interceptors, + ) { + if (multipartUpload) { + context.s3Response = client.withTmBusinessMetric { + it.createMultipartUpload(context.s3Request as CreateMultipartUploadRequest) + } + mpuUploadId = (context.s3Response as CreateMultipartUploadResponse).uploadId!! + } + } + return mpuUploadId +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/TransferBytes.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/TransferBytes.kt new file mode 100644 index 00000000000..7793846310f --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/phases/TransferBytes.kt @@ -0,0 +1,238 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.phases + +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.MutableTransferContext +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferPhase +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.executePhase +import aws.sdk.kotlin.hll.s3transfermanager.model.Part +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toUploadPartRequest +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.ceilDiv +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.nextPartBytes +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.resolvePartSize +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject.resolveSource +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.hll.s3transfermanager.utils.withTmBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.abortMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.sdk.kotlin.services.s3.model.UploadPartResponse +import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.channels.produce +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withLock + +internal suspend fun transferBytes( + multipartUpload: Boolean, + contentLength: Long, + partSizeBytes: Long, + logger: Logger, + uploadObjectRequest: UploadObjectRequest, + context: MutableTransferContext, + mpuUploadId: String?, + interceptors: List, + client: S3Client, + maxInMemoryParts: Int, + maxConcurrentPartUploads: Int, + bufferSemaphore: Semaphore, +): List = coroutineScope { + val uploadedParts = mutableListOf() + + if (multipartUpload) { + try { + val partSize = resolvePartSize( + contentLength, + partSizeBytes, + uploadObjectRequest.key, + logger, + ) + val numberOfParts = ceilDiv(contentLength, partSize).toInt() + val partSource = resolveSource(uploadObjectRequest.body!!) + + val producer = produceParts( + context.transferableBytes!!, + partSource, + partSize, + numberOfParts, + maxInMemoryParts, + bufferSemaphore, + ) + + val mutex = Mutex() + repeat(maxConcurrentPartUploads) { + consumer( + producer, + uploadObjectRequest, + mpuUploadId!!, + context, + interceptors, + client, + uploadedParts, + mutex, + bufferSemaphore, + ) + } + + if (uploadedParts.size != numberOfParts) { + throw S3TransferManagerException("The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}") + } + } catch (uploadPartException: Exception) { + logger.warn { + buildString { + append("Exception occurred while uploading parts for object: ${uploadObjectRequest.key}. ") + append("Aborting multi part upload!") + } + } + + try { + client.withTmBusinessMetric { + it.abortMultipartUpload { + bucket = uploadObjectRequest.bucket + expectedBucketOwner = uploadObjectRequest.expectedBucketOwner + key = uploadObjectRequest.key + requestPayer = uploadObjectRequest.requestPayer + uploadId = mpuUploadId + } + } + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartException) + } catch (abortException: Exception) { + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortException) + .also { it.addSuppressed(uploadPartException) } + } + } + } else { + executePhase( + TransferPhase.BytesTransferred, + context, + interceptors, + ) { + context.s3Response = client.withTmBusinessMetric { + it.putObject(context.s3Request as PutObjectRequest) + } + context.transferredBytes = context.transferableBytes + } + } + + return@coroutineScope uploadedParts +} + +/** + * Produces multipart upload parts to be consumed by [consumer]. + * + * Uses a [kotlinx.coroutines.channels.Channel]. + * Produces until all readable bytes are read. + */ +private fun CoroutineScope.produceParts( + readableBytes: Long, + partSource: Any, + partSize: Long, + numberOfParts: Int, + maxInMemoryParts: Int, + bufferSemaphore: Semaphore, +) = produce( + capacity = maxInMemoryParts, +) { + var readBytes = 0L + var currentPartNumber = 1 + + while (readBytes < readableBytes) { + // +1 part in memory + bufferSemaphore.acquire() + + send( + Part( + currentPartNumber, + nextPartBytes( + partSource, + partSize, + currentPartNumber == numberOfParts, + readBytes, + readableBytes, + ), + ).also { + if (currentPartNumber != numberOfParts && it.bytes.size != partSize) { + throw S3TransferManagerException("Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${it.bytes.size}") + } + }, + ) + + currentPartNumber++ + readBytes += partSize + } +} + +/** + * Launches a coroutine that consumes and uploads multipart upload parts. + * + * It receives mutable shared state that may also be used by other coroutines and is + * intended for use in a [fan-out](https://kotlinlang.org/docs/channels.html#fan-out) pattern, + * where multiple consumers concurrently upload different parts of the same object. + */ +private suspend fun consumer( + channel: ReceiveChannel, + uploadObjectRequest: UploadObjectRequest, + mpuUploadId: String, + context: MutableTransferContext, + interceptors: List, + client: S3Client, + uploadedParts: MutableList, + mutex: Mutex, + semaphore: Semaphore, +) = coroutineScope { + launch { + for (part in channel) { + val partSize = part.bytes.size // Store the original size, as it will shrink when bytes are read + val localContext = context.copy() // Create a separate copy to avoid concurrent modifications + + localContext.s3Request = uploadObjectRequest.toUploadPartRequest( + part.bytes, + part.number, + mpuUploadId, + ) + + executePhase( + TransferPhase.BytesTransferred, + localContext, + interceptors, + ) { + localContext.s3Response = client.withTmBusinessMetric { + it.uploadPart(localContext.s3Request as UploadPartRequest) + } + + // -1 part in memory + semaphore.release() + + localContext.transferredBytes = localContext.transferredBytes!! + partSize + } + + // Update shared state between coroutines + mutex.withLock { + context.s3Request = localContext.s3Request + context.s3Response = localContext.s3Response + context.transferableBytes = localContext.transferableBytes + context.transferredBytes = context.transferredBytes!! + partSize // Don't use transferredBytes from local context as it might be out of date + context.transferableObjects = localContext.transferableObjects + context.transferredObjects = localContext.transferredObjects + + uploadedParts.add( + CompletedPart { + partNumber = part.number + eTag = (localContext.s3Response as UploadPartResponse).eTag + }, + ) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt new file mode 100644 index 00000000000..df00217ac0e --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.withConfig +import aws.smithy.kotlin.runtime.businessmetrics.emitBusinessMetric +import aws.smithy.kotlin.runtime.client.RequestInterceptorContext +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor + +/** + * An interceptor that emits the S3 Transfer Manager business metric + */ +internal object S3TransferManagerBusinessMetricInterceptor : HttpInterceptor { + override suspend fun modifyBeforeSerialization(context: RequestInterceptorContext): Any { + context.executionContext.emitBusinessMetric(AwsBusinessMetric.S3_TRANSFER) + return context.request + } +} + +internal inline fun S3Client.withTmBusinessMetric(block: (S3Client) -> T): T = + withConfig { interceptors += S3TransferManagerBusinessMetricInterceptor }.use(block) diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt new file mode 100644 index 00000000000..c7b09c6738e --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +/** + * Exception thrown when an error occurs during S3 transfer operations. + * + * @param message Description of the error. + * @param cause The underlying cause of the exception, if any. + */ +internal class S3TransferManagerException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/UploadObjectTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/UploadObjectTest.kt new file mode 100644 index 00000000000..a1186499da4 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadobject/UploadObjectTest.kt @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadobject + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.content.ByteStream +import kotlinx.coroutines.runBlocking +import kotlin.invoke +import kotlin.random.Random +import kotlin.test.Ignore +import kotlin.test.Test + +// TODO: Setup e2e test environment - can't run these every build and in CI +class UploadObjectTest { + @Ignore + @Test + fun singleObjectUpload(): Unit = runBlocking { + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadObject { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString("Hello World") + } + } + } + + @Ignore + @Test + fun emptyBody(): Unit = runBlocking { + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadObject { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString("") + } + } + } + + @Ignore + @Test + fun multipartUpload(): Unit = runBlocking { + val messageLength = 10L * 1024L * 1024L // 10 MB + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) { + multipartUploadThresholdBytes = 1 + targetPartSizeBytes = 5L * 1024L * 1024L // 5 MB + }.uploadObject { + bucket = "aoperez" + key = "mpuK" + body = randomBody(messageLength) + } + } + } + + @Ignore + @Test + fun smallLastPart(): Unit = runBlocking { + val messageLength = 12L * 1024L * 1024L // 12 MB (last part will only be 2MB) + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) { + multipartUploadThresholdBytes = 1 + targetPartSizeBytes = 5L * 1024L * 1024L // 5 MB + }.uploadObject { + bucket = "aoperez" + key = "mpuK" + body = randomBody(messageLength) + } + } + } +} + +private fun randomBody(sizeInBytes: Long): ByteStream = + ByteStream.fromBytes( + Random.nextBytes( + ByteArray( + sizeInBytes.toInt(), + ), + ), + ) diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt new file mode 100644 index 00000000000..e8d0460dd30 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.businessmetrics.containsBusinessMetric +import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import kotlin.invoke +import kotlin.test.Test + +class S3TransferManagerBusinessMetricsInterceptorTest { + @Test + fun s3Transfer(): Unit = runBlocking { + val message = "Hello World" + val testInterceptor = object : HttpInterceptor { + override fun readAfterTransmit(context: ProtocolResponseInterceptorContext) { + assert(context.executionContext.containsBusinessMetric(AwsBusinessMetric.S3_TRANSFER)) + } + } + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + interceptors += listOf(testInterceptor) + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadObject { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt new file mode 100644 index 00000000000..0e1fecee787 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt @@ -0,0 +1,104 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.MutableTransferContext +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.interceptors.TransferInterceptor +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.assertThrows +import kotlin.collections.plusAssign +import kotlin.invoke +import kotlin.test.Test +import kotlin.test.assertEquals + +class TransferInterceptorTest { + @Test + fun interceptorsCanReadAndModify(): Unit = runBlocking { + val message = "Hello World" + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager(s3Client) { + interceptors += object : TransferInterceptor { + // Test reads + override fun readBeforeTransferInitiated(context: TransferContext) { + assert(context.transferredBytes == 0L) + assert(context.s3Request is PutObjectRequest) + } + override fun readBeforeTransferCompleted(context: TransferContext) { + assert(context.transferredBytes == message.length.toLong()) + assert(context.s3Response is PutObjectResponse) + } + + // Test modifications + override fun modifyBeforeTransferCompleted(context: MutableTransferContext) { + context.s3Request = CompleteMultipartUploadRequest {} + context.transferredBytes = message.length.toLong() * 10 + } + override fun readAfterTransferCompleted(context: TransferContext) { + assert(context.s3Request is CompleteMultipartUploadRequest) + assert(context.transferredBytes == message.length.toLong() * 10) + } + } + }.uploadObject { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } + + @Test + fun interceptorsExceptionsAreSuppressed(): Unit = runBlocking { + val message = "Hello World" + + val exception = assertThrows { + S3Client { + region = "us-west-2" + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager(s3Client) { + interceptors += listOf( + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferContext): Unit = + throw Exception("1") + }, + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferContext): Unit = + throw Exception("2") + }, + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferContext): Unit = + throw Exception("3") + }, + ) + }.uploadObject { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } + + assertEquals(exception.message, "1") + assertEquals(exception.cause!!.suppressed[0].message, "2") + assertEquals(exception.cause!!.suppressed[1].message, "3") + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 826adedd84d..1412cf2fbe2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -91,6 +91,13 @@ if ("dynamodb".isBootstrappedService) { logger.warn(":services:dynamodb is not bootstrapped, skipping :hll:dynamodb-mapper and subprojects") } +if ("s3".isBootstrappedService) { + include(":hll:s3-transfer-manager") + include(":hll:s3-transfer-manager-codegen") +} else { + logger.warn(":services:s3 is not bootstrapped, skipping :hll:s3-transfer-manager and subprojects") +} + // Service benchmarks project val benchmarkServices = listOf( // keep this list in sync with tests/benchmarks/service-benchmarks/build.gradle.kts