diff --git a/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json new file mode 100644 index 000000000000..402cdd2d7b1c --- /dev/null +++ b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "minor", + "changeLogMessages": [ + "Increasing the default part size for S3 multipart upload from 5MB to 8MB when no part size is specified. This will reduce the number of API calls for multipart uploads." + ] + } + ] +} \ No newline at end of file diff --git a/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adt0.json b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adt0.json new file mode 100644 index 000000000000..bf9e0d53217c --- /dev/null +++ b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adt0.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Update AssemblyInfo to give S3 Unit Test project access to internals." + ] + } + ] +} \ No newline at end of file diff --git a/generator/.DevConfigs/49ef8a70-bb30-4cc4-a8b5-92de4f6068c1.json b/generator/.DevConfigs/49ef8a70-bb30-4cc4-a8b5-92de4f6068c1.json new file mode 100644 index 000000000000..677493821a28 --- /dev/null +++ b/generator/.DevConfigs/49ef8a70-bb30-4cc4-a8b5-92de4f6068c1.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Fixed issue where PartSize and IsLastPart fields were not properly set on Transfer Utility Upload Part Request.", + "Add additional validations for Transfer Utility requests to ensure Upload Parts have the proper Content Length and File Offsets." + ] + } + ] +} \ No newline at end of file diff --git a/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872041.json b/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872041.json new file mode 100644 index 000000000000..cec5b3eb153d --- /dev/null +++ b/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872041.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Create AbortMultipartUploads api that takes in TransferUtilityAbortMultipartUploadRequest." + ] + } + ] +} \ No newline at end of file diff --git a/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f87205d.json b/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f87205d.json new file mode 100644 index 000000000000..9b7c13a4f5ab --- /dev/null +++ b/generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f87205d.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Add missing fields to Transfer Utility request objects. ContentType on TransferUtilityUploadRequest and TransferUtilityUploadDirectoryRequest now directly updates the ContentType header, instead of being a separate field on those objects." + ] + } + ] +} \ No newline at end of file diff --git a/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.cs b/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.cs index ecf1a83dd482..156c2b897efe 100644 --- a/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.cs +++ b/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.cs @@ -15,7 +15,7 @@ namespace ServiceClientGenerator.Generators.SourceFiles /// Class to produce the template output /// - #line 1 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 1 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] public partial class AssemblyInfo : BaseGenerator { @@ -36,35 +36,35 @@ public override string TransformText() // associated with an assembly. [assembly: AssemblyTitle("""); - #line 12 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 12 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyTitle)); #line default #line hidden this.Write("\")]\r\n#if BCL\r\n[assembly: AssemblyDescription(\""); - #line 14 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 14 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: "4.7.2"))); #line default #line hidden this.Write("\")]\r\n#elif NETSTANDARD20\r\n[assembly: AssemblyDescription(\""); - #line 16 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 16 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: "NetStandard 2.0"))); #line default #line hidden this.Write("\")]\r\n#elif NETCOREAPP3_1\r\n[assembly: AssemblyDescription(\""); - #line 18 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 18 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: ".NET Core 3.1"))); #line default #line hidden this.Write("\")]\r\n#elif NET8_0\r\n[assembly: AssemblyDescription(\""); - #line 20 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 20 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: ".NET 8.0"))); #line default @@ -72,7 +72,7 @@ public override string TransformText() this.Write("\")]\r\n#else\r\n#error Unknown platform constant - unable to set correct AssemblyDesc" + "ription\r\n#endif\r\n\r\n"); - #line 25 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 25 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" if (this.Config.AssemblyTitle=="AWSSDK.DynamoDBv2") { #line default @@ -81,7 +81,22 @@ public override string TransformText() [assembly: InternalsVisibleTo(""AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")] "); - #line 28 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 28 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + } + + #line default + #line hidden + + #line 29 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + if (this.Config.AssemblyTitle=="AWSSDK.S3") { + + #line default + #line hidden + this.Write(@"[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")] +[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")] +"); + + #line 32 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" } #line default @@ -110,14 +125,14 @@ public override string TransformText() // [assembly: AssemblyVersion(""1.0.*"")] [assembly: AssemblyVersion("""); - #line 51 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 55 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceVersion)); #line default #line hidden this.Write("\")]\r\n[assembly: AssemblyFileVersion(\""); - #line 52 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" + #line 56 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt" this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceFileVersion)); #line default diff --git a/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.tt b/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.tt index 4a8b9fad751a..ab2cf5d21a23 100644 --- a/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.tt +++ b/generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.tt @@ -26,6 +26,10 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("AWSSDK.UnitTests.DynamoDBv2.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")] [assembly: InternalsVisibleTo("AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")] <# } #> +<# if (this.Config.AssemblyTitle=="AWSSDK.S3") { #> +[assembly: InternalsVisibleTo("AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")] +[assembly: InternalsVisibleTo("AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")] +<# } #> [assembly: AssemblyConfiguration("")] [assembly: AssemblyProduct("Amazon Web Services SDK for .NET")] [assembly: AssemblyCompany("Amazon.com, Inc")] diff --git a/generator/ServiceModels/_manifest.json b/generator/ServiceModels/_manifest.json index 24252ee5bfd6..5d712b04c8c1 100644 --- a/generator/ServiceModels/_manifest.json +++ b/generator/ServiceModels/_manifest.json @@ -60,7 +60,8 @@ "Custom\\Runtime\\TestResponses\\*.txt", "Custom\\Runtime\\EventStreams\\test_vectors\\*", "Custom\\Runtime\\TestEndpoints\\*.json", - "Custom\\TestTools\\ComparerTest.json" + "Custom\\TestTools\\ComparerTest.json", + "..\\Services\\S3\\UnitTests\\Custom\\EmbeddedResource\\*" ], "packageReferences": [ { diff --git a/sdk/src/Services/S3/Custom/Model/HeadersCollection.cs b/sdk/src/Services/S3/Custom/Model/HeadersCollection.cs index ed2945257f9b..32e712cf5018 100644 --- a/sdk/src/Services/S3/Custom/Model/HeadersCollection.cs +++ b/sdk/src/Services/S3/Custom/Model/HeadersCollection.cs @@ -152,6 +152,21 @@ internal bool IsSetContentType() return !string.IsNullOrEmpty(this.ContentType); } + /// + /// The language that the content is in. For more information, + /// see https://www.rfc-editor.org/rfc/rfc9110.html#name-content-language. + /// + public string ContentLanguage + { + get { return this["Content-Language"]; } + set { this["Content-Language"] = value; } + } + + internal bool IsSetContentLanguage() + { + return !string.IsNullOrEmpty(this.ContentLanguage); + } + /// /// /// The date and time at which the object is no longer cacheable. For more information, diff --git a/sdk/src/Services/S3/Custom/Transfer/BaseDownloadRequest.cs b/sdk/src/Services/S3/Custom/Transfer/BaseDownloadRequest.cs index d29a3747f24a..84bc08c29225 100644 --- a/sdk/src/Services/S3/Custom/Transfer/BaseDownloadRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/BaseDownloadRequest.cs @@ -24,6 +24,7 @@ using System.Text; using Amazon.Runtime.Internal; +using Amazon.S3.Model; namespace Amazon.S3.Transfer { @@ -45,6 +46,11 @@ public abstract class BaseDownloadRequest private RequestPayer requestPayer; + private string expectedBucketOwner; + private string ifMatch; + private string ifNoneMatch; + private ResponseHeaderOverrides responseHeaders; + /// /// Gets or sets the name of the bucket. /// @@ -66,7 +72,7 @@ public string BucketName /// internal bool IsSetBucketName() { - return !System.String.IsNullOrEmpty(this.bucketName); + return !String.IsNullOrEmpty(this.bucketName); } @@ -91,7 +97,7 @@ public string Key /// internal bool IsSetKey() { - return !System.String.IsNullOrEmpty(this.key); + return !String.IsNullOrEmpty(this.key); } /// @@ -112,7 +118,7 @@ public string VersionId /// true if VersionId property is set. internal bool IsSetVersionId() { - return !System.String.IsNullOrEmpty(this.versionId); + return !String.IsNullOrEmpty(this.versionId); } /// @@ -220,5 +226,109 @@ public RequestPayer RequestPayer get { return this.requestPayer; } set { this.requestPayer = value; } } + + /// + /// Gets and sets the property ExpectedBucketOwner. + /// + /// The account ID of the expected bucket owner. If the account ID that you provide does + /// not match the actual owner of the bucket, the request fails with the HTTP status code + /// 403 Forbidden (access denied). + /// + /// + public string ExpectedBucketOwner + { + get { return this.expectedBucketOwner; } + set { this.expectedBucketOwner = value; } + } + + /// + /// Checks to see if ExpectedBucketOwner is set. + /// + /// true, if ExpectedBucketOwner property is set. + internal bool IsSetExpectedBucketOwner() + { + return !String.IsNullOrEmpty(this.expectedBucketOwner); + } + + /// + /// Gets and sets the property IfMatch. + /// + /// Return the object only if its entity tag (ETag) is the same as the one specified in this header; + /// otherwise, return a 412 Precondition Failed error. + /// + /// + /// If both of the If-Match and If-Unmodified-Since headers are present in the request as follows: + /// If-Match condition evaluates to true, and; If-Unmodified-Since condition evaluates to false; + /// then, S3 returns 200 OK and the data requested. + /// + /// + /// For more information about conditional requests, see RFC 7232. + /// + /// The property is equivalent to the . + /// + public string IfMatch + { + get { return this.ifMatch; } + set { this.ifMatch = value; } + } + + /// + /// Checks to see if IfMatch is set. + /// + /// true, if IfMatch property is set. + internal bool IsSetIfMatch() + { + return !String.IsNullOrEmpty(this.ifMatch); + } + + /// + /// Gets and sets the property IfNoneMatch. + /// + /// Return the object only if its entity tag (ETag) is different from the one specified in this header; + /// otherwise, return a 304 Not Modified error. + /// + /// + /// If both of the If-None-Match and If-Modified-Since headers are present in the request as follows: + /// If-None-Match condition evaluates to false, and; If-Modified-Since condition evaluates to true; + /// then, S3 returns 304 Not Modified HTTP status code. + /// + /// + /// For more information about conditional requests, see RFC 7232. + /// + /// The property is equivalent to the . + /// + public string IfNoneMatch + { + get { return this.ifNoneMatch; } + set { this.ifNoneMatch = value; } + } + + /// + /// Checks to see if IfNoneMatch is set. + /// + /// true, if IfNoneMatch property is set. + internal bool IsSetIfNoneMatch() + { + return !String.IsNullOrEmpty(this.ifNoneMatch); + } + + /// + /// A set of response headers that should be returned with the object. + /// + public ResponseHeaderOverrides ResponseHeaderOverrides + { + get + { + if (this.responseHeaders == null) + { + this.responseHeaders = new ResponseHeaderOverrides(); + } + return this.responseHeaders; + } + set + { + this.responseHeaders = value; + } + } } } \ No newline at end of file diff --git a/sdk/src/Services/S3/Custom/Transfer/BaseUploadRequest.cs b/sdk/src/Services/S3/Custom/Transfer/BaseUploadRequest.cs index 8a1cec4a957c..d087be435f4f 100644 --- a/sdk/src/Services/S3/Custom/Transfer/BaseUploadRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/BaseUploadRequest.cs @@ -21,6 +21,9 @@ * */ using System; +using System.Collections.Generic; +using Amazon.Runtime.Internal; +using Amazon.S3.Model; namespace Amazon.S3.Transfer { @@ -29,7 +32,28 @@ namespace Amazon.S3.Transfer /// public abstract class BaseUploadRequest { + private string bucketName; private RequestPayer requestPayer; + private ServerSideEncryptionMethod encryption; + private ServerSideEncryptionCustomerMethod serverSideCustomerEncryption; + private string serverSideEncryptionCustomerProvidedKey; + private string serverSideEncryptionCustomerProvidedKeyMD5; + private string serverSideEncryptionKeyManagementServiceKeyId; + private ChecksumAlgorithm checksumAlgorithm; + private S3CannedACL cannedACL; + private S3StorageClass storageClass; + private MetadataCollection metadataCollection = new MetadataCollection(); + private List tagset; + private ObjectLockLegalHoldStatus objectLockLegalHoldStatus; + private ObjectLockMode objectLockMode; + private DateTime? objectLockRetainUntilDate; + private bool? disablePayloadSigning; + private bool? bucketKeyEnabled; + private string expectedBucketOwner; + private string sseKMSEncryptionContext; + private string websiteRedirectLocation; + private HeadersCollection headersCollection = new HeadersCollection(); + private List _grants = AWSConfigs.InitializeCollections ? new List() : null; /// /// Confirms that the requester knows that they will be charged for the request. @@ -40,5 +64,486 @@ public RequestPayer RequestPayer get { return this.requestPayer; } set { this.requestPayer = value; } } + + #region BucketName + + /// + /// Gets or sets the name of the bucket. + /// + /// + /// The name of the bucket. + /// + [AWSProperty(Required = true)] + public string BucketName + { + get { return this.bucketName; } + set { this.bucketName = value; } + } + + + /// + /// Checks if BucketName property is set. + /// + /// true if BucketName property is set. + internal bool IsSetBucketName() + { + return !String.IsNullOrEmpty(this.bucketName); + } + + #endregion + + #region ContentType + /// + /// Gets or sets the content type of the uploaded Amazon S3 object. + /// This is a convenience property for Headers.ContentType. + /// + /// The content type of the uploaded Amazon S3 object. + /// + /// + public string ContentType + { + get { return this.Headers.ContentType; } + set { this.Headers.ContentType = value; } + } + + + /// + /// Checks if ContentType property is set. + /// + /// true if ContentType property is set. + internal bool IsSetContentType() + { + return !String.IsNullOrEmpty(this.Headers.ContentType); + } + + #endregion + + #region ServerSideEncryption + + /// + /// Gets and sets the ServerSideEncryptionMethod property. + /// Specifies the encryption used on the server to + /// store the content. + /// + public ServerSideEncryptionMethod ServerSideEncryptionMethod + { + get { return this.encryption; } + set { this.encryption = value; } + } + + /// + /// The Server-side encryption algorithm to be used with the customer provided key. + /// + public ServerSideEncryptionCustomerMethod ServerSideEncryptionCustomerMethod + { + get { return this.serverSideCustomerEncryption; } + set { this.serverSideCustomerEncryption = value; } + } + + /// + /// The id of the AWS Key Management Service key that Amazon S3 should use to encrypt and decrypt the object. + /// If a key id is not specified, the default key will be used for encryption and decryption. + /// + [AWSProperty(Sensitive=true)] + public string ServerSideEncryptionKeyManagementServiceKeyId + { + get { return this.serverSideEncryptionKeyManagementServiceKeyId; } + set { this.serverSideEncryptionKeyManagementServiceKeyId = value; } + } + + /// + /// Checks if ServerSideEncryptionKeyManagementServiceKeyId property is set. + /// + /// true if ServerSideEncryptionKeyManagementServiceKeyId property is set. + internal bool IsSetServerSideEncryptionKeyManagementServiceKeyId() + { + return !String.IsNullOrEmpty(this.serverSideEncryptionKeyManagementServiceKeyId); + } + + /// + /// The Base64 encoded encryption key for Amazon S3 to use to encrypt the object + /// + /// Using the encryption key you provide as part of your request Amazon S3 manages both the encryption, as it writes + /// to disks, and decryption, when you access your objects. Therefore, you don't need to maintain any data encryption code. The only + /// thing you do is manage the encryption keys you provide. + /// + /// + /// When you retrieve an object, you must provide the same encryption key as part of your request. Amazon S3 first verifies + /// the encryption key you provided matches, and then decrypts the object before returning the object data to you. + /// + /// + /// Important: Amazon S3 does not store the encryption key you provide. + /// + /// + [AWSProperty(Sensitive=true)] + public string ServerSideEncryptionCustomerProvidedKey + { + get { return this.serverSideEncryptionCustomerProvidedKey; } + set { this.serverSideEncryptionCustomerProvidedKey = value; } + } + + /// + /// The MD5 of the customer encryption key specified in the ServerSideEncryptionCustomerProvidedKey property. The MD5 is + /// base 64 encoded. This field is optional, the SDK will calculate the MD5 if this is not set. + /// + public string ServerSideEncryptionCustomerProvidedKeyMD5 + { + get { return this.serverSideEncryptionCustomerProvidedKeyMD5; } + set { this.serverSideEncryptionCustomerProvidedKeyMD5 = value; } + } + + #endregion + + /// + /// Gets and sets the property ChecksumAlgorithm. + /// + /// Indicates the algorithm used to create the checksum for the object. + /// For more information, see + /// Checking object integrity in the Amazon S3 User Guide. + /// + /// + /// + /// If you provide an individual checksum, Amazon S3 will ignore any provided ChecksumAlgorithm. + /// + /// + public ChecksumAlgorithm ChecksumAlgorithm + { + get { return this.checksumAlgorithm; } + set { this.checksumAlgorithm = value; } + } + + #region CannedACL + + /// + /// Gets or sets the canned access control list (ACL) + /// for the uploaded object. + /// Please refer to + /// for + /// information on Amazon S3 canned ACLs. + /// + /// + /// The canned access control list (ACL) + /// for the uploaded object. + /// + public S3CannedACL CannedACL + { + get { return this.cannedACL; } + set { this.cannedACL = value; } + } + + /// + /// Checks if the CannedACL property is set. + /// + /// true if there is the CannedACL property is set. + internal bool IsSetCannedACL() + { + return (cannedACL != null); + } + + /// + /// Removes the canned access control list (ACL) + /// for the uploaded object. + /// + public void RemoveCannedACL() + { + this.cannedACL = null; + } + + #endregion + + #region StorageClass + + /// + /// Gets or sets the storage class for the uploaded Amazon S3 object. + /// Please refer to + /// for + /// information on S3 Storage Classes. + /// + /// + /// The storage class for the uploaded Amazon S3 object. + /// + public S3StorageClass StorageClass + { + get { return this.storageClass; } + set { this.storageClass = value; } + } + + #endregion + + /// + /// The collection of meta data for the request. + /// + public MetadataCollection Metadata + { + get + { + if (this.metadataCollection == null) + this.metadataCollection = new MetadataCollection(); + return this.metadataCollection; + } + internal set { this.metadataCollection = value; } + } + + /// + /// The tag-set for the object. + /// + public List TagSet + { + get { return this.tagset; } + set { this.tagset = value; } + } + + /// + /// Gets and sets the property ObjectLockLegalHoldStatus. + /// + /// Specifies whether a legal hold will be applied to this object. For more information + /// about S3 Object Lock, see Object + /// Lock. + /// + /// + public ObjectLockLegalHoldStatus ObjectLockLegalHoldStatus + { + get { return this.objectLockLegalHoldStatus; } + set { this.objectLockLegalHoldStatus = value; } + } + + /// + /// Gets and sets the property ObjectLockMode. + /// + /// The Object Lock mode that you want to apply to this object. + /// + /// + public ObjectLockMode ObjectLockMode + { + get { return this.objectLockMode; } + set { this.objectLockMode = value; } + } + + /// + /// Gets and sets the property ObjectLockRetainUntilDate. + /// + /// The date and time when you want this object's Object Lock to expire. + /// + /// + public DateTime? ObjectLockRetainUntilDate + { + get { return this.objectLockRetainUntilDate.GetValueOrDefault(); } + set { this.objectLockRetainUntilDate = value; } + } + + // Check to see if ObjectLockRetainUntilDate property is set + internal bool IsSetObjectLockRetainUntilDate() + { + return this.objectLockRetainUntilDate.HasValue; + } + + /// + /// WARNING: Setting DisablePayloadSigning to true disables the SigV4 payload signing + /// data integrity check on this request. + /// If using SigV4, the DisablePayloadSigning flag controls if the payload should be + /// signed on a request by request basis. By default this flag is null which will use the + /// default client behavior. The default client behavior is to sign the payload. When + /// DisablePayloadSigning is true, the request will be signed with an UNSIGNED-PAYLOAD value. + /// Setting DisablePayloadSigning to true requires that the request is sent over a HTTPS + /// connection. + /// Under certain circumstances, such as uploading to S3 while using MD5 hashing, it may + /// be desirable to use UNSIGNED-PAYLOAD to decrease signing CPU usage. This flag only applies + /// to Amazon S3 PutObject and UploadPart requests. + /// MD5Stream, SigV4 payload signing, and HTTPS each provide some data integrity + /// verification. If DisableMD5Stream is true and DisablePayloadSigning is true, then the + /// possibility of data corruption is completely dependent on HTTPS being the only remaining + /// source of data integrity verification. + /// + public bool? DisablePayloadSigning + { + get { return this.disablePayloadSigning; } + set { this.disablePayloadSigning = value; } + } + + /// + /// WARNING: Setting DisableDefaultChecksumValidation to true disables the default data + /// integrity check on upload requests. + /// When true, checksum verification will not be used in upload requests. This may increase upload + /// performance under high CPU loads. Setting DisableDefaultChecksumValidation sets the deprecated property + /// DisableMD5Stream to the same value. The default value is false. + /// Checksums, SigV4 payload signing, and HTTPS each provide some data integrity + /// verification. If DisableDefaultChecksumValidation is true and DisablePayloadSigning is true, then the + /// possibility of data corruption is completely dependent on HTTPS being the only remaining + /// source of data integrity verification. + /// + public bool? DisableDefaultChecksumValidation { get; set; } + + /// + /// Gets and sets the property BucketKeyEnabled. + /// + /// Specifies whether Amazon S3 should use an S3 Bucket Key for object encryption with + /// server-side encryption using Key Management Service (KMS) keys (SSE-KMS). + /// + /// + /// + /// General purpose buckets - Setting this header to true causes Amazon S3 to use an + /// S3 Bucket Key for object encryption with SSE-KMS. + /// Also, specifying this header with a PUT action doesn't affect bucket-level settings for S3 Bucket Key. + /// + /// + /// + /// Directory buckets - S3 Bucket Keys are always enabled for GET and PUT operations in a directory bucket and can't be disabled. + /// S3 Bucket Keys aren't supported, when you copy SSE-KMS encrypted objects from general purpose buckets to directory buckets, + /// from directory buckets to general purpose buckets, or between directory buckets, through + /// CopyObject, + /// UploadPartCopy, + /// the Copy operation in Batch Operations, + /// or the import jobs. + /// + /// In this case, Amazon S3 makes a call to KMS every time a copy request is made for a KMS-encrypted object. + /// + /// + public bool? BucketKeyEnabled + { + get { return this.bucketKeyEnabled; } + set { this.bucketKeyEnabled = value; } + } + + internal bool IsSetBucketKeyEnabled() + { + return bucketKeyEnabled.HasValue; + } + + /// + /// Gets and sets the property ExpectedBucketOwner. + /// + /// The account ID of the expected bucket owner. + /// If the account ID that you provide does not match the actual owner of the bucket, + /// the request fails with the HTTP status code 403 Forbidden (access denied). + /// + /// + public string ExpectedBucketOwner + { + get { return this.expectedBucketOwner; } + set { this.expectedBucketOwner = value; } + } + + /// + /// Checks to see if ExpectedBucketOwner is set. + /// + /// true, if ExpectedBucketOwner property is set. + internal bool IsSetExpectedBucketOwner() + { + return !String.IsNullOrEmpty(this.expectedBucketOwner); + } + + /// + /// Gets the access control lists (ACLs) for this request. + /// Please refer to for information on + /// S3 Grants. + /// + public List Grants + { + get { return _grants; } + set { _grants = value; } + } + + /// + /// Gets and sets the property SSEKMSEncryptionContext. + /// + /// Specifies the Amazon Web Services KMS Encryption Context as + /// an additional encryption context to use for object encryption. + /// The value of this header is a Base64 encoded string of a UTF-8 encoded JSON, + /// which contains the encryption context as key-value pairs. + /// This value is stored as object metadata and automatically gets passed on to + /// Amazon Web Services KMS for future GetObject operations on this object. + /// + /// General purpose buckets + /// - This value must be explicitly added during CopyObject operations + /// if you want an additional encryption context for your object. + /// For more information, see Encryption context + /// in the Amazon S3 User Guide. + /// + /// Directory buckets + /// - You can optionally provide an explicit encryption context value. + /// The value must match the default encryption context + /// - the bucket Amazon Resource Name (ARN). + /// An additional encryption context value is not supported. + /// + /// + public string SSEKMSEncryptionContext + { + get { return this.sseKMSEncryptionContext; } + set { this.sseKMSEncryptionContext = value; } + } + + /// + /// Checks to see if SSEKMSEncryptionContext is set. + /// + /// true, if SSEKMSEncryptionContext property is set. + internal bool IsSetSSEKMSEncryptionContext() + { + return !String.IsNullOrEmpty(this.sseKMSEncryptionContext); + } + + /// + /// Gets and sets the property WebsiteRedirectLocation. + /// + /// If the bucket is configured as a website, + /// redirects requests for this object to another object in the + /// same bucket or to an external URL. + /// Amazon S3 stores the value of this header in the object metadata. + /// For information about object metadata, see Object Key and Metadata + /// in the Amazon S3 User Guide. + /// + /// + /// In the following example, + /// the request header sets the redirect to an object (anotherPage.html) in the same bucket: + /// + /// + /// x-amz-website-redirect-location: /anotherPage.html + /// + /// + /// In the following example, + /// the request header sets the object redirect to another website: + /// + /// + /// x-amz-website-redirect-location: http://www.example.com/ + /// + /// + /// For more information about website hosting in Amazon S3, + /// see Hosting Websites on Amazon S3 + /// and How to Configure Website Page Redirects + /// in the Amazon S3 User Guide. + /// + /// + /// + /// This functionality is not supported for directory buckets. + /// + /// + /// + public string WebsiteRedirectLocation + { + get { return this.websiteRedirectLocation; } + set { this.websiteRedirectLocation = value; } + } + + /// + /// Checks to see if WebsiteRedirectLocation is set. + /// + /// true, if WebsiteRedirectLocation property is set. + internal bool IsSetWebsiteRedirectLocation() + { + return !String.IsNullOrEmpty(this.websiteRedirectLocation); + } + + /// + /// The collection of headers for the request. + /// + public HeadersCollection Headers + { + get + { + if (this.headersCollection == null) + this.headersCollection = new HeadersCollection(); + return this.headersCollection; + } + internal set { this.headersCollection = value; } + } } } \ No newline at end of file diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/AbortMultipartUploadsCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/AbortMultipartUploadsCommand.cs index 09ed0dba2e8f..a0313c75b6c0 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/AbortMultipartUploadsCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/AbortMultipartUploadsCommand.cs @@ -31,36 +31,45 @@ namespace Amazon.S3.Transfer.Internal internal partial class AbortMultipartUploadsCommand : BaseCommand { IAmazonS3 _s3Client; - string _bucketName; - DateTime _initiatedDate; + TransferUtilityAbortMultipartUploadRequest _request; + TransferUtilityConfig _config; - internal AbortMultipartUploadsCommand(IAmazonS3 s3Client, string bucketName, DateTime initiateDate) + internal AbortMultipartUploadsCommand(IAmazonS3 s3Client, TransferUtilityAbortMultipartUploadRequest request, TransferUtilityConfig config) { this._s3Client = s3Client; - this._bucketName = bucketName; - this._initiatedDate = initiateDate; + this._request = request; + this._config = config; } - private ListMultipartUploadsRequest ConstructListMultipartUploadsRequest(ListMultipartUploadsResponse listResponse) + internal ListMultipartUploadsRequest ConstructListMultipartUploadsRequest(ListMultipartUploadsResponse listResponse) { ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest() { - BucketName = this._bucketName, + BucketName = this._request.BucketName, KeyMarker = listResponse.KeyMarker, UploadIdMarker = listResponse.NextUploadIdMarker, + ExpectedBucketOwner = this._request.ExpectedBucketOwner, + RequestPayer = this._request.RequestPayer }; + + + + ((Amazon.Runtime.Internal.IAmazonWebServiceRequest)listRequest).AddBeforeRequestHandler(this.RequestEventHandler); return listRequest; } - private AbortMultipartUploadRequest ConstructAbortMultipartUploadRequest(MultipartUpload upload) + internal AbortMultipartUploadRequest ConstructAbortMultipartUploadRequest(MultipartUpload upload) { var abortRequest = new AbortMultipartUploadRequest() { - BucketName = this._bucketName, + BucketName = this._request.BucketName, Key = upload.Key, UploadId = upload.UploadId, + ExpectedBucketOwner = this._request.ExpectedBucketOwner, + RequestPayer = this._request.RequestPayer }; + ((Amazon.Runtime.Internal.IAmazonWebServiceRequest)abortRequest).AddBeforeRequestHandler(this.RequestEventHandler); return abortRequest; } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/BaseCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/BaseCommand.cs index 5e5e83fdbae2..428758fa54e6 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/BaseCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/BaseCommand.cs @@ -37,7 +37,7 @@ public virtual object Return get { return null; } } - protected GetObjectRequest ConvertToGetObjectRequest(BaseDownloadRequest request) + internal GetObjectRequest ConvertToGetObjectRequest(BaseDownloadRequest request) { GetObjectRequest getRequest = new GetObjectRequest() { @@ -62,6 +62,21 @@ protected GetObjectRequest ConvertToGetObjectRequest(BaseDownloadRequest request getRequest.ChecksumMode = request.ChecksumMode; getRequest.RequestPayer = request.RequestPayer; + if (request.IsSetExpectedBucketOwner()) + { + getRequest.ExpectedBucketOwner = request.ExpectedBucketOwner; + } + if (request.IsSetIfMatch()) + { + getRequest.EtagToMatch = request.IfMatch; + } + if (request.IsSetIfNoneMatch()) + { + getRequest.EtagToNotMatch = request.IfNoneMatch; + } + + getRequest.ResponseHeaderOverrides = request.ResponseHeaderOverrides; + return getRequest; } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadDirectoryCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadDirectoryCommand.cs index 356091b417ee..0140554ded39 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadDirectoryCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/DownloadDirectoryCommand.cs @@ -91,7 +91,7 @@ private void EnsureDirectoryExists(DirectoryInfo directory) directory.Create(); } - private TransferUtilityDownloadRequest ConstructTransferUtilityDownloadRequest(S3Object s3Object, int prefixLength) + internal TransferUtilityDownloadRequest ConstructTransferUtilityDownloadRequest(S3Object s3Object, int prefixLength) { var downloadRequest = new TransferUtilityDownloadRequest(); downloadRequest.BucketName = this._request.BucketName; @@ -102,6 +102,10 @@ private TransferUtilityDownloadRequest ConstructTransferUtilityDownloadRequest(S downloadRequest.ServerSideEncryptionCustomerProvidedKey = this._request.ServerSideEncryptionCustomerProvidedKey; downloadRequest.ServerSideEncryptionCustomerProvidedKeyMD5 = this._request.ServerSideEncryptionCustomerProvidedKeyMD5; downloadRequest.RequestPayer = this._request.RequestPayer; + downloadRequest.ExpectedBucketOwner = this._request.ExpectedBucketOwner; + downloadRequest.IfMatch = this._request.IfMatch; + downloadRequest.IfNoneMatch = this._request.IfNoneMatch; + downloadRequest.ResponseHeaderOverrides = this._request.ResponseHeaderOverrides; //Ensure the target file is a rooted within LocalDirectory. Otherwise error. if(!InternalSDKUtils.IsFilePathRootedWithDirectoryPath(downloadRequest.FilePath, _request.LocalDirectory)) @@ -137,11 +141,12 @@ private ListObjectsV2Request ConstructListObjectRequestV2() } listRequestV2.RequestPayer = this._request.RequestPayer; + listRequestV2.ExpectedBucketOwner = this._request.ExpectedBucketOwner; return listRequestV2; } - private ListObjectsRequest ConstructListObjectRequest() + internal ListObjectsRequest ConstructListObjectRequest() { ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.BucketName = this._request.BucketName; @@ -164,6 +169,7 @@ private ListObjectsRequest ConstructListObjectRequest() } listRequest.RequestPayer = this._request.RequestPayer; + listRequest.ExpectedBucketOwner = this._request.ExpectedBucketOwner; return listRequest; } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs index e31184e6353f..3b9532793578 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs @@ -49,7 +49,6 @@ internal partial class MultipartUploadCommand : BaseCommand long _totalTransferredBytes; Queue _partsToUpload = new Queue(); - long _contentLength; private static Logger Logger { @@ -81,11 +80,12 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config this._s3Client = s3Client; this._fileTransporterRequest = fileTransporterRequest; this._contentLength = this._fileTransporterRequest.ContentLength; + + long targetPartSize = fileTransporterRequest.IsSetPartSize() + ? fileTransporterRequest.PartSize + : S3Constants.DefaultPartSize; - if (fileTransporterRequest.IsSetPartSize()) - this._partSize = fileTransporterRequest.PartSize; - else - this._partSize = calculatePartSize(this._contentLength); + this._partSize = calculatePartSize(this._contentLength, targetPartSize); if (fileTransporterRequest.InputStream != null) { @@ -98,15 +98,9 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config Logger.DebugFormat("Upload part size {0}.", this._partSize); } - private static long calculatePartSize(long fileSize) + private static long calculatePartSize(long contentLength, long targetPartSize) { - double partSize = Math.Ceiling((double)fileSize / S3Constants.MaxNumberOfParts); - if (partSize < S3Constants.MinPartSize) - { - partSize = S3Constants.MinPartSize; - } - - return (long)partSize; + return Math.Max(targetPartSize, contentLength / S3Constants.MaxNumberOfParts); } private string determineContentType() @@ -152,12 +146,12 @@ private int CalculateConcurrentServiceRequests() return threadCount; } - private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse) + internal CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse) { return ConstructCompleteMultipartUploadRequest(initResponse, false, null); } - private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse, bool skipPartValidation, RequestEventHandler requestEventHandler) + internal CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse, bool skipPartValidation, RequestEventHandler requestEventHandler) { if (!skipPartValidation) { @@ -182,6 +176,7 @@ private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(I ChecksumCRC64NVME = this._fileTransporterRequest.ChecksumCRC64NVME, ChecksumSHA1 = this._fileTransporterRequest.ChecksumSHA1, ChecksumSHA256 = this._fileTransporterRequest.ChecksumSHA256, + ExpectedBucketOwner = this._fileTransporterRequest.ExpectedBucketOwner, }; if(this._fileTransporterRequest.ServerSideEncryptionCustomerMethod != null @@ -215,17 +210,30 @@ private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(I return compRequest; } - private UploadPartRequest ConstructUploadPartRequest(int partNumber, long filePosition, InitiateMultipartUploadResponse initiateResponse) + private bool calculateIsLastPart(long remainingBytes) + { + var isLastPart = false; + if (remainingBytes <= this._partSize) + isLastPart = true; + return isLastPart; + } + + internal UploadPartRequest ConstructUploadPartRequest(int partNumber, long filePosition, InitiateMultipartUploadResponse initiateResponse) { UploadPartRequest uploadPartRequest = ConstructGenericUploadPartRequest(initiateResponse); + // Calculating how many bytes are remaining to be uploaded from the current part. + // This is mainly used for the last part scenario. + var remainingBytes = this._contentLength - filePosition; + // We then check based on the remaining bytes and the content length if this is the last part. + var isLastPart = calculateIsLastPart(remainingBytes); uploadPartRequest.PartNumber = partNumber; - uploadPartRequest.PartSize = this._partSize; + uploadPartRequest.PartSize = isLastPart ? remainingBytes : this._partSize; + uploadPartRequest.IsLastPart = isLastPart; - if ((filePosition + this._partSize >= this._contentLength) + if (isLastPart && _s3Client is Amazon.S3.Internal.IAmazonS3Encryption) { - uploadPartRequest.IsLastPart = true; uploadPartRequest.PartSize = 0; } @@ -246,7 +254,7 @@ private UploadPartRequest ConstructUploadPartRequest(int partNumber, long filePo return uploadPartRequest; } - private UploadPartRequest ConstructGenericUploadPartRequest(InitiateMultipartUploadResponse initiateResponse) + internal UploadPartRequest ConstructGenericUploadPartRequest(InitiateMultipartUploadResponse initiateResponse) { UploadPartRequest uploadPartRequest = new UploadPartRequest() { @@ -259,7 +267,8 @@ private UploadPartRequest ConstructGenericUploadPartRequest(InitiateMultipartUpl DisableDefaultChecksumValidation = this._fileTransporterRequest.DisableDefaultChecksumValidation, DisablePayloadSigning = this._fileTransporterRequest.DisablePayloadSigning, ChecksumAlgorithm = this._fileTransporterRequest.ChecksumAlgorithm, - RequestPayer = this._fileTransporterRequest.RequestPayer + RequestPayer = this._fileTransporterRequest.RequestPayer, + ExpectedBucketOwner = this._fileTransporterRequest.ExpectedBucketOwner, }; // If the InitiateMultipartUploadResponse indicates that this upload is using KMS, force SigV4 for each UploadPart request @@ -270,7 +279,7 @@ private UploadPartRequest ConstructGenericUploadPartRequest(InitiateMultipartUpl return uploadPartRequest; } - private UploadPartRequest ConstructUploadPartRequestForNonSeekableStream(Stream inputStream, int partNumber, long partSize, bool isLastPart, InitiateMultipartUploadResponse initiateResponse) + internal UploadPartRequest ConstructUploadPartRequestForNonSeekableStream(Stream inputStream, int partNumber, long partSize, bool isLastPart, InitiateMultipartUploadResponse initiateResponse) { UploadPartRequest uploadPartRequest = ConstructGenericUploadPartRequest(initiateResponse); @@ -290,12 +299,12 @@ private UploadPartRequest ConstructUploadPartRequestForNonSeekableStream(Stream return uploadPartRequest; } - private InitiateMultipartUploadRequest ConstructInitiateMultipartUploadRequest() + internal InitiateMultipartUploadRequest ConstructInitiateMultipartUploadRequest() { return this.ConstructInitiateMultipartUploadRequest(null); } - private InitiateMultipartUploadRequest ConstructInitiateMultipartUploadRequest(RequestEventHandler requestEventHandler) + internal InitiateMultipartUploadRequest ConstructInitiateMultipartUploadRequest(RequestEventHandler requestEventHandler) { var initRequest = new InitiateMultipartUploadRequest() { @@ -309,11 +318,17 @@ private InitiateMultipartUploadRequest ConstructInitiateMultipartUploadRequest(R ServerSideEncryptionCustomerMethod = this._fileTransporterRequest.ServerSideEncryptionCustomerMethod, ServerSideEncryptionCustomerProvidedKey = this._fileTransporterRequest.ServerSideEncryptionCustomerProvidedKey, ServerSideEncryptionCustomerProvidedKeyMD5 = this._fileTransporterRequest.ServerSideEncryptionCustomerProvidedKeyMD5, + ServerSideEncryptionKeyManagementServiceEncryptionContext = this._fileTransporterRequest.SSEKMSEncryptionContext, TagSet = this._fileTransporterRequest.TagSet, ChecksumAlgorithm = this._fileTransporterRequest.ChecksumAlgorithm, ObjectLockLegalHoldStatus = this._fileTransporterRequest.ObjectLockLegalHoldStatus, ObjectLockMode = this._fileTransporterRequest.ObjectLockMode, - RequestPayer = this._fileTransporterRequest.RequestPayer + RequestPayer = this._fileTransporterRequest.RequestPayer, + ExpectedBucketOwner = this._fileTransporterRequest.ExpectedBucketOwner, + Grants = this._fileTransporterRequest.Grants, + Metadata = this._fileTransporterRequest.Metadata, + WebsiteRedirectLocation = this._fileTransporterRequest.WebsiteRedirectLocation, + BucketKeyEnabled = this._fileTransporterRequest.BucketKeyEnabled, }; if (this._fileTransporterRequest.IsSetObjectLockRetainUntilDate()) diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/OpenStreamCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/OpenStreamCommand.cs index e34df962f364..57eab52d3f98 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/OpenStreamCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/OpenStreamCommand.cs @@ -41,7 +41,7 @@ internal OpenStreamCommand(IAmazonS3 s3Client, TransferUtilityOpenStreamRequest this._request = request; } - private GetObjectRequest ConstructRequest() + internal GetObjectRequest ConstructRequest() { if (!this._request.IsSetBucketName()) { diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs index 95a15611d2f5..3f10fa35b1d0 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs @@ -50,7 +50,7 @@ internal SimpleUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config, T var fileName = fileTransporterRequest.FilePath; } - private PutObjectRequest ConstructRequest() + internal PutObjectRequest ConstructRequest() { PutObjectRequest putRequest = new PutObjectRequest() { @@ -78,7 +78,12 @@ private PutObjectRequest ConstructRequest() ChecksumCRC64NVME = this._fileTransporterRequest.ChecksumCRC64NVME, ChecksumSHA1 = this._fileTransporterRequest.ChecksumSHA1, ChecksumSHA256 = this._fileTransporterRequest.ChecksumSHA256, - RequestPayer = this._fileTransporterRequest.RequestPayer + RequestPayer = this._fileTransporterRequest.RequestPayer, + BucketKeyEnabled = this._fileTransporterRequest.BucketKeyEnabled, + ExpectedBucketOwner = this._fileTransporterRequest.ExpectedBucketOwner, + Grants = this._fileTransporterRequest.Grants, + ServerSideEncryptionKeyManagementServiceEncryptionContext = this._fileTransporterRequest.SSEKMSEncryptionContext, + WebsiteRedirectLocation = this._fileTransporterRequest.WebsiteRedirectLocation, }; // Avoid setting ContentType to null, as that may clear diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/UploadDirectoryCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/UploadDirectoryCommand.cs index 816c8ef12e6f..e4be9b27aa74 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/UploadDirectoryCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/UploadDirectoryCommand.cs @@ -50,7 +50,7 @@ internal UploadDirectoryCommand(TransferUtility utility, TransferUtilityConfig c this._config = config; } - private TransferUtilityUploadRequest ConstructRequest(string basePath, string filepath, string prefix) + internal TransferUtilityUploadRequest ConstructRequest(string basePath, string filepath, string prefix) { string key = filepath.Substring(basePath.Length); key = key.Replace(@"\", "/"); @@ -79,6 +79,12 @@ private TransferUtilityUploadRequest ConstructRequest(string basePath, string fi RequestPayer = this._request.RequestPayer, DisableDefaultChecksumValidation = this._request.DisableDefaultChecksumValidation, ChecksumAlgorithm = this._request.ChecksumAlgorithm, + BucketKeyEnabled = this._request.BucketKeyEnabled, + ExpectedBucketOwner = this._request.ExpectedBucketOwner, + SSEKMSEncryptionContext = this._request.SSEKMSEncryptionContext, + WebsiteRedirectLocation = this._request.WebsiteRedirectLocation, + Headers = this._request.Headers, + Grants = this._request.Grants }; if (this._request.IsSetObjectLockRetainUntilDate()) diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/AbortMultipartUploadsCommand.async.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/AbortMultipartUploadsCommand.async.cs index ed3dd81903ea..4e55afcd34e8 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/AbortMultipartUploadsCommand.async.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/AbortMultipartUploadsCommand.async.cs @@ -26,23 +26,19 @@ namespace Amazon.S3.Transfer.Internal { internal partial class AbortMultipartUploadsCommand : BaseCommand { - TransferUtilityConfig _config; - - internal AbortMultipartUploadsCommand(IAmazonS3 s3Client, string bucketName, DateTime initiateDate, TransferUtilityConfig config) - { - this._s3Client = s3Client; - this._bucketName = bucketName; - this._initiatedDate = initiateDate; - this._config = config; - } public override async Task ExecuteAsync(CancellationToken cancellationToken) { - if (string.IsNullOrEmpty(this._bucketName)) + if (string.IsNullOrEmpty(this._request.BucketName)) { throw new InvalidOperationException("The bucketName specified is null or empty!"); } + if (!this._request.IsSetInitiatedDate()) + { + throw new InvalidOperationException("InitiatedDate must be specified!"); + } + SemaphoreSlim asyncThrottler = null; CancellationTokenSource internalCts = null; try @@ -72,7 +68,7 @@ public override async Task ExecuteAsync(CancellationToken cancellationToken) // responses and throw the original exception. break; } - if (upload.Initiated < this._initiatedDate) + if (upload.Initiated < this._request.InitiatedDate.Value) { await asyncThrottler.WaitAsync(cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/MultipartUploadCommand.async.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/MultipartUploadCommand.async.cs index 34bd339dc9a3..8dbb8ba561e7 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/MultipartUploadCommand.async.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/MultipartUploadCommand.async.cs @@ -31,6 +31,8 @@ internal partial class MultipartUploadCommand : BaseCommand { public SemaphoreSlim AsyncThrottler { get; set; } + Dictionary _expectedUploadParts = new Dictionary(); + public override async Task ExecuteAsync(CancellationToken cancellationToken) { if ( (this._fileTransporterRequest.InputStream != null && !this._fileTransporterRequest.InputStream.CanSeek) || this._fileTransporterRequest.ContentLength == -1) @@ -57,6 +59,29 @@ public override async Task ExecuteAsync(CancellationToken cancellationToken) cancellationToken.ThrowIfCancellationRequested(); var uploadRequest = ConstructUploadPartRequest(i, filePosition, initResponse); + + var expectedFileOffset = (i - 1) * this._partSize; + // Calculating how many bytes are remaining to be uploaded from the current part. + // This is mainly used for the last part scenario. + var remainingBytes = this._contentLength - expectedFileOffset; + // We then check based on the remaining bytes and the content length if this is the last part. + var isLastPart = calculateIsLastPart(remainingBytes); + // To maintain the same behavior as the ConstructUploadPartRequest. + // We are setting the remainingBytes/partSize when using the IAmazonS3Encryption client to 0. + if (isLastPart + && _s3Client is Amazon.S3.Internal.IAmazonS3Encryption) + { + remainingBytes = 0; + } + this._expectedUploadParts.Add(i, new ExpectedUploadPart { + PartNumber = i, + ExpectedContentLength = + isLastPart ? + remainingBytes : + this._partSize, + ExpectedFileOffset = expectedFileOffset, + IsLastPart = isLastPart + }); this._partsToUpload.Enqueue(uploadRequest); filePosition += this._partSize; } @@ -133,8 +158,50 @@ private async Task UploadPartAsync(UploadPartRequest uploadR { try { - return await _s3Client.UploadPartAsync(uploadRequest, internalCts.Token) + var response = await _s3Client.UploadPartAsync(uploadRequest, internalCts.Token) .ConfigureAwait(continueOnCapturedContext: false); + + if (response.PartNumber is null) + { + throw new ArgumentNullException(nameof(response.PartNumber)); + } + else + { + if (this._expectedUploadParts.TryGetValue((int) response.PartNumber, out var expectedUploadPart)) + { + var actualContentLength = uploadRequest.PartSize; + if (actualContentLength != expectedUploadPart.ExpectedContentLength) + { + throw new InvalidOperationException($"Cannot complete multipart upload request. The expected content length of part {expectedUploadPart.PartNumber} " + + $"does not equal the actual content length."); + } + + if (expectedUploadPart.IsLastPart) + { + if (actualContentLength < 0 || + actualContentLength > expectedUploadPart.ExpectedContentLength) + { + throw new InvalidOperationException($"Cannot complete multipart upload request. The last part " + + $"has an invalid content length."); + } + } + + var actualFileOsset = uploadRequest.FilePosition; + if (uploadRequest.IsSetFilePath() && + actualFileOsset != expectedUploadPart.ExpectedFileOffset) + { + throw new InvalidOperationException($"Cannot complete multipart upload request. The expected file offset of part {expectedUploadPart.PartNumber} " + + $"does not equal the actual file offset."); + } + } + else + { + throw new InvalidOperationException("Multipart upload request part was unexpected."); + } + } + + + return response; } catch (Exception exception) { @@ -168,17 +235,23 @@ private void Cleanup(string uploadId, List> tasks) AbortMultipartUpload(uploadId); } + internal AbortMultipartUploadRequest ConstructAbortMultipartUploadRequest(string uploadId) + { + return new AbortMultipartUploadRequest() + { + BucketName = this._fileTransporterRequest.BucketName, + ExpectedBucketOwner = this._fileTransporterRequest.ExpectedBucketOwner, + Key = this._fileTransporterRequest.Key, + RequestPayer = this._fileTransporterRequest.RequestPayer, + UploadId = uploadId + }; + } + private void AbortMultipartUpload(string uploadId) { try { - this._s3Client.AbortMultipartUploadAsync(new AbortMultipartUploadRequest() - { - BucketName = this._fileTransporterRequest.BucketName, - Key = this._fileTransporterRequest.Key, - RequestPayer = this._fileTransporterRequest.RequestPayer, - UploadId = uploadId - }).Wait(); + this._s3Client.AbortMultipartUploadAsync(ConstructAbortMultipartUploadRequest(uploadId)).Wait(); } catch (Exception e) { @@ -287,5 +360,13 @@ await _s3Client.AbortMultipartUploadAsync(new AbortMultipartUploadRequest() throw; } } + + private class ExpectedUploadPart + { + public int PartNumber { get; set; } + public long? ExpectedContentLength { get; set; } + public long? ExpectedFileOffset { get; set; } + public bool IsLastPart { get; set; } + } } } diff --git a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityAbortMultipartUploadRequest.cs b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityAbortMultipartUploadRequest.cs new file mode 100644 index 000000000000..d048d128f15f --- /dev/null +++ b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityAbortMultipartUploadRequest.cs @@ -0,0 +1,126 @@ +/******************************************************************************* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * ***************************************************************************** + * __ _ _ ___ + * ( )( \/\/ )/ __) + * /__\ \ / \__ \ + * (_)(_) \/\/ (___/ + * + * AWS SDK for .NET + * API Version: 2006-03-01 + * + */ +using System; +using Amazon.S3.Model; + +namespace Amazon.S3.Transfer +{ + /// + /// Contains all the parameters that can be set when making a request to abort multipart uploads + /// with the TransferUtility method. + /// + public class TransferUtilityAbortMultipartUploadRequest + { + private string _bucketName; + private DateTime? _initiatedDate; + private string _expectedBucketOwner; + private RequestPayer _requestPayer; + + /// + /// Gets or sets the name of the bucket containing multipart uploads. + /// + /// + /// The name of the bucket containing multipart uploads. + /// + public string BucketName + { + get { return this._bucketName; } + set { this._bucketName = value; } + } + + /// + /// Checks if BucketName property is set. + /// + /// true if BucketName property is set. + internal bool IsSetBucketName() + { + return !string.IsNullOrEmpty(this._bucketName); + } + + /// + /// Gets or sets the date before which the multipart uploads were initiated. + /// + /// + /// The date before which the multipart uploads were initiated. + /// + public DateTime? InitiatedDate + { + get { return this._initiatedDate; } + set { this._initiatedDate = value; } + } + + /// + /// Checks if InitiatedDate property is set. + /// + /// true if InitiatedDate property is set. + internal bool IsSetInitiatedDate() + { + return this._initiatedDate.HasValue; + } + + /// + /// Gets or sets the account ID of the expected bucket owner. + /// If the account ID that you provide does not match the actual owner of the bucket, + /// the request fails with the HTTP status code 403 Forbidden (access denied). + /// + /// + /// The account ID of the expected bucket owner. + /// + public string ExpectedBucketOwner + { + get { return this._expectedBucketOwner; } + set { this._expectedBucketOwner = value; } + } + + /// + /// Checks if ExpectedBucketOwner property is set. + /// + /// true if ExpectedBucketOwner property is set. + internal bool IsSetExpectedBucketOwner() + { + return !string.IsNullOrEmpty(this._expectedBucketOwner); + } + + /// + /// Gets or sets the request payer setting for the abort multipart upload operations. + /// Confirms that the requester knows that they will be charged for the request. + /// Bucket owners need not specify this parameter in their requests. + /// + /// + /// The request payer setting for the abort multipart upload operations. + /// + public RequestPayer RequestPayer + { + get { return this._requestPayer; } + set { this._requestPayer = value; } + } + + /// + /// Checks if RequestPayer property is set. + /// + /// true if RequestPayer property is set. + internal bool IsSetRequestPayer() + { + return this._requestPayer != null; + } + } +} diff --git a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadDirectoryRequest.cs b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadDirectoryRequest.cs index 2bffe3c249bb..b0556e92487a 100644 --- a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadDirectoryRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityDownloadDirectoryRequest.cs @@ -52,6 +52,11 @@ public class TransferUtilityDownloadDirectoryRequest private RequestPayer requestPayer; + private string expectedBucketOwner; + private string ifMatch; + private string ifNoneMatch; + private ResponseHeaderOverrides responseHeaders; + /// /// Gets or sets the name of the bucket. /// @@ -255,6 +260,108 @@ public RequestPayer RequestPayer get { return this.requestPayer; } set { this.requestPayer = value; } } + + /// + /// Gets and sets the property ExpectedBucketOwner. + /// + /// The account ID of the expected bucket owner. If the account ID that you provide does + /// not match the actual owner of the bucket, the request fails with the HTTP status code + /// 403 Forbidden (access denied). + /// + /// + public string ExpectedBucketOwner + { + get { return this.expectedBucketOwner; } + set { this.expectedBucketOwner = value; } + } + + /// + /// Checks to see if ExpectedBucketOwner is set. + /// + /// true, if ExpectedBucketOwner property is set. + internal bool IsSetExpectedBucketOwner() + { + return !String.IsNullOrEmpty(this.expectedBucketOwner); + } + + /// + /// Gets and sets the property IfMatch. + /// + /// Return the object only if its entity tag (ETag) is the same as the one specified in this header; + /// otherwise, return a 412 Precondition Failed error. + /// + /// + /// If both of the If-Match and If-Unmodified-Since headers are present in the request as follows: + /// If-Match condition evaluates to true, and; If-Unmodified-Since condition evaluates to false; + /// then, S3 returns 200 OK and the data requested. + /// + /// + /// For more information about conditional requests, see RFC 7232. + /// + /// + public string IfMatch + { + get { return this.ifMatch; } + set { this.ifMatch = value; } + } + + /// + /// Checks to see if IfMatch is set. + /// + /// true, if IfMatch property is set. + internal bool IsSetIfMatch() + { + return !String.IsNullOrEmpty(this.ifMatch); + } + + /// + /// Gets and sets the property IfNoneMatch. + /// + /// Return the object only if its entity tag (ETag) is different from the one specified in this header; + /// otherwise, return a 304 Not Modified error. + /// + /// + /// If both of the If-None-Match and If-Modified-Since headers are present in the request as follows: + /// If-None-Match condition evaluates to false, and; If-Modified-Since condition evaluates to true; + /// then, S3 returns 304 Not Modified HTTP status code. + /// + /// + /// For more information about conditional requests, see RFC 7232. + /// + /// + public string IfNoneMatch + { + get { return this.ifNoneMatch; } + set { this.ifNoneMatch = value; } + } + + /// + /// Checks to see if IfNoneMatch is set. + /// + /// true, if IfNoneMatch property is set. + internal bool IsSetIfNoneMatch() + { + return !String.IsNullOrEmpty(this.ifNoneMatch); + } + + /// + /// A set of response headers that should be returned with the object. + /// + public ResponseHeaderOverrides ResponseHeaderOverrides + { + get + { + if (this.responseHeaders == null) + { + this.responseHeaders = new ResponseHeaderOverrides(); + } + return this.responseHeaders; + } + set + { + this.responseHeaders = value; + } + } /// /// The event for DownloadedDirectoryProgressEvent notifications. All diff --git a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadDirectoryRequest.cs b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadDirectoryRequest.cs index 5588d9ad9327..cf7be9f65437 100644 --- a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadDirectoryRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadDirectoryRequest.cs @@ -38,35 +38,10 @@ namespace Amazon.S3.Transfer public class TransferUtilityUploadDirectoryRequest : BaseUploadRequest { string _directory; - string _bucketname; string _searchPattern = "*"; string _keyPrefix; - private string contentType; private bool _uploadFilesConcurrently = false; SearchOption _searchOption = SearchOption.TopDirectoryOnly; - S3CannedACL _cannedACL; - S3StorageClass _storageClass; - MetadataCollection metadataCollection; - ServerSideEncryptionMethod encryption; - string serverSideEncryptionKeyManagementServiceKeyId; - private ServerSideEncryptionCustomerMethod serverSideCustomerEncryption; - private string serverSideEncryptionCustomerProvidedKey; - private string serverSideEncryptionCustomerProvidedKeyMD5; - private List tagset; - private ObjectLockLegalHoldStatus objectLockLegalHoldStatus; - private ObjectLockMode objectLockMode; - private bool disablePayloadSigning; - private DateTime? objectLockRetainUntilDate; - private ChecksumAlgorithm checksumAlgorithm; - - /// - /// Gets or sets whether the payload should be signed or not - /// - public bool DisablePayloadSigning - { - get { return this.disablePayloadSigning; } - set { this.disablePayloadSigning = value; } - } /// /// Gets or sets the directory where files are uploaded from. @@ -152,226 +127,6 @@ public SearchOption SearchOption set { this._searchOption = value; } } - - /// - /// Gets or sets the name of the bucket. - /// - /// - /// The name of the bucket. - /// - public string BucketName - { - get { return this._bucketname; } - set { this._bucketname = value; } - } - - /// - /// Checks if BucketName property is set. - /// - /// true if BucketName property is set. - internal bool IsSetBucketName() - { - return !System.String.IsNullOrEmpty(this._bucketname); - } - - - /// - /// Gets or sets the canned access control list (ACL) - /// for the uploaded objects. - /// Please refer to - /// for - /// information on Amazon S3 canned ACLs. - /// - /// - /// The canned access control list (ACL) - /// for the uploaded objects. - /// - public S3CannedACL CannedACL - { - get { return this._cannedACL; } - set { this._cannedACL = value; } - } - - - /// - /// Checks if the CannedACL property is set. - /// - /// true if there is the CannedACL property is set. - internal bool IsSetCannedACL() - { - return (_cannedACL != null &&_cannedACL != S3CannedACL.NoACL); - } - - /// - /// Gets or sets the content type for the uploaded Amazon S3 objects. - /// The default behavior when this field is not set is to use the file - /// extension to set the content type. If this field is set to a value it - /// will be applied to all uploaded files in the directory, overriding - /// file extension inspection. - /// - /// - /// The content type for all the uploaded Amazon S3 objects. - /// - public string ContentType - { - get { return this.contentType; } - set { this.contentType = value; } - } - - - /// - /// Gets or sets the storage class for the uploaded Amazon S3 objects. - /// Please refer to - /// for - /// information on S3 Storage Classes. - /// - /// - /// The storage class for the uploaded Amazon S3 objects. - /// - public S3StorageClass StorageClass - { - get { return this._storageClass; } - set - { - this._storageClass = value; - } - } - - - /// - /// The collection of meta data for the request. - /// - public MetadataCollection Metadata - { - get - { - if (this.metadataCollection == null) - this.metadataCollection = new MetadataCollection(); - return this.metadataCollection; - } - internal set { this.metadataCollection = value; } - } - - #region ServerSideEncryption - - /// - /// Gets or sets the ServerSideEncryptionMethod property. - /// Specifies the encryption used on the server to - /// store the content. - /// - public ServerSideEncryptionMethod ServerSideEncryptionMethod - { - get { return this.encryption; } - set { this.encryption = value; } - } - - /// - /// The id of the AWS Key Management Service key that Amazon S3 should use to encrypt and decrypt the object. - /// If a key id is not specified, the default key will be used for encryption and decryption. - /// - [AWSProperty(Sensitive=true)] - public string ServerSideEncryptionKeyManagementServiceKeyId - { - get { return this.serverSideEncryptionKeyManagementServiceKeyId; } - set { this.serverSideEncryptionKeyManagementServiceKeyId = value; } - } - - /// - /// The Server-side encryption algorithm to be used with the customer provided key. - /// - public ServerSideEncryptionCustomerMethod ServerSideEncryptionCustomerMethod - { - get { return this.serverSideCustomerEncryption; } - set { this.serverSideCustomerEncryption = value; } - } - - /// - /// Checks if ServerSideEncryptionKeyManagementServiceKeyId property is set. - /// - /// true if ServerSideEncryptionKeyManagementServiceKeyId property is set. - internal bool IsSetServerSideEncryptionKeyManagementServiceKeyId() - { - return !System.String.IsNullOrEmpty(this.serverSideEncryptionKeyManagementServiceKeyId); - } - - /// - /// The base64-encoded encryption key for Amazon S3 to use to encrypt the object - /// - /// Using the encryption key you provide as part of your request Amazon S3 manages both the encryption, as it writes - /// to disks, and decryption, when you access your objects. Therefore, you don't need to maintain any data encryption code. The only - /// thing you do is manage the encryption keys you provide. - /// - /// - /// When you retrieve an object, you must provide the same encryption key as part of your request. Amazon S3 first verifies - /// the encryption key you provided matches, and then decrypts the object before returning the object data to you. - /// - /// - /// Important: Amazon S3 does not store the encryption key you provide. - /// - /// - [AWSProperty(Sensitive=true)] - public string ServerSideEncryptionCustomerProvidedKey - { - get { return this.serverSideEncryptionCustomerProvidedKey; } - set { this.serverSideEncryptionCustomerProvidedKey = value; } - } - - /// - /// The MD5 of the customer encryption key specified in the ServerSideEncryptionCustomerProvidedKey property. The MD5 is - /// base 64 encoded. This field is optional, the SDK will calculate the MD5 if this is not set. - /// - public string ServerSideEncryptionCustomerProvidedKeyMD5 - { - get { return this.serverSideEncryptionCustomerProvidedKeyMD5; } - set { this.serverSideEncryptionCustomerProvidedKeyMD5 = value; } - } - - #endregion - - /// - /// Gets and sets the property ObjectLockLegalHoldStatus. - /// - /// Specifies whether a legal hold will be applied to this object. For more information - /// about S3 Object Lock, see Object - /// Lock. - /// - /// - public ObjectLockLegalHoldStatus ObjectLockLegalHoldStatus - { - get { return this.objectLockLegalHoldStatus; } - set { this.objectLockLegalHoldStatus = value; } - } - - /// - /// Gets and sets the property ObjectLockMode. - /// - /// The Object Lock mode that you want to apply to this object. - /// - /// - public ObjectLockMode ObjectLockMode - { - get { return this.objectLockMode; } - set { this.objectLockMode = value; } - } - - /// - /// Gets and sets the property ObjectLockRetainUntilDate. - /// - /// The date and time when you want this object's Object Lock to expire. - /// - /// - public DateTime ObjectLockRetainUntilDate - { - get { return this.objectLockRetainUntilDate.GetValueOrDefault(); } - set { this.objectLockRetainUntilDate = value; } - } - - // Check to see if ObjectLockRetainUntilDate property is set - internal bool IsSetObjectLockRetainUntilDate() - { - return this.objectLockRetainUntilDate.HasValue; - } - /// /// Gets or sets the UploadFilesConcurrently property. /// Specifies if multiple files will be uploaded concurrently. @@ -444,41 +199,6 @@ internal void RaiseUploadDirectoryFileRequestEvent(TransferUtilityUploadRequest targetEvent(this, args); } } - - /// - /// Tags that will be applied to all objects in the diretory. - /// - public List TagSet - { - get { return this.tagset; } - set { this.tagset = value; } - } - - /// - /// WARNING: Setting DisableDefaultChecksumValidation to true disables the default data - /// integrity check on upload requests. - /// When true, checksum verification will not be used in upload requests. This may increase upload - /// performance under high CPU loads. The default value is false. - /// Checksums, SigV4 payload signing, and HTTPS each provide some data integrity - /// verification. If DisableDefaultChecksumValidation is true and DisablePayloadSigning is true, then the - /// possibility of data corruption is completely dependent on HTTPS being the only remaining - /// source of data integrity verification. - /// - public bool? DisableDefaultChecksumValidation { get; set; } - - /// - /// Gets and sets the property ChecksumAlgorithm. - /// - /// Indicates the algorithm used to create the checksum for each object in the provided directory. - /// For more information, see - /// Checking object integrity in the Amazon S3 User Guide. - /// - /// - public ChecksumAlgorithm ChecksumAlgorithm - { - get { return this.checksumAlgorithm; } - set { this.checksumAlgorithm = value; } - } } /// diff --git a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs index 868fcf697dd8..b21ab2ae7602 100644 --- a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs @@ -38,20 +38,10 @@ namespace Amazon.S3.Transfer /// public partial class TransferUtilityUploadRequest : BaseUploadRequest { - private string bucketName; private string key; - private S3CannedACL cannedACL; - private string contentType; - private S3StorageClass storageClass; private long? partSize; private bool autoCloseStream = true; private bool autoResetStreamPosition = true; - private ServerSideEncryptionMethod encryption; - private ServerSideEncryptionCustomerMethod serverSideCustomerEncryption; - private string serverSideEncryptionCustomerProvidedKey; - private string serverSideEncryptionCustomerProvidedKeyMD5; - private string serverSideEncryptionKeyManagementServiceKeyId; - private ChecksumAlgorithm checksumAlgorithm; private string _checksumCRC32; private string _checksumCRC32C; private string _checksumCRC64NVME; @@ -61,41 +51,7 @@ public partial class TransferUtilityUploadRequest : BaseUploadRequest private string _ifMatch; private long? _mpuObjectSize; - private HeadersCollection headersCollection = new HeadersCollection(); - private MetadataCollection metadataCollection = new MetadataCollection(); - - private List tagset; - private Stream inputStream; - private ObjectLockLegalHoldStatus objectLockLegalHoldStatus; - private ObjectLockMode objectLockMode; - private DateTime? objectLockRetainUntilDate; - - #region BucketName - - /// - /// Gets or sets the name of the bucket. - /// - /// - /// The name of the bucket. - /// - public string BucketName - { - get { return this.bucketName; } - set { this.bucketName = value; } - } - - - /// - /// Checks if BucketName property is set. - /// - /// true if BucketName property is set. - internal bool IsSetBucketName() - { - return !System.String.IsNullOrEmpty(this.bucketName); - } - - #endregion #region Key /// @@ -122,166 +78,6 @@ internal bool IsSetKey() #endregion - #region CannedACL - - /// - /// Gets or sets the canned access control list (ACL) - /// for the uploaded object. - /// Please refer to - /// for - /// information on Amazon S3 canned ACLs. - /// - /// - /// The canned access control list (ACL) - /// for the uploaded object. - /// - public S3CannedACL CannedACL - { - get { return this.cannedACL; } - set { this.cannedACL = value; } - } - - /// - /// Checks if the CannedACL property is set. - /// - /// true if there is the CannedACL property is set. - internal bool IsSetCannedACL() - { - return (cannedACL != null); - } - - /// - /// Removes the cannned access control list (ACL) - /// for the uploaded object. - /// - public void RemoveCannedACL() - { - this.cannedACL = null; - } - - #endregion - - #region ContentType - /// - /// Gets or sets the content type of the uploaded Amazon S3 object. - /// - /// - /// The content type of the uploaded Amazon S3 object. - /// - public string ContentType - { - get { return this.contentType; } - set { this.contentType = value; } - } - - - /// - /// Checks if ContentType property is set. - /// - /// true if ContentType property is set. - internal bool IsSetContentType() - { - return !System.String.IsNullOrEmpty(this.contentType); - } - - #endregion - - #region StorageClass - - /// - /// Gets or sets the storage class for the uploaded Amazon S3 object. - /// Please refer to - /// for - /// information on S3 Storage Classes. - /// - /// - /// The storage class for the uploaded Amazon S3 object. - /// - public S3StorageClass StorageClass - { - get { return this.storageClass; } - set { this.storageClass = value; } - } - - #endregion - - #region ServerSideEncryption - - /// - /// Gets and sets the ServerSideEncryptionMethod property. - /// Specifies the encryption used on the server to - /// store the content. - /// - public ServerSideEncryptionMethod ServerSideEncryptionMethod - { - get { return this.encryption; } - set { this.encryption = value; } - } - - /// - /// The Server-side encryption algorithm to be used with the customer provided key. - /// - /// - public ServerSideEncryptionCustomerMethod ServerSideEncryptionCustomerMethod - { - get { return this.serverSideCustomerEncryption; } - set { this.serverSideCustomerEncryption = value; } - } - - /// - /// The id of the AWS Key Management Service key that Amazon S3 should use to encrypt and decrypt the object. - /// If a key id is not specified, the default key will be used for encryption and decryption. - /// - [AWSProperty(Sensitive=true)] - public string ServerSideEncryptionKeyManagementServiceKeyId - { - get { return this.serverSideEncryptionKeyManagementServiceKeyId; } - set { this.serverSideEncryptionKeyManagementServiceKeyId = value; } - } - - /// - /// Checks if ServerSideEncryptionKeyManagementServiceKeyId property is set. - /// - /// true if ServerSideEncryptionKeyManagementServiceKeyId property is set. - internal bool IsSetServerSideEncryptionKeyManagementServiceKeyId() - { - return !System.String.IsNullOrEmpty(this.serverSideEncryptionKeyManagementServiceKeyId); - } - - /// - /// The Base64 encoded encryption key for Amazon S3 to use to encrypt the object - /// - /// Using the encryption key you provide as part of your request Amazon S3 manages both the encryption, as it writes - /// to disks, and decryption, when you access your objects. Therefore, you don't need to maintain any data encryption code. The only - /// thing you do is manage the encryption keys you provide. - /// - /// - /// When you retrieve an object, you must provide the same encryption key as part of your request. Amazon S3 first verifies - /// the encryption key you provided matches, and then decrypts the object before returning the object data to you. - /// - /// - /// Important: Amazon S3 does not store the encryption key you provide. - /// - /// - [AWSProperty(Sensitive=true)] - public string ServerSideEncryptionCustomerProvidedKey - { - get { return this.serverSideEncryptionCustomerProvidedKey; } - set { this.serverSideEncryptionCustomerProvidedKey = value; } - } - - /// - /// The MD5 of the customer encryption key specified in the ServerSideEncryptionCustomerProvidedKey property. The MD5 is - /// base 64 encoded. This field is optional, the SDK will calculate the MD5 if this is not set. - /// - public string ServerSideEncryptionCustomerProvidedKeyMD5 - { - get { return this.serverSideEncryptionCustomerProvidedKeyMD5; } - set { this.serverSideEncryptionCustomerProvidedKeyMD5 = value; } - } - - #endregion - /// /// Input stream for the request; content for the request will be read from the stream. /// @@ -345,43 +141,6 @@ internal bool IsSetPartSize() return this.partSize.HasValue; } - /// - /// The collection of headers for the request. - /// - public HeadersCollection Headers - { - get - { - if (this.headersCollection == null) - this.headersCollection = new HeadersCollection(); - return this.headersCollection; - } - internal set { this.headersCollection = value; } - } - - /// - /// The collection of meta data for the request. - /// - public MetadataCollection Metadata - { - get - { - if (this.metadataCollection == null) - this.metadataCollection = new MetadataCollection(); - return this.metadataCollection; - } - internal set { this.metadataCollection = value; } - } - - /// - /// The tag-set for the object. - /// - public List TagSet - { - get { return this.tagset; } - set { this.tagset = value; } - } - /// /// The event for UploadProgressEvent notifications. All /// subscribers will be notified when a new progress @@ -502,100 +261,6 @@ public TransferUtilityUploadRequest WithAutoCloseStream(bool autoCloseStream) } #endregion - /// - /// WARNING: Setting DisableDefaultChecksumValidation to true disables the default data - /// integrity check on upload requests. - /// When true, checksum verification will not be used in upload requests. This may increase upload - /// performance under high CPU loads. Setting DisableDefaultChecksumValidation sets the deprecated property - /// DisableMD5Stream to the same value. The default value is false. - /// Checksums, SigV4 payload signing, and HTTPS each provide some data integrity - /// verification. If DisableDefaultChecksumValidation is true and DisablePayloadSigning is true, then the - /// possibility of data corruption is completely dependent on HTTPS being the only remaining - /// source of data integrity verification. - /// - public bool? DisableDefaultChecksumValidation { get; set; } - - /// - /// WARNING: Setting DisablePayloadSigning to true disables the SigV4 payload signing - /// data integrity check on this request. - /// If using SigV4, the DisablePayloadSigning flag controls if the payload should be - /// signed on a request by request basis. By default this flag is null which will use the - /// default client behavior. The default client behavior is to sign the payload. When - /// DisablePayloadSigning is true, the request will be signed with an UNSIGNED-PAYLOAD value. - /// Setting DisablePayloadSigning to true requires that the request is sent over a HTTPS - /// connection. - /// Under certain circumstances, such as uploading to S3 while using MD5 hashing, it may - /// be desireable to use UNSIGNED-PAYLOAD to decrease signing CPU usage. This flag only applies - /// to Amazon S3 PutObject and UploadPart requests. - /// MD5Stream, SigV4 payload signing, and HTTPS each provide some data integrity - /// verification. If DisableMD5Stream is true and DisablePayloadSigning is true, then the - /// possibility of data corruption is completely dependant on HTTPS being the only remaining - /// source of data integrity verification. - /// - public bool? DisablePayloadSigning { get; set; } - - /// - /// Gets and sets the property ObjectLockLegalHoldStatus. - /// - /// Specifies whether a legal hold will be applied to this object. For more information - /// about S3 Object Lock, see Object - /// Lock. - /// - /// - public ObjectLockLegalHoldStatus ObjectLockLegalHoldStatus - { - get { return this.objectLockLegalHoldStatus; } - set { this.objectLockLegalHoldStatus = value; } - } - - /// - /// Gets and sets the property ObjectLockMode. - /// - /// The Object Lock mode that you want to apply to this object. - /// - /// - public ObjectLockMode ObjectLockMode - { - get { return this.objectLockMode; } - set { this.objectLockMode = value; } - } - - /// - /// Gets and sets the property ObjectLockRetainUntilDate. - /// - /// The date and time when you want this object's Object Lock to expire. - /// - /// - public DateTime ObjectLockRetainUntilDate - { - get { return this.objectLockRetainUntilDate.GetValueOrDefault(); } - set { this.objectLockRetainUntilDate = value; } - } - - // Check to see if ObjectLockRetainUntilDate property is set - internal bool IsSetObjectLockRetainUntilDate() - { - return this.objectLockRetainUntilDate.HasValue; - } - - /// - /// Gets and sets the property ChecksumAlgorithm. - /// - /// Indicates the algorithm used to create the checksum for the object. - /// For more information, see - /// Checking object integrity in the Amazon S3 User Guide. - /// - /// - /// - /// If you provide an individual checksum, Amazon S3 will ignore any provided ChecksumAlgorithm. - /// - /// - public ChecksumAlgorithm ChecksumAlgorithm - { - get { return this.checksumAlgorithm; } - set { this.checksumAlgorithm = value; } - } - /// /// Gets and sets the property ChecksumCRC32. /// diff --git a/sdk/src/Services/S3/Custom/Transfer/_async/TransferUtility.async.cs b/sdk/src/Services/S3/Custom/Transfer/_async/TransferUtility.async.cs index 6622b2622e38..35205ad93f3a 100644 --- a/sdk/src/Services/S3/Custom/Transfer/_async/TransferUtility.async.cs +++ b/sdk/src/Services/S3/Custom/Transfer/_async/TransferUtility.async.cs @@ -238,7 +238,32 @@ public partial class TransferUtility : ITransferUtility using(CreateSpan(nameof(AbortMultipartUploadsAsync), null, Amazon.Runtime.Telemetry.Tracing.SpanKind.CLIENT)) { CheckForBlockedArn(bucketName, "AbortMultipartUploads"); - var command = new AbortMultipartUploadsCommand(this._s3Client, bucketName, initiatedDate, this._config); + var request = new TransferUtilityAbortMultipartUploadRequest + { + BucketName = bucketName, + InitiatedDate = initiatedDate + }; + var command = new AbortMultipartUploadsCommand(this._s3Client, request, this._config); + await command.ExecuteAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Aborts the multipart uploads based on the specified request parameters. + /// + /// + /// Contains all the parameters required to abort multipart uploads. + /// + /// + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// The task object representing the asynchronous operation. + public async Task AbortMultipartUploadsAsync(TransferUtilityAbortMultipartUploadRequest request, CancellationToken cancellationToken = default(CancellationToken)) + { + using(CreateSpan(nameof(AbortMultipartUploadsAsync), null, Amazon.Runtime.Telemetry.Tracing.SpanKind.CLIENT)) + { + CheckForBlockedArn(request.BucketName, "AbortMultipartUploads"); + var command = new AbortMultipartUploadsCommand(this._s3Client, request, this._config); await command.ExecuteAsync(cancellationToken).ConfigureAwait(false); } } diff --git a/sdk/src/Services/S3/Custom/Transfer/_bcl+netstandard/TransferUtility.sync.cs b/sdk/src/Services/S3/Custom/Transfer/_bcl+netstandard/TransferUtility.sync.cs index 659ddc9c7cae..f1ff62ce820d 100644 --- a/sdk/src/Services/S3/Custom/Transfer/_bcl+netstandard/TransferUtility.sync.cs +++ b/sdk/src/Services/S3/Custom/Transfer/_bcl+netstandard/TransferUtility.sync.cs @@ -459,6 +459,24 @@ public void AbortMultipartUploads(string bucketName, DateTime initiatedDate) } } + /// + /// Aborts the multipart uploads based on the specified request parameters. + /// + /// + /// Contains all the parameters required to abort multipart uploads. + /// + public void AbortMultipartUploads(TransferUtilityAbortMultipartUploadRequest request) + { + try + { + AbortMultipartUploadsAsync(request).Wait(); + } + catch (AggregateException e) + { + ExceptionDispatchInfo.Capture(e.InnerException).Throw(); + } + } + #endregion } } diff --git a/sdk/src/Services/S3/Custom/Util/S3Constants.cs b/sdk/src/Services/S3/Custom/Util/S3Constants.cs index becbf30231b7..f85ad35e00e5 100644 --- a/sdk/src/Services/S3/Custom/Util/S3Constants.cs +++ b/sdk/src/Services/S3/Custom/Util/S3Constants.cs @@ -33,6 +33,7 @@ internal static class S3Constants internal const int PutObjectDefaultTimeout = 20 * 60 * 1000; internal static readonly long MinPartSize = 5 * (long)Math.Pow(2, 20); + internal static readonly long DefaultPartSize = 8 * (long)Math.Pow(2, 20); internal const int MaxNumberOfParts = 10000; internal const int DefaultBufferSize = 8192; diff --git a/sdk/src/Services/S3/Properties/AssemblyInfo.cs b/sdk/src/Services/S3/Properties/AssemblyInfo.cs index 1a3aacf94003..7072e132ae1b 100644 --- a/sdk/src/Services/S3/Properties/AssemblyInfo.cs +++ b/sdk/src/Services/S3/Properties/AssemblyInfo.cs @@ -19,6 +19,8 @@ #error Unknown platform constant - unable to set correct AssemblyDescription #endif +[assembly: InternalsVisibleTo("AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")] +[assembly: InternalsVisibleTo("AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyProduct("Amazon Web Services SDK for .NET")] [assembly: AssemblyCompany("Amazon.com, Inc")] diff --git a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs index abc767a0bdd8..cce278d328ae 100644 --- a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs @@ -663,6 +663,25 @@ public void MultipartGetNumberTest() } } + [TestMethod] + [TestCategory("S3")] + public void MultipartValidatePartSize8MbTest() + { + string key = "MultipartValidatePartSizeTest"; + + Upload(key, 20 * MEG_SIZE, null, Client); + + var objectMetadataResponse = Client.GetObjectMetadata(new GetObjectMetadataRequest + { + BucketName = bucketName, + Key = key, + PartNumber = 1, + }); + + Assert.AreEqual(3, objectMetadataResponse.PartsCount); + Assert.AreEqual(8 * MEG_SIZE, objectMetadataResponse.ContentLength); + } + void Upload(string fileName, long size, TransferProgressValidator progressValidator, AmazonS3Client client = null) { diff --git a/sdk/test/Services/S3/UnitTests/AWSSDK.UnitTests.S3.NetFramework.csproj b/sdk/test/Services/S3/UnitTests/AWSSDK.UnitTests.S3.NetFramework.csproj index 3b58730ba499..9a461a902882 100644 --- a/sdk/test/Services/S3/UnitTests/AWSSDK.UnitTests.S3.NetFramework.csproj +++ b/sdk/test/Services/S3/UnitTests/AWSSDK.UnitTests.S3.NetFramework.csproj @@ -75,5 +75,8 @@ + + + \ No newline at end of file diff --git a/sdk/test/Services/S3/UnitTests/Custom/EmbeddedResource/mapping.json b/sdk/test/Services/S3/UnitTests/Custom/EmbeddedResource/mapping.json new file mode 100644 index 000000000000..224a0a35dfdb --- /dev/null +++ b/sdk/test/Services/S3/UnitTests/Custom/EmbeddedResource/mapping.json @@ -0,0 +1,291 @@ +{ + "Definition": { + "UploadRequest": { + "PutObjectRequest": [ + "ACL", + "Bucket", + "BucketKeyEnabled", + "CacheControl", + "ChecksumAlgorithm", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWriteACP", + "IfMatch", + "IfNoneMatch", + "Key", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation" + ] + }, + "UploadResponse": { + "PutObjectResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "DownloadRequest": { + "GetObjectRequest": [ + "Bucket", + "ChecksumMode", + "ExpectedBucketOwner", + "IfMatch", + "IfModifiedSince", + "IfNoneMatch", + "IfUnmodifiedSince", + "Key", + "RequestPayer", + "ResponseCacheControl", + "ResponseContentDisposition", + "ResponseContentEncoding", + "ResponseContentLanguage", + "ResponseContentType", + "ResponseExpires", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "VersionId" + ] + }, + "DownloadResponse": { + "GetObjectResponse": [ + "AcceptRanges", + "BucketKeyEnabled", + "CacheControl", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentLength", + "ContentRange", + "ContentType", + "DeleteMarker", + "ETag", + "Expiration", + "Expires", + "LastModified", + "Metadata", + "MissingMeta", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "PartsCount", + "ReplicationStatus", + "RequestCharged", + "Restore", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "TagCount", + "VersionId", + "WebsiteRedirectLocation" + ] + } + }, + "Conversion": { + "UploadRequest": { + "PutObjectRequest": [ + "Bucket", + "ChecksumAlgorithm", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ExpectedBucketOwner", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "CreateMultipartRequest": [ + "ACL", + "Bucket", + "BucketKeyEnabled", + "CacheControl", + "ChecksumAlgorithm", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWriteACP", + "Key", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation" + ], + "UploadPartRequest": [ + "Bucket", + "ChecksumAlgorithm", + "ExpectedBucketOwner", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "CompleteMultipartRequest": [ + "Bucket", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ExpectedBucketOwner", + "IfMatch", + "IfNoneMatch", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "AbortMultipartRequest": [ + "Bucket", + "ExpectedBucketOwner", + "Key", + "RequestPayer" + ] + }, + "CompleteMultipartResponse": { + "UploadResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "PutObjectResponse": { + "UploadResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "GetObjectResponse": { + "DownloadResponse": [ + "AcceptRanges", + "BucketKeyEnabled", + "CacheControl", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentLength", + "ContentRange", + "ContentType", + "DeleteMarker", + "ETag", + "Expiration", + "Expires", + "ExpiresString", + "LastModified", + "Metadata", + "MissingMeta", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "PartsCount", + "ReplicationStatus", + "RequestCharged", + "Restore", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "TagCount", + "VersionId", + "WebsiteRedirectLocation" + ] + } + } +} \ No newline at end of file diff --git a/sdk/test/Services/S3/UnitTests/Custom/EmbeddedResource/property-aliases.json b/sdk/test/Services/S3/UnitTests/Custom/EmbeddedResource/property-aliases.json new file mode 100644 index 000000000000..e9de0a44b4fe --- /dev/null +++ b/sdk/test/Services/S3/UnitTests/Custom/EmbeddedResource/property-aliases.json @@ -0,0 +1,117 @@ +{ + "PropertyAliases": { + "PutObjectResponse": { + "ServerSideEncryption": "ServerSideEncryptionMethod", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5", + "SSEKMSEncryptionContext": "ServerSideEncryptionKeyManagementServiceEncryptionContext", + "SSEKMSKeyId": "ServerSideEncryptionKeyManagementServiceKeyId" + }, + "TransferUtilityUploadResponse": { + "ServerSideEncryption": "ServerSideEncryptionMethod", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5", + "SSEKMSEncryptionContext": "ServerSideEncryptionKeyManagementServiceEncryptionContext", + "SSEKMSKeyId": "ServerSideEncryptionKeyManagementServiceKeyId" + }, + "PutObjectRequest": { + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5", + "SSEKMSKeyId": "ServerSideEncryptionKeyManagementServiceKeyId", + "ServerSideEncryption": "ServerSideEncryptionMethod", + "SSEKMSEncryptionContext": "ServerSideEncryptionKeyManagementServiceEncryptionContext", + "ACL": "CannedACL", + "Bucket": "BucketName", + "SSECustomerKey": "ServerSideEncryptionCustomerProvidedKey", + "Tagging": "TagSet", + "GrantFullControl": "Grants", + "GrantRead": "Grants", + "GrantReadACP": "Grants", + "GrantWriteACP": "Grants" + }, + "GetObjectRequest": { + "Bucket": "BucketName", + "SSECustomerKey": "ServerSideEncryptionCustomerProvidedKey", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5", + "ResponseCacheControl": "CacheControl", + "ResponseContentDisposition": "ContentDisposition", + "ResponseContentEncoding": "ContentEncoding", + "ResponseContentLanguage": "ContentLanguage", + "ResponseContentType": "ContentType", + "IfMatch": "EtagToMatch", + "IfNoneMatch": "EtagToNotMatch", + "IfModifiedSince": "ModifiedSinceDate", + "IfUnmodifiedSince": "UnmodifiedSinceDate" + }, + "TransferUtilityDownloadRequest": { + "Bucket": "BucketName", + "IfModifiedSince": "ModifiedSinceDate", + "IfUnmodifiedSince": "UnmodifiedSinceDate", + "ResponseCacheControl": "CacheControl", + "ResponseContentDisposition": "ContentDisposition", + "ResponseContentEncoding": "ContentEncoding", + "ResponseContentLanguage": "ContentLanguage", + "ResponseContentType": "ContentType", + "ResponseExpires": "Expires", + "SSECustomerKey": "ServerSideEncryptionCustomerProvidedKey", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5" + }, + "TransferUtilityUploadRequest": { + "ACL": "CannedACL", + "Bucket": "BucketName", + "GrantFullControl": "Grants", + "GrantRead": "Grants", + "GrantReadACP": "Grants", + "GrantWriteACP": "Grants", + "ServerSideEncryption": "ServerSideEncryptionMethod", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKey": "ServerSideEncryptionCustomerProvidedKey", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5", + "SSEKMSKeyId": "ServerSideEncryptionKeyManagementServiceKeyId", + "Tagging": "TagSet" + }, + "TransferUtilityUploadDirectoryRequest": { + "ACL": "CannedACL", + "Bucket": "BucketName", + "GrantFullControl": "Grants", + "GrantRead": "Grants", + "GrantReadACP": "Grants", + "GrantWriteACP": "Grants", + "ServerSideEncryption": "ServerSideEncryptionMethod", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKey": "ServerSideEncryptionCustomerProvidedKey", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5", + "SSEKMSKeyId": "ServerSideEncryptionKeyManagementServiceKeyId", + "Tagging": "TagSet" + }, + "InitiateMultipartUploadRequest": { + "ACL": "CannedACL", + "Bucket": "BucketName", + "GrantFullControl": "Grants", + "GrantRead": "Grants", + "GrantReadACP": "Grants", + "GrantWriteACP": "Grants", + "ServerSideEncryption": "ServerSideEncryptionMethod", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKey": "ServerSideEncryptionCustomerProvidedKey", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5", + "SSEKMSKeyId": "ServerSideEncryptionKeyManagementServiceKeyId", + "Tagging": "TagSet", + "SSEKMSEncryptionContext": "ServerSideEncryptionKeyManagementServiceEncryptionContext" + }, + "UploadPartRequest": { + "Bucket": "BucketName", + "SSECustomerAlgorithm": "ServerSideEncryptionCustomerMethod", + "SSECustomerKey": "ServerSideEncryptionCustomerProvidedKey", + "SSECustomerKeyMD5": "ServerSideEncryptionCustomerProvidedKeyMD5" + }, + "CompleteMultipartUploadRequest": { + "Bucket": "BucketName" + }, + "AbortMultipartUploadRequest": { + "Bucket": "BucketName" + } + } +} \ No newline at end of file diff --git a/sdk/test/Services/S3/UnitTests/Custom/MultipartUploadValidationTests.cs b/sdk/test/Services/S3/UnitTests/Custom/MultipartUploadValidationTests.cs new file mode 100644 index 000000000000..19ea71304c8f --- /dev/null +++ b/sdk/test/Services/S3/UnitTests/Custom/MultipartUploadValidationTests.cs @@ -0,0 +1,133 @@ +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Amazon.S3.Transfer.Internal; +using Amazon.S3.Util; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace AWSSDK.UnitTests +{ + [TestClass] + public class MultipartUploadValidationTests + { + private static string _tempFilePath; + private const long fileSizeInBytes = 40 * 1024 * 1024; + + [ClassInitialize] + public static void ClassInitialize(TestContext context) + { + _tempFilePath = Path.GetTempFileName(); + + CreateFileWithSpecificSize(_tempFilePath, fileSizeInBytes); + } + + [ClassCleanup] + public static void ClassCleanup() + { + if (File.Exists(_tempFilePath)) + { + File.Delete(_tempFilePath); + } + } + + private static void CreateFileWithSpecificSize(string path, long size) + { + using (var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write)) + { + fileStream.SetLength(size); + } + } + + [TestMethod] + [TestCategory("S3")] + public async Task Validation_HappyPath() + { + var initiateMultipartUploadResponse = new InitiateMultipartUploadResponse + { + UploadId = "test" + }; + + var s3Client = new Mock(); + s3Client + .Setup(x => x.InitiateMultipartUploadAsync( + It.IsAny(), + It.IsAny())) + .ReturnsAsync(initiateMultipartUploadResponse); + + s3Client + .Setup(x => x.UploadPartAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((UploadPartRequest request, CancellationToken cancellationToken) => + { + return new UploadPartResponse { PartNumber = request.PartNumber }; + }); + + var uploadRequest = new TransferUtilityUploadRequest + { + FilePath = _tempFilePath, + BucketName = "test-bucket", + Key = "test" + }; + var multipartUpload = new MultipartUploadCommand(s3Client.Object, new TransferUtilityConfig(), uploadRequest); + await multipartUpload.ExecuteAsync(new CancellationToken()); + } + + [TestMethod] + [TestCategory("S3")] + public void Validation_ConstructUploadPartRequest() + { + var initiateMultipartUploadResponse = new InitiateMultipartUploadResponse + { + UploadId = "test" + }; + + var s3Client = new Mock(); + + s3Client + .Setup(x => x.InitiateMultipartUploadAsync( + It.IsAny(), + It.IsAny())) + .ReturnsAsync(initiateMultipartUploadResponse); + + var uploadRequest = new TransferUtilityUploadRequest + { + FilePath = _tempFilePath, + BucketName = "test-bucket", + Key = "test" + }; + + var multipartUpload = new MultipartUploadCommand(s3Client.Object, new TransferUtilityConfig(), uploadRequest); + + var partSize = Math.Max(S3Constants.DefaultPartSize, uploadRequest.ContentLength / S3Constants.MaxNumberOfParts); + + long filePosition = 0; + for (int i = 1; filePosition < uploadRequest.ContentLength; i++) + { + var constructUploadPartRequest = multipartUpload.ConstructUploadPartRequest(i, filePosition, initiateMultipartUploadResponse); + + var expectedFileOffset = (i - 1) * partSize; + var remainingBytes = uploadRequest.ContentLength - expectedFileOffset; + var isLastPart = false; + if (remainingBytes <= partSize) + isLastPart = true; + + Assert.AreEqual(i, constructUploadPartRequest.PartNumber); + Assert.AreEqual(isLastPart, constructUploadPartRequest.IsLastPart); + Assert.AreEqual( + isLastPart ? remainingBytes : partSize, + constructUploadPartRequest.PartSize); + Assert.AreEqual(expectedFileOffset, constructUploadPartRequest.FilePosition); + + filePosition += partSize; + } + + } + } +} diff --git a/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs b/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs new file mode 100644 index 000000000000..80600e9ee078 --- /dev/null +++ b/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs @@ -0,0 +1,820 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Amazon.S3.Transfer.Internal; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Text.Json; + +namespace AWSSDK.UnitTests +{ + [TestClass] + public class ResponseMapperTests + { + private static JsonDocument _mappingJson; + private static JsonDocument _propertyAliasesJson; + private static Dictionary> _propertyAliases; + private static List _s3Grants; + + [ClassInitialize] + public static void ClassInitialize(TestContext context) + { + // Read mapping.json using robust resource loading (same pattern as Utils.cs) + using (var stream = GetResourceStream("mapping.json")) + { + if (stream == null) + { + throw new FileNotFoundException("Could not find embedded resource: mapping.json"); + } + + using (var reader = new StreamReader(stream)) + { + var jsonContent = reader.ReadToEnd(); + _mappingJson = JsonDocument.Parse(jsonContent); + } + } + + // Read property-aliases.json using robust resource loading + using (var stream = GetResourceStream("property-aliases.json")) + { + if (stream != null) + { + using (var reader = new StreamReader(stream)) + { + var aliasContent = reader.ReadToEnd(); + _propertyAliasesJson = JsonDocument.Parse(aliasContent); + + // Convert to dictionary for fast lookup + _propertyAliases = new Dictionary>(); + var objectElement = _propertyAliasesJson.RootElement.GetProperty("PropertyAliases"); + foreach (var objectName in objectElement.EnumerateObject()) + { + var aliases = new Dictionary(); + foreach (var alias in objectName.Value.EnumerateObject()) + { + aliases[alias.Name] = alias.Value.GetString(); + } + _propertyAliases[objectName.Name] = aliases; + } + } + } + else + { + _propertyAliases = new Dictionary>(); + } + } + } + + /// + /// Gets embedded resource stream using partial name matching (same pattern as Utils.cs) + /// + private static Stream GetResourceStream(string resourceName) + { + Assembly assembly = Assembly.GetExecutingAssembly(); + var resource = FindResourceName(assembly, resourceName); + if(resource == null) + { + assembly = Assembly.GetCallingAssembly(); + resource = FindResourceName(assembly, resourceName); + } + + return resource != null ? assembly.GetManifestResourceStream(resource) : null; + } + + /// + /// Finds resource name using case-insensitive partial matching (same pattern as Utils.cs) + /// + private static string FindResourceName(Assembly assembly, string partialName) + { + var resources = FindResourceName(assembly, s => s.IndexOf(partialName, StringComparison.OrdinalIgnoreCase) >= 0); + return resources.FirstOrDefault(); + } + + /// + /// Finds resource names matching predicate (same pattern as Utils.cs) + /// + private static IEnumerable FindResourceName(Assembly assembly, Predicate match) + { + var allResources = assembly.GetManifestResourceNames(); + foreach (var resource in allResources) + { + if (match(resource)) + yield return resource; + } + } + + [ClassCleanup] + public static void ClassCleanup() + { + _mappingJson?.Dispose(); + _propertyAliasesJson?.Dispose(); + } + + [TestMethod] + [TestCategory("S3")] + public void MapUploadRequest_PutObjectRequest_AllMappedProperties_WorkCorrectly() + { + ValidateMappingTransferUtilityAndSdkRequests( + new[] { "Conversion", "UploadRequest", "PutObjectRequest" }, + (sourceRequest) => + { + var simpleUploadCommand = new SimpleUploadCommand(null, null, sourceRequest); + return simpleUploadCommand.ConstructRequest(); + }, + usesHeadersCollection: false); + } + + [TestMethod] + [TestCategory("S3")] + public void MapUploadRequest_CreateMultipartRequest_AllMappedProperties_WorkCorrectly() + { + ValidateMappingTransferUtilityAndSdkRequests( + new[] { "Conversion", "UploadRequest", "CreateMultipartRequest" }, + (sourceRequest) => + { + var multipartUploadCommand = new MultipartUploadCommand(null, null, sourceRequest); + return multipartUploadCommand.ConstructInitiateMultipartUploadRequest(); + }, + usesHeadersCollection: true, + (sourceRequest) => + { + sourceRequest.InputStream = new MemoryStream(1024); + }); + } + + [TestMethod] + [TestCategory("S3")] + public void MapUploadRequest_UploadPartRequest_AllMappedProperties_WorkCorrectly() + { + ValidateMappingTransferUtilityAndSdkRequests( + new[] { "Conversion", "UploadRequest", "UploadPartRequest" }, + (sourceRequest) => + { + var multipartUploadCommand = new MultipartUploadCommand(null, null, sourceRequest); + + var initiateResponse = new InitiateMultipartUploadResponse + { + UploadId = "test-upload-id" + }; + + return multipartUploadCommand.ConstructUploadPartRequest(1, 1024, initiateResponse); + }, + usesHeadersCollection: false, + (sourceRequest) => + { + sourceRequest.InputStream = new MemoryStream(1024); + }); + } + + [TestMethod] + [TestCategory("S3")] + public void MapUploadRequest_CompleteMultipartRequest_AllMappedProperties_WorkCorrectly() + { + ValidateMappingTransferUtilityAndSdkRequests( + new[] { "Conversion", "UploadRequest", "CompleteMultipartRequest" }, + (sourceRequest) => + { + var multipartUploadCommand = new MultipartUploadCommand(null, null, sourceRequest); + + var initiateResponse = new InitiateMultipartUploadResponse + { + UploadId = "test-upload-id", + ChecksumType = ChecksumType.FULL_OBJECT + }; + + return multipartUploadCommand.ConstructCompleteMultipartUploadRequest(initiateResponse); + }, + usesHeadersCollection: false, + (sourceRequest) => + { + sourceRequest.InputStream = new MemoryStream(1024); + sourceRequest.ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256; + }); + } + + [TestMethod] + [TestCategory("S3")] + public void MapUploadRequest_AbortMultipartRequest_AllMappedProperties_WorkCorrectly() + { + ValidateMappingTransferUtilityAndSdkRequests( + new[] { "Conversion", "UploadRequest", "AbortMultipartRequest" }, + (sourceRequest) => + { + var multipartUploadCommand = new MultipartUploadCommand(null, null, sourceRequest); + + return multipartUploadCommand.ConstructAbortMultipartUploadRequest("test-upload-id"); + }, + usesHeadersCollection: false, + (sourceRequest) => + { + sourceRequest.InputStream = new MemoryStream(1024); + sourceRequest.ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256; + }); + } + + [TestMethod] + [TestCategory("S3")] + public void MapAbortMultipartUploadsCommand_ConstructAbortMultipartUploadRequest_AllMappedProperties_WorkCorrectly() + { + // Create a TransferUtilityAbortMultipartUploadRequest with all fields set + var abortRequest = new TransferUtilityAbortMultipartUploadRequest + { + BucketName = "test-bucket", + InitiatedDate = DateTime.UtcNow.AddDays(-1), + ExpectedBucketOwner = "test-bucket-owner", + RequestPayer = RequestPayer.Requester + }; + + // Create the command with the new constructor + var abortCommand = new AbortMultipartUploadsCommand(null, abortRequest, null); + + // Create a test MultipartUpload + var multipartUpload = new MultipartUpload + { + Key = "test-key", + UploadId = "test-upload-id" + }; + + // Call the method we want to test + var result = abortCommand.ConstructAbortMultipartUploadRequest(multipartUpload); + + // Validate all fields are properly mapped + Assert.IsNotNull(result, "Result should not be null"); + Assert.AreEqual("test-bucket", result.BucketName, "BucketName should match"); + Assert.AreEqual("test-key", result.Key, "Key should match from MultipartUpload"); + Assert.AreEqual("test-upload-id", result.UploadId, "UploadId should match from MultipartUpload"); + Assert.AreEqual("test-bucket-owner", result.ExpectedBucketOwner, "ExpectedBucketOwner should be set"); + Assert.AreEqual(RequestPayer.Requester, result.RequestPayer, "RequestPayer should be set"); + } + + [TestMethod] + [TestCategory("S3")] + public void MapAbortMultipartUploadsCommand_ConstructListMultipartUploadsRequest_AllMappedProperties_WorkCorrectly() + { + // Create a TransferUtilityAbortMultipartUploadRequest with all fields set + var abortRequest = new TransferUtilityAbortMultipartUploadRequest + { + BucketName = "test-bucket", + InitiatedDate = DateTime.UtcNow.AddDays(-1), + ExpectedBucketOwner = "test-bucket-owner", + RequestPayer = RequestPayer.Requester + }; + + // Create the command with the new constructor + var abortCommand = new AbortMultipartUploadsCommand(null, abortRequest, null); + + // Create a test ListMultipartUploadsResponse + var listResponse = new ListMultipartUploadsResponse + { + KeyMarker = "test-key-marker", + NextUploadIdMarker = "test-upload-id-marker" + }; + + // Call the method we want to test + var result = abortCommand.ConstructListMultipartUploadsRequest(listResponse); + + // Validate all fields are properly mapped + Assert.IsNotNull(result, "Result should not be null"); + Assert.AreEqual("test-bucket", result.BucketName, "BucketName should match"); + Assert.AreEqual("test-key-marker", result.KeyMarker, "KeyMarker should match from response"); + Assert.AreEqual("test-upload-id-marker", result.UploadIdMarker, "UploadIdMarker should match from response"); + Assert.AreEqual("test-bucket-owner", result.ExpectedBucketOwner, "ExpectedBucketOwner should be set"); + Assert.AreEqual(RequestPayer.Requester, result.RequestPayer, "RequestPayer should be set"); + } + + [TestMethod] + [TestCategory("S3")] + public void MapAbortMultipartUploadsCommand_MinimalRequest_DoesNotSetOptionalFields() + { + // Create a minimal request with only required fields (BucketName and InitiatedDate) + var abortRequest = new TransferUtilityAbortMultipartUploadRequest + { + BucketName = "test-bucket", + InitiatedDate = DateTime.UtcNow.AddDays(-1) + // ExpectedBucketOwner and RequestPayer are not set (null) + }; + + // Create the command with the minimal request + var abortCommand = new AbortMultipartUploadsCommand(null, abortRequest, null); + + // Test ConstructAbortMultipartUploadRequest + var multipartUpload = new MultipartUpload + { + Key = "test-key", + UploadId = "test-upload-id" + }; + + var abortResult = abortCommand.ConstructAbortMultipartUploadRequest(multipartUpload); + + // Validate core fields are set but optional fields are not + Assert.IsNotNull(abortResult, "AbortMultipartUploadRequest should not be null"); + Assert.AreEqual("test-bucket", abortResult.BucketName, "BucketName should match"); + Assert.AreEqual("test-key", abortResult.Key, "Key should match"); + Assert.AreEqual("test-upload-id", abortResult.UploadId, "UploadId should match"); + Assert.IsNull(abortResult.ExpectedBucketOwner, "ExpectedBucketOwner should be null with minimal request"); + Assert.IsNull(abortResult.RequestPayer, "RequestPayer should be null with minimal request"); + + // Test ConstructListMultipartUploadsRequest + var listResponse = new ListMultipartUploadsResponse + { + KeyMarker = "test-key-marker", + NextUploadIdMarker = "test-upload-id-marker" + }; + + var listResult = abortCommand.ConstructListMultipartUploadsRequest(listResponse); + + // Validate core fields are set but optional fields are not + Assert.IsNotNull(listResult, "ListMultipartUploadsRequest should not be null"); + Assert.AreEqual("test-bucket", listResult.BucketName, "BucketName should match"); + Assert.AreEqual("test-key-marker", listResult.KeyMarker, "KeyMarker should match"); + Assert.AreEqual("test-upload-id-marker", listResult.UploadIdMarker, "UploadIdMarker should match"); + Assert.IsNull(listResult.ExpectedBucketOwner, "ExpectedBucketOwner should be null with minimal request"); + Assert.IsNull(listResult.RequestPayer, "RequestPayer should be null with minimal request"); + } + + private void ValidateMappingTransferUtilityAndSdkRequests( + string[] mappingPath, + Func fetchTargetRequest, + bool usesHeadersCollection = false, + Action requestHook = null, + Action additionalValidations = null) + { + // Get the expected mappings from JSON + JsonElement mappingElement = _mappingJson.RootElement; + + foreach (var path in mappingPath) + { + mappingElement = mappingElement.GetProperty(path); + } + + // Get the expected mappings from JSON + var requestMappings = mappingElement + .EnumerateArray() + .Select(prop => prop.GetString()) + .ToList(); + + // Create source object with dynamically generated test data + var sourceRequest = Activator.CreateInstance(); + var sourceType = typeof(TSourceRequest); + var testDataValues = new Dictionary(); + + // Generate test data for each mapped property + foreach (var propertyName in requestMappings) + { + // Resolve alias to actual property name + var resolvedPropertyName = ResolvePropertyName(propertyName, sourceType.Name); + var sourceProperty = sourceType.GetProperty(resolvedPropertyName); + + if (usesHeadersCollection && sourceProperty is null) + { + // Check if source type has a Headers property of type HeadersCollection + var sourceHeadersProperty = sourceType.GetProperty("Headers"); + if (sourceHeadersProperty != null && typeof(HeadersCollection).IsAssignableFrom(sourceHeadersProperty.PropertyType)) + { + var sourceHeadersCollection = sourceHeadersProperty.GetValue(sourceRequest) as HeadersCollection; + var sourceHeadersCollectionProperty = typeof(HeadersCollection).GetProperty(resolvedPropertyName); + + Assert.IsNotNull(sourceHeadersCollectionProperty, $"Source property '{resolvedPropertyName}' in '{nameof(HeadersCollection)}' should not be null"); + + if (sourceHeadersCollectionProperty.CanWrite == true) + { + var testValue = GenerateTestValue(sourceHeadersCollectionProperty.PropertyType, propertyName); + sourceHeadersCollectionProperty.SetValue(sourceHeadersCollection, testValue); + testDataValues[propertyName] = testValue; + } + } + } + else + { + Assert.IsNotNull(sourceProperty, $"Source property '{propertyName}' should not be null"); + + if (sourceProperty.CanWrite == true) + { + var testValue = GenerateTestValue(sourceProperty.PropertyType, propertyName); + sourceProperty.SetValue(sourceRequest, testValue); + testDataValues[propertyName] = testValue; + } + } + } + + requestHook?.Invoke(sourceRequest); + + // Map the response + var mappedRequest = fetchTargetRequest(sourceRequest); + Assert.IsNotNull(mappedRequest, "Mapped request should not be null"); + + // Verify all mapped properties using reflection + var targetType = typeof(TTargetRequest); + var failedAssertions = new List(); + + foreach (var propertyName in requestMappings) + { + // Resolve alias to actual property name for reflection lookups + var resolvedSourcePropertyName = ResolvePropertyName(propertyName, sourceType.Name); + var resolvedTargetPropertyName = ResolvePropertyName(propertyName, targetType.Name); + var sourceProperty = sourceType.GetProperty(resolvedSourcePropertyName); + var targetProperty = targetType.GetProperty(resolvedTargetPropertyName); + + object sourceValue = null; + + if (sourceProperty != null) + { + // Property found directly on source type + sourceValue = sourceProperty.GetValue(sourceRequest); + } + else + { + if (!usesHeadersCollection) + { + failedAssertions.Add($"Source property '{propertyName}' (resolved to: {resolvedSourcePropertyName}) not found in {sourceType.Name}"); + continue; + } + + // Check if source type has a Headers property of type HeadersCollection + var sourceHeadersProperty = sourceType.GetProperty("Headers"); + if (sourceHeadersProperty != null && typeof(HeadersCollection).IsAssignableFrom(sourceHeadersProperty.PropertyType)) + { + var sourceHeadersCollection = sourceHeadersProperty.GetValue(sourceRequest) as HeadersCollection; + if (sourceHeadersCollection != null) + { + var sourceHeadersCollectionProperty = typeof(HeadersCollection).GetProperty(resolvedSourcePropertyName); + if (sourceHeadersCollectionProperty != null) + { + sourceValue = sourceHeadersCollectionProperty.GetValue(sourceHeadersCollection); + } + else + { + failedAssertions.Add($"Source property '{propertyName}' (resolved to: {resolvedSourcePropertyName}) not found in {sourceType.Name} or HeadersCollection"); + continue; + } + } + else + { + failedAssertions.Add($"Source Headers collection is null in {sourceType.Name}"); + continue; + } + } + else + { + failedAssertions.Add($"Source property '{propertyName}' (resolved to: {resolvedSourcePropertyName}) not found in {sourceType.Name}"); + continue; + } + } + + object targetValue = null; + + if (targetProperty != null) + { + // Property found directly on target type + targetValue = targetProperty.GetValue(mappedRequest); + } + else + { + if (!usesHeadersCollection) + { + failedAssertions.Add($"Target property '{propertyName}' (resolved to: {resolvedTargetPropertyName}) not found in {targetType.Name}"); + continue; + } + + // Check if target type has a Headers property of type HeadersCollection + var headersProperty = targetType.GetProperty("Headers"); + if (headersProperty != null && typeof(HeadersCollection).IsAssignableFrom(headersProperty.PropertyType)) + { + var headersCollection = headersProperty.GetValue(mappedRequest) as HeadersCollection; + if (headersCollection != null) + { + var headersCollectionProperty = typeof(HeadersCollection).GetProperty(resolvedTargetPropertyName); + if (headersCollectionProperty != null) + { + targetValue = headersCollectionProperty.GetValue(headersCollection); + } + else + { + failedAssertions.Add($"Target property '{propertyName}' (resolved to: {resolvedTargetPropertyName}) not found in {targetType.Name} or HeadersCollection"); + continue; + } + } + else + { + failedAssertions.Add($"Headers collection is null in {targetType.Name}"); + continue; + } + } + else + { + failedAssertions.Add($"Target property '{propertyName}' (resolved to: {resolvedTargetPropertyName}) not found in {targetType.Name}"); + continue; + } + } + + // Special handling for complex object comparisons + if (!AreValuesEqual(sourceValue, targetValue)) + { + failedAssertions.Add($"{propertyName}: Expected '{sourceValue ?? "null"}', got '{targetValue ?? "null"}'"); + } + } + + additionalValidations?.Invoke(sourceRequest, mappedRequest); + + // Report any failures + if (failedAssertions.Any()) + { + Assert.Fail($"Property mapping failures:\n{string.Join("\n", failedAssertions)}"); + } + } + + [TestMethod] + [TestCategory("S3")] + public void ValidatePutObjectRequestDefinitionCompleteness() + { + ValidateResponseDefinitionCompleteness( + new[] { "Definition", "UploadRequest", "PutObjectRequest" }, + "PutObjectRequest", + () => + { + return typeof(HeadersCollection) + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .Select(p => p.Name) + .ToList(); + }); + } + + [TestMethod] + [TestCategory("S3")] + public void ValidateGetObjectRequestDefinitionCompleteness() + { + ValidateResponseDefinitionCompleteness( + new[] { "Definition", "DownloadRequest", "GetObjectRequest" }, + "GetObjectRequest", + () => + { + return typeof(ResponseHeaderOverrides) + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .Select(p => p.Name) + .ToList(); + }); + } + + [TestMethod] + [TestCategory("S3")] + public void ValidateTransferUtilityDownloadRequestDefinitionCompleteness() + { + ValidateResponseDefinitionCompleteness( + new[] { "Definition", "DownloadRequest", "GetObjectRequest" }, + "TransferUtilityDownloadRequest", + () => + { + return typeof(ResponseHeaderOverrides) + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .Select(p => p.Name) + .ToList(); + }); + } + + [TestMethod] + [TestCategory("S3")] + public void ValidateTransferUtilityUploadRequestDefinitionCompleteness() + { + ValidateResponseDefinitionCompleteness( + new[] { "Definition", "UploadRequest", "PutObjectRequest" }, + "TransferUtilityUploadRequest", + () => + { + return typeof(HeadersCollection) + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .Select(p => p.Name) + .ToList(); + }); + } + + /// + /// Generates appropriate test data for a given property type + /// + private static object GenerateTestValue(Type propertyType, string propertyName) + { + // Handle nullable types + if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + var underlyingType = Nullable.GetUnderlyingType(propertyType); + return GenerateTestValue(underlyingType, propertyName); + } + + // String properties + if (propertyType == typeof(string)) + { + return $"test-{propertyName.ToLower()}"; + } + + // Boolean properties + if (propertyType == typeof(bool)) + { + return true; + } + + // Enum properties + if (propertyType.IsEnum) + { + // For all enums, use the first available value + var enumValues = Enum.GetValues(propertyType); + return enumValues.Length > 0 ? enumValues.GetValue(0) : + throw new InvalidOperationException($"Enum {propertyType.Name} has no values"); + } + + // AWS SDK ConstantClass properties (like ChecksumType, RequestCharged, etc.) + if (typeof(ConstantClass).IsAssignableFrom(propertyType)) + { + // Use reflection to get static readonly fields that are of the same type + var constantFields = propertyType.GetFields(BindingFlags.Public | BindingFlags.Static) + .Where(f => f.IsStatic && f.IsInitOnly && f.FieldType == propertyType); + + var firstConstant = constantFields.FirstOrDefault(); + return firstConstant?.GetValue(null) ?? + throw new InvalidOperationException($"ConstantClass {propertyType.Name} has no static constants"); + } + + // Special object types + if (propertyType == typeof(Expiration)) + { + return new Expiration + { + ExpiryDate = DateTime.UtcNow.AddDays(30), + RuleId = "test-expiration-rule" + }; + } + + // Integer types + if (propertyType == typeof(int) || propertyType == typeof(long)) + { + return 1024; + } + + if (propertyType == typeof(List)) + { + if (_s3Grants is null) + { + _s3Grants = new List { new S3Grant { Grantee = new S3Grantee { DisplayName = "test-s3grantee"} } }; + } + + return _s3Grants; + } + + if (propertyType == typeof(MetadataCollection)) + { + var metadataCollection = new MetadataCollection(); + metadataCollection.Add("x-amz-meta-testkey", "testvalue"); + return metadataCollection; + } + + if (propertyType == typeof(DateTime)) + { + return DateTime.UtcNow; + } + + if (propertyType == typeof(List)) + { + return new List + { + new Tag { Key = "test-key", Value = "test-value" } + }; + } + + // For unknown types, throw an exception instead of returning null + // If we've reached this point it means there is an unhandled scenario/missing mapping in our test code that we need to handle. + throw new NotSupportedException( + $"GenerateTestValue does not support type '{propertyType.FullName}' for property '{propertyName}'. " + + $"Please add support for this type to ensure comprehensive test coverage."); + } + + /// + /// Compares two values for equality with special handling for complex objects + /// + private static bool AreValuesEqual(object sourceValue, object targetValue) + { + // Both null + if (sourceValue == null && targetValue == null) + return true; + + // One null, other not + if (sourceValue == null || targetValue == null) + return false; + + // Special handling for Expiration objects + if (sourceValue is Expiration sourceExpiration && targetValue is Expiration targetExpiration) + { + return sourceExpiration.ExpiryDate == targetExpiration.ExpiryDate && + sourceExpiration.RuleId == targetExpiration.RuleId; + } + + // For most cases, use default equality + return sourceValue.Equals(targetValue); + } + + /// + /// Resolves a property name to its actual class property name, checking aliases if needed + /// + private static string ResolvePropertyName(string propertyName, string responseTypeName) + { + if (_propertyAliases.TryGetValue(responseTypeName, out var objectAliases)) + { + // Check if there's an alias for this property name + if (objectAliases.TryGetValue(propertyName, out var aliasedName)) + { + return aliasedName; + } + } + + // Return the original name if no alias exists + return propertyName; + } + + /// + /// Generic helper method to validate response definition completeness. + /// This method ensures that all properties defined in mapping.json actually exist + /// in the corresponding AWS SDK response classes, supporting property name aliases + /// for backwards compatibility and maintainability. + /// + private static void ValidateResponseDefinitionCompleteness( + string[] jsonPath, + string responseTypeName, + Func> getAdditionalProperties = null) + { + // Get direct properties from response class + var directProperties = typeof(TResponse) + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .Select(p => p.Name) + .ToList(); + + // Get additional properties if provided (e.g., HeadersCollection properties) + var additionalProperties = getAdditionalProperties?.Invoke()?.ToList() ?? new List(); + + // Combine direct and additional properties + var actualProperties = directProperties.Union(additionalProperties) + .OrderBy(name => name) + .ToList(); + + // Navigate to the JSON definition using the provided path + var jsonElement = _mappingJson.RootElement; + foreach (var pathSegment in jsonPath) + { + jsonElement = jsonElement.GetProperty(pathSegment); + } + + var definitionProperties = jsonElement + .EnumerateArray() + .Select(prop => prop.GetString()) + .OrderBy(name => name) + .ToList(); + + // Check each definition property, resolving aliases as needed + var extraInDefinition = new List(); + + foreach (var definitionProperty in definitionProperties) + { + var resolvedPropertyName = ResolvePropertyName(definitionProperty, responseTypeName); + + // Check if the resolved property name exists in the actual class + if (!actualProperties.Contains(resolvedPropertyName)) + { + extraInDefinition.Add($"{definitionProperty} (resolved to: {resolvedPropertyName})"); + } + } + + // Assert no extra properties + if (extraInDefinition.Any()) + { + var additionalContext = additionalProperties.Any() + ? $" or additional properties" + : ""; + + Assert.Fail($"Definition section contains {extraInDefinition.Count} extra properties that don't exist in the actual {responseTypeName} class{additionalContext}: {string.Join(", ", extraInDefinition)}. " + + $"Please verify they exist in the response class{additionalContext}."); + } + } + } +} diff --git a/sdk/test/UnitTests/AWSSDK.UnitTests.NetFramework.csproj b/sdk/test/UnitTests/AWSSDK.UnitTests.NetFramework.csproj index 5d1c7e6f8ba0..b6f3c88dfff9 100644 --- a/sdk/test/UnitTests/AWSSDK.UnitTests.NetFramework.csproj +++ b/sdk/test/UnitTests/AWSSDK.UnitTests.NetFramework.csproj @@ -84,6 +84,7 @@ + \ No newline at end of file