Skip to content

Support for payload signing + chunked encoding in DefaultAwsV4HttpSigner #6466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
+1,246 −376
Open

Support for payload signing + chunked encoding in DefaultAwsV4HttpSigner #6466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7c7941c
Support async V4 payload signing
dagnir Jul 21, 2025
a222719
Review comments
dagnir Aug 6, 2025
1d15408
Combine chunking + encoding
dagnir Aug 7, 2025
3d1d896
Merge remote-tracking branch 'origin/master' into dongie/sra-chunk-en…
dagnir Oct 1, 2025
119cfbf
Merge remote-tracking branch 'origin/master' into dongie/sra-chunk-en…
dagnir Oct 7, 2025
e52ea62
Fixes after merge
dagnir Oct 2, 2025
e1df197
Merge branch 'master' into dongie/sra-chunk-encoding-pt1
dagnir Oct 8, 2025
7cd278b
Add store test for AsyncSigningStage
dagnir Oct 9, 2025
0a236dc
Review comments
dagnir Oct 10, 2025
d480132
Additional test for no content-length
dagnir Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/next-release/feature-AWSSDKforJavav2-e386e28.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"type": "feature",
"category": "AWS SDK for Java v2",
"contributor": "",
"description": "Add support for payload signing of async streaming requests signed with SigV4 using default `AwsV4HttpSigner` (using `AwsV4HttpSigner.create()`). Note, requests using the `http` URI scheme will not be signed regardless of the value of `AwsV4FamilyHttpSigner.PAYLOAD_SIGNING_ENABLED` to remain consistent with existing behavior. This may change in a future release."
}
4 changes: 2 additions & 2 deletions .../software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_ECDSA_SIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils.moveContentLength;

import java.io.InputStream;
import java.nio.charset.StandardCharsets;
Expand All @@ -41,6 +40,7 @@
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils;
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
import software.amazon.awssdk.utils.BinaryUtils;
import software.amazon.awssdk.utils.Logger;
Expand Down Expand Up @@ -115,7 +115,7 @@ public ContentStreamProvider sign(ContentStreamProvider payload, V4aRequestSigni
@Override
public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider payload, String checksum) {
long encodedContentLength = 0;
long contentLength = moveContentLength(request, payload);
long contentLength = SignerUtils.computeAndMoveContentLength(request, payload);
setupPreExistingTrailers(request);

// pre-existing trailers
Expand Down
145 changes: 99 additions & 46 deletions .../java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,28 +23,35 @@
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_SIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_CONTENT_SHA256;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_DECODED_CONTENT_LENGTH;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils.moveContentLength;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils.computeAndMoveContentLength;

import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import org.reactivestreams.Publisher;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.checksums.SdkChecksum;
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
import software.amazon.awssdk.http.ContentStreamProvider;
import software.amazon.awssdk.http.Header;
import software.amazon.awssdk.http.SdkHttpRequest;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.AsyncChunkEncodedPayload;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedPayload;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedPublisher;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SigV4ChunkExtensionProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SigV4TrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SyncChunkEncodedPayload;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils;
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
import software.amazon.awssdk.utils.BinaryUtils;
import software.amazon.awssdk.utils.Logger;
Expand Down Expand Up @@ -79,81 +86,140 @@ public static Builder builder() {

@Override
public ContentStreamProvider sign(ContentStreamProvider payload, V4RequestSigningResult requestSigningResult) {
SdkHttpRequest.Builder request = requestSigningResult.getSignedRequest();

String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
);

ChunkedEncodedInputStream.Builder chunkedEncodedInputStreamBuilder = ChunkedEncodedInputStream
.builder()
.inputStream(payload.newStream())
.chunkSize(chunkSize)
.header(chunk -> Integer.toHexString(chunk.remaining()).getBytes(StandardCharsets.UTF_8));

preExistingTrailers.forEach(trailer -> chunkedEncodedInputStreamBuilder.addTrailer(() -> trailer));
SyncChunkEncodedPayload chunkedPayload = new SyncChunkEncodedPayload(chunkedEncodedInputStreamBuilder);
signCommon(chunkedPayload, requestSigningResult);

return new ResettableContentStreamProvider(chunkedEncodedInputStreamBuilder::build);
}

@Override
public Publisher<ByteBuffer> signAsync(Publisher<ByteBuffer> payload, V4RequestSigningResult requestSigningResult) {
ChunkedEncodedPublisher.Builder chunkedStreamBuilder = ChunkedEncodedPublisher.builder()
.publisher(payload)
.chunkSize(chunkSize)
.addEmptyTrailingChunk(true);

AsyncChunkEncodedPayload chunkedPayload = new AsyncChunkEncodedPayload(chunkedStreamBuilder);
signCommon(chunkedPayload, requestSigningResult);

return chunkedStreamBuilder.build();
}

private void signCommon(ChunkedEncodedPayload payload, V4RequestSigningResult requestSigningResult) {
preExistingTrailers.forEach(t -> payload.addTrailer(() -> t));

SdkHttpRequest.Builder request = requestSigningResult.getSignedRequest();

payload.decodedContentLength(request.firstMatchingHeader(X_AMZ_DECODED_CONTENT_LENGTH)
.map(Long::parseLong)
.orElseThrow(() -> {
String msg = String.format("Expected header '%s' to be present",
X_AMZ_DECODED_CONTENT_LENGTH);
return new RuntimeException(msg);
}));

String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
);

switch (checksum) {
case STREAMING_SIGNED_PAYLOAD: {
RollingSigner rollingSigner = new RollingSigner(requestSigningResult.getSigningKey(),
requestSigningResult.getSignature());
chunkedEncodedInputStreamBuilder.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
payload.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
break;
}
case STREAMING_UNSIGNED_PAYLOAD_TRAILER:
setupChecksumTrailerIfNeeded(chunkedEncodedInputStreamBuilder);
setupChecksumTrailerIfNeeded(payload);
break;
case STREAMING_SIGNED_PAYLOAD_TRAILER: {
setupChecksumTrailerIfNeeded(payload);
RollingSigner rollingSigner = new RollingSigner(requestSigningResult.getSigningKey(),
requestSigningResult.getSignature());
chunkedEncodedInputStreamBuilder.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
setupChecksumTrailerIfNeeded(chunkedEncodedInputStreamBuilder);
chunkedEncodedInputStreamBuilder.addTrailer(
new SigV4TrailerProvider(chunkedEncodedInputStreamBuilder.trailers(), rollingSigner, credentialScope)
payload.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
payload.addTrailer(
new SigV4TrailerProvider(payload.trailers(), rollingSigner, credentialScope)
);
break;
}
default:
throw new UnsupportedOperationException();
}

return new ResettableContentStreamProvider(chunkedEncodedInputStreamBuilder::build);
}

@Override
public Publisher<ByteBuffer> signAsync(Publisher<ByteBuffer> payload, V4RequestSigningResult requestSigningResult) {
// TODO(sra-identity-and-auth): implement this first and remove addFlexibleChecksumInTrailer logic in HttpChecksumStage
throw new UnsupportedOperationException();
}

@Override
public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider payload) {
long encodedContentLength = 0;
long contentLength = moveContentLength(request, payload);
long contentLength = SignerUtils.computeAndMoveContentLength(request, payload);
setupPreExistingTrailers(request);

// pre-existing trailers
encodedContentLength = calculateEncodedContentLength(request, contentLength);

if (checksumAlgorithm != null) {
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
}
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
}

@Override
public CompletableFuture<Pair<SdkHttpRequest.Builder, Optional<Publisher<ByteBuffer>>>> beforeSigningAsync(
SdkHttpRequest.Builder request, Publisher<ByteBuffer> payload) {
return computeAndMoveContentLength(request, payload)
.thenApply(p -> {
SdkHttpRequest.Builder requestBuilder = p.left();
setupPreExistingTrailers(requestBuilder);

long decodedContentLength = requestBuilder.firstMatchingHeader(X_AMZ_DECODED_CONTENT_LENGTH)
.map(Long::parseLong)
// should not happen, this header is added by moveContentLength
.orElseThrow(() -> new RuntimeException(X_AMZ_DECODED_CONTENT_LENGTH
+ " header not present"));

long encodedContentLength = calculateEncodedContentLength(request, decodedContentLength);

if (checksumAlgorithm != null) {
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
}
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
return Pair.of(requestBuilder, p.right());
});
}

private long calculateEncodedContentLength(SdkHttpRequest.Builder requestBuilder, long decodedContentLength) {
long encodedContentLength = 0;

encodedContentLength += calculateExistingTrailersLength();

String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
String checksum = requestBuilder.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
);

switch (checksum) {
case STREAMING_SIGNED_PAYLOAD: {
long extensionsLength = 81; // ;chunk-signature:<sigv4 hex signature, 64 bytes>
encodedContentLength += calculateChunksLength(contentLength, extensionsLength);
encodedContentLength += calculateChunksLength(decodedContentLength, extensionsLength);
break;
}
case STREAMING_UNSIGNED_PAYLOAD_TRAILER:
if (checksumAlgorithm != null) {
encodedContentLength += calculateChecksumTrailerLength(checksumHeaderName(checksumAlgorithm));
}
encodedContentLength += calculateChunksLength(contentLength, 0);
encodedContentLength += calculateChunksLength(decodedContentLength, 0);
break;
case STREAMING_SIGNED_PAYLOAD_TRAILER: {
long extensionsLength = 81; // ;chunk-signature:<sigv4 hex signature, 64 bytes>
encodedContentLength += calculateChunksLength(contentLength, extensionsLength);
encodedContentLength += calculateChunksLength(decodedContentLength, extensionsLength);
if (checksumAlgorithm != null) {
encodedContentLength += calculateChecksumTrailerLength(checksumHeaderName(checksumAlgorithm));
}
Expand All @@ -167,12 +233,7 @@ public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider
// terminating \r\n
encodedContentLength += 2;

if (checksumAlgorithm != null) {
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
}
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
return encodedContentLength;
}

/**
Expand Down Expand Up @@ -256,12 +317,7 @@ private long calculateChecksumTrailerLength(String checksumHeaderName) {
return lengthInBytes + 2;
}

/**
* Add the checksum as a trailer to the chunk-encoded stream.
* <p>
* If the checksum-algorithm is not present, then nothing is done.
*/
private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder builder) {
private void setupChecksumTrailerIfNeeded(ChunkedEncodedPayload payload) {
if (checksumAlgorithm == null) {
return;
}
Expand All @@ -273,20 +329,17 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil
if (cachedChecksum != null) {
LOG.debug(() -> String.format("Cached payload checksum available for algorithm %s: %s. Using cached value",
checksumAlgorithm.algorithmId(), checksumHeaderName));
builder.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum)));
payload.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum)));
return;
}

SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm);
ChecksumInputStream checksumInputStream = new ChecksumInputStream(
builder.inputStream(),
Collections.singleton(sdkChecksum)
);

TrailerProvider checksumTrailer =
new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName, checksumAlgorithm, payloadChecksumStore);

builder.inputStream(checksumInputStream).addTrailer(checksumTrailer);
payload.checksumPayload(sdkChecksum);
payload.addTrailer(checksumTrailer);
}

private String getCachedChecksum() {
Expand Down
Loading
Loading