From fcc779b81f09af90deb38ced77b9a58bdbc48ed0 Mon Sep 17 00:00:00 2001 From: rohangoli <27857671+rohangoli@users.noreply.github.com> Date: Mon, 13 Oct 2025 14:20:33 -0700 Subject: [PATCH 1/5] Ignore SSL Verification --- .../admin/model/CatalogSerializationTest.java | 1 + getting-started/minio-https/README.md | 94 ++++++++ .../minio-https/docker-compose.yml | 150 ++++++++++++ .../polaris/core/entity/CatalogEntity.java | 2 + .../aws/AwsCredentialsStorageIntegration.java | 12 +- .../aws/AwsStorageConfigurationInfo.java | 3 + .../core/storage/aws/StsClientProvider.java | 12 +- .../catalog/io/DefaultFileIOFactory.java | 60 ++++- .../io/s3/ReflectionS3ClientInjector.java | 224 ++++++++++++++++++ .../service/config/ServiceProducers.java | 53 ++++- .../SigV4ConnectionCredentialVendor.java | 5 +- .../service/storage/aws/StsClientsPool.java | 42 +++- .../service/admin/ManagementServiceTest.java | 55 +++++ .../ReflectionS3ClientInjectorConfigTest.java | 150 ++++++++++++ .../io/s3/ReflectionS3ClientInjectorTest.java | 58 +++++ spec/polaris-management-service.yml | 8 + 16 files changed, 917 insertions(+), 12 deletions(-) create mode 100644 getting-started/minio-https/README.md create mode 100644 getting-started/minio-https/docker-compose.yml create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjector.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorConfigTest.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorTest.java diff --git a/api/management-model/src/test/java/org/apache/polaris/core/admin/model/CatalogSerializationTest.java b/api/management-model/src/test/java/org/apache/polaris/core/admin/model/CatalogSerializationTest.java index c4210486ba..04cd2cbbb0 100644 --- a/api/management-model/src/test/java/org/apache/polaris/core/admin/model/CatalogSerializationTest.java +++ b/api/management-model/src/test/java/org/apache/polaris/core/admin/model/CatalogSerializationTest.java @@ -71,6 +71,7 @@ public void testJsonFormat() throws JsonProcessingException { + "\"storageConfigInfo\":{" + "\"roleArn\":\"arn:aws:iam::123456789012:role/test-role\"," + "\"pathStyleAccess\":false," + + "\"ignoreSSLVerification\":false," + "\"storageType\":\"S3\"," + "\"allowedLocations\":[]" + "}}"); diff --git a/getting-started/minio-https/README.md b/getting-started/minio-https/README.md new file mode 100644 index 0000000000..3df789c6f8 --- /dev/null +++ b/getting-started/minio-https/README.md @@ -0,0 +1,94 @@ + + +# Getting Started with Apache Polaris and MinIO (HTTPS) + +## Overview + +This example uses MinIO (HTTPS) as a storage provider with Polaris, it can be used for other S3-Compatible Storage API with Ignoring SSL Verification for Development/Test Purposes + +Spark is used as a query engine. This example assumes a local Spark installation. +See the [Spark Notebooks Example](../spark/README.md) for a more advanced Spark setup. + +## Starting the Example + +1. Build the Polaris server image if it's not already present locally: + + ```shell + ./gradlew \ + :polaris-server:assemble \ + :polaris-server:quarkusAppPartsBuild --rerun \ + -Dquarkus.container-image.build=true + ``` + +2. Start the docker compose group by running the following command from the root of the repository: + + ```shell + docker compose -f getting-started/minio/docker-compose.yml up + ``` + +## Connecting From Spark + +```shell +bin/spark-sql \ + --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.iceberg:iceberg-aws-bundle:1.9.0 \ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ + --conf spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.polaris.type=rest \ + --conf spark.sql.catalog.polaris.uri=http://localhost:8181/api/catalog \ + --conf spark.sql.catalog.polaris.token-refresh-enabled=false \ + --conf spark.sql.catalog.polaris.warehouse=quickstart_catalog \ + --conf spark.sql.catalog.polaris.scope=PRINCIPAL_ROLE:ALL \ + --conf spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation=vended-credentials \ + --conf spark.sql.catalog.polaris.credential=root:s3cr3t \ + --conf spark.sql.catalog.polaris.client.region=irrelevant +``` + +Note: `s3cr3t` is defined as the password for the `root` users in the `docker-compose.yml` file. + +Note: The `client.region` configuration is required for the AWS S3 client to work, but it is not used in this example +since MinIO does not require a specific region. + +## Running Queries + +Run inside the Spark SQL shell: + +``` +spark-sql (default)> use polaris; +Time taken: 0.837 seconds + +spark-sql ()> create namespace ns; +Time taken: 0.374 seconds + +spark-sql ()> create table ns.t1 as select 'abc'; +Time taken: 2.192 seconds + +spark-sql ()> select * from ns.t1; +abc +Time taken: 0.579 seconds, Fetched 1 row(s) +``` + +## MinIO Endpoints + +Note that the catalog configuration defined in the `docker-compose.yml` contains +different endpoints for the Polaris Server and the client (Spark). Specifically, +the client endpoint is `https://localhost:9000`, but `endpointInternal` is `https://minio:9000`. + +This is necessary because clients running on `localhost` do not normally see service +names (such as `minio`) that are internal to the docker compose environment. diff --git a/getting-started/minio-https/docker-compose.yml b/getting-started/minio-https/docker-compose.yml new file mode 100644 index 0000000000..f65bce149e --- /dev/null +++ b/getting-started/minio-https/docker-compose.yml @@ -0,0 +1,150 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +services: + + minio: + image: quay.io/minio/minio:latest + ports: + # API port + - "9000:9000" + # UI port + - "9001:9001" + environment: + MINIO_ROOT_USER: minio_root + MINIO_ROOT_PASSWORD: m1n1opwd + command: + - "server" + - "/data" + - "--console-address" + - ":9001" + volumes: + # Mount the generated certs volume into MinIO + - minio_certs:/root/.minio/certs + depends_on: + certs-init: + condition: service_completed_successfully + healthcheck: + # Use HTTPS healthcheck now that MinIO will serve TLS on 9000 + test: ["CMD", "curl", "-k", "https://127.0.0.1:9000/minio/health/live"] + interval: 1s + timeout: 10s + + polaris: + image: apache/polaris:latest + ports: + # API port + - "8181:8181" + # Optional, allows attaching a debugger to the Polaris JVM + - "5005:5005" + depends_on: + minio: + condition: service_healthy + setup_bucket: + condition: service_completed_successfully + environment: + JAVA_DEBUG: true + JAVA_DEBUG_PORT: "*:5005" + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: minio_root + AWS_SECRET_ACCESS_KEY: m1n1opwd + POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t + polaris.realm-context.realms: POLARIS + quarkus.otel.sdk.disabled: "true" + quarkus.log.category."org.apache.polaris".level: DEBUG + quarkus.log.category."org.apache.polaris.catalog".level: DEBUG + quarkus.log.category."org.apache.polaris.table".level: DEBUG + healthcheck: + test: ["CMD", "curl", "http://localhost:8182/q/health"] + interval: 2s + timeout: 10s + retries: 10 + start_period: 10s + + setup_bucket: + image: quay.io/minio/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: "/bin/sh" + command: + - "-c" + - >- + echo Creating MinIO bucket...; + mc alias set pol https://minio:9000 minio_root m1n1opwd --insecure; + mc mb pol/bucket123 --insecure; + mc ls pol --insecure; + echo Bucket setup complete.; + + # TLS certs are created at `docker compose up` by the certs-init one-shot service + certs-init: + image: alpine:3.18 + entrypoint: "/bin/sh" + command: + - "-c" + - >- + apk add --no-cache openssl; + printf '%s\n' "[req]" "ndistinguished_name = req_distinguished_name" "req_extensions = v3_req" "prompt = no" "" \ + "[req_distinguished_name]" "CN = localhost" "" "[v3_req]" "subjectAltName = @alt_names" "" "[alt_names]" "DNS.1 = localhost" "DNS.2 = minio" "IP.1 = 127.0.0.1" \ + > /openssl.cnf; + openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout /certs/private.key -out /certs/public.crt \ + -subj "/C=US/ST=State/L=City/O=Org/CN=localhost" -extensions v3_req -config /openssl.cnf; + chmod 600 /certs/private.key || true; chmod 644 /certs/public.crt || true; + echo Generated certs.; + volumes: + - minio_certs:/certs + restart: "no" + + polaris-setup: + image: alpine/curl + depends_on: + polaris: + condition: service_healthy + environment: + CLIENT_ID: root + CLIENT_SECRET: s3cr3t + # Use HTTPS endpoints and indicate to Polaris to ignore SSL verification for this self-signed setup + STORAGE_CONFIG_INFO: '{"storageType":"S3","endpoint":"https://localhost:9000","endpointInternal":"https://minio:9000","pathStyleAccess":true,"ignoreSSLVerification":true, "stsUnavailable":true}' + STORAGE_LOCATION: 's3://bucket123' + volumes: + - ../assets/polaris/:/polaris + entrypoint: "/bin/sh" + command: + - "-c" + - >- + chmod +x /polaris/create-catalog.sh; + chmod +x /polaris/obtain-token.sh; + source /polaris/obtain-token.sh; + echo Creating catalog...; + /polaris/create-catalog.sh POLARIS $$TOKEN; + echo Extra grants...; + curl -H "Authorization: Bearer $$TOKEN" -H 'Content-Type: application/json' \ + -X PUT \ + http://polaris:8181/api/management/v1/catalogs/quickstart_catalog/catalog-roles/catalog_admin/grants \ + -d '{"type":"catalog", "privilege":"CATALOG_MANAGE_CONTENT"}'; + echo Done.; + +volumes: + minio_certs: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + o: "size=1m,mode=1777" diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index 055ccd8959..a6cc3b7dbf 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -166,6 +166,7 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) .setRegion(awsConfig.getRegion()) .setEndpoint(awsConfig.getEndpoint()) .setStsEndpoint(awsConfig.getStsEndpoint()) + .setIgnoreSSLVerification(awsConfig.getIgnoreSSLVerification()) .setPathStyleAccess(awsConfig.getPathStyleAccess()) .setStsUnavailable(awsConfig.getStsUnavailable()) .setEndpointInternal(awsConfig.getEndpointInternal()) @@ -315,6 +316,7 @@ public Builder setStorageConfigurationInfo( .pathStyleAccess(awsConfigModel.getPathStyleAccess()) .stsUnavailable(awsConfigModel.getStsUnavailable()) .endpointInternal(awsConfigModel.getEndpointInternal()) + .ignoreSSLVerification(awsConfigModel.getIgnoreSSLVerification()) .build(); config = awsConfig; break; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index 8023f7a607..451ce1b922 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -102,7 +102,11 @@ public AccessConfig getSubscopedCreds( @SuppressWarnings("resource") // Note: stsClientProvider returns "thin" clients that do not need closing StsClient stsClient = - stsClientProvider.stsClient(StsDestination.of(storageConfig.getStsEndpointUri(), region)); + stsClientProvider.stsClient( + StsDestination.of( + storageConfig.getStsEndpointUri(), + region, + storageConfig.getIgnoreSSLVerification())); AssumeRoleResponse response = stsClient.assumeRole(request.build()); accessConfig.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId()); @@ -139,6 +143,12 @@ public AccessConfig getSubscopedCreds( StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), internalEndpointUri.toString()); } + // Propagate ignore SSL verification flag so callers (e.g., FileIOFactory) can honor it when + // constructing S3 clients for metadata ops. + if (Boolean.TRUE.equals(storageConfig.getIgnoreSSLVerification())) { + accessConfig.putInternalProperty("polaris.ignore-ssl-verification", "true"); + } + if (Boolean.TRUE.equals(storageConfig.getPathStyleAccess())) { accessConfig.put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, Boolean.TRUE.toString()); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java index b3d7d60790..982ab7f828 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java @@ -108,6 +108,9 @@ public URI getInternalEndpointUri() { @Nullable public abstract String getStsEndpoint(); + /** Flag indicating whether SSL certificate verification should be disabled */ + public abstract @Nullable Boolean getIgnoreSSLVerification(); + /** Returns the STS endpoint if set, defaulting to {@link #getEndpointUri()} otherwise. */ @JsonIgnore @Nullable diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsClientProvider.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsClientProvider.java index b5a1fefe14..3329659be4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsClientProvider.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsClientProvider.java @@ -52,8 +52,16 @@ interface StsDestination { @Value.Parameter(order = 2) Optional region(); - static StsDestination of(@Nullable URI endpoint, @Nullable String region) { - return ImmutableStsDestination.of(Optional.ofNullable(endpoint), Optional.ofNullable(region)); + /** Whether to ignore SSL certificate verification */ + @Value.Parameter(order = 3) + Optional ignoreSSLVerification(); + + static StsDestination of( + @Nullable URI endpoint, @Nullable String region, @Nullable Boolean ignoreSSLVerification) { + return ImmutableStsDestination.of( + Optional.ofNullable(endpoint), + Optional.ofNullable(region), + Optional.ofNullable(ignoreSSLVerification)); } } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index 44f038d72f..5ec52c4be6 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -39,6 +39,12 @@ import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.apache.polaris.service.catalog.io.s3.ReflectionS3ClientInjector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; /** * A default FileIO factory implementation for creating Iceberg {@link FileIO} instances with @@ -52,6 +58,8 @@ @Identifier("default") public class DefaultFileIOFactory implements FileIOFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultFileIOFactory.class); + private final StorageCredentialCache storageCredentialCache; private final MetaStoreManagerFactory metaStoreManagerFactory; @@ -63,6 +71,14 @@ public DefaultFileIOFactory( this.metaStoreManagerFactory = metaStoreManagerFactory; } + @Inject + @Identifier("aws-sdk-http-client") + SdkHttpClient sdkHttpClient; + + @Inject + @Identifier("aws-sdk-http-client-insecure") + SdkHttpClient insecureSdkHttpClient; + @Override public FileIO loadFileIO( @Nonnull CallContext callContext, @@ -109,6 +125,48 @@ public FileIO loadFileIO( @VisibleForTesting FileIO loadFileIOInternal( @Nonnull String ioImplClassName, @Nonnull Map properties) { - return new ExceptionMappingFileIO(CatalogUtil.loadFileIO(ioImplClassName, properties, null)); + FileIO fileIO = CatalogUtil.loadFileIO(ioImplClassName, properties, null); + + // If this is Iceberg's S3FileIO and the storage config requested ignoring SSL verification, + // try to construct and inject an S3 client that uses our insecure SDK HTTP client. This is + // a best-effort reflective integration to make the 'ignoreSSLVerification' flag effective + // for the S3 client used by Iceberg. + try { + boolean ignoreSsl = "true".equals(properties.get("polaris.ignore-ssl-verification")); + if (ignoreSsl && "org.apache.iceberg.aws.s3.S3FileIO".equals(ioImplClassName)) { + try { + // Build prebuilt S3 clients using the insecure SDK HTTP client + S3Client prebuilt = + ReflectionS3ClientInjector.buildS3Client(insecureSdkHttpClient, properties); + S3AsyncClient prebuiltAsync = + ReflectionS3ClientInjector.buildS3AsyncClient(insecureSdkHttpClient, properties); + boolean supplierInjected = + ReflectionS3ClientInjector.injectSupplierIntoS3FileIO( + fileIO, prebuilt, prebuiltAsync); + if (supplierInjected) { + LOGGER.info( + "Injected SerializableSupplier for insecure S3 client into Iceberg S3FileIO for ioImpl={}", + ioImplClassName); + } else { + LOGGER.warn( + "Requested ignore-ssl-verification but failed to inject insecure S3Client supplier into {}.\n" + + "Consider importing the storage certificate into Polaris JVM truststore or using HTTP endpoints for local testing.", + ioImplClassName); + } + } catch (Throwable t) { + LOGGER.warn( + "Failed to build or inject prebuilt S3 client for {}: {}", + ioImplClassName, + t.getMessage()); + } + } + } catch (Exception e) { + LOGGER.warn( + "Exception while attempting to inject S3 client for {}: {}", + ioImplClassName, + e.toString()); + } + + return new ExceptionMappingFileIO(fileIO); } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjector.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjector.java new file mode 100644 index 0000000000..75708f67e8 --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjector.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.catalog.io.s3; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URI; +import java.util.Map; +import org.apache.polaris.core.storage.StorageAccessProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.S3Configuration; + +/** + * Best-effort reflective injector that attempts to wire a pre-built S3Client into Iceberg's + * S3FileIO instance. This avoids needing to modify Iceberg. + */ +public final class ReflectionS3ClientInjector { + private static final Logger LOGGER = LoggerFactory.getLogger(ReflectionS3ClientInjector.class); + + private ReflectionS3ClientInjector() {} + + /** + * For Iceberg's S3FileIO implementation, PrefixedS3Client holds SerializableSupplier + * fields named 's3' and 's3Async' which are used to lazily construct the real clients. To ensure + * Iceberg will use our prebuilt S3Client (configured with the provided SdkHttpClient), replace + * those supplier fields with suppliers that return the provided instances. + */ + public static boolean injectSupplierIntoS3FileIO( + Object s3FileIOInstance, S3Client prebuiltS3Client, S3AsyncClient prebuiltS3AsyncClient) { + if (s3FileIOInstance == null) { + return false; + } + + Class clazz = s3FileIOInstance.getClass(); + boolean changed = false; + while (clazz != null) { + try { + Field s3Field = null; + Field s3AsyncField = null; + try { + s3Field = clazz.getDeclaredField("s3"); + } catch (NoSuchFieldException ignored) { + } + try { + s3AsyncField = clazz.getDeclaredField("s3Async"); + } catch (NoSuchFieldException ignored) { + } + + if (s3Field != null) { + s3Field.setAccessible(true); + // Build a SerializableSupplier that returns our prebuilt S3Client. + // Use a simple SerializableSupplier implementation used by Iceberg: + // org.apache.iceberg.util.SerializableSupplier + try { + Class serializableSupplierClazz = + Class.forName("org.apache.iceberg.util.SerializableSupplier"); + Object serializableSupplier = + java.lang.reflect.Proxy.newProxyInstance( + serializableSupplierClazz.getClassLoader(), + new Class[] {serializableSupplierClazz, java.io.Serializable.class}, + (proxy, method, args) -> { + if ("get".equals(method.getName())) { + return prebuiltS3Client; + } + // default proxy behavior + return method.invoke(proxy, args); + }); + + s3Field.set(s3FileIOInstance, serializableSupplier); + changed = true; + } catch (ClassNotFoundException cnfe) { + // Fallback: try to set any field assignable from java.util.function.Supplier + Object simple = (java.util.function.Supplier) () -> prebuiltS3Client; + s3Field.set(s3FileIOInstance, simple); + changed = true; + } + } + + if (s3AsyncField != null && prebuiltS3AsyncClient != null) { + s3AsyncField.setAccessible(true); + try { + Class serializableSupplierClazz = + Class.forName("org.apache.iceberg.util.SerializableSupplier"); + Object serializableSupplierAsync = + java.lang.reflect.Proxy.newProxyInstance( + serializableSupplierClazz.getClassLoader(), + new Class[] {serializableSupplierClazz, java.io.Serializable.class}, + (proxy, method, args) -> { + if ("get".equals(method.getName())) { + return prebuiltS3AsyncClient; + } + return method.invoke(proxy, args); + }); + + s3AsyncField.set(s3FileIOInstance, serializableSupplierAsync); + changed = true; + } catch (ClassNotFoundException cnfe) { + Object simpleAsync = + (java.util.function.Supplier) () -> prebuiltS3AsyncClient; + s3AsyncField.set(s3FileIOInstance, simpleAsync); + changed = true; + } + } + } catch (Throwable t) { + LOGGER.debug("Failed to set supplier fields on {}: {}", clazz, t.getMessage()); + } + + clazz = clazz.getSuperclass(); + } + + return changed; + } + + public static S3Client buildS3Client(SdkHttpClient httpClient, Map properties) { + S3ClientBuilder builder = S3Client.builder(); + if (httpClient != null) { + builder.httpClient(httpClient); + } + AwsCredentialsProvider creds = credentialsProviderFrom(properties); + if (creds != null) builder.credentialsProvider(creds); + + applyRegionAndEndpoint(builder, properties); + builder.serviceConfiguration(s3ConfigurationFrom(properties)); + + return builder.build(); + } + + public static S3AsyncClient buildS3AsyncClient( + SdkHttpClient httpClient, Map properties) { + try { + // Attempt to build an async client. If the provided httpClient is an instance that also + // implements the async HTTP client interface, use it. Otherwise fall back to the + // default async client builder. + software.amazon.awssdk.http.async.SdkAsyncHttpClient asyncHttpClient = null; + if (httpClient instanceof software.amazon.awssdk.http.async.SdkAsyncHttpClient async) { + asyncHttpClient = async; + } + + software.amazon.awssdk.services.s3.S3AsyncClientBuilder asyncBuilder = + software.amazon.awssdk.services.s3.S3AsyncClient.builder(); + + if (asyncHttpClient != null) { + asyncBuilder.httpClient(asyncHttpClient); + } + + AwsCredentialsProvider creds = credentialsProviderFrom(properties); + if (creds != null) asyncBuilder.credentialsProvider(creds); + + applyRegionAndEndpoint(asyncBuilder, properties); + asyncBuilder.serviceConfiguration(s3ConfigurationFrom(properties)); + + return asyncBuilder.build(); + } catch (Exception e) { + LOGGER.debug("Failed to build S3AsyncClient: {}", e.toString()); + return null; + } + } + + private static AwsCredentialsProvider credentialsProviderFrom(Map properties) { + String accessKey = properties.get(StorageAccessProperty.AWS_KEY_ID.getPropertyName()); + String secretKey = properties.get(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName()); + if (accessKey != null && secretKey != null) { + AwsBasicCredentials creds = AwsBasicCredentials.create(accessKey, secretKey); + return StaticCredentialsProvider.create(creds); + } + return null; + } + + private static void applyRegionAndEndpoint(Object builder, Map properties) { + String region = properties.get(StorageAccessProperty.CLIENT_REGION.getPropertyName()); + if (region != null) { + try { + Method m = builder.getClass().getMethod("region", Region.class); + m.invoke(builder, Region.of(region)); + } catch (Exception ignored) { + LOGGER.debug("Unable to apply region to builder {}", builder.getClass().getName()); + } + } + + String endpoint = properties.get(StorageAccessProperty.AWS_ENDPOINT.getPropertyName()); + if (endpoint != null) { + try { + Method m = builder.getClass().getMethod("endpointOverride", URI.class); + m.invoke(builder, URI.create(endpoint)); + } catch (Exception ignored) { + LOGGER.debug( + "Unable to apply endpointOverride to builder {}", builder.getClass().getName()); + } + } + } + + private static S3Configuration s3ConfigurationFrom(Map properties) { + String pathStyle = + properties.get(StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName()); + boolean forcePathStyle = "true".equals(pathStyle); + return S3Configuration.builder().pathStyleAccessEnabled(forcePathStyle).build(); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java b/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java index 6e25dbfc18..70344225a4 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java @@ -34,6 +34,10 @@ import jakarta.ws.rs.core.Context; import java.time.Clock; import java.util.stream.Collectors; +import javax.net.ssl.SSLContext; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.ssl.SSLContextBuilder; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; @@ -230,6 +234,50 @@ public SdkHttpClient sdkHttpClient(S3AccessConfig config) { return httpClient.build(); } + /** + * Producer that creates an insecure SDK HTTP client (trusts all certs). This allows other + * components to explicitly request the insecure client instance when wiring clients that need to + * ignore TLS verification (for development/test setups only). + */ + @Produces + @Singleton + @Identifier("aws-sdk-http-client-insecure") + public SdkHttpClient insecureSdkHttpClient(S3AccessConfig config) { + return createInsecureHttpClient(config); + } + + /** + * Creates an HTTP client that bypasses SSL certificate verification. WARNING: This should only be + * used for development and testing environments. + */ + public SdkHttpClient createInsecureHttpClient(S3AccessConfig config) { + try { + SSLContext sslContext = + SSLContextBuilder.create().loadTrustMaterial(null, (chain, authType) -> true).build(); + + ApacheHttpClient.Builder httpClient = + ApacheHttpClient.builder() + .socketFactory( + new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE)); + + // Apply configuration options + config.maxHttpConnections().ifPresent(httpClient::maxConnections); + config.readTimeout().ifPresent(httpClient::socketTimeout); + config.connectTimeout().ifPresent(httpClient::connectionTimeout); + config.connectionAcquisitionTimeout().ifPresent(httpClient::connectionAcquisitionTimeout); + config.connectionMaxIdleTime().ifPresent(httpClient::connectionMaxIdleTime); + config.connectionTimeToLive().ifPresent(httpClient::connectionTimeToLive); + config.expectContinueEnabled().ifPresent(httpClient::expectContinueEnabled); + + LOGGER.warn( + "Creating HTTP client with SSL certificate verification disabled. Use only in development!"); + return httpClient.build(); + } catch (Exception e) { + LOGGER.error("Failed to create insecure HTTP client, using secure client instead", e); + return sdkHttpClient(config); + } + } + public void closeSdkHttpClient( @Disposes @Identifier("aws-sdk-http-client") SdkHttpClient client) { client.close(); @@ -241,7 +289,10 @@ public StsClientsPool stsClientsPool( @Identifier("aws-sdk-http-client") SdkHttpClient httpClient, StorageConfiguration config, MeterRegistry meterRegistry) { - return new StsClientsPool(config.effectiveClientsCacheMaxSize(), httpClient, meterRegistry); + // Create insecure HTTP client for SSL bypass scenarios + SdkHttpClient insecureHttpClient = createInsecureHttpClient(config); + return new StsClientsPool( + config.effectiveClientsCacheMaxSize(), httpClient, insecureHttpClient, meterRegistry); } /** diff --git a/runtime/service/src/main/java/org/apache/polaris/service/credentials/connection/SigV4ConnectionCredentialVendor.java b/runtime/service/src/main/java/org/apache/polaris/service/credentials/connection/SigV4ConnectionCredentialVendor.java index d85d318cfe..3caf634b94 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/credentials/connection/SigV4ConnectionCredentialVendor.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/credentials/connection/SigV4ConnectionCredentialVendor.java @@ -144,7 +144,8 @@ StsClient getStsClient(@Nonnull SigV4AuthenticationParametersDpo sigv4Params) { // Get STS client from the provider (potentially pooled) // The Polaris service identity credentials are set on the AssumeRole request via // overrideConfiguration, not on the STS client itself - // TODO: Configure proper StsDestination with region/endpoint from sigv4Params - return stsClientProvider.stsClient(StsClientProvider.StsDestination.of(null, null)); + // TODO: Configure proper StsDestination with region/endpoint/ignoreSSLVerification from + // sigv4Params + return stsClientProvider.stsClient(StsClientProvider.StsDestination.of(null, null, null)); } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java index 170274d40a..07f9a6c234 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java @@ -45,8 +45,8 @@ public class StsClientsPool implements StsClientProvider { private static final String CACHE_NAME = "sts-clients"; - private final Cache clients; - private final Function clientBuilder; + private final Cache clients; + private final Function clientBuilder; public StsClientsPool( int clientsCacheMaxSize, SdkHttpClient sdkHttpClient, MeterRegistry meterRegistry) { @@ -56,10 +56,21 @@ public StsClientsPool( Optional.ofNullable(meterRegistry)); } + public StsClientsPool( + int clientsCacheMaxSize, + SdkHttpClient sdkHttpClient, + SdkHttpClient insecureHttpClient, + MeterRegistry meterRegistry) { + this( + clientsCacheMaxSize, + key -> createStsClient(key, sdkHttpClient, insecureHttpClient), + Optional.of(meterRegistry)); + } + @VisibleForTesting StsClientsPool( int maxSize, - Function clientBuilder, + Function clientBuilder, Optional meterRegistry) { this.clientBuilder = clientBuilder; this.clients = @@ -70,11 +81,12 @@ public StsClientsPool( } @Override - public StsClient stsClient(StsDestination destination) { + public StsClient stsClient(StsClientProvider.StsDestination destination) { return clients.get(destination, clientBuilder); } - private static StsClient defaultStsClient(StsDestination parameters, SdkHttpClient sdkClient) { + private static StsClient defaultStsClient( + StsClientProvider.StsDestination parameters, SdkHttpClient sdkClient) { StsClientBuilder builder = StsClient.builder(); builder.httpClient(sdkClient); if (parameters.endpoint().isPresent()) { @@ -98,4 +110,24 @@ static StatsCounter statsCounter(Optional meterRegistry, int maxS } return StatsCounter.disabledStatsCounter(); } + + private static StsClient createStsClient( + StsClientProvider.StsDestination parameters, + SdkHttpClient secureClient, + SdkHttpClient insecureClient) { + StsClientBuilder builder = StsClient.builder(); + + boolean ignoreSSL = parameters.ignoreSSLVerification().orElse(false); + builder.httpClient(ignoreSSL ? insecureClient : secureClient); + + if (parameters.endpoint().isPresent()) { + CompletableFuture endpointFuture = + completedFuture(Endpoint.builder().url(parameters.endpoint().get()).build()); + builder.endpointProvider(params -> endpointFuture); + } + + parameters.region().ifPresent(r -> builder.region(Region.of(r))); + + return builder.build(); + } } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index e13d6fe080..af6bd08791 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -529,4 +529,59 @@ public void testCreateCatalogReturnErrorOnFailure() { resultWithError.getReturnStatus(), resultWithError.getExtraInformation())); } + + @Test + public void testCreateCatalogWithIgnoreSSLVerification() { + // Create a new test service that allows setting S3 endpoints + TestServices sslTestServices = + TestServices.builder() + .config(Map.of("SUPPORTED_CATALOG_STORAGE_TYPES", List.of("S3", "GCS", "AZURE"))) + .config(Map.of("ALLOW_SETTING_S3_ENDPOINTS", Boolean.TRUE)) + .build(); + + AwsStorageConfigInfo awsConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::123456789012:role/my-role") + .setExternalId("externalId") + .setUserArn("userArn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://my-bucket/path/to/data")) + .setStsEndpoint("https://sts.example.com:4443") + .setIgnoreSSLVerification(true) + .build(); + + String catalogName = "ssl-ignore-catalog"; + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) + .setStorageConfigInfo(awsConfigModel) + .build(); + + // Verify catalog creation succeeds with ignoreSSLVerification flag + try (Response response = + sslTestServices + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalog), + sslTestServices.realmContext(), + sslTestServices.securityContext())) { + assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + + // Verify the flag is persisted and returned in GET + try (Response response = + sslTestServices + .catalogsApi() + .getCatalog( + catalogName, sslTestServices.realmContext(), sslTestServices.securityContext())) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + Catalog fetchedCatalog = (Catalog) response.getEntity(); + AwsStorageConfigInfo storageConfig = + (AwsStorageConfigInfo) fetchedCatalog.getStorageConfigInfo(); + assertThat(storageConfig.getIgnoreSSLVerification()).isTrue(); + assertThat(storageConfig.getStsEndpoint()).isEqualTo("https://sts.example.com:4443"); + } + } } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorConfigTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorConfigTest.java new file mode 100644 index 0000000000..94dfe07ec9 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorConfigTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.catalog.io.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import org.apache.polaris.core.storage.StorageAccessProperty; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Configuration; + +/** Tests that S3 client builders apply region/endpoint/path-style/credentials as expected. */ +@SuppressWarnings("unused") +public class ReflectionS3ClientInjectorConfigTest { + + private Map makeProps(String region, String endpoint, boolean pathStyle) { + Map p = new HashMap<>(); + p.put(StorageAccessProperty.CLIENT_REGION.getPropertyName(), region); + p.put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), endpoint); + p.put( + StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName(), Boolean.toString(pathStyle)); + p.put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), "AKIA_TEST"); + p.put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), "SECRET_TEST"); + return p; + } + + // helper removed: deep traversal not used after switching to reflective checks + + @Test + public void testBuildS3Client_appliesConfiguration() throws Exception { + Map props = makeProps("us-west-2", "https://custom.example:9000", true); + + // Verify credentials helper + Method credsMethod = + ReflectionS3ClientInjector.class.getDeclaredMethod("credentialsProviderFrom", Map.class); + credsMethod.setAccessible(true); + Object credsProv = credsMethod.invoke(null, props); + assertTrue(credsProv instanceof AwsCredentialsProvider); + AwsCredentialsProvider prov = (AwsCredentialsProvider) credsProv; + var creds = prov.resolveCredentials(); + assertTrue("AKIA_TEST".equals(creds.accessKeyId())); + assertTrue("SECRET_TEST".equals(creds.secretAccessKey())); + + // Verify S3Configuration helper + Method s3ConfMethod = + ReflectionS3ClientInjector.class.getDeclaredMethod("s3ConfigurationFrom", Map.class); + s3ConfMethod.setAccessible(true); + Object s3Conf = s3ConfMethod.invoke(null, props); + assertTrue(s3Conf instanceof S3Configuration); + assertTrue(((S3Configuration) s3Conf).pathStyleAccessEnabled()); + + // Verify applyRegionAndEndpoint applies to arbitrary builder-like objects + @SuppressWarnings("unused") + class TestBuilder { + public Region regionVal; + public URI endpointVal; + + public TestBuilder region(Region r) { + this.regionVal = r; + return this; + } + + public TestBuilder endpointOverride(URI u) { + this.endpointVal = u; + return this; + } + } + + TestBuilder tb = new TestBuilder(); + Method applyMethod = + ReflectionS3ClientInjector.class.getDeclaredMethod( + "applyRegionAndEndpoint", Object.class, Map.class); + applyMethod.setAccessible(true); + applyMethod.invoke(null, tb, props); + assertEquals("us-west-2", tb.regionVal.id()); + assertEquals(new URI("https://custom.example:9000"), tb.endpointVal); + } + + @Test + public void testBuildS3AsyncClient_appliesConfiguration() throws Exception { + Map props = makeProps("eu-central-1", "https://async.example:9000", false); + + // credentials + Method credsMethod = + ReflectionS3ClientInjector.class.getDeclaredMethod("credentialsProviderFrom", Map.class); + credsMethod.setAccessible(true); + Object credsProv = credsMethod.invoke(null, props); + assertTrue(credsProv instanceof AwsCredentialsProvider); + AwsCredentialsProvider prov = (AwsCredentialsProvider) credsProv; + var creds = prov.resolveCredentials(); + assertEquals("AKIA_TEST", creds.accessKeyId()); + + // s3 config + Method s3ConfMethod = + ReflectionS3ClientInjector.class.getDeclaredMethod("s3ConfigurationFrom", Map.class); + s3ConfMethod.setAccessible(true); + Object s3Conf = s3ConfMethod.invoke(null, props); + assertTrue(s3Conf instanceof S3Configuration); + assertTrue(!((S3Configuration) s3Conf).pathStyleAccessEnabled()); + + // applyRegionAndEndpoint on TestBuilder + @SuppressWarnings("unused") + class TestBuilder { + public Region regionVal; + public URI endpointVal; + + public TestBuilder region(Region r) { + this.regionVal = r; + return this; + } + + public TestBuilder endpointOverride(URI u) { + this.endpointVal = u; + return this; + } + } + + TestBuilder tb = new TestBuilder(); + Method applyMethod = + ReflectionS3ClientInjector.class.getDeclaredMethod( + "applyRegionAndEndpoint", Object.class, Map.class); + applyMethod.setAccessible(true); + applyMethod.invoke(null, tb, props); + assertEquals("eu-central-1", tb.regionVal.id()); + assertEquals(new URI("https://async.example:9000"), tb.endpointVal); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorTest.java new file mode 100644 index 0000000000..ae4da15b0a --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/s3/ReflectionS3ClientInjectorTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.catalog.io.s3; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; + +/** Unit tests for ReflectionS3ClientInjector. */ +public class ReflectionS3ClientInjectorTest { + + private static class DummyS3FileIO { + // Simulate Iceberg's SerializableSupplier and SerializableSupplier + public Supplier s3; + public Supplier s3Async; + } + + @Test + public void testInjectSupplierIntoS3FileIO_replacesSuppliers() { + DummyS3FileIO fileIO = new DummyS3FileIO(); + + // Build simple clients with default builders (they won't be used to actually call network) + S3Client prebuilt = S3Client.builder().build(); + S3AsyncClient prebuiltAsync = S3AsyncClient.builder().build(); + + boolean injected = + ReflectionS3ClientInjector.injectSupplierIntoS3FileIO(fileIO, prebuilt, prebuiltAsync); + assertTrue(injected, "Expected supplier injection to return true"); + + // The injected suppliers should return the prebuilt instances + Supplier s3Supplier = fileIO.s3; + Supplier s3AsyncSupplier = fileIO.s3Async; + + assertSame(prebuilt, s3Supplier.get()); + assertSame(prebuiltAsync, s3AsyncSupplier.get()); + } +} diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index 59baaf99d8..dbef556159 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -1139,6 +1139,14 @@ components: Whether S3 requests to files in this catalog should use 'path-style addressing for buckets'. example: true default: false + ignoreSSLVerification: + type: boolean + description: >- + Whether SSL certificate verification should be disabled for STS and S3 endpoints (optional). + WARNING: This should only be used for development and testing environments with self-signed certificates. + Disabling SSL verification in production environments compromises security. + example: false + default: false AzureStorageConfigInfo: type: object From 331ce313290d11d574d1a4262354df85dcf19b8d Mon Sep 17 00:00:00 2001 From: rohangoli <27857671+rohangoli@users.noreply.github.com> Date: Mon, 13 Oct 2025 21:20:12 -0700 Subject: [PATCH 2/5] Update STS Role Pattern to match ECS --- .../aws/AwsStorageConfigurationInfo.java | 12 ++++++--- .../service/entity/CatalogEntityTest.java | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java index 982ab7f828..ca8f2c1c43 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java @@ -46,7 +46,11 @@ public static ImmutableAwsStorageConfigurationInfo.Builder builder() { // Technically, it should be ^arn:(aws|aws-cn|aws-us-gov):iam::(\d{12}):role/.+$, @JsonIgnore - public static final String ROLE_ARN_PATTERN = "^arn:(aws|aws-us-gov):iam::(\\d{12}):role/.+$"; + // Account id may be a 12-digit AWS account number or a vendor-specific namespace that must + // not be purely numeric (must start with a letter, underscore or hyphen followed by allowed + // chars). + public static final String ROLE_ARN_PATTERN = + "^(arn|urn):(aws|aws-us-gov|ecs):iam::((\\d{12})|([a-zA-Z_-][a-zA-Z0-9_-]*)):role/.+$"; private static final Pattern ROLE_ARN_PATTERN_COMPILED = Pattern.compile(ROLE_ARN_PATTERN); @@ -125,7 +129,8 @@ public String getAwsAccountId() { if (arn != null) { Matcher matcher = ROLE_ARN_PATTERN_COMPILED.matcher(arn); checkState(matcher.matches()); - return matcher.group(2); + // group(3) is the account identifier (either 12-digit AWS account or vendor namespace) + return matcher.group(3); } return null; } @@ -137,7 +142,8 @@ public String getAwsPartition() { if (arn != null) { Matcher matcher = ROLE_ARN_PATTERN_COMPILED.matcher(arn); checkState(matcher.matches()); - return matcher.group(1); + // group(2) captures the partition (e.g. aws, aws-us-gov, ecs) + return matcher.group(2); } return null; } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java b/runtime/service/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java index 47e61f5734..d0b2445872 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/entity/CatalogEntityTest.java @@ -285,6 +285,31 @@ public void testInvalidArn(String roleArn) { .hasMessage(expectedMessage); } + @Test + public void testUrnRoleArnAccepted() { + String urnRole = "urn:ecs:iam::test-namespace:role/test-role"; + String baseLocation = "s3://externally-owned-bucket"; + AwsStorageConfigInfo awsStorageConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn(urnRole) + .setExternalId("externalId") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(baseLocation)) + .build(); + + CatalogProperties prop = new CatalogProperties(baseLocation); + Catalog awsCatalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName("name") + .setProperties(prop) + .setStorageConfigInfo(awsStorageConfigModel) + .build(); + + Assertions.assertThatCode(() -> CatalogEntity.fromCatalog(realmConfig, awsCatalog)) + .doesNotThrowAnyException(); + } + @Test public void testCatalogTypeDefaultsToInternal() { String baseLocation = "s3://test-bucket/path"; From 2f06810182d3245aa4ec79d793d019791470d70f Mon Sep 17 00:00:00 2001 From: rohangoli <27857671+rohangoli@users.noreply.github.com> Date: Mon, 13 Oct 2025 21:54:28 -0700 Subject: [PATCH 3/5] Fix AssumeRole Invalid Policy Resource for ECS S3 Bucket --- .../aws/AwsCredentialsStorageIntegration.java | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index 451ce1b922..935952e323 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -189,14 +189,24 @@ private IamPolicy policyString( Map bucketGetLocationStatementBuilder = new HashMap<>(); String arnPrefix = arnPrefixForPartition(awsPartition); + boolean isEcsPartition = "ecs".equals(awsPartition); Stream.concat(readLocations.stream(), writeLocations.stream()) .distinct() .forEach( location -> { URI uri = URI.create(location); - allowGetObjectStatementBuilder.addResource( - IamResource.create( - arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + // Some on-prem S3/STSc implementations (for example ECS) do not accept object ARNs + // that include the path portion (bucket/key/*). For those, scope object permissions + // to + // the whole bucket (bucket/*) and rely on s3:prefix conditions for finer granularity. + if (isEcsPartition) { + allowGetObjectStatementBuilder.addResource( + IamResource.create(arnPrefix + StorageUtil.getBucket(uri) + "/*")); + } else { + allowGetObjectStatementBuilder.addResource( + IamResource.create( + arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + } final var bucket = arnPrefix + StorageUtil.getBucket(uri); if (allowList) { bucketListStatementBuilder @@ -230,9 +240,14 @@ private IamPolicy policyString( writeLocations.forEach( location -> { URI uri = URI.create(location); - allowPutObjectStatementBuilder.addResource( - IamResource.create( - arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + if (isEcsPartition) { + allowPutObjectStatementBuilder.addResource( + IamResource.create(arnPrefix + StorageUtil.getBucket(uri) + "/*")); + } else { + allowPutObjectStatementBuilder.addResource( + IamResource.create( + arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + } }); policyBuilder.addStatement(allowPutObjectStatementBuilder.build()); } @@ -253,7 +268,13 @@ private IamPolicy policyString( } private static String arnPrefixForPartition(String awsPartition) { - return String.format("arn:%s:s3:::", awsPartition != null ? awsPartition : "aws"); + // Some on-prem S3 compatible systems (e.g. ECS) use a non-standard partition value + // but expect S3 resource ARNs to use the 'aws' partition form (arn:aws:s3:::bucket). + String partition = awsPartition != null ? awsPartition : "aws"; + if ("ecs".equals(partition)) { + partition = "aws"; + } + return String.format("arn:%s:s3:::", partition); } private static @Nonnull String parseS3Path(URI uri) { From 5dc10508a2ef6c2d02d1f24f3f72d1075992649d Mon Sep 17 00:00:00 2001 From: rohangoli <27857671+rohangoli@users.noreply.github.com> Date: Tue, 14 Oct 2025 20:13:10 -0700 Subject: [PATCH 4/5] Handle Non-Standard STS XML Response --- .../aws/AwsCredentialsStorageIntegration.java | 48 +++++--- .../core/storage/aws/StsResponseCapture.java | 42 +++++++ .../aws/StsResponseCaptureInterceptor.java | 95 ++++++++++++++++ .../core/storage/aws/StsXmlParser.java | 104 ++++++++++++++++++ .../StsResponseCaptureInterceptorTest.java | 96 ++++++++++++++++ .../storage/aws/StsResponseCaptureTest.java | 38 +++++++ .../core/storage/aws/StsXmlParserTest.java | 68 ++++++++++++ .../PolarisConfigurationStoreTest.java | 2 +- .../service/storage/aws/StsClientsPool.java | 11 ++ .../config/DefaultConfigurationStoreTest.java | 3 +- 10 files changed, 492 insertions(+), 15 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCapture.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptor.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsXmlParser.java create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptorTest.java create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureTest.java create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsXmlParserTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index 935952e323..1a005ae1e2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -109,19 +109,41 @@ public AccessConfig getSubscopedCreds( storageConfig.getIgnoreSSLVerification())); AssumeRoleResponse response = stsClient.assumeRole(request.build()); - accessConfig.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId()); - accessConfig.put( - StorageAccessProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); - accessConfig.put(StorageAccessProperty.AWS_TOKEN, response.credentials().sessionToken()); - Optional.ofNullable(response.credentials().expiration()) - .ifPresent( - i -> { - accessConfig.put( - StorageAccessProperty.EXPIRATION_TIME, String.valueOf(i.toEpochMilli())); - accessConfig.put( - StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, - String.valueOf(i.toEpochMilli())); - }); + if (response != null && response.credentials() != null) { + accessConfig.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + accessConfig.put( + StorageAccessProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + accessConfig.put(StorageAccessProperty.AWS_TOKEN, response.credentials().sessionToken()); + Optional.ofNullable(response.credentials().expiration()) + .ifPresent( + i -> { + accessConfig.put( + StorageAccessProperty.EXPIRATION_TIME, String.valueOf(i.toEpochMilli())); + accessConfig.put( + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + String.valueOf(i.toEpochMilli())); + }); + } else { + // Try to recover by reading raw STS body captured by interceptor + try { + String raw = org.apache.polaris.core.storage.aws.StsResponseCapture.getLastBody(); + if (raw != null && !raw.isBlank()) { + try { + var parsed = + org.apache.polaris.core.storage.aws.StsXmlParser.parseToAccessConfig(raw); + // merge parsed credentials into accessConfig builder + parsed.credentials().forEach((k, v) -> accessConfig.putCredential(k, v)); + parsed.internalProperties().forEach((k, v) -> accessConfig.putInternalProperty(k, v)); + parsed.extraProperties().forEach((k, v) -> accessConfig.putExtraProperty(k, v)); + parsed.expiresAt().ifPresent(accessConfig::expiresAt); + } catch (Exception ignore) { + // parsing failed - ignore and fallthrough + } + } + } finally { + org.apache.polaris.core.storage.aws.StsResponseCapture.clear(); + } + } } if (region != null) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCapture.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCapture.java new file mode 100644 index 0000000000..68ee143b67 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCapture.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.aws; + +/** + * Simple thread-local holder for the last raw STS HTTP response body captured by an + * ExecutionInterceptor. This is intended as a pragmatic bridge so synchronous SDK calls can consult + * the raw response body when unmarshalling produced an unexpected null result. + */ +public final class StsResponseCapture { + private static final ThreadLocal LAST_BODY = new ThreadLocal<>(); + + private StsResponseCapture() {} + + public static void setLastBody(String body) { + LAST_BODY.set(body); + } + + public static String getLastBody() { + return LAST_BODY.get(); + } + + public static void clear() { + LAST_BODY.remove(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptor.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptor.java new file mode 100644 index 0000000000..f83d99687c --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptor.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.aws; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import software.amazon.awssdk.core.interceptor.Context.AfterTransmission; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; + +/** + * ExecutionInterceptor that captures the raw HTTP response body for STS calls and saves it into + * StsResponseCapture (thread-local). This allows calling code to inspect the raw response when the + * SDK's unmarshalling yields null credentials. + */ +public class StsResponseCaptureInterceptor implements ExecutionInterceptor { + + @Override + public void afterTransmission( + AfterTransmission context, ExecutionAttributes executionAttributes) { + try { + // Use reflection to call context.httpResponse() because SDK versions expose different + // types/APIs + Method httpRespMethod = context.getClass().getMethod("httpResponse"); + Object httpResp = httpRespMethod.invoke(context); + if (httpResp != null) { + try { + Optional content = OptionalUtils.safeGetContent(httpResp); + if (content.isPresent()) { + try (InputStream in = content.get(); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] buf = new byte[8192]; + int r; + while ((r = in.read(buf)) != -1) { + out.write(buf, 0, r); + } + String resp = new String(out.toByteArray(), StandardCharsets.UTF_8); + StsResponseCapture.setLastBody(resp); + } + } + } catch (Exception e) { + // best-effort; don't fail the call because of capture problems + } + } + } catch (Throwable t) { + // swallow - capture is non-fatal + } + } +} + +// Small utility to safely read content from SdkHttpResponse across SDK versions. +final class OptionalUtils { + static Optional safeGetContent(Object httpResp) { + try { + Method contentMethod = httpResp.getClass().getMethod("content"); + Object val = contentMethod.invoke(httpResp); + if (val == null) return Optional.empty(); + if (val instanceof Optional) { + Optional anyOpt = (Optional) val; + if (anyOpt.isPresent() && anyOpt.get() instanceof InputStream) { + return Optional.of((InputStream) anyOpt.get()); + } + return Optional.empty(); + } + // Some SDKs may return an InputStream directly + if (val instanceof InputStream) { + return Optional.of((InputStream) val); + } + } catch (NoSuchMethodException nsme) { + // ignore + } catch (Exception e) { + // ignore other reflection errors + } + return Optional.empty(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsXmlParser.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsXmlParser.java new file mode 100644 index 0000000000..766b025f1b --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/StsXmlParser.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.aws; + +import java.time.Instant; +import java.util.Objects; +import org.apache.polaris.core.storage.AccessConfig; + +/** Utility to parse STS AssumeRoleResponse XML (namespaced or not) into an AccessConfig. */ +public final class StsXmlParser { + private StsXmlParser() {} + + public static AccessConfig parseToAccessConfig(String xml) throws Exception { + Objects.requireNonNull(xml); + var dbf = javax.xml.parsers.DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + var db = dbf.newDocumentBuilder(); + try (java.io.ByteArrayInputStream in = + new java.io.ByteArrayInputStream(xml.getBytes(java.nio.charset.StandardCharsets.UTF_8))) { + org.w3c.dom.Document doc = db.parse(in); + javax.xml.xpath.XPath xPath = javax.xml.xpath.XPathFactory.newInstance().newXPath(); + + String accessKeyId = + (String) + xPath.evaluate( + "//*[local-name() = 'Credentials']/*[local-name() = 'AccessKeyId']/text()", + doc, + javax.xml.xpath.XPathConstants.STRING); + String secretAccessKey = + (String) + xPath.evaluate( + "//*[local-name() = 'Credentials']/*[local-name() = 'SecretAccessKey']/text()", + doc, + javax.xml.xpath.XPathConstants.STRING); + String sessionToken = + (String) + xPath.evaluate( + "//*[local-name() = 'Credentials']/*[local-name() = 'SessionToken']/text()", + doc, + javax.xml.xpath.XPathConstants.STRING); + String expiration = + (String) + xPath.evaluate( + "//*[local-name() = 'Credentials']/*[local-name() = 'Expiration']/text()", + doc, + javax.xml.xpath.XPathConstants.STRING); + + if (accessKeyId == null || accessKeyId.isBlank()) { + throw new IllegalArgumentException("No AccessKeyId found in STS response XML"); + } + + AccessConfig.Builder builder = AccessConfig.builder(); + builder.putCredential( + org.apache.polaris.core.storage.StorageAccessProperty.AWS_KEY_ID.getPropertyName(), + accessKeyId.trim()); + if (secretAccessKey != null && !secretAccessKey.isBlank()) { + builder.putCredential( + org.apache.polaris.core.storage.StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), + secretAccessKey.trim()); + } + if (sessionToken != null && !sessionToken.isBlank()) { + builder.putCredential( + org.apache.polaris.core.storage.StorageAccessProperty.AWS_TOKEN.getPropertyName(), + sessionToken.trim()); + } + if (expiration != null && !expiration.isBlank()) { + try { + Instant i = Instant.parse(expiration.trim()); + builder.putCredential( + org.apache.polaris.core.storage.StorageAccessProperty.EXPIRATION_TIME + .getPropertyName(), + String.valueOf(i.toEpochMilli())); + builder.putCredential( + org.apache.polaris.core.storage.StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS + .getPropertyName(), + String.valueOf(i.toEpochMilli())); + builder.expiresAt(i); + } catch (Exception e) { + builder.putExtraProperty( + org.apache.polaris.core.storage.StorageAccessProperty.EXPIRATION_TIME + .getPropertyName(), + expiration.trim()); + } + } + return builder.build(); + } + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptorTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptorTest.java new file mode 100644 index 0000000000..0c94a9e66b --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureInterceptorTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.aws; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayInputStream; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; + +public class StsResponseCaptureInterceptorTest { + + @BeforeEach + public void before() { + StsResponseCapture.clear(); + } + + // Local interface used by the dynamic proxy to provide content() + @SuppressWarnings("unused") + private interface ContentHolder { + java.util.Optional content(); + } + + @Test + public void testAfterTransmissionCapturesBody() { + StsResponseCaptureInterceptor interceptor = new StsResponseCaptureInterceptor(); + try { + Class afterCls = software.amazon.awssdk.core.interceptor.Context.AfterTransmission.class; + Class httpRespType = afterCls.getMethod("httpResponse").getReturnType(); + + // Build a response proxy that implements both the SDK response return type and ContentHolder + Object respProxy = + java.lang.reflect.Proxy.newProxyInstance( + httpRespType.getClassLoader(), + new Class[] {httpRespType, ContentHolder.class}, + (proxy, method, args) -> { + if ("content".equals(method.getName())) { + return Optional.of( + new ByteArrayInputStream( + "raw-body-xyz".getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + return null; + }); + + // Now build an AfterTransmission proxy that returns the respProxy from httpResponse() + Object ctxProxy = + java.lang.reflect.Proxy.newProxyInstance( + afterCls.getClassLoader(), + new Class[] {afterCls}, + (proxy, method, args) -> { + if ("httpResponse".equals(method.getName())) { + return respProxy; + } + return null; + }); + + interceptor.afterTransmission( + (software.amazon.awssdk.core.interceptor.Context.AfterTransmission) ctxProxy, + new ExecutionAttributes()); + } catch (Throwable t) { + throw new AssertionError(t); + } + // Because we cast via reflection, ensure the captured body is set + assertThat(StsResponseCapture.getLastBody()).isEqualTo("raw-body-xyz"); + } + + @Test + public void testAfterTransmissionSilentlyIgnoresUnknownContext() { + StsResponseCaptureInterceptor interceptor = new StsResponseCaptureInterceptor(); + // Create a mock AfterTransmission that returns null for httpResponse() + software.amazon.awssdk.core.interceptor.Context.AfterTransmission nullRespContext = + org.mockito.Mockito.mock( + software.amazon.awssdk.core.interceptor.Context.AfterTransmission.class); + org.mockito.Mockito.when(nullRespContext.httpResponse()).thenReturn(null); + interceptor.afterTransmission(nullRespContext, new ExecutionAttributes()); + assertThat(StsResponseCapture.getLastBody()).isNull(); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureTest.java new file mode 100644 index 0000000000..ce724ef13e --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsResponseCaptureTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.aws; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +public class StsResponseCaptureTest { + + @Test + public void testThreadLocalSetGetClear() { + StsResponseCapture.clear(); + assertThat(StsResponseCapture.getLastBody()).isNull(); + + StsResponseCapture.setLastBody("hello"); + assertThat(StsResponseCapture.getLastBody()).isEqualTo("hello"); + + StsResponseCapture.clear(); + assertThat(StsResponseCapture.getLastBody()).isNull(); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsXmlParserTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsXmlParserTest.java new file mode 100644 index 0000000000..03c45ec9c0 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/StsXmlParserTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.aws; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.polaris.core.storage.AccessConfig; +import org.junit.jupiter.api.Test; + +public class StsXmlParserTest { + + private static final String SAMPLE_XML = + "" + + "" + + "" + + "" + + "ASIA9BC3BEA6F9D5B811" + + "2025-10-15T02:38:50Z" + + "sekrit" + + "the-token-value" + + "" + + "" + + ""; + + @Test + public void testParseValidXml() throws Exception { + AccessConfig cfg = StsXmlParser.parseToAccessConfig(SAMPLE_XML); + assertThat(cfg.get(org.apache.polaris.core.storage.StorageAccessProperty.AWS_KEY_ID)) + .isEqualTo("ASIA9BC3BEA6F9D5B811"); + assertThat(cfg.get(org.apache.polaris.core.storage.StorageAccessProperty.AWS_SECRET_KEY)) + .isEqualTo("sekrit"); + assertThat(cfg.get(org.apache.polaris.core.storage.StorageAccessProperty.AWS_TOKEN)) + .isEqualTo("the-token-value"); + assertThat(cfg.expiresAt()).isPresent(); + } + + @Test + public void testParseMissingAccessKeyThrows() { + String bad = + ""; + try { + StsXmlParser.parseToAccessConfig(bad); + } catch (IllegalArgumentException e) { + assertThat(e).hasMessageContaining("No AccessKeyId"); + return; + } catch (Exception e) { + // fail + throw new AssertionError("unexpected exception", e); + } + throw new AssertionError("expected IllegalArgumentException for missing AccessKeyId"); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/PolarisConfigurationStoreTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/PolarisConfigurationStoreTest.java index 612b8716bf..4a2f5c61e8 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/PolarisConfigurationStoreTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/PolarisConfigurationStoreTest.java @@ -163,10 +163,10 @@ public void testEntityOverrides() { PolarisConfigurationStore store = new PolarisConfigurationStore() { + @SuppressWarnings("unchecked") @Override public @Nullable T getConfiguration( @Nonnull RealmContext realmContext, String configName) { - //noinspection unchecked return (T) Map.of("key2", "config-value2").get(configName); } }; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java index 07f9a6c234..c47196e667 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/StsClientsPool.java @@ -88,6 +88,11 @@ public StsClient stsClient(StsClientProvider.StsDestination destination) { private static StsClient defaultStsClient( StsClientProvider.StsDestination parameters, SdkHttpClient sdkClient) { StsClientBuilder builder = StsClient.builder(); + // capture raw STS HTTP responses for troubleshooting non-standard endpoints + builder.overrideConfiguration( + b -> + b.addExecutionInterceptor( + new org.apache.polaris.core.storage.aws.StsResponseCaptureInterceptor())); builder.httpClient(sdkClient); if (parameters.endpoint().isPresent()) { CompletableFuture endpointFuture = @@ -117,6 +122,12 @@ private static StsClient createStsClient( SdkHttpClient insecureClient) { StsClientBuilder builder = StsClient.builder(); + // capture raw STS HTTP responses for troubleshooting non-standard endpoints + builder.overrideConfiguration( + b -> + b.addExecutionInterceptor( + new org.apache.polaris.core.storage.aws.StsResponseCaptureInterceptor())); + boolean ignoreSSL = parameters.ignoreSSLVerification().orElse(false); builder.httpClient(ignoreSSL ? insecureClient : secureClient); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java b/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java index e1f76417aa..e1031d53a7 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java @@ -176,7 +176,8 @@ public void testInjectedFeaturesConfiguration() { } @Test - public void testRegisterAndUseFeatureConfigurations() { + @SuppressWarnings("deprecation") + public void testRegisterAndUseFeatureConfigurations() { String prefix = "testRegisterAndUseFeatureConfigurations"; FeatureConfiguration safeConfig = From 1e027a66d0c5daba7569f3f3cccf46b38802e7d4 Mon Sep 17 00:00:00 2001 From: rohangoli <27857671+rohangoli@users.noreply.github.com> Date: Tue, 14 Oct 2025 22:29:38 -0700 Subject: [PATCH 5/5] fix spotlessApply --- .../polaris/service/config/DefaultConfigurationStoreTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java b/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java index e1031d53a7..fbecfbed46 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/config/DefaultConfigurationStoreTest.java @@ -176,8 +176,8 @@ public void testInjectedFeaturesConfiguration() { } @Test - @SuppressWarnings("deprecation") - public void testRegisterAndUseFeatureConfigurations() { + @SuppressWarnings("deprecation") + public void testRegisterAndUseFeatureConfigurations() { String prefix = "testRegisterAndUseFeatureConfigurations"; FeatureConfiguration safeConfig =