Handle Non-Standard STS XML Response - S3 Compatible STS Endpoint

api/management-model/src/test/java/org/apache/polaris/core/admin/model/CatalogSerializationTest.java

@@ -71,6 +71,7 @@ public void testJsonFormat() throws JsonProcessingException {
                 + "\"storageConfigInfo\":{"
                 + "\"roleArn\":\"arn:aws:iam::123456789012:role/test-role\","
                 + "\"pathStyleAccess\":false,"
+                + "\"ignoreSSLVerification\":false,"
                 + "\"storageType\":\"S3\","
                 + "\"allowedLocations\":[]"
                 + "}}");

getting-started/minio-https/README.md

@@ -0,0 +1,94 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Getting Started with Apache Polaris and MinIO (HTTPS)
+
+## Overview
+
+This example uses MinIO (HTTPS) as a storage provider with Polaris, it can be used for other S3-Compatible Storage API with Ignoring SSL Verification for Development/Test Purposes
+
+Spark is used as a query engine. This example assumes a local Spark installation.
+See the [Spark Notebooks Example](../spark/README.md) for a more advanced Spark setup.
+
+## Starting the Example
+
+1. Build the Polaris server image if it's not already present locally:
+
+    ```shell
+    ./gradlew \
+       :polaris-server:assemble \
+       :polaris-server:quarkusAppPartsBuild --rerun \
+       -Dquarkus.container-image.build=true
+    ```
+
+2. Start the docker compose group by running the following command from the root of the repository:
+
+    ```shell
+    docker compose -f getting-started/minio/docker-compose.yml up
+    ```
+
+## Connecting From Spark
+
+```shell
+bin/spark-sql \
+    --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.iceberg:iceberg-aws-bundle:1.9.0 \
+    --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
+    --conf spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog \
+    --conf spark.sql.catalog.polaris.type=rest \
+    --conf spark.sql.catalog.polaris.uri=http://localhost:8181/api/catalog \
+    --conf spark.sql.catalog.polaris.token-refresh-enabled=false \
+    --conf spark.sql.catalog.polaris.warehouse=quickstart_catalog \
+    --conf spark.sql.catalog.polaris.scope=PRINCIPAL_ROLE:ALL \
+    --conf spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation=vended-credentials \
+    --conf spark.sql.catalog.polaris.credential=root:s3cr3t \
+    --conf spark.sql.catalog.polaris.client.region=irrelevant
+```
+
+Note: `s3cr3t` is defined as the password for the `root` users in the `docker-compose.yml` file.
+
+Note: The `client.region` configuration is required for the AWS S3 client to work, but it is not used in this example
+since MinIO does not require a specific region.
+
+## Running Queries
+
+Run inside the Spark SQL shell:
+
+```
+spark-sql (default)> use polaris;
+Time taken: 0.837 seconds
+
+spark-sql ()> create namespace ns;
+Time taken: 0.374 seconds
+
+spark-sql ()> create table ns.t1 as select 'abc';
+Time taken: 2.192 seconds
+
+spark-sql ()> select * from ns.t1;
+abc
+Time taken: 0.579 seconds, Fetched 1 row(s)
+```
+
+## MinIO Endpoints
+
+Note that the catalog configuration defined in the `docker-compose.yml` contains
+different endpoints for the Polaris Server and the client (Spark). Specifically,
+the client endpoint is `https://localhost:9000`, but `endpointInternal` is `https://minio:9000`.
+
+This is necessary because clients running on `localhost` do not normally see service
+names (such as `minio`) that are internal to the docker compose environment.

getting-started/minio-https/docker-compose.yml

@@ -0,0 +1,150 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+services:
+
+  minio:
+    image: quay.io/minio/minio:latest
+    ports:
+      # API port
+      - "9000:9000"
+      # UI port
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: minio_root
+      MINIO_ROOT_PASSWORD: m1n1opwd
+    command:
+      - "server"
+      - "/data"
+      - "--console-address"
+      - ":9001"
+    volumes:
+      # Mount the generated certs volume into MinIO
+      - minio_certs:/root/.minio/certs
+    depends_on:
+      certs-init:
+        condition: service_completed_successfully
+    healthcheck:
+      # Use HTTPS healthcheck now that MinIO will serve TLS on 9000
+      test: ["CMD", "curl", "-k", "https://127.0.0.1:9000/minio/health/live"]
+      interval: 1s
+      timeout: 10s
+
+  polaris:
+    image: apache/polaris:latest
+    ports:
+      # API port
+      - "8181:8181"
+      # Optional, allows attaching a debugger to the Polaris JVM
+      - "5005:5005"
+    depends_on:
+      minio:
+        condition: service_healthy
+      setup_bucket:
+        condition: service_completed_successfully
+    environment:
+      JAVA_DEBUG: true
+      JAVA_DEBUG_PORT: "*:5005"
+      AWS_REGION: us-west-2
+      AWS_ACCESS_KEY_ID: minio_root
+      AWS_SECRET_ACCESS_KEY: m1n1opwd
+      POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t
+      polaris.realm-context.realms: POLARIS
+      quarkus.otel.sdk.disabled: "true"
+      quarkus.log.category."org.apache.polaris".level: DEBUG
+      quarkus.log.category."org.apache.polaris.catalog".level: DEBUG
+      quarkus.log.category."org.apache.polaris.table".level: DEBUG
+    healthcheck:
+      test: ["CMD", "curl", "http://localhost:8182/q/health"]
+      interval: 2s
+      timeout: 10s
+      retries: 10
+      start_period: 10s
+
+  setup_bucket:
+    image: quay.io/minio/mc:latest
+    depends_on:
+      minio:
+        condition: service_healthy
+    entrypoint: "/bin/sh"
+    command:
+      - "-c"
+      - >-
+        echo Creating MinIO bucket...;
+        mc alias set pol https://minio:9000 minio_root m1n1opwd --insecure;
+        mc mb pol/bucket123 --insecure;
+        mc ls pol --insecure;
+        echo Bucket setup complete.;
+
+  # TLS certs are created at `docker compose up` by the certs-init one-shot service
+  certs-init:
+    image: alpine:3.18
+    entrypoint: "/bin/sh"
+    command:
+      - "-c"
+      - >-
+        apk add --no-cache openssl;
+        printf '%s\n' "[req]" "ndistinguished_name = req_distinguished_name" "req_extensions = v3_req" "prompt = no" "" \
+          "[req_distinguished_name]" "CN = localhost" "" "[v3_req]" "subjectAltName = @alt_names" "" "[alt_names]" "DNS.1 = localhost" "DNS.2 = minio" "IP.1 = 127.0.0.1" \
+          > /openssl.cnf;
+        openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+          -keyout /certs/private.key -out /certs/public.crt \
+          -subj "/C=US/ST=State/L=City/O=Org/CN=localhost" -extensions v3_req -config /openssl.cnf;
+        chmod 600 /certs/private.key || true; chmod 644 /certs/public.crt || true;
+        echo Generated certs.;
+    volumes:
+      - minio_certs:/certs
+    restart: "no"
+
+  polaris-setup:
+    image: alpine/curl
+    depends_on:
+      polaris:
+        condition: service_healthy
+    environment:
+      CLIENT_ID: root
+      CLIENT_SECRET: s3cr3t
+      # Use HTTPS endpoints and indicate to Polaris to ignore SSL verification for this self-signed setup
+      STORAGE_CONFIG_INFO: '{"storageType":"S3","endpoint":"https://localhost:9000","endpointInternal":"https://minio:9000","pathStyleAccess":true,"ignoreSSLVerification":true, "stsUnavailable":true}'
+      STORAGE_LOCATION: 's3://bucket123'
+    volumes:
+      - ../assets/polaris/:/polaris
+    entrypoint: "/bin/sh"
+    command:
+      - "-c"
+      - >-
+        chmod +x /polaris/create-catalog.sh;
+        chmod +x /polaris/obtain-token.sh;
+        source /polaris/obtain-token.sh;
+        echo Creating catalog...;
+        /polaris/create-catalog.sh POLARIS $$TOKEN;
+        echo Extra grants...;
+        curl -H "Authorization: Bearer $$TOKEN" -H 'Content-Type: application/json' \
+          -X PUT \
+          http://polaris:8181/api/management/v1/catalogs/quickstart_catalog/catalog-roles/catalog_admin/grants \
+          -d '{"type":"catalog", "privilege":"CATALOG_MANAGE_CONTENT"}';
+        echo Done.;
+
+volumes:
+  minio_certs:
+    driver: local
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: "size=1m,mode=1777"

polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java

@@ -166,6 +166,7 @@ private StorageConfigInfo getStorageInfo(Map<String, String> internalProperties)
             .setRegion(awsConfig.getRegion())
             .setEndpoint(awsConfig.getEndpoint())
             .setStsEndpoint(awsConfig.getStsEndpoint())
+            .setIgnoreSSLVerification(awsConfig.getIgnoreSSLVerification())
             .setPathStyleAccess(awsConfig.getPathStyleAccess())
             .setStsUnavailable(awsConfig.getStsUnavailable())
             .setEndpointInternal(awsConfig.getEndpointInternal())
@@ -315,6 +316,7 @@ public Builder setStorageConfigurationInfo(
                     .pathStyleAccess(awsConfigModel.getPathStyleAccess())
                     .stsUnavailable(awsConfigModel.getStsUnavailable())
                     .endpointInternal(awsConfigModel.getEndpointInternal())
+                    .ignoreSSLVerification(awsConfigModel.getIgnoreSSLVerification())
                     .build();
             config = awsConfig;
             break;