Add S3 path style access support to exchange spooling

trinodb · Oct 17, 2022 · 0c21fef · 0c21fef
1 parent 9da2a42
commit 0c21fef
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 0 deletions.
diff --git a/...nge-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/ExchangeS3Config.java b/...nge-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/ExchangeS3Config.java
@@ -49,6 +49,7 @@ public class ExchangeS3Config
     private int asyncClientConcurrency = 100;
     private int asyncClientMaxPendingConnectionAcquires = 10000;
     private Duration connectionAcquisitionTimeout = new Duration(1, MINUTES);
+    private boolean s3PathStyleAccess;
     private Optional<String> gcsJsonKeyFilePath = Optional.empty();
     private Optional<String> gcsJsonKey = Optional.empty();
 
@@ -223,6 +224,19 @@ public ExchangeS3Config setConnectionAcquisitionTimeout(Duration connectionAcqui
         return this;
     }
 
+    public boolean isS3PathStyleAccess()
+    {
+        return s3PathStyleAccess;
+    }
+
+    @Config("exchange.s3.path-style-access")
+    @ConfigDescription("Use path-style access for all request to S3")
+    public ExchangeS3Config setS3PathStyleAccess(boolean s3PathStyleAccess)
+    {
+        this.s3PathStyleAccess = s3PathStyleAccess;
+        return this;
+    }
+
     public Optional<String> getGcsJsonKeyFilePath()
     {
         return gcsJsonKeyFilePath;

diff --git a/...tem/src/main/java/io/trino/plugin/exchange/filesystem/s3/S3FileSystemExchangeStorage.java b/...tem/src/main/java/io/trino/plugin/exchange/filesystem/s3/S3FileSystemExchangeStorage.java
@@ -53,6 +53,7 @@
 import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.S3Configuration;
 import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.AbortMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
@@ -171,6 +172,7 @@ public S3FileSystemExchangeStorage(S3FileSystemExchangeStorageStats stats, Excha
         S3AsyncClient client = createS3AsyncClient(
                 credentialsProvider,
                 overrideConfig,
+                config.isS3PathStyleAccess(),
                 config.getAsyncClientConcurrency(),
                 config.getAsyncClientMaxPendingConnectionAcquires(),
                 config.getConnectionAcquisitionTimeout());
@@ -471,13 +473,17 @@ private S3Client createS3Client(AwsCredentialsProvider credentialsProvider, Clie
     private S3AsyncClient createS3AsyncClient(
             AwsCredentialsProvider credentialsProvider,
             ClientOverrideConfiguration overrideConfig,
+            boolean isS3PathStyleAccess,
             int maxConcurrency,
             int maxPendingConnectionAcquires,
             Duration connectionAcquisitionTimeout)
     {
         S3AsyncClientBuilder clientBuilder = S3AsyncClient.builder()
                 .credentialsProvider(credentialsProvider)
                 .overrideConfiguration(overrideConfig)
+                .serviceConfiguration(S3Configuration.builder()
+                        .pathStyleAccessEnabled(isS3PathStyleAccess)
+                        .build())
                 .httpClientBuilder(NettyNioAsyncHttpClient.builder()
                         .maxConcurrency(maxConcurrency)
                         .maxPendingConnectionAcquires(maxPendingConnectionAcquires)

diff --git a/...filesystem/src/test/java/io/trino/plugin/exchange/filesystem/s3/TestExchangeS3Config.java b/...filesystem/src/test/java/io/trino/plugin/exchange/filesystem/s3/TestExchangeS3Config.java
@@ -47,6 +47,7 @@ public void testDefaults()
                 .setAsyncClientConcurrency(100)
                 .setAsyncClientMaxPendingConnectionAcquires(10000)
                 .setConnectionAcquisitionTimeout(new Duration(1, MINUTES))
+                .setS3PathStyleAccess(false)
                 .setGcsJsonKeyFilePath(null)
                 .setGcsJsonKey(null));
     }
@@ -68,6 +69,7 @@ public void testExplicitPropertyMappings()
                 .put("exchange.s3.async-client-concurrency", "202")
                 .put("exchange.s3.async-client-max-pending-connection-acquires", "999")
                 .put("exchange.s3.async-client-connection-acquisition-timeout", "5m")
+                .put("exchange.s3.path-style-access", "true")
                 .put("exchange.gcs.json-key-file-path", "/path/to/gcs_keyfile.json")
                 .put("exchange.gcs.json-key", "{}")
                 .buildOrThrow();
@@ -86,6 +88,7 @@ public void testExplicitPropertyMappings()
                 .setAsyncClientConcurrency(202)
                 .setAsyncClientMaxPendingConnectionAcquires(999)
                 .setConnectionAcquisitionTimeout(new Duration(5, MINUTES))
+                .setS3PathStyleAccess(true)
                 .setGcsJsonKeyFilePath("/path/to/gcs_keyfile.json")
                 .setGcsJsonKey("{}");