Add configurable application id to native filesystems

docs/src/main/sphinx/object-storage/file-system-azure.md

@@ -47,6 +47,9 @@ system support:
     Azure from every node. Defaults to double the number of processors on the
     node. Minimum `1`. Use this property to reduce the number of requests when
     you encounter rate limiting issues.
+* - `azure.application-id`
+  - Specify the application identifier appended to the `User-Agent` header
+    for all requests sent to Azure Storage. Defaults to `Trino`. 
 :::
 
 (azure-access-key-authentication)=

docs/src/main/sphinx/object-storage/file-system-gcs.md

@@ -48,6 +48,9 @@ Storage file system support:
 * - `gcs.batch-size`
   - Number of blobs to delete per batch. Defaults to 100. [Recommended batch
     size](https://cloud.google.com/storage/docs/batch) is 100.
+* - `gcs.application-id`
+  - Specify the application identifier appended to the `User-Agent` header
+    for all requests sent to Google Cloud Storage. Defaults to `Trino`.
 :::
 
 ## Authentication

docs/src/main/sphinx/object-storage/file-system-s3.md

@@ -102,6 +102,9 @@ support:
     Trino on Amazon EKS and using [IAM roles for service accounts
     (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html)
     Defaults to `false`.
+* - `s3.application-id`
+  - Specify the application identifier appended to the `User-Agent` header 
+    for all requests sent to S3. Defaults to `Trino`.
 :::
 
 ## Authentication

lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureFileSystemConfig.java

@@ -20,6 +20,7 @@
 import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
+import jakarta.validation.constraints.Size;
 
 public class AzureFileSystemConfig
 {
@@ -37,6 +38,7 @@ public enum AuthType
     private int maxWriteConcurrency = 8;
     private DataSize maxSingleUploadSize = DataSize.of(4, Unit.MEGABYTE);
     private Integer maxHttpRequests = 2 * Runtime.getRuntime().availableProcessors();
+    private String applicationId = "Trino";
 
     @NotNull
     public AuthType getAuthType()
@@ -128,4 +130,19 @@ public AzureFileSystemConfig setMaxHttpRequests(int maxHttpRequests)
         this.maxHttpRequests = maxHttpRequests;
         return this;
     }
+
+    @Size(max = 50)
+    @NotNull
+    public String getApplicationId()
+    {
+        return applicationId;
+    }
+
+    @Config("azure.application-id")
+    @ConfigDescription("Suffix that will be added to HTTP User-Agent header to identify the application")
+    public AzureFileSystemConfig setApplicationId(String applicationId)
+    {
+        this.applicationId = applicationId;
+        return this;
+    }
 }

lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureFileSystemFactory.java

@@ -57,7 +57,8 @@ public AzureFileSystemFactory(OpenTelemetry openTelemetry, AzureAuth azureAuth,
                 config.getWriteBlockSize(),
                 config.getMaxWriteConcurrency(),
                 config.getMaxSingleUploadSize(),
-                config.getMaxHttpRequests());
+                config.getMaxHttpRequests(),
+                config.getApplicationId());
     }
 
     public AzureFileSystemFactory(
@@ -68,7 +69,8 @@ public AzureFileSystemFactory(
             DataSize writeBlockSize,
             int maxWriteConcurrency,
             DataSize maxSingleUploadSize,
-            int maxHttpRequests)
+            int maxHttpRequests,
+            String applicationId)
     {
         this.auth = requireNonNull(azureAuth, "azureAuth is null");
         this.endpoint = requireNonNull(endpoint, "endpoint is null");
@@ -87,6 +89,7 @@ public AzureFileSystemFactory(
                 .build();
         HttpClientOptions clientOptions = new HttpClientOptions();
         clientOptions.setTracingOptions(tracingOptions);
+        clientOptions.setApplicationId(applicationId);
         httpClient = createAzureHttpClient(okHttpClient, clientOptions);
     }
 

lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureFileSystemConfig.java

@@ -37,7 +37,8 @@ void testDefaults()
                 .setWriteBlockSize(DataSize.of(4, Unit.MEGABYTE))
                 .setMaxWriteConcurrency(8)
                 .setMaxSingleUploadSize(DataSize.of(4, Unit.MEGABYTE))
-                .setMaxHttpRequests(2 * Runtime.getRuntime().availableProcessors()));
+                .setMaxHttpRequests(2 * Runtime.getRuntime().availableProcessors())
+                .setApplicationId("Trino"));
     }
 
     @Test
@@ -51,6 +52,7 @@ public void testExplicitPropertyMappings()
                 .put("azure.max-write-concurrency", "7")
                 .put("azure.max-single-upload-size", "7MB")
                 .put("azure.max-http-requests", "128")
+                .put("azure.application-id", "application id")
                 .buildOrThrow();
 
         AzureFileSystemConfig expected = new AzureFileSystemConfig()
@@ -60,7 +62,8 @@ public void testExplicitPropertyMappings()
                 .setWriteBlockSize(DataSize.of(5, Unit.MEGABYTE))
                 .setMaxWriteConcurrency(7)
                 .setMaxSingleUploadSize(DataSize.of(7, Unit.MEGABYTE))
-                .setMaxHttpRequests(128);
+                .setMaxHttpRequests(128)
+                .setApplicationId("application id");
 
         assertFullMapping(properties, expected);
     }

lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java

@@ -24,6 +24,7 @@
 import jakarta.validation.constraints.AssertTrue;
 import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotNull;
+import jakarta.validation.constraints.Size;
 
 import java.util.concurrent.TimeUnit;
 
@@ -48,6 +49,7 @@ public class GcsFileSystemConfig
     private Duration minBackoffDelay = new Duration(10, TimeUnit.MILLISECONDS);
     // Note: there is no benefit to setting this much higher as the rpc quota is 1x per second: https://cloud.google.com/storage/docs/retry-strategy#java
     private Duration maxBackoffDelay = new Duration(2000, TimeUnit.MILLISECONDS);
+    private String applicationId = "Trino";
 
     @NotNull
     public DataSize getReadBlockSize()
@@ -231,6 +233,21 @@ public GcsFileSystemConfig setMaxBackoffDelay(Duration maxBackoffDelay)
         return this;
     }
 
+    @Size(max = 50)
+    @NotNull
+    public String getApplicationId()
+    {
+        return applicationId;
+    }
+
+    @Config("gcs.application-id")
+    @ConfigDescription("Suffix that will be added to HTTP User-Agent header to identify the application")
+    public GcsFileSystemConfig setApplicationId(String applicationId)
+    {
+        this.applicationId = applicationId;
+        return this;
+    }
+
     @AssertTrue(message = "gcs.client.min-backoff-delay must be less than or equal to gcs.client.max-backoff-delay")
     public boolean isRetryDelayValid()
     {

lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java

@@ -28,10 +28,12 @@
 import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 import static com.google.cloud.storage.StorageRetryStrategy.getUniformStorageRetryStrategy;
 import static com.google.common.base.Strings.nullToEmpty;
+import static com.google.common.net.HttpHeaders.USER_AGENT;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 public class GcsStorageFactory
@@ -46,6 +48,7 @@ public class GcsStorageFactory
     private final Duration maxRetryTime;
     private final Duration minBackoffDelay;
     private final Duration maxBackoffDelay;
+    private final String applicationId;
 
     @Inject
     public GcsStorageFactory(GcsFileSystemConfig config)
@@ -75,6 +78,7 @@ else if (jsonKeyFilePath != null) {
         this.maxRetryTime = Duration.ofMillis(config.getMaxRetryTime().toMillis());
         this.minBackoffDelay = Duration.ofMillis(config.getMinBackoffDelay().toMillis());
         this.maxBackoffDelay = Duration.ofMillis(config.getMaxBackoffDelay().toMillis());
+        this.applicationId = config.getApplicationId();
     }
 
     public Storage create(ConnectorIdentity identity)
@@ -113,6 +117,7 @@ public Storage create(ConnectorIdentity identity)
                             .setInitialRetryDelay(minBackoffDelay)
                             .setMaxRetryDelay(maxBackoffDelay)
                             .build())
+                    .setHeaderProvider(() -> Map.of(USER_AGENT, StorageOptions.getLibraryName() + "/" + StorageOptions.version() + " " + applicationId))
                     .build()
                     .getService();
         }

lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java

@@ -51,7 +51,8 @@ void testDefaults()
                 .setBackoffScaleFactor(3.0)
                 .setMaxRetryTime(new Duration(25, SECONDS))
                 .setMinBackoffDelay(new Duration(10, MILLISECONDS))
-                .setMaxBackoffDelay(new Duration(2000, MILLISECONDS)));
+                .setMaxBackoffDelay(new Duration(2000, MILLISECONDS))
+                .setApplicationId("Trino"));
     }
 
     @Test
@@ -74,6 +75,7 @@ void testExplicitPropertyMappings()
                 .put("gcs.client.max-retry-time", "10s")
                 .put("gcs.client.min-backoff-delay", "20ms")
                 .put("gcs.client.max-backoff-delay", "20ms")
+                .put("gcs.application-id", "application id")
                 .buildOrThrow();
 
         GcsFileSystemConfig expected = new GcsFileSystemConfig()
@@ -89,7 +91,8 @@ void testExplicitPropertyMappings()
                 .setBackoffScaleFactor(4.0)
                 .setMaxRetryTime(new Duration(10, SECONDS))
                 .setMinBackoffDelay(new Duration(20, MILLISECONDS))
-                .setMaxBackoffDelay(new Duration(20, MILLISECONDS));
+                .setMaxBackoffDelay(new Duration(20, MILLISECONDS))
+                .setApplicationId("application id");
         assertFullMapping(properties, expected);
     }
 

lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java

@@ -26,6 +26,7 @@
 import jakarta.validation.constraints.AssertTrue;
 import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotNull;
+import jakarta.validation.constraints.Size;
 import software.amazon.awssdk.retries.api.RetryStrategy;
 import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
 
@@ -117,6 +118,7 @@ public static RetryStrategy getRetryStrategy(RetryMode retryMode)
     private RetryMode retryMode = RetryMode.LEGACY;
     private int maxErrorRetries = 10;
     private boolean supportsExclusiveCreate = true;
+    private String applicationId = "Trino";
 
     public String getAwsAccessKey()
     {
@@ -536,4 +538,19 @@ public S3FileSystemConfig setSupportsExclusiveCreate(boolean supportsExclusiveCr
         this.supportsExclusiveCreate = supportsExclusiveCreate;
         return this;
     }
+
+    @Size(max = 50)
+    @NotNull
+    public String getApplicationId()
+    {
+        return applicationId;
+    }
+
+    @Config("s3.application-id")
+    @ConfigDescription("Suffix that will be added to HTTP User-Agent header to identify the application")
+    public S3FileSystemConfig setApplicationId(String applicationId)
+    {
+        this.applicationId = applicationId;
+        return this;
+    }
 }

lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java

@@ -271,6 +271,7 @@ private static ClientOverrideConfiguration createOverrideConfiguration(OpenTelem
                 .retryStrategy(getRetryStrategy(config.getRetryMode()).toBuilder()
                         .maxAttempts(config.getMaxErrorRetries())
                         .build())
+                .appId(config.getApplicationId())
                 .addMetricPublisher(metricPublisher)
                 .build();
     }

lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java

@@ -70,7 +70,8 @@ public void testDefaults()
                 .setHttpProxyUsername(null)
                 .setHttpProxyPassword(null)
                 .setHttpProxyPreemptiveBasicProxyAuth(false)
-                .setSupportsExclusiveCreate(true));
+                .setSupportsExclusiveCreate(true)
+                .setApplicationId("Trino"));
     }
 
     @Test
@@ -109,6 +110,7 @@ public void testExplicitPropertyMappings()
                 .put("s3.http-proxy.password", "test")
                 .put("s3.http-proxy.preemptive-basic-auth", "true")
                 .put("s3.exclusive-create", "false")
+                .put("s3.application-id", "application id")
                 .buildOrThrow();
 
         S3FileSystemConfig expected = new S3FileSystemConfig()
@@ -143,7 +145,8 @@ public void testExplicitPropertyMappings()
                 .setHttpProxyUsername("test")
                 .setHttpProxyPassword("test")
                 .setHttpProxyPreemptiveBasicProxyAuth(true)
-                .setSupportsExclusiveCreate(false);
+                .setSupportsExclusiveCreate(false)
+                .setApplicationId("application id");
 
         assertFullMapping(properties, expected);
     }