GH-937 Improve S3 client configuration & initialization (#936)

* MDP-1259: Zeppelin + reposilite PoC * reorder params * update documentation and tests * Junit test fix - add missing prefixes Co-authored-by: evompa <evompa@twilio.com>
dzikoysk · Nov 22, 2021 · 500abe2 · 500abe2
1 parent 6ea9e32
commit 500abe2
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 41 deletions.
diff --git a/...ackend/src/integration/kotlin/com/reposilite/ReposiliteRemoteIntegrationJunitExtension.kt b/...ackend/src/integration/kotlin/com/reposilite/ReposiliteRemoteIntegrationJunitExtension.kt
@@ -49,7 +49,7 @@ internal class ReposiliteRemoteIntegrationJunitExtension : Extension, BeforeEach
 
             type.getField("_extensionInitialized").set(instance, true)
             type.getField("_database").set(instance, "mysql ${mariaDb.host}:${mariaDb.getMappedPort(3306)} ${mariaDb.databaseName} ${mariaDb.username} ${mariaDb.password}")
-            type.getField("_storageProvider").set(instance, "s3 -e ${localstack.getEndpointOverride(S3)} ${localstack.accessKey} ${localstack.secretKey} ${localstack.region} {repository}")
+            type.getField("_storageProvider").set(instance, "s3 -e ${localstack.getEndpointOverride(S3)} -a ${localstack.accessKey} -s ${localstack.secretKey} -r ${localstack.region} {repository}")
         }
     }
 

diff --git a/...egration/kotlin/com/reposilite/storage/infrastructure/S3StorageProviderIntegrationTest.kt b/...egration/kotlin/com/reposilite/storage/infrastructure/S3StorageProviderIntegrationTest.kt
@@ -44,7 +44,7 @@ internal class S3StorageProviderIntegrationTest : StorageProviderIntegrationTest
             InMemoryLogger(),
             rootDirectory.toPath(),
             "test-repository",
-            "s3 --endpoint ${localstack.getEndpointOverride(S3).toURL()} ${localstack.accessKey} ${localstack.secretKey} ${localstack.region} test-repository"
+            "s3 test-repository --endpoint ${localstack.getEndpointOverride(S3).toURL()} --access-key ${localstack.accessKey} --secret-key ${localstack.secretKey} --region ${localstack.region}"
         )
     }
 

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/settings/SharedConfiguration.kt b/reposilite-backend/src/main/kotlin/com/reposilite/settings/SharedConfiguration.kt
@@ -88,9 +88,13 @@ class SharedConfiguration : Serializable, DeserializationHandler<SharedConfigura
         @Description("# Example usage:")
         @Description("# storageProvider: fs --quota 50GB")
         @Description("# > S3 provider. Supported flags:")
-        @Description("# --endpoint = custom endpoint with which the S3 provider should communicate (optional)")
+        @Description("# --endpoint = overwrite the AWS endpoint (optional)")
+        @Description("# --access-key = overwrite AWS access-key used to authenticate (optional)")
+        @Description("# --secret-key = overwrite AWS secret-key used to authenticate (optional)")
+        @Description("# --region = overwrite AWS region (optional)")
+        @Description("# See software.amazon.awssdk.services.s3.S3Client for default values")
         @Description("# Example usage:")
-        @Description("# storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name")
+        @Description("# storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region")
         @JvmField
         var storageProvider = "fs --quota 100%"
 
@@ -104,16 +108,16 @@ class SharedConfiguration : Serializable, DeserializationHandler<SharedConfigura
 
         @Command(name = "s3", description = ["Amazon S3 storage provider settings"])
         internal class S3StorageProviderSettings : Validator() {
+            @Parameters(index = "0", paramLabel = "<bucket-name>")
+            lateinit var bucketName: String
             @Option(names = ["-e", "--endpoint"], defaultValue = "")
             lateinit var endpoint: String
-            @Parameters(index = "0", paramLabel = "<access-key>")
+            @Option(names = ["-a", "--access-key"], defaultValue = "")
             lateinit var accessKey: String
-            @Parameters(index = "1", paramLabel = "<secret-key>")
+            @Option(names = ["-s", "--secret-key"], defaultValue = "")
             lateinit var secretKey: String
-            @Parameters(index = "2", paramLabel = "<region>")
+            @Option(names = ["-r", "--region"], defaultValue = "")
             lateinit var region: String
-            @Parameters(index = "3", paramLabel = "<bucket-name>")
-            lateinit var bucketName: String
         }
 
         @Description("")

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/storage/StorageProviderFactory.kt b/reposilite-backend/src/main/kotlin/com/reposilite/storage/StorageProviderFactory.kt
@@ -57,8 +57,15 @@ object StorageProviderFactory {
         val settings = loadCommandBasedConfiguration(S3StorageProviderSettings(), storageDescription).configuration
 
         val client = S3Client.builder()
-            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(settings.accessKey, settings.secretKey)))
-            .region(Region.of(settings.region))
+
+        if (settings.accessKey.isNotEmpty() && settings.secretKey.isNotEmpty()) {
+            client.credentialsProvider(StaticCredentialsProvider
+                    .create(AwsBasicCredentials.create(settings.accessKey, settings.secretKey)))
+        }
+
+        if (settings.region.isNotEmpty()) {
+            client.region(Region.of(settings.region))
+        }
 
         if (settings.endpoint.isNotEmpty()) {
             client.endpointOverride(URI.create(settings.endpoint))

diff --git a/...silite-backend/src/main/kotlin/com/reposilite/storage/infrastructure/S3StorageProvider.kt b/...silite-backend/src/main/kotlin/com/reposilite/storage/infrastructure/S3StorageProvider.kt
@@ -18,13 +18,7 @@ package com.reposilite.storage.infrastructure
 
 import com.reposilite.journalist.Journalist
 import com.reposilite.journalist.Logger
-import com.reposilite.shared.fs.DirectoryInfo
-import com.reposilite.shared.fs.DocumentInfo
-import com.reposilite.shared.fs.FileDetails
-import com.reposilite.shared.fs.SimpleDirectoryInfo
-import com.reposilite.shared.fs.getExtension
-import com.reposilite.shared.fs.getSimpleName
-import com.reposilite.shared.fs.safeResolve
+import com.reposilite.shared.fs.*
 import com.reposilite.storage.StorageProvider
 import com.reposilite.web.http.ErrorResponse
 import com.reposilite.web.http.errorResponse
@@ -38,15 +32,7 @@ import panda.std.Result.ok
 import panda.std.asSuccess
 import software.amazon.awssdk.core.sync.RequestBody
 import software.amazon.awssdk.services.s3.S3Client
-import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException
-import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException
-import software.amazon.awssdk.services.s3.model.DeleteObjectRequest
-import software.amazon.awssdk.services.s3.model.GetObjectRequest
-import software.amazon.awssdk.services.s3.model.HeadObjectRequest
-import software.amazon.awssdk.services.s3.model.HeadObjectResponse
-import software.amazon.awssdk.services.s3.model.ListObjectsRequest
-import software.amazon.awssdk.services.s3.model.NoSuchKeyException
-import software.amazon.awssdk.services.s3.model.PutObjectRequest
+import software.amazon.awssdk.services.s3.model.*
 import java.io.File
 import java.io.IOException
 import java.io.InputStream
@@ -61,14 +47,18 @@ class S3StorageProvider(
 ) : StorageProvider, Journalist {
 
     init {
+        val headBucketRequest = HeadBucketRequest.builder().bucket(bucket).build()
+
         try {
-            s3.createBucket {
-                it.bucket(bucket)
+            s3.headBucket(headBucketRequest)
+        } catch (noSuchBucket: NoSuchBucketException) {
+            try {
+                s3.createBucket { it.bucket(bucket) }
+            } catch (bucketExists: BucketAlreadyExistsException) {
+                // ignored
+            } catch (buckedOwned: BucketAlreadyOwnedByYouException) {
+                // ignored
             }
-        } catch (bucketExists: BucketAlreadyExistsException) {
-            // ignored
-        } catch (buckedOwned: BucketAlreadyOwnedByYouException) {
-            // ignored
         }
     }
 

diff --git a/reposilite-backend/src/test/workspace/configuration.shared.cdn b/reposilite-backend/src/test/workspace/configuration.shared.cdn
diff --git a/reposilite-site/docs/configuration.md b/reposilite-site/docs/configuration.md
@@ -76,9 +76,13 @@ repositories {
     # Example usage:
     # storageProvider: fs --quota 50GB
     # > S3 provider. Supported flags:
-    # --endpoint = custom endpoint with which the S3 provider should communicate (optional)
+    # --endpoint = overwrite the AWS endpoint (optional)
+    # --access-key = overwrite AWS access-key used to authenticate (optional)
+    # --secret-key = overwrite AWS secret-key used to authenticate (optional)
+    # --region = overwrite AWS region (optional)
+    # See software.amazon.awssdk.services.s3.S3Client for default values
     # Example usage:
-    # storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
+    # storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
     storageProvider: fs --quota 100%
 
     # List of proxied repositories associated with this repository.
@@ -105,9 +109,13 @@ repositories {
     # Example usage:
     # storageProvider: fs --quota 50GB
     # > S3 provider. Supported flags:
-    # --endpoint = custom endpoint with which the S3 provider should communicate (optional)
+    # --endpoint = overwrite the AWS endpoint (optional)
+    # --access-key = overwrite AWS access-key used to authenticate (optional)
+    # --secret-key = overwrite AWS secret-key used to authenticate (optional)
+    # --region = overwrite AWS region (optional)
+    # See software.amazon.awssdk.services.s3.S3Client for default values
     # Example usage:
-    # storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
+    # storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
     storageProvider: fs --quota 100%
 
     # List of proxied repositories associated with this repository.

diff --git a/reposilite-site/docs/proxy.md b/reposilite-site/docs/proxy.md
@@ -31,9 +31,13 @@ Caching may allocate thousands of artifacts, especially at the beginning - for t
 # Example usage:
 # storageProvider: fs --quota 50GB
 # > S3 provider. Supported flags:
-# --endpoint = custom endpoint with which the S3 provider should communicate (optional)
+# --endpoint = overwrite the AWS endpoint (optional)
+# --access-key = overwrite AWS access-key used to authenticate (optional)
+# --secret-key = overwrite AWS secret-key used to authenticate (optional)
+# --region = overwrite AWS region (optional)
+# See software.amazon.awssdk.services.s3.S3Client for default values
 # Example usage:
-# storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
+# storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
 storageProvider: fs --quota 85%
 ```
 

diff --git a/reposilite-site/docs/repositories.md b/reposilite-site/docs/repositories.md
@@ -36,9 +36,13 @@ repositories {
     # Example usage:
     # storageProvider: fs --quota 50GB
     # > S3 provider. Supported flags:
-    # --endpoint = custom endpoint with which the S3 provider should communicate (optional)
+    # --endpoint = overwrite the AWS endpoint (optional)
+    # --access-key = overwrite AWS access-key used to authenticate (optional)
+    # --secret-key = overwrite AWS secret-key used to authenticate (optional)
+    # --region = overwrite AWS region (optional)
+    # See software.amazon.awssdk.services.s3.S3Client for default values
     # Example usage:
-    # storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
+    # storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
     storageProvider: fs --quota 100%
 
     # List of proxied repositories associated with this repository.