feat(executor): Auto create s3 bucket if not present. Closes #3586

api/openapi-spec/swagger.json

@@ -2472,6 +2472,24 @@
         }
       }
     },
+    "io.argoproj.workflow.v1alpha1.CreateS3BucketOptions": {
+      "description": "CreateS3BucketOptions options used to determine automatic automatic bucket-creation process",
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "description": "Enabled tells the driver whether to attempt to create the S3 bucket or not",
+          "type": "boolean"
+        },
+        "objectLocking": {
+          "description": "ObjectLocking Enable object locking",
+          "type": "boolean"
+        },
+        "region": {
+          "description": "Region Bucket location",
+          "type": "string"
+        }
+      }
+    },
     "io.argoproj.workflow.v1alpha1.CronWorkflow": {
       "description": "CronWorkflow is the definition of a scheduled workflow resource",
       "type": "object",
@@ -3576,6 +3594,10 @@
           "description": "Bucket is the name of the bucket",
           "type": "string"
         },
+        "createBucketIfNotPresent": {
+          "description": "CreateBucketIfNotPresent tells the driver to attempt to create the S3 bucket for output artifacts, if it doesn't exist",
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.CreateS3BucketOptions"
+        },
         "endpoint": {
           "description": "Endpoint is the hostname of the bucket endpoint",
           "type": "string"

docs/fields.md

@@ -3586,6 +3586,7 @@ S3Artifact is the location of an S3 artifact
 |:----------:|:----------:|---------------|
 |`accessKeySecret`|[`SecretKeySelector`](#secretkeyselector)|AccessKeySecret is the secret selector to the bucket's access key|
 |`bucket`|`string`|Bucket is the name of the bucket|
+|`createBucketIfNotPresent`|[`CreateS3BucketOptions`](#creates3bucketoptions)|CreateBucketIfNotPresent tells the driver to attempt to create the S3 bucket for output artifacts, if it doesn't exist|
 |`endpoint`|`string`|Endpoint is the hostname of the bucket endpoint|
 |`insecure`|`boolean`|Insecure will connect to the service with TLS|
 |`key`|`string`|Key is the key in the bucket where the artifact resides|
@@ -3962,6 +3963,17 @@ Header indicate a key-value request header to be used when fetching artifacts ov
 |`name`|`string`|Name is the header name|
 |`value`|`string`|Value is the literal value to use for the header|
 
+## CreateS3BucketOptions
+
+CreateS3BucketOptions options used to determine automatic automatic bucket-creation process
+
+### Fields
+| Field Name | Field Type | Description   |
+|:----------:|:----------:|---------------|
+|`enabled`|`boolean`|Enabled tells the driver whether to attempt to create the S3 bucket or not|
+|`objectLocking`|`boolean`|ObjectLocking Enable object locking|
+|`region`|`string`|Region Bucket location|
+
 ## SuppliedValueFrom
 
 SuppliedValueFrom is a placeholder for a value to be filled in directly, either through the CLI, API, etc.

docs/swagger.md

@@ -1,3 +1,3 @@
 # Argo Server API
 
-[Open the Swagger API docs](https://raw.githubusercontent.com/argoproj/argo/master/api/openapi-spec/swagger.json).
+[Open the Swagger API docs](https://raw.githubusercontent.com/argoproj/argo/master/api/openapi-spec/swagger.json).

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
 	github.com/aliyun/aliyun-oss-go-sdk v2.0.6+incompatible
 	github.com/antonmedv/expr v1.8.2
-	github.com/argoproj/pkg v0.2.0
+	github.com/argoproj/pkg v0.3.0
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blushft/go-diagrams v0.0.0-20201006005127-c78c821223d9
@@ -37,6 +37,7 @@ require (
 	github.com/klauspost/compress v1.9.7 // indirect
 	github.com/lib/pq v1.3.0 // indirect
 	github.com/mattn/goreman v0.3.5
+	github.com/minio/minio-go/v7 v7.0.2
 	github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.0.0

go.sum

@@ -81,8 +81,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
 github.com/antonmedv/expr v1.8.2 h1:BfkVHGudYqq7jp3Ji33kTn+qZ9D19t/Mndg0ag/Ycq4=
 github.com/antonmedv/expr v1.8.2/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8=
-github.com/argoproj/pkg v0.2.0 h1:ETgC600kr8WcAi3MEVY5sA1H7H/u1/IysYOobwsZ8No=
-github.com/argoproj/pkg v0.2.0/go.mod h1:F4TZgInLUEjzsWFB/BTJBsewoEy0ucnKSq6vmQiD/yc=
+github.com/argoproj/pkg v0.3.0 h1:0xxNHc9duXZt1TlDVVsBzcGm8S/V5Bc3eNqawz3wFo8=
+github.com/argoproj/pkg v0.3.0/go.mod h1:F4TZgInLUEjzsWFB/BTJBsewoEy0ucnKSq6vmQiD/yc=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=

manifests/base/crds/full/argoproj.io_clusterworkflowtemplates.yaml

@@ -559,6 +559,15 @@ spec:
                             type: object
                           bucket:
                             type: string
+                          createBucketIfNotPresent:
+                            properties:
+                              enabled:
+                                type: boolean
+                              objectLocking:
+                                type: boolean
+                              region:
+                                type: string
+                            type: object
                           endpoint:
                             type: string
                           insecure:
@@ -1421,6 +1430,15 @@ spec:
                             type: object
                           bucket:
                             type: string
+                          createBucketIfNotPresent:
+                            properties:
+                              enabled:
+                                type: boolean
+                              objectLocking:
+                                type: boolean
+                              region:
+                                type: string
+                            type: object
                           endpoint:
                             type: string
                           insecure:
@@ -1721,6 +1739,15 @@ spec:
                                   type: object
                                 bucket:
                                   type: string
+                                createBucketIfNotPresent:
+                                  properties:
+                                    enabled:
+                                      type: boolean
+                                    objectLocking:
+                                      type: boolean
+                                    region:
+                                      type: string
+                                  type: object
                                 endpoint:
                                   type: string
                                 insecure:
@@ -2610,6 +2637,15 @@ spec:
                                             type: object
                                           bucket:
                                             type: string
+                                          createBucketIfNotPresent:
+                                            properties:
+                                              enabled:
+                                                type: boolean
+                                              objectLocking:
+                                                type: boolean
+                                              region:
+                                                type: string
+                                            type: object
                                           endpoint:
                                             type: string
                                           insecure:
@@ -3572,6 +3608,15 @@ spec:
                                   type: object
                                 bucket:
                                   type: string
+                                createBucketIfNotPresent:
+                                  properties:
+                                    enabled:
+                                      type: boolean
+                                    objectLocking:
+                                      type: boolean
+                                    region:
+                                      type: string
+                                  type: object
                                 endpoint:
                                   type: string
                                 insecure:
@@ -4018,6 +4063,15 @@ spec:
                                   type: object
                                 bucket:
                                   type: string
+                                createBucketIfNotPresent:
+                                  properties:
+                                    enabled:
+                                      type: boolean
+                                    objectLocking:
+                                      type: boolean
+                                    region:
+                                      type: string
+                                  type: object
                                 endpoint:
                                   type: string
                                 insecure:

manifests/base/crds/full/argoproj.io_cronworkflows.yaml

@@ -580,6 +580,15 @@ spec:
                                 type: object
                               bucket:
                                 type: string
+                              createBucketIfNotPresent:
+                                properties:
+                                  enabled:
+                                    type: boolean
+                                  objectLocking:
+                                    type: boolean
+                                  region:
+                                    type: string
+                                type: object
                               endpoint:
                                 type: string
                               insecure:
@@ -1442,6 +1451,15 @@ spec:
                                 type: object
                               bucket:
                                 type: string
+                              createBucketIfNotPresent:
+                                properties:
+                                  enabled:
+                                    type: boolean
+                                  objectLocking:
+                                    type: boolean
+                                  region:
+                                    type: string
+                                type: object
                               endpoint:
                                 type: string
                               insecure:
@@ -1742,6 +1760,15 @@ spec:
                                       type: object
                                     bucket:
                                       type: string
+                                    createBucketIfNotPresent:
+                                      properties:
+                                        enabled:
+                                          type: boolean
+                                        objectLocking:
+                                          type: boolean
+                                        region:
+                                          type: string
+                                      type: object
                                     endpoint:
                                       type: string
                                     insecure:
@@ -2631,6 +2658,15 @@ spec:
                                                 type: object
                                               bucket:
                                                 type: string
+                                              createBucketIfNotPresent:
+                                                properties:
+                                                  enabled:
+                                                    type: boolean
+                                                  objectLocking:
+                                                    type: boolean
+                                                  region:
+                                                    type: string
+                                                type: object
                                               endpoint:
                                                 type: string
                                               insecure:
@@ -3593,6 +3629,15 @@ spec:
                                       type: object
                                     bucket:
                                       type: string
+                                    createBucketIfNotPresent:
+                                      properties:
+                                        enabled:
+                                          type: boolean
+                                        objectLocking:
+                                          type: boolean
+                                        region:
+                                          type: string
+                                      type: object
                                     endpoint:
                                       type: string
                                     insecure:
@@ -4039,6 +4084,15 @@ spec:
                                       type: object
                                     bucket:
                                       type: string
+                                    createBucketIfNotPresent:
+                                      properties:
+                                        enabled:
+                                          type: boolean
+                                        objectLocking:
+                                          type: boolean
+                                        region:
+                                          type: string
+                                      type: object
                                     endpoint:
                                       type: string
                                     insecure:

manifests/base/crds/full/argoproj.io_workfloweventbindings.yaml

@@ -302,6 +302,15 @@ spec:
                                 type: object
                               bucket:
                                 type: string
+                              createBucketIfNotPresent:
+                                properties:
+                                  enabled:
+                                    type: boolean
+                                  objectLocking:
+                                    type: boolean
+                                  region:
+                                    type: string
+                                type: object
                               endpoint:
                                 type: string
                               insecure: