feat(controller): Enhanced artifact repository ref. See #3184 (#4458)

Signed-off-by: Alex Collins <alex_collins@intuit.com>
argoproj · Nov 30, 2020 · c84d56b · c84d56b
1 parent 5c538d7
commit c84d56b
Show file tree

Hide file tree

Showing 32 changed files with 1,499 additions and 749 deletions.
diff --git a/api/jsonschema/schema.json b/api/jsonschema/schema.json
@@ -212,9 +212,32 @@
     "io.argoproj.workflow.v1alpha1.ArtifactRepositoryRef": {
       "properties": {
         "configMap": {
+          "description": "The name of the config map. Defaults to \"artifact-repositories\".",
           "type": "string"
         },
         "key": {
+          "description": "The config map key. Defaults to the value of the \"workflows.argoproj.io/default-artifact-repository\" annotation.",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "io.argoproj.workflow.v1alpha1.ArtifactRepositoryRefStatus": {
+      "properties": {
+        "configMap": {
+          "description": "The name of the config map. Defaults to \"artifact-repositories\".",
+          "type": "string"
+        },
+        "default": {
+          "description": "If this ref represents the default artifact repository, rather than a config map.",
+          "type": "boolean"
+        },
+        "key": {
+          "description": "The config map key. Defaults to the value of the \"workflows.argoproj.io/default-artifact-repository\" annotation.",
+          "type": "string"
+        },
+        "namespace": {
+          "description": "The namespace of the config map. Defaults to the workflow's namespace, or the controller's namespace (if found).",
           "type": "string"
         }
       },
@@ -2739,6 +2762,10 @@
     "io.argoproj.workflow.v1alpha1.WorkflowStatus": {
       "description": "WorkflowStatus contains overall status information about a workflow",
       "properties": {
+        "artifactRepositoryRef": {
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactRepositoryRefStatus",
+          "description": "ArtifactRepositoryRef is used to cache the repository to use so we do not need to determine it everytime we reconcile."
+        },
         "compressedNodes": {
           "description": "Compressed and base64 decoded Nodes map",
           "type": "string"

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -2269,9 +2269,32 @@
       "type": "object",
       "properties": {
         "configMap": {
+          "description": "The name of the config map. Defaults to \"artifact-repositories\".",
           "type": "string"
         },
         "key": {
+          "description": "The config map key. Defaults to the value of the \"workflows.argoproj.io/default-artifact-repository\" annotation.",
+          "type": "string"
+        }
+      }
+    },
+    "io.argoproj.workflow.v1alpha1.ArtifactRepositoryRefStatus": {
+      "type": "object",
+      "properties": {
+        "configMap": {
+          "description": "The name of the config map. Defaults to \"artifact-repositories\".",
+          "type": "string"
+        },
+        "default": {
+          "description": "If this ref represents the default artifact repository, rather than a config map.",
+          "type": "boolean"
+        },
+        "key": {
+          "description": "The config map key. Defaults to the value of the \"workflows.argoproj.io/default-artifact-repository\" annotation.",
+          "type": "string"
+        },
+        "namespace": {
+          "description": "The namespace of the config map. Defaults to the workflow's namespace, or the controller's namespace (if found).",
           "type": "string"
         }
       }
@@ -4788,6 +4811,10 @@
       "description": "WorkflowStatus contains overall status information about a workflow",
       "type": "object",
       "properties": {
+        "artifactRepositoryRef": {
+          "description": "ArtifactRepositoryRef is used to cache the repository to use so we do not need to determine it everytime we reconcile.",
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactRepositoryRefStatus"
+        },
         "compressedNodes": {
           "description": "Compressed and base64 decoded Nodes map",
           "type": "string"

diff --git a/docs/artifact-repository-ref.md b/docs/artifact-repository-ref.md
@@ -6,15 +6,19 @@
 
 You can reduce duplication in your templates by configuring repositories that can be accessed by any workflow. This can also remove sensitive information from your templates.
 
-Create a suitable config map in either (a) your workflows namespace or (b) in the Argo's namespace, the default name is `artifact-repositories`:
+Create a suitable config map in either (a) your workflows namespace or (b) in the managed namespace:
 
-```
+```yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
+    # if you want to use this config map by default - name it "artifact-repositories" 
   name: artifact-repositories
+  annotations:
+    # if you want to use a specific key, put that's key into this annotation 
+    workflows.argoproj.io/default-artifact-repository: default-v1
 data:
-  minio: |
+  default-v1: |
     s3:
       bucket: my-bucket
       endpoint: minio:9000
@@ -29,10 +33,11 @@ data:
 
 You can override the repository for a workflow as follows:
 
-```
+```yaml
 spec:
   artifactRepositoryRef:
-    key: minio
+    configMap: my-cm # default is "artifact-repositories"
+    key: my-key # default can be set by the annotation
 ```
 
 Reference: [fields.md#artifactrepositoryref](fields.md#artifactrepositoryref).
diff --git a/docs/configure-artifact-repository.md b/docs/configure-artifact-repository.md
@@ -16,6 +16,12 @@ Subsequent sections will show how to use it.
 | Raw | Yes | No | 5% |
 | S3 | Yes | Yes | 86% |
 
+The actual repository used by a workflow is choose by the following rules:
+
+1. Anything explicitly configured using [Artifact Repository Ref](artifact-repository-ref.md). This is the most flexible, safe, and secure option.
+2. From a config map named `artifact-repositories` if it has the `workflows.argoproj.io/default-artifact-repository` annotation in the workflow's namespace.                                        
+3. From a workflow controller configmap.
+
 ## Configuring Minio
 
 ```

diff --git a/docs/fields.md b/docs/fields.md
@@ -662,6 +662,7 @@ WorkflowStatus contains overall status information about a workflow
 ### Fields
 | Field Name | Field Type | Description   |
 |:----------:|:----------:|---------------|
+|`artifactRepositoryRef`|[`ArtifactRepositoryRefStatus`](#artifactrepositoryrefstatus)|ArtifactRepositoryRef is used to cache the repository to use so we do not need to determine it everytime we reconcile.|
 |`compressedNodes`|`string`|Compressed and base64 decoded Nodes map|
 |`conditions`|`Array<`[`Condition`](#condition)`>`|Conditions is a list of conditions the Workflow may have|
 |`estimatedDuration`|`int32`|EstimatedDuration in seconds.|
@@ -1492,8 +1493,8 @@ _No description available_
 ### Fields
 | Field Name | Field Type | Description   |
 |:----------:|:----------:|---------------|
-|`configMap`|`string`|_No description available_|
-|`key`|`string`|_No description available_|
+|`configMap`|`string`|The name of the config map. Defaults to "artifact-repositories".|
+|`key`|`string`|The config map key. Defaults to the value of the "workflows.argoproj.io/default-artifact-repository" annotation.|
 
 ## ExecutorConfig
 
@@ -1963,6 +1964,25 @@ WorkflowTemplateRef is a reference to a WorkflowTemplate resource.
 |`clusterScope`|`boolean`|ClusterScope indicates the referred template is cluster scoped (i.e. a ClusterWorkflowTemplate).|
 |`name`|`string`|Name is the resource name of the workflow template.|
 
+## ArtifactRepositoryRefStatus
+
+_No description available_
+
+<details>
+<summary>Examples with this field (click to open)</summary>
+<br>
+
+- [`artifact-repository-ref.yaml`](https://github.com/argoproj/argo/blob/master/examples/artifact-repository-ref.yaml)
+</details>
+
+### Fields
+| Field Name | Field Type | Description   |
+|:----------:|:----------:|---------------|
+|`configMap`|`string`|The name of the config map. Defaults to "artifact-repositories".|
+|`default`|`boolean`|If this ref represents the default artifact repository, rather than a config map.|
+|`key`|`string`|The config map key. Defaults to the value of the "workflows.argoproj.io/default-artifact-repository" annotation.|
+|`namespace`|`string`|The namespace of the config map. Defaults to the workflow's namespace, or the controller's namespace (if found).|
+
 ## Condition
 
 _No description available_

diff --git a/examples/artifact-repository-ref.yaml b/examples/artifact-repository-ref.yaml
@@ -5,13 +5,13 @@ metadata:
 spec:
   entrypoint: main
   artifactRepositoryRef:
-    key: minio
+    key: my-key
   templates:
     - name: main
       container:
         image: docker/whalesay:latest
-        command: [sh, -c]
-        args: ["cowsay hello world | tee /tmp/hello_world.txt"]
+        command: [ sh, -c ]
+        args: [ "cowsay hello world | tee /tmp/hello_world.txt" ]
       outputs:
         artifacts:
           - name: hello_world

diff --git a/manifests/base/crds/full/argoproj.io_workflows.yaml b/manifests/base/crds/full/argoproj.io_workflows.yaml
@@ -6702,6 +6702,17 @@ spec:
           type: object
         status:
           properties:
+            artifactRepositoryRef:
+              properties:
+                configMap:
+                  type: string
+                default:
+                  type: boolean
+                key:
+                  type: string
+                namespace:
+                  type: string
+              type: object
             compressedNodes:
               type: string
             conditions:

diff --git a/manifests/quick-start-minimal.yaml b/manifests/quick-start-minimal.yaml
@@ -449,7 +449,20 @@ subjects:
 ---
 apiVersion: v1
 data:
-  minio: |
+  default-v1: |
+    archiveLogs: true
+    s3:
+      bucket: my-bucket
+      endpoint: minio:9000
+      insecure: true
+      accessKeySecret:
+        name: my-minio-cred
+        key: accesskey
+      secretKeySecret:
+        name: my-minio-cred
+        key: secretkey
+  my-key: |
+    archiveLogs: true
     s3:
       bucket: my-bucket
       endpoint: minio:9000
@@ -462,6 +475,8 @@ data:
         key: secretkey
 kind: ConfigMap
 metadata:
+  annotations:
+    workflows.argoproj.io/default-artifact-repository: default-v1
   name: artifact-repositories
 ---
 apiVersion: v1

diff --git a/manifests/quick-start-mysql.yaml b/manifests/quick-start-mysql.yaml
@@ -449,7 +449,20 @@ subjects:
 ---
 apiVersion: v1
 data:
-  minio: |
+  default-v1: |
+    archiveLogs: true
+    s3:
+      bucket: my-bucket
+      endpoint: minio:9000
+      insecure: true
+      accessKeySecret:
+        name: my-minio-cred
+        key: accesskey
+      secretKeySecret:
+        name: my-minio-cred
+        key: secretkey
+  my-key: |
+    archiveLogs: true
     s3:
       bucket: my-bucket
       endpoint: minio:9000
@@ -462,6 +475,8 @@ data:
         key: secretkey
 kind: ConfigMap
 metadata:
+  annotations:
+    workflows.argoproj.io/default-artifact-repository: default-v1
   name: artifact-repositories
 ---
 apiVersion: v1

diff --git a/manifests/quick-start-postgres.yaml b/manifests/quick-start-postgres.yaml
@@ -449,7 +449,20 @@ subjects:
 ---
 apiVersion: v1
 data:
-  minio: |
+  default-v1: |
+    archiveLogs: true
+    s3:
+      bucket: my-bucket
+      endpoint: minio:9000
+      insecure: true
+      accessKeySecret:
+        name: my-minio-cred
+        key: accesskey
+      secretKeySecret:
+        name: my-minio-cred
+        key: secretkey
+  my-key: |
+    archiveLogs: true
     s3:
       bucket: my-bucket
       endpoint: minio:9000
@@ -462,6 +475,8 @@ data:
         key: secretkey
 kind: ConfigMap
 metadata:
+  annotations:
+    workflows.argoproj.io/default-artifact-repository: default-v1
   name: artifact-repositories
 ---
 apiVersion: v1

diff --git a/manifests/quick-start/base/artifact-repositories-configmap.yaml b/manifests/quick-start/base/artifact-repositories-configmap.yaml
@@ -1,6 +1,14 @@
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: artifact-repositories
+  annotations:
+    # you'll want to change the default over time, e.g. when you move to new storage solution,
+    # so we recommend you version them from the outset by suffixing the version
+    workflows.argoproj.io/default-artifact-repository: default-v1
 data:
-  minio: |
+  default-v1: |
+    archiveLogs: true
     s3:
       bucket: my-bucket
       endpoint: minio:9000
@@ -11,6 +19,15 @@ data:
       secretKeySecret:
         name: my-minio-cred
         key: secretkey
-kind: ConfigMap
-metadata:
-  name: artifact-repositories
+  my-key: |
+    archiveLogs: true
+    s3:
+      bucket: my-bucket
+      endpoint: minio:9000
+      insecure: true
+      accessKeySecret:
+        name: my-minio-cred
+        key: accesskey
+      secretKeySecret:
+        name: my-minio-cred
+        key: secretkey