feat: update argo image to v2.12.9 and automate update process. Fixes #…

…5232 (#5266) * add notices and licenses for argo 2.12 * feat: upgrade argo images to v2.12.9 * update all refs to argo image version * add NOTICES generation script * upgrade argo cli to latest * fix * fix * add license_info.csv back * make release process safer * add back third_party/license.txt * refactor(deployment): move argo manifests to third-party, updates for 2.12.9 * update marketplace snapshots * set up marketplace presubmit test * add comment
kubeflow · Mar 10, 2021 · be64b32 · be64b32
1 parent 6b1eba7
commit be64b32
Show file tree

Hide file tree

Showing 50 changed files with 900 additions and 12,244 deletions.
diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
@@ -192,10 +192,10 @@ steps:
   args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.14']
   id:   'pullCloudsqlProxy'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.12.9-license-compliance']
   id:   'pullArgoExecutor'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.12.9-license-compliance']
   id:   'pullArgoWorkflowController'
 
 # Tag for Hosted - SemVersion to Major.Minor parsing

diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml
@@ -479,14 +479,14 @@ steps:
     docker push gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$(cat /workspace/mm.ver)
 
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.12.9-license-compliance']
   id:   'pullArgoExecutor'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.12.9-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
   id:   'tagArgoExecutorForMarketplace'
   waitFor: ['pullArgoExecutor']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.12.9-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
   id:   'tagArgoExecutorForMarketplaceTest'
   waitFor: ['pullArgoExecutor']
 - id:   'tagArgoExecutorForMarketplaceMajorMinor'
@@ -496,20 +496,20 @@ steps:
   args:
   - -ceux
   - |
-    docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
-    docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/argoexec:v2.12.9-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/argoexec:v2.12.9-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
 
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.12.9-license-compliance']
   id:   'pullArgoWorkflowController'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.12.9-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
   id:   'tagArgoWorkflowControllerForMarketplace'
   waitFor: ['pullArgoWorkflowController']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.12.9-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
   id:   'tagArgoWorkflowControllerForMarketplaceTest'
   waitFor: ['pullArgoWorkflowController']
 - id:   'tagArgoWorkflowControllerForMarketplaceMajorMinor'
@@ -519,8 +519,8 @@ steps:
   args:
   - -ceux
   - |
-    docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
-    docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/workflow-controller:v2.12.9-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/workflow-controller:v2.12.9-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
 

diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -15,9 +15,11 @@ COPY backend/requirements.txt .
 RUN python3 -m pip install -r requirements.txt --no-cache-dir
 
 # Downloading Argo CLI so that the samples are validated
-#ADD  https://github.com/argoproj/argo/releases/download/v2.7.5/argo-linux-amd64 /usr/local/bin/argo
-ADD  https://github.com/argoproj/argo/releases/download/v2.4.3/argo-linux-amd64 /usr/local/bin/argo
-RUN chmod +x /usr/local/bin/argo
+ENV ARGO_VERSION v2.12.9
+RUN curl -sLO https://github.com/argoproj/argo/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz && \
+  gunzip argo-linux-amd64.gz && \
+  chmod +x argo-linux-amd64 && \
+  mv ./argo-linux-amd64 /usr/local/bin/argo
 
 WORKDIR /go/src/github.com/kubeflow/pipelines
 COPY sdk sdk

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml
@@ -3,7 +3,7 @@ kind: CustomResourceDefinition
 metadata:
   name: workflows.argoproj.io
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 spec:
   group: argoproj.io
   names:
@@ -19,7 +19,7 @@ kind: CustomResourceDefinition
 metadata:
   name: clusterworkflowtemplates.argoproj.io
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 spec:
   group: argoproj.io
   names:
@@ -36,7 +36,7 @@ kind: CustomResourceDefinition
 metadata:
   name: cronworkflows.argoproj.io
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 spec:
   group: argoproj.io
   names:
@@ -53,7 +53,7 @@ kind: CustomResourceDefinition
 metadata:
   name: workflowtemplates.argoproj.io
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 spec:
   group: argoproj.io
   names:
@@ -69,14 +69,14 @@ kind: ServiceAccount
 metadata:
   name: argo
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: argo-role
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 rules:
   - apiGroups:
       - ""
@@ -128,22 +128,21 @@ kind: RoleBinding
 metadata:
   name: argo-binding
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: argo-role
 subjects:
   - kind: ServiceAccount
     name: argo
-    namespace: {{ .Release.Namespace }}
+    namespace: "{{ .Release.Namespace }}"
 ---
 apiVersion: v1
 data:
   config: |
     {
     namespace: {{ .Release.Namespace }},
-    executorImage: {{ .Values.images.argoexecutor }},
     artifactRepository:
     {
         s3: {
@@ -167,7 +166,7 @@ kind: ConfigMap
 metadata:
   name: workflow-controller-configmap
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 ---
 apiVersion: v1
 data:
@@ -177,15 +176,15 @@ kind: Secret
 metadata:
   name: mlpipeline-minio-artifact
   labels:
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
 type: Opaque
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
     app: workflow-controller
-    app.kubernetes.io/name: {{ .Release.Name }}
+    app.kubernetes.io/name: "{{ .Release.Name }}"
   name: workflow-controller
 spec:
   progressDeadlineSeconds: 600
@@ -194,7 +193,7 @@ spec:
   selector:
     matchLabels:
       app: workflow-controller
-      app.kubernetes.io/name: {{ .Release.Name }}
+      app.kubernetes.io/name: "{{ .Release.Name }}"
   strategy:
     rollingUpdate:
       maxSurge: 25%
@@ -205,14 +204,14 @@ spec:
       creationTimestamp: null
       labels:
         app: workflow-controller
-        app.kubernetes.io/name: {{ .Release.Name }}
+        app.kubernetes.io/name: "{{ .Release.Name }}"
     spec:
       containers:
         - args:
             - --configmap
             - workflow-controller-configmap
             - --executor-image
-            - {{ .Values.images.argoexecutor }}
+            - "{{ .Values.images.argoexecutor }}"
           command:
             - workflow-controller
           env:
@@ -221,7 +220,7 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: metadata.namespace
-          image: {{ .Values.images.argoworkflowcontroller }}
+          image: "{{ .Values.images.argoworkflowcontroller }}"
           imagePullPolicy: IfNotPresent
           name: workflow-controller
           resources:

diff --git a/manifests/gcp_marketplace/test/presubmit.sh b/manifests/gcp_marketplace/test/presubmit.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Copyright 2020-2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# When running this in MacOS, we can run `OS=darwin-amd64 ./presubmit.sh`
+OS="${OS:-linux-amd64}"
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+if ! which helm; then
+    HELM_VERSION=v2.17.0
+    curl -sLO "https://get.helm.sh/helm-${HELM_VERSION}-${OS}.tar.gz"
+    tar xvf "helm-${HELM_VERSION}-${OS}.tar.gz"
+    mv ./${OS}/helm /usr/local/bin/helm
+    helm version
+fi
+
+"${DIR}/snapshots.sh"
+# exit 1 if snapshots are not up-to-date.
+git diff --exit-code -- "${DIR}" || \
+    (echo "gcp_marketplace test snapshots are not tidy, run 'manifests/gcp_marketplace/test/snapshots.sh' to update them." \
+    && exit 1)