Merge branch 'master' into subset-experiment-flag

.flux.yaml

@@ -5,23 +5,86 @@ metadata:
   labels:
     sandboxId: FILLED_IN_BY_CI
 ---
-apiVersion: helm.fluxcd.io/v1
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: GitRepository
+metadata:
+  name: FILLED_IN_BY_CI
+  namespace: FILLED_IN_BY_CI
+spec:
+  interval: 5m0s
+  url: https://github.com/biomage-org/pipeline
+  ref:
+    branch: FILLED_IN_BY_CI
+---
+apiVersion: image.toolkit.fluxcd.io/v1beta1
+kind: ImageRepository
+metadata:
+  name: FILLED_IN_BY_CI
+  namespace: FILLED_IN_BY_CI
+spec:
+  image: FILLED_IN_BY_CI
+  interval: 2m0s
+---
+apiVersion: image.toolkit.fluxcd.io/v1beta1
+kind: ImagePolicy
+metadata:
+  name: FILLED_IN_BY_CI
+  namespace: FILLED_IN_BY_CI
+spec:
+  imageRepositoryRef:
+    name: FILLED_IN_BY_CI
+    namespace: FILLED_IN_BY_CI
+  filterTags:
+    pattern: FILLED_IN_BY_CI
+    extract: FILLED_IN_BY_CI
+---
+apiVersion: image.toolkit.fluxcd.io/v1beta1
+kind: ImageUpdateAutomation
+metadata:
+  name: FILLED_IN_BY_CI
+  namespace: FILLED_IN_BY_CI
+spec:
+  interval: 2m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  git:
+    commit:
+      author:
+        name: Flux - Pipeline image update
+        email: ci@biomage.net
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
   name: FILLED_IN_BY_CI
   namespace: FILLED_IN_BY_CI
-  annotations:
-    fluxcd.io/automated: "true"
   labels:
     sandboxId: FILLED_IN_BY_CI
 spec:
   releaseName: FILLED_IN_BY_CI
+  interval: 2m0s
   chart:
-    git: https://github.com/biomage-org/pipeline
-    path: chart-infra/
-    ref: FILLED_IN_BY_CI
+    spec:
+      chart: chart-infra/
+      sourceRef:
+        kind: GitRepository
+        name: FILLED_IN_BY_CI
+        namespace: FILLED_IN_BY_CI
+  values:
+    replicaCount: FILLED_IN_BY_CI
+    clusterEnv: FILLED_IN_BY_CI
+    sandboxId: FILLED_IN_BY_CI
+    image:
+      registry: FILLED_IN_BY_CI
+      repository: FILLED_IN_BY_CI
+      tag: FILLED_IN_BY_CI
+    memoryRequest: FILLED_IN_BY_CI
+    serviceAccount:
+      iamRole: FILLED_IN_BY_CI
   valuesFrom:
-  - configMapKeyRef:
+    - kind: ConfigMap
+      valuesKey: account-config.yaml
       name: account-config
-      namespace: flux
-      key: account-config.yaml
+      optional: false

.github/workflows/ci.yaml

@@ -59,6 +59,7 @@ jobs:
       repo-name: ${{ steps.ref.outputs.repo-name }}
       image-tag: ${{ steps.ref.outputs.image-tag }}
       ref-id: ${{ steps.ref.outputs.ref-id }}
+      timestamp: ${{ steps.ref.outputs.timestamp }}
     defaults:
       run:
         working-directory: ${{ matrix.project }}
@@ -84,22 +85,32 @@ jobs:
           # in push events, the latest commit of the master branch is GITHUB_SHA
           # in PR synch the latest commit of the branch is found in github.event.pull_request.head.sha instead
 
+          TIMESTAMP=$(date +%s)
+
           if [ "${{ github.event_name }}" = "release" ] && [ "${{ github.event.action }}" = "released" ]; then
             COMMIT_SHA=""
             IMAGE_TAG=$REF_ID
+            TIMESTAMP=""
           elif [ "${{ github.event_name }}" = "pull_request" ]; then
             COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
             IMAGE_TAG="$REF_ID-$COMMIT_SHA"
+            TIMESTAMP=$TIMESTAMP
           else
             COMMIT_SHA=$GITHUB_SHA
             IMAGE_TAG="$REF_ID-$COMMIT_SHA"
+            TIMESTAMP=$TIMESTAMP
           fi
 
           # IMAGE_TAG is used in the Build Docker Image step.
           # We can easily build the image-tag from REF_ID and COMMIT_SHA for non-production releases.
           # But we can not easily create the image tag for production releases, so we're bulding it here
           echo "::set-output name=image-tag::$IMAGE_TAG"
 
+          # TIMESTAMP is used to postfix images in the "push docker images to ECR" step.
+          # The timestamp is used by Flux to auto update images for staging environments.
+          # Images for production uses semantic versioning to determine the latest image.
+          echo "::set-output name=timestamp::$TIMESTAMP"
+
           # This will take a GitHub repo name like `biomage-org/releases`
           # and turns it into `releases`. This will be the name of the
           # ECR repository.
@@ -205,8 +216,16 @@ jobs:
 
           echo Pushing image $IMAGE_NAME to ECR.
           docker push $IMAGE_NAME
+
+          if [ ! -z "$TIMESTAMP" ]; then
+            echo Pushing timestamped image $IMAGE_NAME-$TIMESTAMP to ECR
+
+            docker tag $IMAGE_NAME $IMAGE_NAME-$TIMESTAMP
+            docker push $IMAGE_NAME-$TIMESTAMP
+          fi
         env:
           IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag, matrix.project) }}
+          TIMESTAMP: ${{ steps.ref.outputs.timestamp }}
 
   deploy:
     name: Deploy to Kubernetes
@@ -242,58 +261,111 @@ jobs:
           # Deployment config for `production-default`
           if [ "${{ matrix.environment }}" = "production" ]; then
             export SANDBOX_ID="default"
+            export CHART_REF_TYPE="commit"
             export CHART_REF="$GITHUB_SHA"
             export KUBERNETES_ENV="production"
+            export IMAGE_NAME=$IMAGE_TAG-pipeline-runner
             export REPLICA_COUNT="3"
             export VERSION_NUMBER=${REF_ID/refs-tags-/}
-            export IMAGE_GLOB="${REF_ID/$VERSION_NUMBER/*}"
+            export IMAGE_PATTERN="^refs-tags-(?P<version>[0-9]+\.[0-9]+\.[0-9]+)-pipeline-runner$"
+            export IMAGE_EXTRACT='$version'
+            export IMAGE_POLICY_TYPE="semver"
+            export IMAGE_POLICY_KEY="range"
+            export IMAGE_POLICY_VALUE=">=0.0.0"
             export MEMORY_REQUEST="29Gi"
           fi
 
           # Deployment config for `staging-default`
           if [ "${{ matrix.environment }}" = "develop" ]; then
             export SANDBOX_ID="default"
-            export CHART_REF="$GITHUB_SHA"
+            export CHART_REF_TYPE="branch"
+            export CHART_REF="master"
             export KUBERNETES_ENV="staging"
+            export IMAGE_NAME="$IMAGE_TAG-pipeline-runner-$TIMESTAMP"
             export REPLICA_COUNT="0"
-            export IMAGE_GLOB="$REF_ID-*"
+            export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-pipeline-runner-(?P<timestamp>[0-9]+)"
+            export IMAGE_EXTRACT='$timestamp'
+            export IMAGE_POLICY_TYPE="numerical"
+            export IMAGE_POLICY_KEY="order"
+            export IMAGE_POLICY_VALUE="asc"
             export MEMORY_REQUEST="29Gi"
           fi
 
           # Deployment config for other staging env i.e. non `default`
           if [ "${{ matrix.environment }}" = "staging" ]; then
+
+            export CHART_REF="$GITHUB_HEAD_REF"
+
+            # $GITHUB_HEAD_REF references the source branch of the pull request in a workflow run
+            # $GITHUB_HEAD_REF does not have a value when we merge, because merging is counted as a "push" event to master branch
+            # Therefore we set this value to "master" manually
+            if [ -z "$CHART_REF" ]; then
+              export CHART_REF="master"
+            fi
+
             export SANDBOX_ID="STAGING_SANDBOX_ID"
-            export CHART_REF="STAGING_CHART_REF"
+            export CHART_REF_TYPE="branch"
             export KUBERNETES_ENV="staging"
+            export IMAGE_NAME="$IMAGE_TAG-pipeline-runner-$TIMESTAMP"
             export REPLICA_COUNT="0"
-            export IMAGE_GLOB="$REF_ID-*"
+            export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-pipeline-runner-(?P<timestamp>[0-9]+)"
+            export IMAGE_EXTRACT='$timestamp'
+            export IMAGE_POLICY_TYPE="numerical"
+            export IMAGE_POLICY_KEY="order"
+            export IMAGE_POLICY_VALUE="asc"
             export MEMORY_REQUEST="4Gi"
           fi
 
           echo "::set-output name=sandbox-id::$SANDBOX_ID"
           echo "::set-output name=kubernetes-env::$KUBERNETES_ENV"
 
-          yq 'select(di == 0).metadata.name = strenv(DEPLOYMENT_NAME) + "-" + strenv(SANDBOX_ID) |
+          export NAMESPACE="$DEPLOYMENT_NAME-$SANDBOX_ID"
+          export CHART_CRD_NAME="$DEPLOYMENT_NAME-chart"
+          export IMAGE_POLICY_TAG="{\"\$imagepolicy\": \"$NAMESPACE:$DEPLOYMENT_NAME:tag\"}"
+
+          yq '
+            select(di == 0).metadata.name = strenv(NAMESPACE) |
             select(di == 0).metadata.labels.sandboxId = strenv(SANDBOX_ID) |
-            select(di == 1).metadata.name = strenv(DEPLOYMENT_NAME) |
-            select(di == 1).metadata.namespace = strenv(DEPLOYMENT_NAME) + "-" + strenv(SANDBOX_ID) |
-            select(di == 1).metadata.labels.sandboxId = strenv(SANDBOX_ID) |
-            select(di == 1).spec.chart.ref = strenv(CHART_REF) |
-            select(di == 1).spec.releaseName = strenv(DEPLOYMENT_NAME) |
-            select(di == 1).spec.values.clusterEnv = strenv(KUBERNETES_ENV) |
-            select(di == 1).spec.values.pipelineRunner.image = strenv(IMAGE_NAME) + "-pipeline-runner" |
-            select(di == 1).spec.values.sandboxId = strenv(SANDBOX_ID) |
-            select(di == 1).spec.values.memoryRequest = strenv(MEMORY_REQUEST) |
-            select(di == 1).spec.values.replicaCount = env(REPLICA_COUNT) |
-            select(di == 1).spec.values.serviceAccount.iamRole = "pipeline-role-" + strenv(KUBERNETES_ENV) |
-            select(di == 1).spec.values.datadogTags = "kube_namespace:" + strenv(DEPLOYMENT_NAME) + "-" + strenv(SANDBOX_ID) |
-            select(di == 1).metadata.annotations["filter.fluxcd.io/pipelineRunner"] = "glob:" + strenv(IMAGE_GLOB) + "-pipeline-runner"
+            select(di == 1).metadata.name = strenv(CHART_CRD_NAME) |
+            select(di == 1).metadata.namespace = strenv(NAMESPACE) |
+            select(di == 1).spec.ref.[strenv(CHART_REF_TYPE)] = strenv(CHART_REF) |
+            select(di == 2).metadata.name = strenv(DEPLOYMENT_NAME) |
+            select(di == 2).metadata.namespace = strenv(NAMESPACE) |
+            select(di == 2).spec.image = strenv(REGISTRY) + "/" + strenv(DEPLOYMENT_NAME) |
+            select(di == 3).metadata.name = strenv(DEPLOYMENT_NAME) |
+            select(di == 3).metadata.namespace = strenv(NAMESPACE) |
+            select(di == 3).spec.imageRepositoryRef.name = strenv(DEPLOYMENT_NAME) |
+            select(di == 3).spec.imageRepositoryRef.namespace = strenv(NAMESPACE) |
+            select(di == 3).spec.filterTags.pattern = strenv(IMAGE_PATTERN) |
+            select(di == 3).spec.filterTags.extract = strenv(IMAGE_EXTRACT) |
+            select(di == 3).spec.policy.[strenv(IMAGE_POLICY_TYPE)].[strenv(IMAGE_POLICY_KEY)] = strenv(IMAGE_POLICY_VALUE) |
+            select(di == 4).metadata.name = strenv(DEPLOYMENT_NAME)  + "-image-update" |
+            select(di == 4).metadata.namespace = strenv(NAMESPACE) |
+            select(di == 5).metadata.name = strenv(DEPLOYMENT_NAME) |
+            select(di == 5).metadata.namespace = strenv(NAMESPACE) |
+            select(di == 5).metadata.labels.sandboxId = strenv(SANDBOX_ID) |
+            select(di == 5).spec.releaseName = strenv(DEPLOYMENT_NAME) |
+            select(di == 5).spec.chart.spec.sourceRef.name = strenv(CHART_CRD_NAME) |
+            select(di == 5).spec.chart.spec.sourceRef.namespace = strenv(NAMESPACE) |
+            select(di == 5).spec.values.clusterEnv = strenv(KUBERNETES_ENV) |
+            select(di == 5).spec.values.image.registry = strenv(REGISTRY) |
+            select(di == 5).spec.values.image.repository = strenv(REPOSITORY) |
+            select(di == 5).spec.values.image.tag = strenv(IMAGE_NAME) |
+            select(di == 5).spec.values.image.tag line_comment = strenv(IMAGE_POLICY_TAG) |
+            select(di == 5).spec.values.sandboxId = strenv(SANDBOX_ID) |
+            select(di == 5).spec.values.memoryRequest = strenv(MEMORY_REQUEST) |
+            select(di == 5).spec.values.replicaCount = env(REPLICA_COUNT) |
+            select(di == 5).spec.values.serviceAccount.iamRole = "pipeline-role-" + strenv(KUBERNETES_ENV) |
+            select(di == 5).spec.values.datadogTags = "kube_namespace:" + strenv(NAMESPACE)
           ' .flux.yaml > $DEPLOYMENT_NAME.yaml
 
           cat $DEPLOYMENT_NAME.yaml
         env:
-          IMAGE_NAME: ${{ format('{0}/{1}:{2}', steps.login-ecr.outputs.registry, needs.build-docker.outputs.repo-name, needs.build-docker.outputs.image-tag) }}
+          IMAGE_TAG: ${{ needs.build-docker.outputs.image-tag }}
           REF_ID: ${{ needs.build-docker.outputs.ref-id }}
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: ${{ needs.build-docker.outputs.repo-name  }}
+          TIMESTAMP: ${{ needs.build-docker.outputs.timestamp }}
 
       - name: Push production/develop template to releases
         if:

chart-infra/templates/pod-template.tpl

@@ -11,7 +11,7 @@
       serviceAccountName: 'deployment-runner'
       containers:
       - name: "{{ .Release.Name }}"
-        image: "{{ .Values.pipelineRunner.image }}"
+        image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         env:
           - name: CLUSTER_ENV
             value: "{{ .Values.clusterEnv }}"