feat(argo): enable ARC build for oci images

.argo/nix.yaml

@@ -0,0 +1,44 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  namespace: argo
+spec:
+  entrypoint: main
+  serviceAccountName: argo-server
+  arguments:
+    parameters:
+      - name: appName
+        value: pyrovelocity
+      - name: gitUrlNoProtocol
+        value: github.com/pinellolab
+      - name: images
+      - name: version
+      - name: branch
+      - name: gitShaShort
+      - name: gitSha
+
+  templates:
+    - name: main
+      steps:
+        - - name: publish-pyrovelocity-image
+            templateRef:
+              name: nix-workflow-template
+              template: main
+              clusterScope: true
+            arguments:
+              parameters:
+                - name: appName
+                  value: "{{workflow.parameters.appName}}"
+                - name: gitUrlNoProtocol
+                  value: "{{workflow.parameters.gitUrlNoProtocol}}"
+                - name: imageName
+                  value: "{{item}}"
+                - name: version
+                  value: "{{workflow.parameters.version}}"
+                - name: branch
+                  value: "{{workflow.parameters.branch}}"
+                - name: gitShaShort
+                  value: "{{workflow.parameters.gitShaShort}}"
+                - name: gitSha
+                  value: "{{workflow.parameters.gitSha}}"
+            withParam: "{{workflow.parameters.images}}"

.github/actions/build_nix_image/action.yml

@@ -137,3 +137,120 @@ runs:
         ENCODED_GAR_SA_CREDS: ${{ inputs.encodedGARCredentials }}
         ARTIFACT_REGISTRY_PASSWORD: ${{ inputs.artifactRegistryPassword }}
         VERSION: ${{ inputs.releaseVersion }}
+
+  # build-nix-images:
+  #   runs-on: ubuntu-latest
+  #   needs: [set-variables]
+  #   if: ${{ needs.set-variables.outputs.skip_ci != 'true' && ( contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images) || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow) ) }}
+  #   concurrency:
+  #     group: bni-${{ matrix.image }}-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }}
+  #     cancel-in-progress: true
+  #   permissions:
+  #     contents: read
+  #     packages: write
+  #   strategy:
+  #     matrix:
+  #       image:
+  #         [
+  #           "pyrovelocity",
+  #           "pyrovelocitydev",
+  #           "pyrovelocitycode",
+  #           "pyrovelocityjupyter",
+  #         ]
+  #   steps:
+  #     - name: Maximize build space
+  #       # uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # ratchet:easimon/maximize-build-space@v10
+  #       uses: cameronraysmith/maximize-build-space@fdf0c06b18d92be98aa64cb68ae4ea4c9bc4794d # ratchet:cameronraysmith/maximize-build-space@print-usage
+  #       if: ${{ needs.set-variables.outputs.mode != 'dev' }}
+  #       with:
+  #         build-mount-path: /nix
+  #         build-mount-path-ownership: "root:root"
+  #         root-reserve-mb: 34816
+  #         swap-size-mb: 2048
+  #         remove-dotnet: "true"
+  #         remove-android: "true"
+  #         remove-codeql: "true"
+  #         remove-docker-images: "true"
+
+  #     - name: Setup build environment
+  #       uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # ratchet:actions/checkout@v4
+  #       if: ${{ needs.set-variables.outputs.mode != 'dev' }}
+  #       with:
+  #         ref: ${{ needs.set-variables.outputs.checkout_ref }}
+
+  #     - name: Build ${{ matrix.image }} image
+  #       uses: ./.github/actions/build_nix_image
+  #       if: ${{ needs.set-variables.outputs.mode != 'dev' }}
+  #       with:
+  #         triggerEventName: ${{ github.event_name }}
+  #         imageName: "${{ matrix.image }}"
+  #         nixCommand: "nix run .#${{ matrix.image }}Manifest --impure --accept-flake-config --show-trace"
+  #         releaseVersion: ""
+  #         gcpProjectId: ${{ secrets.GCP_PROJECT_ID }}
+  #         encodedGARCredentials: ${{ secrets.ENCODED_GAR_SA_CREDS }}
+  #         artifactRegistryPassword: ${{ secrets.ARTIFACT_REGISTRY_PASSWORD }}
+  #         nixSubstituterName: ${{ vars.NIX_SUBSTITUTER }}
+  #         # nixSubstituterName: ${{ vars.S3_NIX_SUBSTITUTER }}
+  #         nixPrivateKey: ${{ secrets.NIX_PRIVATE_KEY }}
+  #         gacd: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_DATA }}
+  #         gcsHmacAccessID: ${{ secrets.GCS_HMAC_ACCESS_ID }}
+  #         gcsHmacSecretAccessKey: ${{ secrets.GCS_HMAC_SECRET_ACCESS_KEY }}
+  #         cachixCacheName: ${{ vars.CACHIX_CACHE_NAME }}
+  #         cachixAuthToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+  # publish-release-images:
+  #   runs-on: ubuntu-latest
+  #   needs: [set-variables, release]
+  #   if: ${{ needs.release.outputs.released == 'true' }}
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       image:
+  #         [
+  #           "pyrovelocity",
+  #           "pyrovelocitydev",
+  #           "pyrovelocitycode",
+  #           "pyrovelocityjupyter",
+  #         ]
+  #   concurrency:
+  #     group: pri-${{ matrix.image }}-${{ github.workflow }}-${{ github.ref }}-${{ github.sha }}
+  #     cancel-in-progress: true
+  #   permissions:
+  #     contents: read
+  #     packages: write
+  #   # environment:
+  #   #   name: release
+  #   #   url: ${{ matrix.url }}
+  #   steps:
+  #     - name: maximize build space
+  #       # uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # ratchet:easimon/maximize-build-space@v10
+  #       uses: cameronraysmith/maximize-build-space@fdf0c06b18d92be98aa64cb68ae4ea4c9bc4794d # ratchet:cameronraysmith/maximize-build-space@print-usage
+  #       with:
+  #         build-mount-path: /nix
+  #         build-mount-path-ownership: "root:root"
+  #         root-reserve-mb: 34816
+  #         swap-size-mb: 2048
+  #         remove-dotnet: "true"
+  #         remove-android: "true"
+  #         remove-codeql: "true"
+  #         remove-docker-images: "true"
+  #     - name: Setup build environment
+  #       uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # ratchet:actions/checkout@v4
+  #       with:
+  #         ref: ${{ needs.release.outputs.git-tag }}
+  #     - name: Build ${{ matrix.image }} image
+  #       uses: ./.github/actions/build_nix_image
+  #       if: ${{ needs.set-variables.outputs.mode != 'dev' }}
+  #       with:
+  #         triggerEventName: ${{ github.event_name }}
+  #         imageName: "${{ matrix.image }}"
+  #         nixCommand: "nix run .#${{ matrix.image }}Manifest --impure --accept-flake-config --show-trace"
+  #         releaseVersion: ${{ needs.release.outputs.version }}
+  #         gcpProjectId: ${{ secrets.GCP_PROJECT_ID }}
+  #         encodedGARCredentials: ${{ secrets.ENCODED_GAR_SA_CREDS }}
+  #         artifactRegistryPassword: ${{ secrets.ARTIFACT_REGISTRY_PASSWORD }}
+  #         nixSubstituterName: ${{ vars.NIX_SUBSTITUTER }}
+  #         nixPrivateKey: ${{ secrets.NIX_PRIVATE_KEY }}
+  #         gacd: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_DATA }}
+  #         cachixCacheName: ${{ vars.CACHIX_CACHE_NAME }}
+  #         cachixAuthToken: ${{ secrets.CACHIX_AUTH_TOKEN }}

.github/mergify.yml

@@ -25,8 +25,3 @@ queue_rules:
     update_method: rebase
     update_bot_account: pyrovelocity-bot
     merge_bot_account: pyrovelocity-bot
-    conditions:
-      - check-success=test-python
-      - check-success=test-nix
-      - check-success=test-bazel
-      - check-success=test-docs-build

.github/workflows/build-images.yaml

@@ -98,3 +98,16 @@ jobs:
             -p containerRegistryURL2="${CONTAINER_REGISTRY_URL2}" \
             -p gitUrlNoProtocol="github.com/${CI_GITHUB_REPOSITORY_OWNER_PART_SLUG}" \
             --wait --log
+
+  # build-docker-images:
+  #   needs: [set-variables]
+  #   if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.mode != 'dev' && (contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images)) }}
+  #   uses: ./.github/workflows/build-images.yaml
+  #   # concurrency:
+  #   #   group: bi-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }}
+  #   #   cancel-in-progress: true
+  #   secrets:
+  #     GCP_ARTIFACT_REGISTRY_PATH: ${{ secrets.GCP_ARTIFACT_REGISTRY_PATH }}
+  #   with:
+  #     debug_enabled: ${{ needs.set-variables.outputs.debug }}
+  #     mode: ${{ needs.set-variables.outputs.mode }}

.github/workflows/build-nix-images.yaml

@@ -0,0 +1,124 @@
+name: Build nix images
+
+on:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: "Run with tmate.io debugging enabled"
+        required: true
+        type: boolean
+        default: false
+      version:
+        description: "Version of the container image"
+        required: true
+        type: string
+      images:
+        description: "List of images to build"
+        required: true
+        type: string
+      branch:
+        description: "Git branch from which to build the container image"
+        required: true
+        type: string
+      revision:
+        description: "Git revision from which to build the container image"
+        required: true
+        type: string
+      mode:
+        description: "Container build mode ('dev' uses branch name image tags and 'prod' uses short sha.)"
+        required: true
+        type: choice
+        default: "dev"
+        options:
+          - dev
+          - prod
+  workflow_call:
+    inputs:
+      debug_enabled:
+        description: "Run with tmate.io debugging enabled"
+        required: true
+        type: string
+        default: "false"
+      version:
+        description: "Version of the container image"
+        required: true
+        type: string
+      images:
+        description: 'Stringified list ''["i1", "i2", ...]'' of image names to build'
+        required: true
+        type: string
+      branch:
+        description: "Git branch from which to build the container image"
+        required: true
+        type: string
+      revision:
+        description: "Git revision from which to build the container image"
+        required: true
+        type: string
+      mode:
+        description: "Container build mode ('dev' skips and 'prod' builds.)"
+        required: true
+        type: string
+        default: "dev"
+
+env:
+  ARGO_NAMESPACE: argo
+  ARGO_VERSION: v3.5.8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  actions: write
+  contents: read
+
+jobs:
+  build:
+    runs-on: pinellolab-runners
+    steps:
+      # - name: Cache argo CLI
+      #   uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # ratchet:actions/cache@v4
+      #   with:
+      #     path: /usr/local/bin/argo
+      #     key: argo-${{ env.ARGO_VERSION }}
+      - name: Install argo CLI
+        if: ${{ inputs.mode != 'dev' }}
+        run: |
+          if [ ! -f /usr/local/bin/argo ]; then
+            echo "Installing Argo CLI"
+            cd $HOME
+            sudo apt-get update && sudo apt-get install -y curl
+            curl -sLO https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz
+            gunzip argo-linux-amd64.gz
+            chmod +x argo-linux-amd64
+            sudo mv ./argo-linux-amd64 /usr/local/bin/argo
+          else
+            echo "Using cached Argo CLI"
+          fi
+          argo version
+      - name: Check out repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # ratchet:actions/checkout@v4
+      - name: Setup tmate debug session
+        if: ${{ inputs.debug_enabled == 'true' }}
+        uses: mxschmitt/action-tmate@a283f9441d2d96eb62436dc46d7014f5d357ac22 # ratchet:mxschmitt/action-tmate@v3
+      - name: Build images
+        if: ${{ inputs.mode != 'dev' }}
+        env:
+          CI_GITHUB_SHA: ${{ inputs.revision }}
+        run: |
+          echo "Building images: '${{ inputs.images }}'"
+          CI_GITHUB_SHA_SHORT="${CI_GITHUB_SHA:0:7}"
+          echo "CI_GITHUB_SHA: $CI_GITHUB_SHA"
+          echo "CI_GITHUB_SHA_SHORT: $CI_GITHUB_SHA_SHORT"
+          echo "Branch: ${{ inputs.branch }}"
+          echo "Version: ${{ inputs.version }}"
+
+          argo submit .argo/nix.yaml \
+            --generate-name="pyrovelocity-nix-${CI_GITHUB_SHA_SHORT}-" \
+            -p images='${{ inputs.images }}' \
+            -p version="${{ inputs.version }}" \
+            -p branch="${{ inputs.branch }}" \
+            -p gitShaShort="${CI_GITHUB_SHA_SHORT}" \
+            -p gitSha="${CI_GITHUB_SHA}" \
+            --wait --log