Merge remote-tracking branch 'origin/main' into canton-3.4

.envrc.vars

@@ -57,15 +57,7 @@ export CLOUDSDK_COMPUTE_REGION="us-central1"
 export DB_CLOUDSDK_COMPUTE_ZONE="${CLOUDSDK_COMPUTE_REGION}-a"
 # Default to the scratch environment
 export CLOUDSDK_CORE_PROJECT="da-cn-scratchnet"
-# Default cluster sizing
-export GCP_CLUSTER_NODE_TYPE=e2-standard-16
-export GCP_CLUSTER_MIN_NODES=0
-# A high max-nodes by default to support large deployments and hard migrations
-# Should be set to a lower number (currently 8) on CI clusters that do neither of those.
-export GCP_CLUSTER_MAX_NODES=20
-# The logging variant supports default, that google recommends for up to 100kb/s logs (https://cloud.google.com/kubernetes-engine/docs/how-to/adjust-log-throughput)
-# The max throughput variant supports multiple tens of MB/s of logs, but also the agents require 2CPUs and therefore we lose 2 CPUs per node
-export GCP_CLUSTER_LOGGING_VARIANT="DEFAULT"
+
 export GCP_DNS_PROJECT="da-gcp-canton-domain"
 export GCP_DNS_SA_SECRET="clouddns-dns01-solver-svc-acct"
 # DNS Service Account information

.github/actions/nix/setup_nix/action.yml

@@ -0,0 +1,235 @@
+name: "Setup Nix"
+description: "Setup Nix"
+inputs:
+  artifactory_user:
+    description: "The Artifactory user"
+    required: true
+  artifactory_password:
+    description: "The Artifactory password"
+    required: true
+  oss_only:
+    description: "Restrict upstream dependencies (e.g. Canton) to OSS versions (the equivalent of OSS_ONLY=1 in local checkouts)"
+    default: "false"
+    required: false
+  cache_version:
+    description: "The cache version"
+    required: true
+  should_save:
+    description: "If the nix cache should be saved"
+    # this should be run just from one job to ensure we avoid multi write conflicts, which makes everything worse
+    default: "false"
+  should_save_gcp:
+    description: "If the nix cache should be saved to the public GCP bucket"
+    default: "false"
+  upload_workload_identity_provider:
+    description: "The workload identity provider to use for uploading the cache"
+    required: false
+    default: ""
+  upload_service_account:
+    description: "The service account to use for uploading the cache"
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: Compute cache Key
+      id: cache_key
+      shell: bash
+      run: |
+        set -euxo pipefail
+        git ls-files nix/ | grep -v '[.]md$' | LC_ALL=C sort | xargs sha256sum -b > /tmp/nix-cache-key
+        uname -m >> /tmp/nix-cache-key # Add architecture to the cache key
+        echo "gh_cache_version: ${{ inputs.cache_version }}" >> /tmp/nix-cache-key # Add cache version to the cache key
+        if [ "${{ inputs.oss_only }}" == true ]; then
+          echo "Using OSS only dependencies"
+          echo "oss_only: ${{ inputs.oss_only }}" >> /tmp/nix-cache-key
+          touch /tmp/oss-only # Create a file to indicate that we are using OSS only dependencies (so we don't need to re-specifify oss_only to run_bash_command_in_nix)
+        fi
+        cat /tmp/nix-cache-key
+        cache_key=($(md5sum "/tmp/nix-cache-key"))
+        echo "cache_key=$cache_key" >> $GITHUB_ENV
+
+    - name: Download cache (for non-self-hosted)
+      if: ${{ !startsWith(runner.name, 'self-hosted') }}
+      shell: bash
+      run: |
+        set -euxo pipefail
+
+        if [ ${{ inputs.oss_only }} != 'true' ]; then
+          echo "Must use OSS only dependencies in GitHub-hosted runners"
+          exit 1
+        fi
+
+        echo "Latest nix cache:"
+
+        wget -q "https://storage.googleapis.com/splice-nix-cache-public/${cache_key}.tar.gz" -O cache.tar.gz || true
+        if [ ! -s cache.tar.gz ]; then
+          echo "Cache not found, fetching latest instead"
+          latest=$(curl https://storage.googleapis.com/storage/v1/b/splice-nix-cache-public/o | jq -r '.items | sort_by(.updated) | .[-1].name')
+          wget -q "https://storage.googleapis.com/splice-nix-cache-public/${latest}" -O cache.tar.gz
+
+        fi
+
+        sudo mkdir -p /cache/nix/${cache_key}
+        sudo tar -xzf cache.tar.gz -C /cache/nix/${cache_key}
+
+    - name: Restore nix
+      id: restore_nix
+      shell: bash
+      run: |
+        set -euxo pipefail
+        sudo mkdir -p /nix/store
+        sudo chown -R $(whoami):$(whoami) /nix
+        if [ -f "/cache/nix/$cache_key/cached" ]; then
+          echo "Restoring nix cache (key $cache_key)"
+          # we use rsync here because it's simply faster to install
+          rsync -avi /cache/nix/$cache_key/.nix-* $HOME/
+          rsync -avi "/cache/nix/$cache_key/nix" $HOME/.config/
+          rsync -avi "/cache/nix/$cache_key/nix_store/var/" /nix/var
+          sudo mount --bind /cache/nix/$cache_key/nix_store/store /nix/store
+        else
+          sudo mkdir -p "/cache/nix/$cache_key"
+          sudo chown $(whoami):$(whoami) "/cache/nix/$cache_key"
+          sudo chown $(whoami):$(whoami) "/cache/nix"
+        fi
+    - name: Setup Nix
+      shell: bash
+      run: |
+        set -exuo pipefail
+        echo 'source ~/.nix-profile/etc/profile.d/nix.sh' > nix.rc
+        if [[ -f ~/.config/nix/nix.conf && -f ~/.nix-profile/etc/profile.d/nix.sh ]]; then
+          echo "nix.conf or nix.sh already exists, skipping Nix setup"
+          exit 0
+        else
+          # Disabling sandbox because:
+          # 1. It doesn't work on CircleCI (sethostname is not allowed)
+          # 2. We don't plan to build anything, so the risk is fairly low
+          mkdir -p ~/.config/nix
+          if [ true ]; then
+            cat <<EOF > ~/.config/nix/nix.conf
+            sandbox = false
+            netrc-file = /etc/nix/netrc
+            extra-experimental-features = nix-command flakes
+            substituters = file:///cache/nix/binary_cache?trusted=1 https://cache.nixos.org/
+            trusted-substituters = file:///cache/nix/binary_cache?trusted=1
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+            cores = 4
+            max-jobs = 16
+        EOF
+          else
+            cat <<EOF > ~/.config/nix/nix.conf
+            sandbox = false
+            netrc-file = /etc/nix/netrc
+            extra-experimental-features = nix-command flakes
+            cores = 4
+            max-jobs = 16
+        EOF
+          fi
+          sh <(curl -fsSL --retry 8 https://releases.nixos.org/nix/nix-2.13.3/install) --no-daemon
+          sudo mkdir -p /etc/nix
+          sudo chmod a+rw /etc/nix
+          if [[ "${{ inputs.oss_only }}" == true ]]; then
+            echo "Using OSS only dependencies, not setting up Artifactory credentials"
+          else
+            cat <<EOF > /etc/nix/netrc
+            machine digitalasset.jfrog.io
+            login ${{ inputs.artifactory_user }}
+            password ${{ inputs.artifactory_password }}
+        EOF
+          fi
+          export USER=$(whoami)
+          echo "Running nix.sh"
+          . ~/.nix-profile/etc/profile.d/nix.sh
+          if [[ "${{ inputs.oss_only }}" == true ]]; then
+            target="oss"
+          else
+            target="default"
+          fi
+          nix develop path:nix#${target} -v --profile "$HOME/.nix-shell" --command echo "Done loading packages"
+          echo "Garbage collecting to reduce cache size"
+          nix-store --gc
+        fi
+
+    - name: Invoke nix before saving cache
+      uses: ./.github/actions/nix/run_bash_command_in_nix
+      with:
+        cmd: |
+          echo "Validated nix"
+          ls -al
+
+    # The nix cache does not change in the workflow, so we can save it immediately, rather than splitting it into pre-&post- steps
+    - name: Save nix cache
+      shell: bash
+      if: ${{ inputs.should_save == 'true' }}
+      run: |
+        set -euxo pipefail
+        echo ~
+        chown -R $(whoami):$(whoami) ~
+        cat /tmp/nix-cache-key
+        if [ ! -f "/cache/nix/$cache_key/cached" ]; then
+          echo "Saving nix"
+
+          sudo -v ; curl https://rclone.org/install.sh | sudo bash
+
+          echo "sourcing nix profile"
+          export USER=$(whoami)
+          . ~/.nix-profile/etc/profile.d/nix.sh
+
+          nix copy --all --to 'file:///cache/nix/binary_cache?trusted=1' -v
+
+          CLONE_COMMAND="rclone --no-update-dir-modtime --no-update-modtime --size-only --multi-thread-streams=32 --transfers=32 --ignore-existing --links --create-empty-src-dirs --fast-list --metadata --order-by name,mixed --retries 10 copy"
+          ${CLONE_COMMAND} "$HOME/" "/cache/nix/$cache_key/" --include ".nix-*/**" --include ".nix-*"
+          ${CLONE_COMMAND} $HOME/.config/nix "/cache/nix/$cache_key/nix"
+
+          mkdir -p "/cache/nix/$cache_key/nix_store/store"
+          mkdir -p "/cache/nix/$cache_key/nix_store/var"
+
+          #requires to preserve read only during clone
+          sudo ${CLONE_COMMAND} /nix/store/ /cache/nix/$cache_key/nix_store/store
+          sudo ${CLONE_COMMAND} /nix/var/ "/cache/nix/$cache_key/nix_store/var"
+
+          echo "done" > "/cache/nix/$cache_key/cached"
+        fi
+
+    - name: Check if cache already exists in GCP
+      id: already_exists
+      if: ${{ inputs.should_save_gcp == 'true' }}
+      shell: bash
+      run: |
+        if curl -Isf https://storage.googleapis.com/splice-nix-cache-public/${cache_key}.tar.gz &> /dev/null; then
+          echo "Cache with key ${cache_key} already exists in GCP, not uploading again"
+          echo "already_exists=true" >> $GITHUB_OUTPUT;
+        fi
+    - name: Authenticate to GCP
+      id: auth
+      if: ${{ inputs.should_save_gcp == 'true' && steps.already_exists.outputs.already_exists != 'true' }}
+      uses: "google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193" #v2.1.10
+      with:
+        workload_identity_provider: "${{ inputs.upload_workload_identity_provider }}"
+        service_account: "${{ inputs.upload_service_account }}"
+
+    - name: tar-gz the cache
+      shell: bash
+      if: ${{ inputs.should_save_gcp == 'true' && steps.already_exists.outputs.already_exists != 'true' }}
+      id: prep_cache_upload
+      run: |
+        set -euxo pipefail
+        echo "Compressing nix cache to /cache/nix/${cache_key}.tar.gz"
+        mkdir -p /tmp/nix-upload
+
+        tar -czf "/tmp/nix-upload/${cache_key}.tar.gz" -C "/cache/nix/$cache_key" .
+
+        echo "Cache compressed to /tmp/nix-upload/${cache_key}.tar.gz"
+        ls /tmp/nix-upload
+        echo "cache_file=/tmp/nix-upload/${cache_key}.tar.gz" >> $GITHUB_OUTPUT
+
+    - name: Upload nix cache
+      if: ${{ inputs.should_save_gcp == 'true' && steps.already_exists.outputs.already_exists != 'true' }}
+      uses: google-github-actions/upload-cloud-storage@v2
+      with:
+        destination: splice-nix-cache-public
+        path: "${{ steps.prep_cache_upload.outputs.cache_file }}"
+        parent: false # upload to root of the bucket
+        process_gcloudignore: false # no gcloud ignore file in this repo, must set this to false
+        gzip: false # it's already gzipped