Migrate to Dagger Runners v2 (dagger#7082)

* Use Dagger v0.11.2 via the new CI setup for all workflows except dev-engine This one requires Docker with specific fixes that we don't yet have in the new CI setup. Signed-off-by: Gerhard Lazu <gerhard@dagger.io> * Setup CI for new, legacy and vertical scaling The setup we want for production is: - For all <LANG> SDK jobs, run them on the new CI only - For testdev, run them on the docker-fix legacy CI - For test/dagger-runner, run them on both legacy CI and new CI - For all the rest, run them on the new CI and github runners for the really simple jobs Signed-off-by: Matias Pan <matias@dagger.io> * Rename concurrency group Signed-off-by: Matias Pan <matias@dagger.io> * Install curl on production vertical scaling runner Signed-off-by: Matias Pan <matias@dagger.io> * Add customizable runner for separate perf tests Signed-off-by: Matias Pan <matias@dagger.io> * Rename to _async_hack_make Signed-off-by: Matias Pan <matias@dagger.io> * Upgrade missing workflow to v0.11.1 Signed-off-by: Matias Pan <matias@dagger.io> * Target nvme Signed-off-by: Matias Pan <matias@dagger.io> * CI: Default to 4CPUs & NVMe disks Otherwise the workflows are too slow on the new CI runners and are blocking the migration off the legacy CI runners. Signed-off-by: Gerhard Lazu <gerhard@dagger.io> * Bump to v0.11.2 & capture extra details in comments Signed-off-by: Gerhard Lazu <gerhard@dagger.io> * Debug dagger-engine.dev in large GitHub Runner Signed-off-by: Gerhard Lazu <gerhard@dagger.io> * Continuer running engine:testdev in dagger-runner-docker-fix runner Large GitHub Runners are failing consistently, not worth debugging at this point since we know this works on a vanilla Ubuntu 24.04 instance with Docker - must be an issue related to GitHub Large Runners. FTR: dagger#7223 (comment) Signed-off-by: Gerhard Lazu <gerhard@dagger.io> --------- Signed-off-by: Gerhard Lazu <gerhard@dagger.io> Signed-off-by: Matias Pan <matias@dagger.io> Co-authored-by: Gerhard Lazu <gerhard@dagger.io> Co-authored-by: Matias Pan <matias@dagger.io>
vikram-dagger · May 8, 2024 · 184d6db · 184d6db
1 parent 7672599
commit 184d6db
Show file tree

Hide file tree

Showing 4 changed files with 123 additions and 25 deletions.
diff --git a/.github/workflows/_async_hack_make.yml b/.github/workflows/_async_hack_make.yml
@@ -0,0 +1,67 @@
+# This is run asynchronously, triggered from "hack/make".
+# The purpose is to compare the behaviour of different runner configurations.
+name: "async/hack/make"
+
+on:
+  workflow_dispatch:
+    inputs:
+      mage-targets:
+        description: "The mage target(s) to execute"
+        type: string
+        required: true
+      dev-engine:
+        description: "Run against a dev Engine"
+        type: boolean
+        default: false
+        required: false
+      dagger-cloud-cache:
+        description: "Use Dagger Cloud Cache"
+        type: boolean
+        default: false
+        required: false
+      runner:
+        description: "Runner"
+        type: string
+        default: "dagger-v011-on-k8s-gerhard-production"
+        required: false
+      concurrency-group:
+        description: "Concurrency group"
+        type: string
+        default: "global"
+        required: false
+
+jobs:
+  depot-runner:
+    if: ${{ github.repository == 'dagger/dagger' }}
+    runs-on: ${{ inputs.runner }}
+    name: ${{ inputs.mage-targets }} on ${{ inputs.runner }} dev-engine:${{ inputs.dev-engine }} dagger-cloud-cache:${{ inputs.dagger-cloud-cache }}
+    concurrency:
+      group: ${{ inputs.concurrency-group }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+          cache-dependency-path: "internal/mage/go.sum"
+      - name: Waiting for Dagger Engine to be ready...
+        run: |
+          if [ "${{ inputs.dev-engine }}" == "true" ]
+          then
+            ./hack/dev
+            export _EXPERIMENTAL_DAGGER_CLI_BIN="$PWD/bin/dagger"
+            chmod +x $_EXPERIMENTAL_DAGGER_CLI_BIN
+            echo "_EXPERIMENTAL_DAGGER_CLI_BIN=${_EXPERIMENTAL_DAGGER_CLI_BIN}" >> "$GITHUB_ENV"
+            export _EXPERIMENTAL_DAGGER_RUNNER_HOST=docker-container://dagger-engine.dev
+            echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=${_EXPERIMENTAL_DAGGER_RUNNER_HOST}" >> "$GITHUB_ENV"
+          fi
+          ./hack/make engine:connect
+      - name: ${{ inputs.mage-targets }}
+        run: |
+          if [ "${{ inputs.dagger-cloud-cache }}" == "true" ]
+          then
+            export DAGGER_CLOUD_TOKEN="${{ secrets.DAGGER_CLOUD_TOKEN }}"
+          fi
+          ./hack/make ${{ inputs.mage-targets }}
+        env:
+          _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICIwYzhmMGY2Yy00YjY1LTRhODktYTI0YS0yN2NjNWNhNzNmNTcifQ.Em92UDP-KmcNd80Y-euTS2IrRkIE8qTisP3SXYkQi3c"
diff --git a/.github/workflows/_hack_make.yml b/.github/workflows/_hack_make.yml
@@ -21,7 +21,7 @@ on:
         default: 60  
       size:
         type: string
-        default: "dagger-runner-2c-8g"
+        default: "dagger-v0-11-2-4c-nvme"
         required: false
       dagger-version:
         type: string
@@ -62,9 +62,9 @@ jobs:
           ./hack/make ${{ inputs.mage-targets }}
         timeout-minutes: ${{ inputs.timeout }}
 
-  # Use our own Dagger runner when running in the dagger/dagger repo (including PRs)
-  dagger-runner:
-    if: ${{ github.repository == 'dagger/dagger' }}
+  # Use new Dagger runners when running in the dagger/dagger repo (including PRs)
+  dagger-runner-v2:
+    if: ${{ !inputs.dev-engine && github.repository == 'dagger/dagger' }}
     runs-on: ${{ inputs.size }}
     concurrency:
       group: ${{github.workflow}}-${{ inputs.mage-targets }}-${{ github.head_ref || github.run_id }}
@@ -75,32 +75,66 @@ jobs:
         with:
           go-version: "1.22"
           cache-dependency-path: "ci/go.sum"
-      - name: Install dagger
+      - name: Waiting for Dagger Engine to be ready...
         run: |
-          curl -L https://dl.dagger.io/dagger/install.sh | DAGGER_VERSION=${{ inputs.dagger-version }} BIN_DIR=/usr/local/bin/ sudo -E sh
-      - name: Starting Dagger Engine
+          ./hack/make engine:connect
+        env:
+          DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICIwYzhmMGY2Yy00YjY1LTRhODktYTI0YS0yN2NjNWNhNzNmNTcifQ.Em92UDP-KmcNd80Y-euTS2IrRkIE8qTisP3SXYkQi3c"
+      - name: ${{ inputs.mage-targets }}
         run: |
-          if [ "${{ inputs.dev-engine }}" == "true" ]
-          then
-            ./hack/dev
-
-            export _EXPERIMENTAL_DAGGER_CLI_BIN="$PWD/bin/dagger"
-            chmod +x $_EXPERIMENTAL_DAGGER_CLI_BIN
-            echo "_EXPERIMENTAL_DAGGER_CLI_BIN=${_EXPERIMENTAL_DAGGER_CLI_BIN}" >> "$GITHUB_ENV"
+          ./hack/make ${{ inputs.mage-targets }}
+        env:
+          DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICIwYzhmMGY2Yy00YjY1LTRhODktYTI0YS0yN2NjNWNhNzNmNTcifQ.Em92UDP-KmcNd80Y-euTS2IrRkIE8qTisP3SXYkQi3c"
 
-            export _EXPERIMENTAL_DAGGER_RUNNER_HOST=docker-container://dagger-engine.dev
-          fi
+  # The job will succesfully trigger, but the async job is expected to fail.
+  # We confirmed that this requires at least 16CPUs & 32GB of RAM: https://github.com/dagger/dagger/pull/7223#issuecomment-2091059448
+  # The current machine only has 12CPUs, so it is likely to fail. The value is in capturing measurements & seeing them improve over time.
+  dagger-runner-gerhard-production:
+    if: ${{ !inputs.dev-engine && github.repository == 'dagger/dagger' && inputs.mage-targets == 'engine:testrace' }}
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{github.workflow}}-${{ inputs.mage-targets }}-${{ github.head_ref || github.run_id }}-gp
+      cancel-in-progress: true
+    steps:
+      # https://github.com/benc-uk/workflow-dispatch/tree/v1
+      - name: Trigger ${{ inputs.mage-targets }} on dagger-v011-on-k8s-gerhard-production
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: ./.github/workflows/_async_hack_make.yml
+          inputs: '{ "mage-targets": "${{ inputs.mage-targets }}", "concurrency-group": "${{ inputs.mage-targets }}-${{ github.head_ref || github.run_id }}-gp", "runner": "dagger-v011-on-k8s-gerhard-production" }'
 
-          echo "_EXPERIMENTAL_DAGGER_RUNNER_HOST=${_EXPERIMENTAL_DAGGER_RUNNER_HOST}" >> "$GITHUB_ENV"
+  # Use Legacy Dagger Runners until either:
+  # 1. Dagger in Dagger: https://github.com/dagger/dagger/pull/7223
+  # 2. Docker on v2 runners gets set up
+  dagger-runner-v1:
+    if: ${{ inputs.dev-engine && github.repository == 'dagger/dagger' }}
+    runs-on: dagger-runner-docker-fix
+    concurrency:
+      group: ${{github.workflow}}-${{ inputs.mage-targets }}-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.21"
+          cache-dependency-path: "internal/mage/go.sum"
+      - name: Install Dagger CLI
+        run: |
+          curl -L https://dl.dagger.io/dagger/install.sh | DAGGER_VERSION=${{ inputs.dagger-version }} BIN_DIR=/usr/local/bin/ sudo -E sh
+      - name: Build Dagger Dev Engine...
+        run: |
+          ./hack/dev
+          docker ps -a
+          docker image ls
         env:
-          _EXPERIMENTAL_DAGGER_RUNNER_HOST: "unix:///var/run/buildkit/buildkitd.sock"
-      - name: Waiting for Dagger Engine to be ready...
+          DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICIwYzhmMGY2Yy00YjY1LTRhODktYTI0YS0yN2NjNWNhNzNmNTcifQ.Em92UDP-KmcNd80Y-euTS2IrRkIE8qTisP3SXYkQi3c"
+      - name: Waiting for Dagger Dev Engine to be ready...
         run: |
-          ./hack/make engine:connect
+          ./hack/with-dev ./hack/make engine:connect
         env:
           DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICIwYzhmMGY2Yy00YjY1LTRhODktYTI0YS0yN2NjNWNhNzNmNTcifQ.Em92UDP-KmcNd80Y-euTS2IrRkIE8qTisP3SXYkQi3c"
       - name: ${{ inputs.mage-targets }}
         run: |
-          ./hack/make ${{ inputs.mage-targets }}
+          ./hack/with-dev ./hack/make ${{ inputs.mage-targets }}
         env:
           DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICIwYzhmMGY2Yy00YjY1LTRhODktYTI0YS0yN2NjNWNhNzNmNTcifQ.Em92UDP-KmcNd80Y-euTS2IrRkIE8qTisP3SXYkQi3c"
diff --git a/.github/workflows/engine-and-cli.yml b/.github/workflows/engine-and-cli.yml
@@ -26,7 +26,7 @@ jobs:
     secrets: inherit
     with:
       mage-targets: engine:testrace
-      size: dagger-runner-16c-64g
+      size: "dagger-v0-11-2-16c-nvme"
 
   # Run Engine tests in dev Engine so that we can spot integration failures early
   # Only run a subset of important test cases since we just need to verify basic
@@ -57,4 +57,3 @@ jobs:
     secrets: inherit
     with:
       mage-targets: engine:scan
-
diff --git a/.github/workflows/sdk-rust.yml b/.github/workflows/sdk-rust.yml
@@ -22,13 +22,11 @@ jobs:
     secrets: inherit
     with:
       mage-targets: sdk:rust:lint
-      size: dagger-runner-16c-64g
       timeout: 10
 
   test:
     uses: ./.github/workflows/_hack_make.yml
     secrets: inherit
     with:
       mage-targets: sdk:rust:test
-      size: dagger-runner-16c-64g
       timeout: 10