airbyte-ci: run on github hosted runners (#34316)

airbytehq · Jan 29, 2024 · 57b43a4 · 57b43a4
1 parent 3e6d842
commit 57b43a4
Show file tree

Hide file tree

Showing 21 changed files with 370 additions and 480 deletions.
diff --git a/.github/actions/get-dagger-engine-image/action.yml b/.github/actions/get-dagger-engine-image/action.yml
@@ -0,0 +1,53 @@
+name: "Get Dagger Engine Image"
+description: "Pulls the Dagger Engine Image or load from cache"
+
+inputs:
+  dagger_engine_image:
+    description: "Image name of the Dagger Engine"
+    required: true
+  path_to_dagger_engine_image_cache:
+    description: "Path to the Dagger Engine image cache"
+    required: false
+    default: "/home/runner/dagger-engine-image-cache"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create local image cache directory
+      id: create-dagger-engine-image-cache-dir
+      shell: bash
+      run: mkdir -p ${{ inputs.path_to_dagger_engine_image_cache }}
+
+    - name: Restore dagger engine image cache
+      id: dagger-engine-image-cache-restore
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ inputs.path_to_dagger_engine_image_cache }}
+        key: ${{ inputs.dagger_engine_image }}
+
+    # If no GitHub Action cache hit, pull the image and save it locally as tar to the cache directory
+    - name: Pull dagger engine image
+      id: pull-dagger-engine-image
+      if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        set -x
+        docker pull ${{ inputs.dagger_engine_image }}
+        docker save -o ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar ${{ inputs.dagger_engine_image }}
+
+    # If no GitHub Action  cache hit, save the path to the image cache directory to the Github Action cache
+    - name: Save dagger engine image cache
+      id: dagger-engine-image-cache-save
+      if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ inputs.path_to_dagger_engine_image_cache }}
+        key: ${{ inputs.dagger_engine_image }}
+
+    # If GitHub Action cache hit, load the image tar restored from the cache
+    - name: Load dagger engine image from cache
+      if: steps.dagger-engine-image-cache-restore.outputs.cache-hit == 'true'
+      shell: bash
+      run: |
+        set -x
+        docker load -i ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar
diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml
@@ -0,0 +1,81 @@
+name: "Install Airbyte CI"
+description: "Install Airbyte CI from source or from a binary according to changed files. Pulls the Dagger Engine image according to the dagger version used in airbyte-ci."
+
+inputs:
+  airbyte_ci_binary_url:
+    description: "URL to airbyte-ci binary"
+    required: false
+    default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+  path_to_airbyte_ci_source:
+    description: "Path to airbyte-ci source"
+    required: false
+    default: airbyte-ci/connectors/pipelines
+runs:
+  using: "composite"
+  steps:
+    - name: Get changed files
+      uses: tj-actions/changed-files@v39
+      id: changes
+      with:
+        files_yaml: |
+          pipelines:
+            - '${{ inputs.path_to_airbyte_ci_source }}/**'
+
+    - name: Determine how Airbyte CI should be installed
+      shell: bash
+      id: determine-install-mode
+      run: |
+        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
+          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
+          echo "install-mode=source" >> $GITHUB_OUTPUT
+        else
+          echo "install-mode=binary" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install Airbyte CI from binary
+      id: install-airbyte-ci-binary
+      if: steps.determine-install-mode.outputs.install-mode == 'binary'
+      shell: bash
+      run: |
+        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
+        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+        sudo chmod +x /usr/local/bin/airbyte-ci
+
+    - name: Install Python 3.10
+      id: install-python-3-10
+      uses: actions/setup-python@v4
+      if: steps.determine-install-mode.outputs.install-mode == 'source'
+      with:
+        python-version: "3.10"
+        token: ${{ inputs.github_token }}
+
+    - name: Install Airbyte CI from source
+      id: install-airbyte-ci-source
+      if: steps.determine-install-mode.outputs.install-mode == 'source'
+      shell: bash
+      run: |
+        pip install --upgrade pip
+        pip install pipx
+        pipx ensurepath
+        pipx install ${{ inputs.path_to_airbyte_ci_source }}
+
+    - name: Get dagger engine image name
+      id: get-dagger-engine-image-name
+      shell: bash
+      run: |
+        dagger_engine_image=$(airbyte-ci --ci-requirements | tail -n 1 | jq -r '.dagger_engine_image')
+        echo "dagger_engine_image=${dagger_engine_image}" >> "$GITHUB_OUTPUT"
+
+    - name: Get dagger engine image
+      id: get-dagger-engine-image
+      uses: ./.github/actions/get-dagger-engine-image
+      with:
+        dagger_engine_image: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}
+
+outputs:
+  install_mode:
+    description: "Whether Airbyte CI was installed from source or from a binary"
+    value: ${{ steps.determine-install-mode.outputs.install-mode }}
+  dagger_engine_image_name:
+    description: "Dagger engine image name"
+    value: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}
diff --git a/...ub/actions/run-dagger-pipeline/action.yml → .github/actions/run-airbyte-ci/action.yml b/...ub/actions/run-dagger-pipeline/action.yml → .github/actions/run-airbyte-ci/action.yml
@@ -19,11 +19,6 @@ inputs:
   docker_hub_password:
     description: "Dockerhub password"
     required: true
-  docker_registry_mirror_url:
-    description: "Docker registry mirror URL (not including http or https)"
-    required: false
-    # Do not use http or https here
-    default: "ci-dockerhub-registry.airbyte.com"
   options:
     description: "Options for the subcommand"
     required: false
@@ -77,8 +72,6 @@ inputs:
   s3_build_cache_secret_key:
     description: "Gradle S3 Build Cache AWS secret key"
     required: false
-  tailscale_auth_key:
-    description: "Tailscale auth key"
   airbyte_ci_binary_url:
     description: "URL to airbyte-ci binary"
     required: false
@@ -90,7 +83,13 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Get start timestamp
+      id: get-start-timestamp
+      shell: bash
+      run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
+
     - name: Check if PR is from a fork
+      id: check-if-pr-is-from-fork
       if: github.event_name == 'pull_request'
       shell: bash
       run: |
@@ -99,64 +98,23 @@ runs:
           exit 78
         fi
 
-    - name: Get changed files
-      uses: tj-actions/changed-files@v39
-      id: changes
-      with:
-        files_yaml: |
-          pipelines:
-            - 'airbyte-ci/connectors/pipelines/**'
-
-    - name: Determine how Airbyte CI should be installed
-      shell: bash
-      id: determine-install-mode
-      run: |
-        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
-          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
-          echo "install-mode=dev" >> $GITHUB_OUTPUT
-        else
-          echo "install-mode=production" >> $GITHUB_OUTPUT
-        fi
-
     - name: Docker login
-      uses: docker/login-action@v1
+      id: docker-login
+      uses: docker/login-action@v3
       with:
         username: ${{ inputs.docker_hub_username }}
         password: ${{ inputs.docker_hub_password }}
 
-    - name: Get start timestamp
-      id: get-start-timestamp
-      shell: bash
-      run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
-
-    - name: Install airbyte-ci binary
+    - name: Install Airbyte CI
       id: install-airbyte-ci
-      if: steps.determine-install-mode.outputs.install-mode == 'production'
-      shell: bash
-      run: |
-        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
-        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
-        sudo chmod +x /usr/local/bin/airbyte-ci
-
-    - name: Install Python 3.10
-      uses: actions/setup-python@v4
-      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      uses: ./.github/actions/install-airbyte-ci
       with:
-        python-version: "3.10"
-        token: ${{ inputs.github_token }}
-
-    - name: Install ci-connector-ops package
-      if: steps.determine-install-mode.outputs.install-mode == 'dev'
-      shell: bash
-      run: |
-        pip install pipx
-        pipx ensurepath
-        pipx install airbyte-ci/connectors/pipelines/
+        airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url }}
 
     - name: Run airbyte-ci
+      id: run-airbyte-ci
       shell: bash
       run: |
-        export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
         airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
         CI_CONTEXT: "${{ inputs.context }}"
@@ -166,23 +124,27 @@ runs:
         CI_JOB_KEY: ${{ inputs.ci_job_key }}
         CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
         CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }}
+        CI: "True"
         DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}"
+        DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }}
+        DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }}
         GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }}
         GCS_CREDENTIALS: ${{ inputs.gcs_credentials }}
         METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }}
         METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }}
         PRODUCTION: ${{ inputs.production }}
         PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+        PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
+        S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
+        S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
         SENTRY_DSN: ${{ inputs.sentry_dsn }}
         SENTRY_ENVIRONMENT: ${{ steps.determine-install-mode.outputs.install-mode }}
         SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
         SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
         SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
-        DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }}
-        DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }}
-        S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
-        S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
-        CI: "True"
-        TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }}
-        DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }}
-        PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
+    # give the Dagger Engine more time to push cache data to Dagger Cloud
+    - name: Stop Engine
+      id: stop-engine
+      if: always()
+      shell: bash
+      run: docker stop --time 300 $(docker ps --filter name="dagger-engine-*" -q)