airbyte-ci: upgrade to dagger 0.9.5 (#33582)

airbytehq · Jan 5, 2024 · 617075d · 617075d
1 parent f90d8db
commit 617075d
Show file tree

Hide file tree

Showing 32 changed files with 1,472 additions and 1,342 deletions.
diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml
@@ -10,6 +10,9 @@ inputs:
   github_token:
     description: "GitHub token"
     required: true
+  dagger_cloud_token:
+    description: "Dagger Cloud token"
+    required: true
   docker_hub_username:
     description: "Dockerhub username"
     required: true
@@ -153,14 +156,14 @@ runs:
         export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
         airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
-        _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
         CI_CONTEXT: "${{ inputs.context }}"
         CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }}
         CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }}
         CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }}
         CI_JOB_KEY: ${{ inputs.ci_job_key }}
         CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
         CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }}
+        DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}"
         GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }}
         GCS_CREDENTIALS: ${{ inputs.gcs_credentials }}
         METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }}

diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml
@@ -19,7 +19,7 @@ on:
 jobs:
   run-airbyte-ci-tests:
     name: Run Airbyte CI tests
-    runs-on: "ci-runner-connector-test-large-dagger-0-6-4"
+    runs-on: "ci-runner-connector-test-large-dagger-0-9-5"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -71,6 +71,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
@@ -86,6 +87,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
@@ -101,6 +103,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
@@ -117,6 +120,7 @@ jobs:
         with:
           subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'"
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
@@ -130,6 +134,7 @@ jobs:
         with:
           subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'"
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}

diff --git a/.github/workflows/cat-tests.yml b/.github/workflows/cat-tests.yml
@@ -16,7 +16,7 @@ on:
 jobs:
   run-cat-unit-tests:
     name: Run CAT unit tests
-    runs-on: "ci-runner-connector-test-large-dagger-0-6-4"
+    runs-on: "ci-runner-connector-test-large-dagger-0-9-5"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -25,6 +25,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml
@@ -8,19 +8,23 @@ on:
     inputs:
       runs-on:
         type: string
-        default: ci-runner-connector-nightly-xlarge-dagger-0-6-4
+        default: ci-runner-connector-nightly-xlarge-dagger-0-9-5
         required: true
       test-connectors-options:
         default: --concurrency=5 --support-level=certified
         required: true
+      airbyte-ci-binary-url:
+        description: "URL to airbyte-ci binary"
+        required: false
+        default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
 
 jobs:
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
     timeout-minutes: 720 # 12 hours
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}
+    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -36,6 +40,7 @@ jobs:
         with:
           context: "master"
           ci_job_key: "nightly_builds"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -45,3 +50,4 @@ jobs:
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=8 --support-level=certified' }} test"
+          airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }}
diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml
@@ -19,7 +19,7 @@ on:
         default: "--modified"
       runner:
         description: "The runner to use for this job"
-        default: "ci-runner-connector-test-large-dagger-0-6-4"
+        default: "ci-runner-connector-test-large-dagger-0-9-5"
       airbyte_ci_binary_url:
         description: "The URL to download the airbyte-ci binary from"
         required: false
@@ -33,7 +33,7 @@ jobs:
   connectors_ci:
     name: Connectors CI
     timeout-minutes: 1440 # 24 hours
-    runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-6-4'}}
+    runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-9-5'}}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -60,6 +60,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "manual"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -77,6 +78,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml
@@ -8,19 +8,19 @@ on:
     inputs:
       runs-on:
         type: string
-        default: ci-runner-connector-nightly-xlarge-dagger-0-6-4
+        default: ci-runner-connector-nightly-xlarge-dagger-0-9-5
         required: true
       test-connectors-options:
         default: --concurrency=3 --support-level=community
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
 
 jobs:
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
     timeout-minutes: 8640 # 6 days
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}
+    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -36,6 +36,7 @@ jobs:
         with:
           context: "master"
           ci_job_key: "weekly_alpha_test"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml
@@ -2,14 +2,19 @@ name: Check for formatting errors
 run-name: Check for formatting errors on ${{ github.ref }}
 on:
   workflow_dispatch:
+    inputs:
+      airbyte-ci-binary-url:
+        description: "URL to airbyte-ci binary"
+        required: false
+        default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
   push:
     branches:
       - master
   pull_request:
 
 jobs:
   format-check:
-    runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
+    runs-on: "ci-runner-connector-format-medium-dagger-0-9-5"
     # IMPORTANT: This name must match the require check name on the branch protection settings
     name: "Check for formatting errors"
     steps:
@@ -26,6 +31,7 @@ jobs:
         continue-on-error: true
         with:
           context: "master"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
@@ -41,6 +47,7 @@ jobs:
         continue-on-error: false
         with:
           context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
@@ -56,13 +63,15 @@ jobs:
         continue-on-error: false
         with:
           context: "manual"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
           subcommand: "format check all"
+          airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }}
 
       - name: Match GitHub User to Slack User [MASTER]
         if: github.ref == 'refs/heads/master'

diff --git a/.github/workflows/format_fix.yml b/.github/workflows/format_fix.yml
@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
 jobs:
   format-fix:
-    runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
+    runs-on: "ci-runner-connector-format-medium-dagger-0-9-5"
     name: "Run airbyte-ci format fix all"
     steps:
       - name: Checkout Airbyte
@@ -26,6 +26,7 @@ jobs:
         continue-on-error: true
         with:
           context: "manual"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}

diff --git a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml
@@ -10,7 +10,7 @@ on:
 jobs:
   connector_metadata_service_deploy_orchestrator:
     name: Connector metadata service deploy orchestrator
-    runs-on: ci-runner-connector-test-large-dagger-0-6-4
+    runs-on: ci-runner-connector-test-large-dagger-0-9-5
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
@@ -20,6 +20,7 @@ jobs:
         with:
           subcommand: "metadata deploy orchestrator"
           context: "master"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}

diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml
@@ -16,7 +16,7 @@ on:
         default: "--pre-release"
       runs-on:
         type: string
-        default: ci-runner-connector-publish-large-dagger-0-6-4
+        default: ci-runner-connector-publish-large-dagger-0-9-5
         required: true
       airbyte-ci-binary-url:
         description: "URL to airbyte-ci binary"
@@ -25,7 +25,7 @@ on:
 jobs:
   publish_connectors:
     name: Publish connectors
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-6-4' }}
+    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-9-5' }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -35,6 +35,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "master"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -55,6 +56,7 @@ jobs:
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "manual"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml
@@ -5,12 +5,12 @@ on:
     inputs:
       runs-on:
         type: string
-        default: ci-runner-connector-publish-large-dagger-0-6-4
+        default: ci-runner-connector-publish-large-dagger-0-9-5
         required: true
 
 jobs:
   no-op:
     name: No-op
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-6-4' }}
+    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-9-5' }}
     steps:
       - run: echo 'hi!'
diff --git a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py
@@ -76,11 +76,7 @@ async def check_python_image_has_expected_env_vars(python_image_container: dagge
         "PATH",
         "LANG",
         "GPG_KEY",
-        "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL",
         "PYTHON_SETUPTOOLS_VERSION",
-        "OTEL_TRACES_EXPORTER",
-        "OTEL_TRACE_PARENT",
-        "TRACEPARENT",
     }
     # It's not suboptimal to call printenv multiple times because the printenv output is cached.
     for expected_env_var in expected_env_vars: