connectors-ci: no concurrency on java connectors for nightly builds (#…

…27003)
airbytehq · Jun 5, 2023 · 4e88af0 · 4e88af0
1 parent 43879c1
commit 4e88af0
Show file tree

Hide file tree

Showing 2 changed files with 96 additions and 9 deletions.
diff --git a/...flows/connector_nightly_builds_dagger.yml → ...hub/workflows/connectors_manual_tests.yml b/...flows/connector_nightly_builds_dagger.yml → ...hub/workflows/connectors_manual_tests.yml
@@ -1,9 +1,6 @@
-name: POC Connectors CI - nightly builds
+name: Connectors CI - Manual tests
 
 on:
-  schedule:
-    # 11AM UTC is 12AM CET, 4AM PST.
-    - cron: "0 11 * * *"
   workflow_dispatch:
     inputs:
       runs-on:
@@ -14,11 +11,11 @@ on:
         default: --concurrency=10 --release-stage=generally_available --release-stage=beta
         required: true
 
-run-name: Test connectors ${{ inputs.test-connectors-options || 'Nightly builds GA and Beta connectors' }}
+run-name: "Test connectors ${{ inputs.test-connectors-options}} on ${{ inputs.runs-on }}"
 
 jobs:
   test_connectors:
-    name: Test connectors ${{ inputs.test-connectors-options || 'Nightly builds GA and Beta connectors' }}
+    name: "Test connectors ${{ inputs.test-connectors-options}} on ${{ inputs.runs-on }}"
     timeout-minutes: 600 # 10 hours
     runs-on: ${{ inputs.runs-on || 'dev-large-runner' }}
     steps:
@@ -47,7 +44,7 @@ jobs:
           token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
       - name: Install ci-connector-ops package
         run: pip install ./tools/ci_connector_ops\[pipelines]\
-      - name: Test connectors
+      - name: "Test connectors ${{ inputs.test-connectors-options}}"
         run: |
           export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
           DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0"
@@ -57,7 +54,7 @@ jobs:
             mkdir -p "$DAGGER_TMP_BINDIR"
             curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options || '--concurrency=10 --release-stage=generally_available --release-stage=beta' }} test
+          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options }} test
         env:
           _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -67,5 +64,5 @@ jobs:
           TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status"
           CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
-          CI_CONTEXT: "nightly_builds"
+          CI_CONTEXT: "manual"
           CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
diff --git a/.github/workflows/connectors_nightly_builds.yml b/.github/workflows/connectors_nightly_builds.yml
@@ -0,0 +1,90 @@
+name: Connectors CI - nightly builds for GA and Beta connectors
+
+on:
+  schedule:
+    # 11AM UTC is 12AM CET, 4AM PST.
+    - cron: "0 11 * * *"
+  workflow_dispatch:
+    inputs:
+      runs-on:
+        type: string
+        default: dev-large-runner
+        required: true
+
+run-name: "Connectors CI - nightly builds for GA and Beta connectors - ${{ inputs.runs-on }}"
+
+jobs:
+  test_connectors:
+    name: Test GA and Beta connectors
+    timeout-minutes: 600 # 10 hours
+    runs-on: ${{ inputs.runs-on || 'dev-large-runner' }}
+    steps:
+      - name: Get start timestamp
+        id: get-start-timestamp
+        run: echo "::set-output name=start-timestamp::$(date +%s)"
+      - name: Login to DockerHub
+        run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}"
+        env:
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ github.event.inputs.repo }}
+          ref: ${{ github.event.inputs.gitref }}
+      - name: Extract branch name
+        shell: bash
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+      - name: Install ci-connector-ops package
+        run: pip install ./tools/ci_connector_ops\[pipelines]\
+      - name: Test Python connectors
+        run: |
+          export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
+          DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0"
+          DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}"
+          export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger"
+          if [ ! -f  "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then
+            mkdir -p "$DAGGER_TMP_BINDIR"
+            curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
+          fi
+          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=8 --release-stage=generally_available --release-stage=beta --language=python --languague=low-code test
+        env:
+          _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: "us-east-2"
+          TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status"
+          CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
+          CI_CONTEXT: "nightly_builds"
+          CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
+      - name: Test Java connectors
+        run: |
+          export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
+          DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0"
+          DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}"
+          export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger"
+          if [ ! -f  "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then
+            mkdir -p "$DAGGER_TMP_BINDIR"
+            curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
+          fi
+          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=1 --release-stage=generally_available --release-stage=beta --language=java test
+        env:
+          _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: "us-east-2"
+          TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status"
+          CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
+          CI_CONTEXT: "nightly_builds"
+          CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}