.github: fix java cdk publish workflow (#35533)

.github/workflows/gradle.yml

@@ -80,9 +80,12 @@ jobs:
         env:
           CI: true
         with:
+          job-id: gradle-check
           read-only: ${{ github.ref != 'refs/heads/master' }}
+          gradle-distribution-sha-256-sum-warning: false
+          concurrent: true
           # TODO: be able to remove the skipSlowTests property
-          arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true
+          arguments: --scan check -DskipSlowTests=true
 
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure

.github/workflows/publish-java-cdk-command.yml

@@ -8,17 +8,10 @@
 #    /publish-java-cdk force=true     # Force-publish if needing to replace an already published version
 name: Publish Java CDK
 on:
-  # Temporarily run on commits to the 'java-cdk/publish-workflow' branch.
-  # TODO: Remove this 'push' trigger before merging to master.
-  push:
-    branches:
-      - java-cdk/publish-workflow
-
   workflow_dispatch:
     inputs:
       repo:
         description: "Repo to check out code from. Defaults to the main airbyte repo."
-        # TODO: If publishing from forks is needed, we'll need to revert type to `string` of `choice`.
         type: choice
         required: true
         default: airbytehq/airbyte
@@ -30,7 +23,7 @@ on:
         type: boolean
         default: false
       force:
-        description: "Force release (ignore existing)"
+        description: "Force release (overwrite existing)"
         required: true
         type: boolean
         default: false
@@ -40,69 +33,39 @@ on:
       comment-id:
         description: "Optional comment-id of the slash command. Ignore if not applicable."
         required: false
-      # uuid:
-      #   description: "Custom UUID of workflow run. Used because GitHub dispatches endpoint does not return workflow run id."
-      #   required: false
 
 concurrency:
   group: publish-airbyte-cdk
   cancel-in-progress: false
 
 env:
   # Use the provided GITREF or default to the branch triggering the workflow.
-  REPO: ${{ github.event.inputs.repo }}
   GITREF: ${{ github.event.inputs.gitref || github.ref }}
   FORCE: "${{ github.event.inputs.force == null && 'false' || github.event.inputs.force }}"
   DRY_RUN: "${{ github.event.inputs.dry-run == null && 'true' || github.event.inputs.dry-run }}"
   CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties"
+  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
 
 jobs:
-  # We are using these runners because they are the same as the one for `publish-command.yml`
-  # One problem we had using `ubuntu-latest` for example is that the user is not root and some commands would fail in
-  # `manage.sh` (specifically `apt-get`)
-  start-publish-docker-image-runner-0:
-    name: Start Build EC2 Runner 0
-    runs-on: ubuntu-latest
-    outputs:
-      label: ${{ steps.start-ec2-runner.outputs.label }}
-      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-        with:
-          repository: airbytehq/airbyte
-          ref: master
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Start AWS Runner
-        id: start-ec2-runner
-        uses: ./.github/actions/start-aws-runner
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ env.PAT }}
-          label: ${{ github.run_id }}-publisher
-
   publish-cdk:
     name: Publish Java CDK
-    needs: start-publish-docker-image-runner-0
-    runs-on: ubuntu-latest
+    runs-on: connector-test-large
+    timeout-minutes: 30
     steps:
-      - name: Link comment to workflow run
+      - name: Link comment to Workflow Run
         if: github.event.inputs.comment-id
         uses: peter-evans/create-or-update-comment@v1
         with:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
             > :clock2: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
-          repository: ${{ env.REPO }}
           ref: ${{ env.GITREF }}
+
       - name: Read Target Java CDK version
         id: read-target-java-cdk-version
         run: |
@@ -112,28 +75,57 @@ jobs:
             exit 1
           fi
           echo "CDK_VERSION=${cdk_version}" >> $GITHUB_ENV
+
       - name: Setup Java
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
           java-version: "21"
-      - name: Check for already-published version (${{ env.CDK_VERSION }}, FORCE=${{ env.FORCE }})
-        if: ${{ !(env.FORCE == 'true') }}
-        run: ./gradlew :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished
+
+      - name: Docker login
+        # Some tests use testcontainers which pull images from DockerHub.
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
       - name: Build Java CDK
-        run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:build
-      - name: Upload jars as artifacts
-        if: ${{ !(env.DRY_RUN == 'false') }}
-        uses: actions/upload-artifact@v2
+        uses: burrunan/gradle-cache-action@v1
+        env:
+          CI: true
         with:
-          name: mavenlocal-jars
-          path: ~/.m2/repository/io/airbyte/
-      - name: Publish Java Modules to CloudRepo
+          job-id: cdk-publish
+          read-only: ${{ !(env.DRY_RUN == 'false') }}
+          concurrent: true
+          gradle-distribution-sha-256-sum-warning: false
+          arguments: --scan :airbyte-cdk:java:airbyte-cdk:cdkBuild
+
+      - name: Check for Existing Version
+        if: ${{ !(env.FORCE == 'true') }}
+        uses: burrunan/gradle-cache-action@v1
+        env:
+          CI: true
+        with:
+          job-id: cdk-publish
+          read-only: true
+          concurrent: true
+          gradle-distribution-sha-256-sum-warning: false
+          arguments: --scan :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished
+
+      - name: Publish Poms and Jars to CloudRepo
         if: ${{ env.DRY_RUN == 'false' }}
-        run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:publish
+        uses: burrunan/gradle-cache-action@v1
         env:
+          CI: true
           CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }}
           CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
+        with:
+          job-id: cdk-publish
+          read-only: true
+          concurrent: true
+          execution-only-caches: true
+          gradle-distribution-sha-256-sum-warning: false
+          arguments: --scan :airbyte-cdk:java:airbyte-cdk:cdkPublish
 
       - name: Add Success Comment
         if: github.event.inputs.comment-id && success()
@@ -143,6 +135,7 @@ jobs:
           edit-mode: append
           body: |
             > :white_check_mark: Successfully published Java CDK ${{ env.CDK_VERSION }}!
+
       - name: Add Failure Comment
         if: github.event.inputs.comment-id && failure()
         uses: peter-evans/create-or-update-comment@v1
@@ -151,7 +144,8 @@ jobs:
           edit-mode: append
           body: |
             > :x: Publish Java CDK ${{ env.CDK_VERSION }} failed!
-      - name: "Post failure to Slack channel `#dev-connectors-extensibility-releases`"
+
+      - name: "Post failure to Slack channel"
         if: ${{ env.DRY_RUN == 'false' && failure() }}
         uses: slackapi/slack-github-action@v1.23.0
         continue-on-error: true
@@ -179,7 +173,8 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
-      - name: "Post success to Slack channel `#dev-connectors-extensibility-releases`"
+
+      - name: "Post success to Slack channel"
         if: ${{ env.DRY_RUN == 'false' && !failure() }}
         uses: slackapi/slack-github-action@v1.23.0
         continue-on-error: true
@@ -207,33 +202,3 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
-
-  # In case of self-hosted EC2 errors, remove this block.
-  stop-publish-docker-image-runner-0:
-    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-    name: Stop Build EC2 Runner
-    needs:
-      - start-publish-docker-image-runner-0 # required to get output from the start-runner job
-      - publish-cdk # required to wait when the main job is done
-    runs-on: ubuntu-latest
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Stop EC2 runner
-        uses: airbytehq/ec2-github-runner@base64v1.1.0
-        with:
-          mode: stop
-          github-token: ${{ env.PAT }}
-          label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }}
-          ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }}

airbyte-cdk/java/airbyte-cdk/build.gradle

@@ -1,3 +1,9 @@
+final var cdkVersion = {
+    var props = new Properties()
+    file("core/src/main/resources/version.properties").withInputStream(props::load)
+    return props.getProperty('version', 'undefined')
+}()
+
 allprojects {
     apply plugin: 'java-library'
     apply plugin: 'maven-publish'
@@ -8,11 +14,14 @@ allprojects {
     def artifactBaseName = 'airbyte-cdk-' + project.name
     // E.g. airbyte-cdk-core, airbyte-cdk-db-sources, airbyte-cdk-db-destinations, etc.
 
+    project.version = cdkVersion
+
     publishing {
         publications {
             main(MavenPublication) {
                 groupId = 'io.airbyte.cdk'
                 artifactId = artifactBaseName
+                version = project.version
                 from components.java
             }
             testFixtures(MavenPublication) {
@@ -46,17 +55,18 @@ allprojects {
 
 description = "Airbyte Connector Development Kit (CDK) for Java."
 
+tasks.register('cdkBuild').configure {
+    dependsOn subprojects.collect { it.tasks.named('build') }
+}
+tasks.register('cdkPublish').configure {
+    dependsOn subprojects.collect { it.tasks.named('publish') }
+}
 tasks.register('assertCdkVersionNotPublished') {
     doLast {
-        var props = new Properties()
-        file("core/src/main/resources/version.properties").withInputStream(props::load)
 
-        var checkGroupId = "io.airbyte.cdk"
-        var checkArtifactId = "airbyte-cdk-core"
-        var checkVersion = props.getProperty('version') ?: 'undefined'
         var repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars"
-        var groupIdUrl = "${repoUrl}/${checkGroupId.replace('.', '/')}"
-        var artifactUrl = "${groupIdUrl}/${checkArtifactId}/${checkVersion}/${checkArtifactId}-${checkVersion}.pom"
+        var groupIdUrl = "${repoUrl}/io/airbyte/cdk"
+        var artifactUrl = "${groupIdUrl}/airbyte-cdk-core/${project.version}/airbyte-cdk-core-${project.version}.pom"
 
         var connection = artifactUrl.toURL().openConnection() as HttpURLConnection
         connection.setRequestMethod("HEAD")
@@ -65,11 +75,11 @@ tasks.register('assertCdkVersionNotPublished') {
         var responseCode = connection.getResponseCode()
 
         if (responseCode == 200) {
-            throw new GradleException("Java CDK '${checkVersion}' already published at ${groupIdUrl}")
+            throw new GradleException("Java CDK '${project.version}' already published at ${groupIdUrl}")
         } else if (responseCode == 404) {
-            logger.lifecycle("Java CDK '${checkVersion}' not yet published at ${groupIdUrl}.")
+            logger.lifecycle("Java CDK '${project.version}' not yet published at ${groupIdUrl}")
         } else {
-            throw new GradleException("Unexpected HTTP response code ${responseCode} from ${artifactUrl}, expected either 200 or 404.")
+            throw new GradleException("Unexpected HTTP response code ${responseCode} from ${artifactUrl} : expected either 200 or 404.")
         }
     }
 }