gradle: overall simplification

.github/workflows/gradle.yml

@@ -1,4 +1,4 @@
-name: Airbyte CI - Repository Health Check
+name: Connector Ops CI - Gradle Check
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -20,47 +20,18 @@ on:
       - synchronize
 
 jobs:
-  # In case of self-hosted EC2 errors, remove this block.
-  start-check-runner:
-    name: Start EC2 Runner
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    outputs:
-      label: ${{ steps.start-ec2-runner.outputs.label }}
-      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Start AWS Runner
-        id: start-ec2-runner
-        uses: ./.github/actions/start-aws-runner
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          # Use a beefier instance type than the default c5.2xlarge, but with the same per-core cost.
-          # When gradle runs on this instance, it will use up all the available cores anyway.
-          # There should be little to no difference in total cost, however the job latency will be improved.
-          # At the time of this writing, the latency doesn't improve much beyond this instance size (approx 5 minutes).
-          # This is largely thanks to the gradle cache.
-          ec2-instance-type: "c5.4xlarge"
-          github-token: ${{ env.PAT }}
-
   run-check:
-    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: start-check-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner
+    # The gradle check task which we will run is embarrassingly parallelizable.
+    # We therefore run this on a machine with a maximum number of cores.
+    # We pay per time and per core, so there should be little difference in total cost.
+    # The latency overhead of setting up gradle prior to running the actual task adds up to about a minute.
+    runs-on: connector-test-xxlarge
     name: Gradle Check
     timeout-minutes: 30
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
-
-      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
+      # IMPORTANT! This is necessary to make sure that a status is reported on the PR
       # even if the workflow is skipped. If we used github actions filters, the workflow
       # would not be reported as skipped, but instead would be forever pending.
       #
@@ -106,39 +77,6 @@ jobs:
           # TODO: be able to remove the skipSlowTests property
           arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true
 
-  # In case of self-hosted EC2 errors, remove this block.
-  stop-check-runner:
-    name: Stop EC2 Runner
-    timeout-minutes: 10
-    needs:
-      - start-check-runner # required to get output from the start-runner job
-      - run-check # required to wait when the main job is done
-    runs-on: ubuntu-latest
-    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
-    # Thus, we check for skipped here.
-    if: ${{ always() && needs.start-check-runner.result != 'skipped'}}
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Stop EC2 runner
-        uses: supertopher/ec2-github-runner@base64v1.0.10
-        with:
-          mode: stop
-          github-token: ${{ env.PAT }}
-          label: ${{ needs.start-check-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-check-runner.outputs.ec2-instance-id }}
-
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure
     runs-on: ubuntu-latest

.github/workflows/legacy-test-command.yml

@@ -113,9 +113,6 @@ jobs:
           fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
-      - name: Build Java CDK Snapshot if Needed
-        # If a snapshot version is specified for the Java CDK, build publish locally. Otherwise, do nothing.
-        run: ./gradlew :airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded
       - name: Test ${{ github.event.inputs.connector }}
         id: test
         env:

.github/workflows/publish-java-cdk-command.yml

@@ -122,9 +122,6 @@ jobs:
         run: ./gradlew :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished
       - name: Build Java CDK
         run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:build
-      - name: Publish Java Modules to MavenLocal (Dry-Run)
-        if: ${{ !(env.DRY_RUN == 'false') }}
-        run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:publishToMavenLocal
       - name: Upload jars as artifacts
         if: ${{ !(env.DRY_RUN == 'false') }}
         uses: actions/upload-artifact@v2

airbyte-cdk/java/airbyte-cdk/build.gradle

@@ -1,17 +1,11 @@
 allprojects {
     apply plugin: 'java-library'
     apply plugin: 'maven-publish'
-    apply plugin: 'airbyte-java-cdk'
-    apply plugin: 'airbyte-integration-test-java'
-    apply plugin: 'airbyte-performance-test-java'
     apply plugin: 'java-test-fixtures'
 
     group 'io.airbyte.cdk'
-    version = getCdkTargetVersion()
-}
 
-subprojects { subproject ->
-    def artifactBaseName = 'airbyte-cdk-' + subproject.name
+    def artifactBaseName = 'airbyte-cdk-' + project.name
     // E.g. airbyte-cdk-core, airbyte-cdk-db-sources, airbyte-cdk-db-destinations, etc.
 
     publishing {
@@ -25,7 +19,7 @@ subprojects { subproject ->
                 groupId = 'io.airbyte.cdk'
                 artifactId = artifactBaseName + '-test-fixtures'
                 version = project.version
-                artifact subproject.tasks.testFixturesJar
+                artifact project.tasks.testFixturesJar
             }
         }
         // This repository is only defined and used in the context of an artifact publishing
@@ -52,69 +46,30 @@ subprojects { subproject ->
 
 description = "Airbyte Connector Development Kit (CDK) for Java."
 
-def recursiveTasks = [
-    'assemble',
-    'build',
-    'integrationTestJava',
-    'publish',
-    'publishToMavenLocal',
-    'test',
-]
-recursiveTasks.each { taskName ->
-    tasks.named(taskName).configure {
-        dependsOn subprojects.collect { it.tasks.named(taskName) }
-    }
-}
-
-// The `publishSnapshotIfNeeded` task will be a no-op if CDK version does not end with '-SNAPSHOT'.
-task publishSnapshotIfNeeded {}
-
-if (version.endsWith("-SNAPSHOT")) {
-    logger.lifecycle("Version ${version} ends with '-SNAPSHOT'. Enqueing 'publishToMavenLocal'...")
-    publishSnapshotIfNeeded.dependsOn publishToMavenLocal
-} else {
-    // Uncomment as needed for debugging:
-    // logger.lifecycle("Version ${version} does not end with '-SNAPSHOT'. Skipping task 'publishToMavenLocal'.")
-}
-
-task assertCdkVersionNotPublished {
+tasks.register('assertCdkVersionNotPublished') {
     doLast {
-        def checkGroupId = "io.airbyte.cdk"
-        def checkArtifactId = "airbyte-cdk-core"
-        def checkVersion = getCdkTargetVersion()
-        def repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars"
-        def groupIdUrl = checkGroupId.replace('.', '/')
-        def artifactUrl = "${repoUrl}/${groupIdUrl}/${checkArtifactId}/${checkVersion}/${checkArtifactId}-${checkVersion}.pom"
+        var props = new Properties()
+        file("core/src/main/resources/version.properties").withInputStream(props::load)
 
-        def connection = artifactUrl.toURL().openConnection() as HttpURLConnection
+        var checkGroupId = "io.airbyte.cdk"
+        var checkArtifactId = "airbyte-cdk-core"
+        var checkVersion = props.getProperty('version') ?: 'undefined'
+        var repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars"
+        var groupIdUrl = "${repoUrl}/${checkGroupId.replace('.', '/')}"
+        var artifactUrl = "${groupIdUrl}/${checkArtifactId}/${checkVersion}/${checkArtifactId}-${checkVersion}.pom"
+
+        var connection = artifactUrl.toURL().openConnection() as HttpURLConnection
         connection.setRequestMethod("HEAD")
         connection.connect()
 
-        def responseCode = connection.getResponseCode()
+        var responseCode = connection.getResponseCode()
 
         if (responseCode == 200) {
-            throw new GradleException("Assert failed. Java CDK '${checkVersion}' already published at: ${artifactUrl}")
+            throw new GradleException("Java CDK '${checkVersion}' already published at ${groupIdUrl}")
         } else if (responseCode == 404) {
-            logger.lifecycle(
-                "Assert succeeded. Version ${checkVersion} of ${checkArtifactId} has not been published. " +
-                "Checked: ${artifactUrl}"
-            )
+            logger.lifecycle("Java CDK '${checkVersion}' not yet published at ${groupIdUrl}.")
         } else {
-            logger.error("Received unexpected HTTP response code ${responseCode}. Ensure the repository is accessible.")
-            throw new GradleException("Error during assertion. Received unexpected HTTP response code ${responseCode}.")
+            throw new GradleException("Unexpected HTTP response code ${responseCode} from ${artifactUrl}, expected either 200 or 404.")
         }
     }
 }
-
-def cleanLocalCache = tasks.register('cleanLocalCache') {
-    def userHome = System.getProperty("user.home")
-    doLast {
-        delete '.gradle'
-        delete '${userHome}/.m2/repository/io/airbyte/'
-        delete '${userHome}/.gradle/caches/modules-2/files-2.1/io.airbyte.cdk/'
-    }
-}
-cleanLocalCache.configure {
-    dependsOn tasks.named('clean')
-    dependsOn subprojects.collect { it.tasks.named('clean') }
-}

airbyte-integrations/connectors/build.gradle

@@ -0,0 +1,104 @@
+plugins {
+    id 'base'
+    id 'ru.vyarus.use-python' version '2.3.0'
+}
+
+// Pyenv support.
+try {
+    def pyenvRoot = "pyenv root".execute()
+    if (pyenvRoot.waitFor() == 0) {
+        ext.pyenvRoot = pyenvRoot.text.trim()
+    }
+} catch (IOException _) {
+    // Swallow exception if pyenv is not installed.
+}
+
+def pythonBin = layout.buildDirectory.file('.venv/bin/python').get().asFile.absolutePath
+
+// python is required by the connectors project to run airbyte-ci from source to build connector images.
+python {
+    envPath = layout.buildDirectory.file('.venv').get().asFile
+    minPythonVersion = '3.10' // should be 3.10 for local development
+
+    // Pyenv support.
+    try {
+        def pyenvRoot = "pyenv root".execute()
+        def pyenvLatest = "pyenv latest ${minPythonVersion}".execute()
+        // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases.
+        pythonBinary "python"
+        if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) {
+            pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin"
+        }
+    } catch (IOException _) {
+        // Swallow exception if pyenv is not installed.
+    }
+
+    scope = 'VIRTUALENV'
+    installVirtualenv = true
+    // poetry is required for installing and running airbyte-ci
+    pip 'poetry:1.5.1'
+}
+
+def poetryInstallAirbyteCI = tasks.register('poetryInstallAirbyteCI', Exec) {
+    workingDir rootProject.file('airbyte-ci/connectors/pipelines')
+    commandLine pythonBin
+    args "-m", "poetry", "install", "--no-cache"
+}
+poetryInstallAirbyteCI.configure {
+    dependsOn tasks.named('pipInstall')
+}
+def poetryCleanVirtualenv = tasks.register('cleanPoetryVirtualenv', Exec) {
+    workingDir rootProject.file('airbyte-ci/connectors/pipelines')
+    commandLine pythonBin
+    args "-m", "poetry", "env", "remove", "--all"
+    onlyIf {
+        layout.buildDirectory.file('venv/bin/python').get().asFile.exists()
+    }
+}
+tasks.named('clean').configure {
+    dependsOn poetryCleanVirtualenv
+}
+
+allprojects {
+    // Evaluate CDK project before evaluating the connector.
+    evaluationDependsOn(':airbyte-cdk:java:airbyte-cdk')
+
+    // Adds airbyte-ci task.
+    def airbyteCIConnectorsTask = { String taskName, String... connectorsArgs ->
+        def task = tasks.register(taskName, Exec) {
+            workingDir rootDir
+            environment "CI", "1" // set to use more suitable logging format
+            commandLine pythonBin
+            args "-m", "poetry"
+            args "--directory", "${rootProject.file('airbyte-ci/connectors/pipelines').absolutePath}"
+            args "run"
+            args "airbyte-ci", "connectors", "--name=${project.name}"
+            args connectorsArgs
+            // Forbid these kinds of tasks from running concurrently.
+            // We can induce serial execution by giving them all a common output directory.
+            outputs.dir rootProject.file("${rootProject.buildDir}/airbyte-ci-lock")
+            outputs.upToDateWhen { false }
+        }
+        task.configure { dependsOn poetryInstallAirbyteCI }
+        return task
+    }
+
+    // Build connector image as part of 'assemble' task.
+    // This is required for local 'integrationTest' execution.
+    def buildConnectorImage = airbyteCIConnectorsTask(
+            'buildConnectorImage', '--disable-report-auto-open', 'build', '--use-host-gradle-dist-tar')
+    buildConnectorImage.configure {
+        // Images for java projects always rely on the distribution tarball.
+        dependsOn tasks.matching { it.name == 'distTar' }
+        // Ensure that all files exist beforehand.
+        dependsOn tasks.matching { it.name == 'generate' }
+    }
+    tasks.named('assemble').configure {
+        // We may revisit the dependency on assemble but the dependency should always be on a base task.
+        dependsOn buildConnectorImage
+    }
+
+    // Convenience tasks for local airbyte-ci execution.
+    airbyteCIConnectorsTask('airbyteCIConnectorBuild', 'build')
+    airbyteCIConnectorsTask('airbyteCIConnectorTest', 'test')
+}