Remove shared getDockerBuildTask to enable parallel docker builds (#…

…16384)

Define logic in root build.gradle to create a task in each subproject if the subproject contains dockerImageName in the gradle.properties file.

Some caveats:
- We also define a copyGeneratedTar task that is applied to all subprojects with the dockerImageName property.
- This does not need to be used in each subproject. We leave it up to each subproject to define what task is depends on since some projects have more custom copy logic.

airbyte-bootloader/build.gradle

@@ -24,25 +24,15 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-bootloader-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("bootloader", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 // produce reproducible archives
 // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
 tasks.withType(AbstractArchiveTask) {
     preserveFileTimestamps = false
     reproducibleFileOrder = true
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-bootloader/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=bootloader

airbyte-cli/build.gradle

@@ -1,3 +1,3 @@
-Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyDocker)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyDocker
+}

airbyte-cli/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=cli

airbyte-config/init/build.gradle

@@ -22,8 +22,8 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("init", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyScripts)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyScripts
+}
 
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-config/init/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=init

airbyte-container-orchestrator/build.gradle

@@ -36,16 +36,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-container-orchestrator-*.tar'
-    }
-    into 'build/docker/bin'
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
 }
-
-Task dockerBuildTask = getDockerBuildTask("container-orchestrator", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-container-orchestrator/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=container-orchestrator

airbyte-cron/build.gradle

@@ -22,16 +22,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-cron-*.tar'
-    }
-    into 'build/docker/bin'
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
 }
-
-Task dockerBuildTask = getDockerBuildTask("cron", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-cron/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=cron

airbyte-db/db-lib/build.gradle

@@ -1,3 +1,5 @@
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+
 plugins {
     id 'java-library'
 }
@@ -92,8 +94,8 @@ task copyInitSql(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("db", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyInitSql)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyInitSql
+}
 
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-db/db-lib/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=db

airbyte-metrics/reporter/build.gradle

@@ -17,16 +17,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-metrics-reporter-*.tar'
-    }
-    into 'build/docker/bin'
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
 }
-
-Task dockerBuildTask = getDockerBuildTask("metrics-reporter", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-metrics/reporter/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=metrics-reporter

airbyte-server/build.gradle

@@ -84,25 +84,15 @@ run {
     environment "TEMPORAL_HOST", "localhost:7233"
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-server-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 // produce reproducible archives
 // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
 tasks.withType(AbstractArchiveTask) {
     preserveFileTimestamps = false
     reproducibleFileOrder = true
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-server/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=server

airbyte-temporal/build.gradle

@@ -5,6 +5,6 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyScripts)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyScripts
+}

airbyte-temporal/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=temporal

airbyte-webapp/build.gradle

@@ -117,10 +117,10 @@ check.dependsOn validateLinks, licenseCheck, test, stylelint
 
 build.dependsOn buildStorybook
 
-Task dockerBuildTask = getDockerBuildTask("webapp", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyDocker)
-dockerBuildTask.dependsOn(copyBuildOutput)
-dockerBuildTask.dependsOn(copyNginx)
-dockerBuildTask.dependsOn(copyDocs)
-dockerBuildTask.dependsOn(copyDocAssets)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyDocker
+    dependsOn copyBuildOutput
+    dependsOn copyNginx
+    dependsOn copyDocs
+    dependsOn copyDocAssets
+}

airbyte-webapp/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=webapp

airbyte-workers/build.gradle

@@ -73,20 +73,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-workers-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("worker", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 task cloudStorageIntegrationTest(type: Test) {
     useJUnitPlatform {
         includeTags cloudStorageTestTagName
@@ -96,4 +82,8 @@ task cloudStorageIntegrationTest(type: Test) {
     }
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-workers/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=worker

build.gradle

@@ -157,33 +157,6 @@ spotless {
 }
 check.dependsOn 'spotlessApply'
 
-@SuppressWarnings('GroovyAssignabilityCheck')
-def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) {
-    return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) {
-        // This is currently only used for connectors.
-        def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'
-
-        def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase()
-        def isArm64 = arch == "aarch64" || arch == "arm64"
-
-        def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64'
-        def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'
-        def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine'
-        def openjdkImage = System.getenv('JDK_IMAGE') ?: isArm64 ? "arm64v8/openjdk:19-slim-bullseye" : "amd64/openjdk:19-slim-bullseye"
-        def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64'
-
-        inputDir = file("$projectDir/build/docker")
-        platform = buildPlatform
-        images.add("airbyte/$artifactName:$buildTag")
-        buildArgs.put('JDK_VERSION', jdkVersion)
-        buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
-        buildArgs.put('ALPINE_IMAGE', alpineImage)
-        buildArgs.put('NGINX_IMAGE', nginxImage)
-        buildArgs.put('JDK_IMAGE', openjdkImage)
-        buildArgs.put('VERSION', buildVersion)
-    })
-}
-
 @SuppressWarnings('GroovyAssignabilityCheck')
 def Task getPublishArtifactsTask(String buildVersion, Project subproject) {
     // generate a unique task name based on the directory name.
@@ -230,9 +203,62 @@ allprojects {
 }
 
 // Java projects common configurations
-subprojects {
+subprojects { subproj ->
+
+    // Common Docker Configuration:
+    // If subprojects have the dockerImageName property configured in their gradle.properties file,
+    // register:
+    // 1) A copyGeneratedTar task to copy generated TARs. Subprojects that produce TARs can use this
+    //    to copy the produced tar into the docker image.
+    // 2) A buildDockerImage task to build the docker image.
+    // 3) Make the docker image task depend on the default assemble task.
+    if (subproj.hasProperty('dockerImageName')) {
+        project.logger.lifecycle("configuring docker task for $subproj.name")
+
+        // Although this task is defined for every subproject with the dockerImageName property,
+        // It is not necessarily used for all subprojects. Non-TAR producing projects can ignore this.
+        tasks.register("copyGeneratedTar", Copy) {
+            dependsOn copyDocker
+            dependsOn distTar
+            from('build/distributions') {
+                // Assume that tars are named <parent-project-name>-<child-project-name>-*.tar.
+                // Because we only have a handle to the child project, and to keep things simple,
+                // use a * regex to catch all prefixes.
+                include "*$subproj.name-*.tar"
+            }
+            into 'build/docker/bin'
+        }
+
+        tasks.register("buildDockerImage", DockerBuildImage) {
+            // This is currently only used for connectors.
+            def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'
+
+            def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase()
+            def isArm64 = arch == "aarch64" || arch == "arm64"
+
+            def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64'
+            def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'
+            def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine'
+            def openjdkImage = System.getenv('JDK_IMAGE') ?: isArm64 ? "arm64v8/openjdk:19-slim-bullseye" : "amd64/openjdk:19-slim-bullseye"
+            def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64'
+
+            platform = buildPlatform
+            images.add("airbyte/$subproj.dockerImageName:$rootProject.ext.image_tag")
+            buildArgs.put('JDK_VERSION', jdkVersion)
+            buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
+            buildArgs.put('ALPINE_IMAGE', alpineImage)
+            buildArgs.put('NGINX_IMAGE', nginxImage)
+            buildArgs.put('JDK_IMAGE', openjdkImage)
+            buildArgs.put('VERSION', rootProject.ext.version)
+
+        }
+
+        tasks.named("assemble") {
+            dependsOn buildDockerImage
+        }
+    }
 
-    if (project.name == 'airbyte-webapp' || project.name == 'airbyte-webapp-e2e-tests') {
+    if (subproj.name == 'airbyte-webapp' || subproj.name == 'airbyte-webapp-e2e-tests') {
         return
     }
 
@@ -362,7 +388,7 @@ subprojects {
     }
 
     dependencies {
-        if (project.name != 'airbyte-commons') {
+        if (subproj.name != 'airbyte-commons') {
             implementation project(':airbyte-commons')
         }