Bmoric/remove docker compose for build (airbytehq#7500)

This making the build using a gradle plugin instead of using docker-compose build.
It aims to make the build to be more incremental as described in airbytehq#7306

Building the docker image don't rely on docker-compose anymore.
The docker build step is isolated into a dedicated folder (in order to make sure that gradle plugin don't recompute the build of the docker container)
Gradle is responsible for copying the files that docker needs to build its image.
That removes the need of having a dockerignore file.
This might not be effective until airbytehq#7539 is solved.

.bumpversion.cfg

@@ -10,6 +10,8 @@ serialize =
 
 [bumpversion:file:.env]
 
+[bumpversion:file:airbyte-migration/Dockerfile]
+
 [bumpversion:file:airbyte-server/Dockerfile]
 
 [bumpversion:file:airbyte-workers/Dockerfile]

.github/workflows/gradle.yml

@@ -339,7 +339,7 @@ jobs:
           EOF
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan
+        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
 
       - name: Run End-to-End Frontend Tests
         run: ./tools/bin/e2e_test.sh
@@ -457,7 +457,7 @@ jobs:
           HOME: /home/runner
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
+        run: SUB_BUILD=PLATFORM ./gradlew assemble --scan
 
       - name: Run Kubernetes End-to-End Acceptance Tests
         env:

airbyte-cli/build.gradle

@@ -1,3 +1,3 @@
-plugins {
-    id "airbyte-docker"
-}
+Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir")
+dockerBuildTask.dependsOn(copyDocker)
+assemble.dependsOn(dockerBuildTask)

airbyte-config/init/Dockerfile

@@ -4,4 +4,4 @@ WORKDIR /app
 
 # the sole purpose of this image is to seed the data volume with the default data
 # that the app should have when it is first installed.
-COPY scripts scripts
+COPY bin/scripts scripts

airbyte-config/init/build.gradle

@@ -11,3 +11,14 @@ dependencies {
     implementation project(':airbyte-commons-docker')
     implementation project(':airbyte-json-validation')
 }
+
+task copyScripts(type: Copy) {
+    dependsOn copyDocker
+
+    from('scripts')
+    into 'build/docker/bin/scripts'
+}
+
+Task dockerBuildTask = getDockerBuildTask("init",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyScripts)
+assemble.dependsOn(dockerBuildTask)

airbyte-db/lib/Dockerfile

@@ -1,3 +1,3 @@
 FROM postgres:13-alpine
 
-COPY src/main/resources/init.sql /docker-entrypoint-initdb.d/000_init.sql
+COPY bin/init.sql /docker-entrypoint-initdb.d/000_init.sql

airbyte-db/lib/build.gradle

@@ -69,3 +69,16 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
     args 'jobs', 'dump_schema'
 }
+
+task copyInitSql(type: Copy) {
+    dependsOn copyDocker
+
+    from('src/main/resources') {
+        include 'init.sql'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("db",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyInitSql)
+assemble.dependsOn(dockerBuildTask)

airbyte-migration/Dockerfile

@@ -6,7 +6,7 @@ ENV APPLICATION airbyte-migration
 WORKDIR /app
 
 # Move and run scheduler
-COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+COPY bin/${APPLICATION}-0.30.31-alpha.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 

airbyte-migration/build.gradle

@@ -15,3 +15,16 @@ application {
     mainClass = 'io.airbyte.migrate.MigrationRunner'
 }
 
+task copyGeneratedTar(type: Copy) {
+    dependsOn distTar
+    dependsOn copyDocker
+
+    from('build/distributions') {
+        include 'airbyte-migration-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("migration",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java

@@ -26,7 +26,6 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.http.client.utils.URIBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

airbyte-scheduler/app/Dockerfile

@@ -5,7 +5,7 @@ ENV APPLICATION airbyte-scheduler
 
 WORKDIR /app
 
-ADD build/distributions/${APPLICATION}-0.30.31-alpha.tar /app
+ADD bin/${APPLICATION}-0.30.31-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.31-alpha/bin/${APPLICATION}"]

airbyte-scheduler/app/build.gradle

@@ -57,3 +57,17 @@ run {
     environment "TEMPORAL_HOST", "localhost:7233"
 
 }
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-scheduler-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("scheduler",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-server/Dockerfile

@@ -7,7 +7,7 @@ ENV APPLICATION airbyte-server
 
 WORKDIR /app
 
-ADD build/distributions/${APPLICATION}-0.30.31-alpha.tar /app
+ADD bin/${APPLICATION}-0.30.31-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.31-alpha/bin/${APPLICATION}"]

airbyte-server/build.gradle

@@ -127,3 +127,17 @@ run {
     environment "AIRBYTE_ROLE", System.getenv('AIRBYTE_ROLE')
     environment "TEMPORAL_HOST", "localhost:7233"
 }
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-server-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("server",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-webapp/Dockerfile

@@ -2,8 +2,6 @@ FROM nginx:1.19-alpine as webapp
 
 EXPOSE 80
 
-COPY build/docs docs/
-# docs get copied twice because npm gradle plugin ignores output dir.
-COPY build /usr/share/nginx/html
-RUN rm -rf /usr/share/nginx/html/docs
-COPY nginx/default.conf.template /etc/nginx/templates/default.conf.template
+COPY bin/docs docs/
+COPY bin/build /usr/share/nginx/html
+COPY bin/nginx/default.conf.template /etc/nginx/templates/default.conf.template

airbyte-webapp/build.gradle

@@ -32,11 +32,38 @@ task test(type: NpmTask) {
 assemble.dependsOn npm_run_build
 build.finalizedBy test
 
+task copyBuild(type: Copy) {
+    dependsOn copyDocker
+
+    from "${project.projectDir}/build"
+    into "build/docker/bin/build"
+    exclude ".docker"
+    exclude "docker"
+}
+
 task copyDocs(type: Copy) {
-    from "${System.getProperty("user.dir")}/docs/integrations"
-    into "${buildDir}/docs/integrations"
+    dependsOn copyDocker
+
+    from "${project.rootProject.projectDir}/docs/integrations"
+    into "build/docker/bin/docs/integrations"
     duplicatesStrategy DuplicatesStrategy.INCLUDE
 }
 
+task copyNginx(type: Copy) {
+    dependsOn copyDocker
+
+    from "${project.projectDir}/nginx"
+    into "build/docker/bin/nginx"
+}
+
+copyBuild.dependsOn npm_run_build
+copyNginx.dependsOn npm_run_build
 copyDocs.dependsOn npm_run_build
 assemble.dependsOn copyDocs
+copyDocker.dependsOn(npm_run_build)
+
+Task dockerBuildTask = getDockerBuildTask("webapp",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyBuild)
+dockerBuildTask.dependsOn(copyNginx)
+dockerBuildTask.dependsOn(copyDocs)
+assemble.dependsOn(dockerBuildTask)

airbyte-workers/Dockerfile

@@ -23,7 +23,7 @@ ENV APPLICATION airbyte-workers
 WORKDIR /app
 
 # Move worker app
-ADD build/distributions/${APPLICATION}-0.30.31-alpha.tar /app
+ADD bin/${APPLICATION}-0.30.31-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.31-alpha/bin/${APPLICATION}"]

airbyte-workers/build.gradle

@@ -62,3 +62,17 @@ application {
     mainClass = mainClassName
     applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0']
 }
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-workers-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("worker",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

build.gradle

@@ -1,3 +1,16 @@
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+
+buildscript {
+    repositories {
+        maven {
+            url "https://plugins.gradle.org/m2/"
+        }
+    }
+    dependencies {
+        classpath 'com.bmuschko:gradle-docker-plugin:7.1.0'
+    }
+}
+
 plugins {
     id 'base'
     id 'pmd'
@@ -112,27 +125,36 @@ spotless {
 }
 check.dependsOn 'spotlessApply'
 
+@SuppressWarnings('GroovyAssignabilityCheck')
+def Task getDockerBuildTask(String artifactName, String projectDir) {
+    return task ("buildDockerImage-$artifactName" (type: DockerBuildImage) {
+        def buildTag = System.getenv('VERSION') ?: 'dev'
+        def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64'
+        def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2'
+        def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64'
+
+        inputDir = file("$projectDir/build/docker")
+        platform = buildPlatform
+        images.add("airbyte/$artifactName:$buildTag")
+        buildArgs.put('JDK_VERSION', jdkVersion)
+        buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
+    })
+}
+
 allprojects {
-    apply plugin: 'base'
+    apply plugin: 'com.bmuschko.docker-remote-api'
 
-    afterEvaluate { project ->
-        def composeDeps = [
-                ":airbyte-config:init",
-                ":airbyte-db:lib",
-                ":airbyte-migration",
-                ":airbyte-scheduler:app",
-                ":airbyte-workers",
-                ":airbyte-server",
-                ":airbyte-webapp",
-        ].toSet().asImmutable()
-
-        if (project.getPath() in composeDeps) {
-            composeBuild.dependsOn(project.getPath() + ':assemble')
-        }
+    task copyDocker(type: Copy) {
+        delete "build/docker"
+
+        from "${project.projectDir}/Dockerfile"
+        into "build/docker/"
     }
 }
 
 allprojects {
+    apply plugin: 'base'
+
     // by default gradle uses directory as the project name. That works very well in a single project environment but
     // projects clobber each other in an environments with subprojects when projects are in directories named identically.
     def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
@@ -234,6 +256,7 @@ subprojects {
         testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
         testImplementation 'org.mockito:mockito-junit-jupiter:3.12.4'
         testImplementation 'org.assertj:assertj-core:3.21.0'
+
     }
 
     tasks.withType(Tar) {
@@ -245,28 +268,6 @@ subprojects {
     }
 }
 
-task composeBuild {
-    def buildTag = System.getenv('VERSION') ?: 'dev'
-    def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64'
-    def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64'
-    def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2'
-    def dockerComposeFile = buildArch == 'arm64' ? 'docker-compose.build-m1.yaml' : 'docker-compose.build.yaml'
-    doFirst {
-        exec {
-            workingDir rootDir
-            commandLine 'docker-compose', '-f', dockerComposeFile, 'build', '--parallel', '--quiet'
-            environment 'VERSION', buildTag
-            environment 'DOCKER_BUILD_PLATFORM', buildPlatform
-            environment 'DOCKER_BUILD_ARCH', buildArch
-            environment 'JDK_VERSION', jdkVersion
-        }
-    }
-}
-
-if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") {
-    build.dependsOn(composeBuild)
-}
-
 task('generate') {
     dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
     dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }

docs/SUMMARY.md

@@ -197,6 +197,7 @@
 * [Contributing to Airbyte](contributing-to-airbyte/README.md)
   * [Code of Conduct](contributing-to-airbyte/code-of-conduct.md)
   * [Developing Locally](contributing-to-airbyte/developing-locally.md)
+  * [Developing on Docker](contributing-to-airbyte/developing-on-docker.md)
   * [Developing on Kubernetes](contributing-to-airbyte/developing-on-kubernetes.md)
   * [Monorepo Python Development](contributing-to-airbyte/monorepo-python-development.md)
   * [Code Style](contributing-to-airbyte/code-style.md)