Revert "Revert "Bmoric/remove docker compose for build (#7500)" (#7698)…

…" (#7746)

This reverts commit 797d11a.

Restore the removal of the docker compose file

.bumpversion.cfg

@@ -10,6 +10,8 @@ serialize =
 
 [bumpversion:file:.env]
 
+[bumpversion:file:airbyte-migration/Dockerfile]
+
 [bumpversion:file:airbyte-server/Dockerfile]
 
 [bumpversion:file:airbyte-workers/Dockerfile]

.github/workflows/gradle.yml

@@ -339,7 +339,7 @@ jobs:
           EOF
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan
+        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
 
       - name: Run End-to-End Frontend Tests
         run: ./tools/bin/e2e_test.sh
@@ -457,7 +457,7 @@ jobs:
           HOME: /home/runner
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
+        run: SUB_BUILD=PLATFORM ./gradlew assemble --scan
 
       - name: Run Kubernetes End-to-End Acceptance Tests
         env:

airbyte-cli/build.gradle

@@ -1,3 +1,3 @@
-plugins {
-    id "airbyte-docker"
-}
+Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir")
+dockerBuildTask.dependsOn(copyDocker)
+assemble.dependsOn(dockerBuildTask)

airbyte-config/init/Dockerfile

@@ -5,4 +5,4 @@ WORKDIR /app
 
 # the sole purpose of this image is to seed the data volume with the default data
 # that the app should have when it is first installed.
-COPY scripts scripts
+COPY bin/scripts scripts

airbyte-config/init/build.gradle

@@ -11,3 +11,14 @@ dependencies {
     implementation project(':airbyte-commons-docker')
     implementation project(':airbyte-json-validation')
 }
+
+task copyScripts(type: Copy) {
+    dependsOn copyDocker
+
+    from('scripts')
+    into 'build/docker/bin/scripts'
+}
+
+Task dockerBuildTask = getDockerBuildTask("init",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyScripts)
+assemble.dependsOn(dockerBuildTask)

airbyte-db/lib/Dockerfile

@@ -1,3 +1,3 @@
 FROM postgres:13-alpine
 
-COPY src/main/resources/init.sql /docker-entrypoint-initdb.d/000_init.sql
+COPY bin/init.sql /docker-entrypoint-initdb.d/000_init.sql

airbyte-db/lib/build.gradle

@@ -69,3 +69,16 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
     args 'jobs', 'dump_schema'
 }
+
+task copyInitSql(type: Copy) {
+    dependsOn copyDocker
+
+    from('src/main/resources') {
+        include 'init.sql'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("db",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyInitSql)
+assemble.dependsOn(dockerBuildTask)

airbyte-migration/Dockerfile

@@ -6,7 +6,7 @@ ENV APPLICATION airbyte-migration
 WORKDIR /app
 
 # Move and run scheduler
-COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+COPY bin/${APPLICATION}-0.30.34-alpha.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 

airbyte-migration/build.gradle

@@ -15,3 +15,16 @@ application {
     mainClass = 'io.airbyte.migrate.MigrationRunner'
 }
 
+task copyGeneratedTar(type: Copy) {
+    dependsOn distTar
+    dependsOn copyDocker
+
+    from('build/distributions') {
+        include 'airbyte-migration-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("migration",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-scheduler/app/Dockerfile

@@ -5,7 +5,7 @@ ENV APPLICATION airbyte-scheduler
 
 WORKDIR /app
 
-ADD build/distributions/${APPLICATION}-0.30.34-alpha.tar /app
+ADD bin/${APPLICATION}-0.30.34-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.34-alpha/bin/${APPLICATION}"]

airbyte-scheduler/app/build.gradle

@@ -57,3 +57,17 @@ run {
     environment "TEMPORAL_HOST", "localhost:7233"
 
 }
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-scheduler-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("scheduler",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-server/Dockerfile

@@ -7,7 +7,7 @@ ENV APPLICATION airbyte-server
 
 WORKDIR /app
 
-ADD build/distributions/${APPLICATION}-0.30.34-alpha.tar /app
+ADD bin/${APPLICATION}-0.30.34-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.34-alpha/bin/${APPLICATION}"]

airbyte-server/build.gradle

@@ -127,3 +127,17 @@ run {
     environment "AIRBYTE_ROLE", System.getenv('AIRBYTE_ROLE')
     environment "TEMPORAL_HOST", "localhost:7233"
 }
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-server-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("server",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-webapp/Dockerfile

@@ -2,5 +2,6 @@ FROM nginx:1.19-alpine as webapp
 
 EXPOSE 80
 
-COPY build /usr/share/nginx/html
-COPY nginx/default.conf.template /etc/nginx/templates/default.conf.template
+COPY bin/docs docs/
+COPY bin/build /usr/share/nginx/html
+COPY bin/nginx/default.conf.template /etc/nginx/templates/default.conf.template

airbyte-webapp/build.gradle

@@ -32,11 +32,38 @@ task test(type: NpmTask) {
 assemble.dependsOn npm_run_build
 build.finalizedBy test
 
+task copyBuild(type: Copy) {
+    dependsOn copyDocker
+
+    from "${project.projectDir}/build"
+    into "build/docker/bin/build"
+    exclude ".docker"
+    exclude "docker"
+}
+
 task copyDocs(type: Copy) {
-    from "${System.getProperty("user.dir")}/docs/integrations"
-    into "${buildDir}/docs/integrations"
+    dependsOn copyDocker
+
+    from "${project.rootProject.projectDir}/docs/integrations"
+    into "build/docker/bin/docs/integrations"
     duplicatesStrategy DuplicatesStrategy.INCLUDE
 }
 
+task copyNginx(type: Copy) {
+    dependsOn copyDocker
+
+    from "${project.projectDir}/nginx"
+    into "build/docker/bin/nginx"
+}
+
+copyBuild.dependsOn npm_run_build
+copyNginx.dependsOn npm_run_build
 copyDocs.dependsOn npm_run_build
 assemble.dependsOn copyDocs
+copyDocker.dependsOn(npm_run_build)
+
+Task dockerBuildTask = getDockerBuildTask("webapp",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyBuild)
+dockerBuildTask.dependsOn(copyNginx)
+dockerBuildTask.dependsOn(copyDocs)
+assemble.dependsOn(dockerBuildTask)

airbyte-workers/Dockerfile

@@ -23,7 +23,7 @@ ENV APPLICATION airbyte-workers
 WORKDIR /app
 
 # Move worker app
-ADD build/distributions/${APPLICATION}-0.30.34-alpha.tar /app
+ADD bin/${APPLICATION}-0.30.34-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.34-alpha/bin/${APPLICATION}"]

airbyte-workers/build.gradle

@@ -62,3 +62,17 @@ application {
     mainClass = mainClassName
     applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0']
 }
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-workers-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("worker",  "$project.projectDir")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

build.gradle

@@ -1,3 +1,16 @@
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+
+buildscript {
+    repositories {
+        maven {
+            url "https://plugins.gradle.org/m2/"
+        }
+    }
+    dependencies {
+        classpath 'com.bmuschko:gradle-docker-plugin:7.1.0'
+    }
+}
+
 plugins {
     id 'base'
     id 'pmd'
@@ -133,26 +146,19 @@ def Task getDockerBuildTask(String artifactName, String projectDir) {
 }
 
 allprojects {
-    apply plugin: 'base'
+    apply plugin: 'com.bmuschko.docker-remote-api'
 
-    afterEvaluate { project ->
-        def composeDeps = [
-                ":airbyte-config:init",
-                ":airbyte-db:lib",
-                ":airbyte-migration",
-                ":airbyte-scheduler:app",
-                ":airbyte-workers",
-                ":airbyte-server",
-                ":airbyte-webapp",
-        ].toSet().asImmutable()
-
-        if (project.getPath() in composeDeps) {
-            composeBuild.dependsOn(project.getPath() + ':assemble')
-        }
+    task copyDocker(type: Copy) {
+        delete "build/docker"
+
+        from "${project.projectDir}/Dockerfile"
+        into "build/docker/"
     }
 }
 
 allprojects {
+    apply plugin: 'base'
+
     // by default gradle uses directory as the project name. That works very well in a single project environment but
     // projects clobber each other in an environments with subprojects when projects are in directories named identically.
     def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
@@ -254,6 +260,7 @@ subprojects {
         testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
         testImplementation 'org.mockito:mockito-junit-jupiter:3.12.4'
         testImplementation 'org.assertj:assertj-core:3.21.0'
+
     }
 
     tasks.withType(Tar) {
@@ -265,28 +272,6 @@ subprojects {
     }
 }
 
-task composeBuild {
-    def buildTag = System.getenv('VERSION') ?: 'dev'
-    def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64'
-    def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64'
-    def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2'
-    def dockerComposeFile = buildArch == 'arm64' ? 'docker-compose.build-m1.yaml' : 'docker-compose.build.yaml'
-    doFirst {
-        exec {
-            workingDir rootDir
-            commandLine 'docker-compose', '-f', dockerComposeFile, 'build', '--parallel', '--quiet'
-            environment 'VERSION', buildTag
-            environment 'DOCKER_BUILD_PLATFORM', buildPlatform
-            environment 'DOCKER_BUILD_ARCH', buildArch
-            environment 'JDK_VERSION', jdkVersion
-        }
-    }
-}
-
-if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") {
-    build.dependsOn(composeBuild)
-}
-
 task('generate') {
     dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
     dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }

docs/SUMMARY.md

@@ -199,6 +199,7 @@
 * [Contributing to Airbyte](contributing-to-airbyte/README.md)
   * [Code of Conduct](contributing-to-airbyte/code-of-conduct.md)
   * [Developing Locally](contributing-to-airbyte/developing-locally.md)
+  * [Developing on Docker](contributing-to-airbyte/developing-on-docker.md)
   * [Developing on Kubernetes](contributing-to-airbyte/developing-on-kubernetes.md)
   * [Monorepo Python Development](contributing-to-airbyte/monorepo-python-development.md)
   * [Code Style](contributing-to-airbyte/code-style.md)

docs/contributing-to-airbyte/developing-on-docker.md

@@ -0,0 +1,43 @@
+# Developing on docker
+
+## Incrementality 
+
+The docker build is fully incremental for the platform build, which means that it will only build an image if it is needed. We need to keep it that 
+way.
+A task generator, `getDockerBuildTask`, is available for building a docker image for any given module. Behind the scene, it will generate a 
+task which will run the build of a docker image in a specific folder. The goal is to make sure that we have an isolated 
+context which helps with incrementality. All files that need to be present in the docker image will need to be copy into this folder. The generate 
+method takes 2 arguments:
+- The image name, for example if `foo` is given as an image name, the image `airbyte/foo` will be created
+- The project directory folder. It is needed because the `getDockerBuildTask` is declared in the rootProject
+
+## Adding a new docker build
+
+Once you have a `Dockerfile`, generating the docker image is done in the following way:
+- specify the artifact name and the project directory,
+- make sure that the Dockerfile is properly copied to the docker context dir before building the image
+- make the build docker task to depend on the `assemble` task.
+
+For example:
+```groovy
+Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir)
+dockerBuildTask.dependsOn(copyDocker)
+assemble.dependsOn(dockerBuildTask)
+```
+
+If you need to add files in your image you need to copy them in `build/docker/bin` first. The need to happen after the `copyDocker` task.
+The `copyDocker` task clean up the `build/docker` folder as a first step.
+
+For example:
+```groovy
+task copyScripts(type: Copy) {
+    dependsOn copyDocker
+
+    from('scripts')
+    into 'build/docker/bin/scripts'
+}
+
+Task dockerBuildTask = getDockerBuildTask("init", project.projectDir)
+dockerBuildTask.dependsOn(copyScripts)
+assemble.dependsOn(dockerBuildTask)
+```