🎉 Support build on MacOS M1 (Apple Silicon) (#7104)

- See this doc for details: https://github.com/airbytehq/airbyte/blob/master/docs/contributing-to-airbyte/developing-locally.md
- Unit test does not work yet.

airbyte-config/persistence/build.gradle

@@ -14,6 +14,6 @@ dependencies {
     implementation project(':airbyte-config:init')
     implementation project(':airbyte-json-validation')
     implementation 'com.google.cloud:google-cloud-secretmanager:1.7.2'
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
     integrationTestJavaImplementation project(':airbyte-config:persistence')
 }

airbyte-db/jooq/build.gradle

@@ -13,7 +13,7 @@ dependencies {
 
     // jOOQ code generation
     implementation 'org.jooq:jooq-codegen:3.13.4'
-    implementation "org.testcontainers:postgresql:1.15.1"
+    implementation "org.testcontainers:postgresql:1.15.3"
     // The jOOQ code generator only has access to classes added to the jooqGenerator configuration
     jooqGenerator project(':airbyte-db:lib')
 }

airbyte-db/lib/build.gradle

@@ -11,7 +11,7 @@ dependencies {
     implementation project(':airbyte-protocol:models')
     implementation project(':airbyte-json-validation')
     implementation "org.flywaydb:flyway-core:7.14.0"
-    implementation "org.testcontainers:postgresql:1.15.1"
+    implementation "org.testcontainers:postgresql:1.15.3"
 
     testImplementation project(':airbyte-test-utils')
     testImplementation 'org.apache.commons:commons-lang3:3.11'

airbyte-e2e-testing/build.gradle

@@ -3,9 +3,11 @@ plugins {
     id "com.github.node-gradle.node" version "2.2.4"
 }
 
+def nodeVersion = System.getenv('NODE_VERSION') ?: '14.11.0'
+
 node {
     download = true
-    version = "14.11.0"
+    version = nodeVersion
 }
 
 

airbyte-integrations/bases/base-java/Dockerfile

@@ -1,4 +1,5 @@
-FROM openjdk:14.0.2-slim
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim
 COPY --from=airbyte/integration-base:dev /airbyte /airbyte
 
 WORKDIR /airbyte

airbyte-integrations/bases/base-java/build.gradle

@@ -14,8 +14,8 @@ dependencies {
 
     implementation project(':airbyte-protocol:models')
     implementation project(":airbyte-json-validation")
-    implementation "org.testcontainers:testcontainers:1.15.1"
-    implementation "org.testcontainers:jdbc:1.15.1"
+    implementation "org.testcontainers:testcontainers:1.15.3"
+    implementation "org.testcontainers:jdbc:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs)
 }

airbyte-integrations/bases/base-standard-source-test-file/Dockerfile

@@ -1,4 +1,5 @@
-FROM openjdk:14.0.2-slim
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim
 
 # Install Docker to launch worker images. Eventually should be replaced with Docker-java.
 # See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java

airbyte-integrations/bases/standard-source-test/Dockerfile

@@ -1,4 +1,5 @@
-FROM openjdk:14.0.2-slim
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim
 
 # Install Docker to launch worker images. Eventually should be replaced with Docker-java.
 # See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java

airbyte-integrations/connector-templates/generator/build.gradle

@@ -3,9 +3,11 @@ plugins {
     id "com.github.node-gradle.node" version "2.2.4"
 }
 
+def nodeVersion = System.getenv('NODE_VERSION') ?: '14.11.0'
+
 node {
     download = true
-    version = "14.11.0"
+    version = nodeVersion
 }
 
 assemble.dependsOn(npmInstall)

airbyte-integrations/connectors/destination-e2e-test/build.gradle

@@ -16,11 +16,11 @@ dependencies {
 
     testImplementation project(':airbyte-test-utils')
 
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
 
-    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)

airbyte-integrations/connectors/destination-jdbc/build.gradle

@@ -17,10 +17,10 @@ dependencies {
     implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2'
     implementation 'com.fasterxml.jackson.core:jackson-databind'
 
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
-    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)

airbyte-integrations/connectors/destination-meilisearch/build.gradle

@@ -19,7 +19,7 @@ dependencies {
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
     integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-meilisearch')
 
-    integrationTestJavaImplementation "org.testcontainers:testcontainers:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:testcontainers:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 }

airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle

@@ -20,7 +20,7 @@ dependencies {
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
     integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mysql')
-    integrationTestJavaImplementation "org.testcontainers:mysql:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:mysql:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)

airbyte-integrations/connectors/destination-mysql/build.gradle

@@ -19,7 +19,7 @@ dependencies {
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
     integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mysql')
-    integrationTestJavaImplementation "org.testcontainers:mysql:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:mysql:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)

airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle

@@ -18,7 +18,7 @@ dependencies {
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
 
-    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)

airbyte-integrations/connectors/destination-postgres/build.gradle

@@ -17,12 +17,12 @@ dependencies {
 
     testImplementation project(':airbyte-test-utils')
 
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
     integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-postgres')
 
-    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)

airbyte-integrations/connectors/source-e2e-test/build.gradle

@@ -24,7 +24,7 @@ dependencies {
     testImplementation project(":airbyte-json-validation")
     testImplementation project(':airbyte-test-utils')
 
-    testImplementation 'org.testcontainers:postgresql:1.15.1'
+    testImplementation 'org.testcontainers:postgresql:1.15.3'
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
 

airbyte-integrations/connectors/source-jdbc/build.gradle

@@ -28,10 +28,10 @@ dependencies {
     testImplementation project(':airbyte-test-utils')
 
     testImplementation "org.postgresql:postgresql:42.2.18"
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
-    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.1"
+    integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3"
 
     testFixturesImplementation "org.hamcrest:hamcrest-all:1.3"
     testFixturesImplementation project(':airbyte-protocol:models')

airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle

@@ -21,7 +21,7 @@ dependencies {
     testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
 
     testImplementation 'org.apache.commons:commons-lang3:3.11'
-    testImplementation "org.testcontainers:mssqlserver:1.15.1"
+    testImplementation "org.testcontainers:mssqlserver:1.15.3"
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
     integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mssql-strict-encrypt')

airbyte-integrations/connectors/source-mssql/build.gradle

@@ -26,7 +26,7 @@ dependencies {
     testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
 
     testImplementation 'org.apache.commons:commons-lang3:3.11'
-    testImplementation "org.testcontainers:mssqlserver:1.15.1"
+    testImplementation "org.testcontainers:mssqlserver:1.15.3"
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
     integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mssql')

airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle

@@ -20,7 +20,7 @@ dependencies {
     testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
     testImplementation project(':airbyte-test-utils')
 
-    testImplementation 'org.testcontainers:mysql:1.15.1'
+    testImplementation 'org.testcontainers:mysql:1.15.3'
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
 

airbyte-integrations/connectors/source-mysql/build.gradle

@@ -23,7 +23,7 @@ dependencies {
     testImplementation testFixtures(project(':airbyte-integrations:bases:debezium'))
     testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
     testImplementation 'org.apache.commons:commons-lang3:3.11'
-    testImplementation 'org.testcontainers:mysql:1.15.1'
+    testImplementation 'org.testcontainers:mysql:1.15.3'
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
     integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mysql')

airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle

@@ -27,7 +27,7 @@ dependencies {
     testImplementation project(':airbyte-test-utils')
 
     testImplementation 'org.apache.commons:commons-lang3:3.11'
-    testImplementation 'org.testcontainers:oracle-xe:1.15.2'
+    testImplementation 'org.testcontainers:oracle-xe:1.15.3'
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
 

airbyte-integrations/connectors/source-oracle/build.gradle

@@ -26,7 +26,7 @@ dependencies {
     testImplementation project(':airbyte-test-utils')
 
     testImplementation 'org.apache.commons:commons-lang3:3.11'
-    testImplementation 'org.testcontainers:oracle-xe:1.15.2'
+    testImplementation 'org.testcontainers:oracle-xe:1.15.3'
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
 

airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle

@@ -22,7 +22,7 @@ dependencies {
     testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
     testImplementation project(':airbyte-test-utils')
 
-    testImplementation 'org.testcontainers:postgresql:1.15.1'
+    testImplementation 'org.testcontainers:postgresql:1.15.3'
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
 

airbyte-integrations/connectors/source-postgres/build.gradle

@@ -25,7 +25,7 @@ dependencies {
     testImplementation project(":airbyte-json-validation")
     testImplementation project(':airbyte-test-utils')
 
-    testImplementation 'org.testcontainers:postgresql:1.15.1'
+    testImplementation 'org.testcontainers:postgresql:1.15.3'
 
     integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
 

airbyte-integrations/connectors/source-relational-db/build.gradle

@@ -19,7 +19,7 @@ dependencies {
     testImplementation project(':airbyte-test-utils')
 
     testImplementation "org.postgresql:postgresql:42.2.18"
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 }

airbyte-migration/Dockerfile

@@ -1,4 +1,5 @@
-FROM openjdk:14.0.2-slim AS migrate
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim AS migrate
 
 ENV APPLICATION airbyte-migration
 

airbyte-scheduler/app/Dockerfile

@@ -1,4 +1,5 @@
-FROM openjdk:14.0.2-slim AS scheduler
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim AS scheduler
 
 ENV APPLICATION airbyte-scheduler
 

airbyte-scheduler/app/build.gradle

@@ -21,7 +21,7 @@ dependencies {
     implementation project(':airbyte-scheduler:persistence')
     implementation project(':airbyte-workers')
 
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 }
 
 application {

airbyte-scheduler/persistence/build.gradle

@@ -15,5 +15,5 @@ dependencies {
     implementation project(':airbyte-scheduler:models')
 
     testImplementation "org.flywaydb:flyway-core:7.14.0"
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 }

airbyte-secrets-migration/Dockerfile

@@ -1,4 +1,5 @@
-FROM openjdk:14.0.2-slim
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim
 
 ENV APPLICATION airbyte-secrets-migration
 WORKDIR /airbyte

airbyte-secrets-migration/build.gradle

@@ -16,7 +16,7 @@ dependencies {
     implementation project(':airbyte-config:persistence')
     testImplementation project(':airbyte-json-validation')
     testImplementation 'org.apache.commons:commons-lang3:3.11'
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
     integrationTestJavaImplementation project(':airbyte-secrets-migration')
 }
 

airbyte-server/Dockerfile

@@ -1,4 +1,5 @@
-FROM openjdk:14.0.2-slim AS server
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim AS server
 
 EXPOSE 8000
 

airbyte-server/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id 'application'
     id 'maven-publish'
-    id 'com.github.johnrengelman.shadow' version '6.1.0'
+    id 'com.github.johnrengelman.shadow' version '7.1.0'
 }
 
 shadowJar {
@@ -81,7 +81,7 @@ dependencies {
 
     testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1'
 
-    testImplementation "org.testcontainers:postgresql:1.15.1"
+    testImplementation "org.testcontainers:postgresql:1.15.3"
 }
 
 // we want to be able to access the generated db files from config/init when we build the server docker image.

airbyte-test-utils/build.gradle

@@ -5,6 +5,6 @@ plugins {
 dependencies {
     implementation project(':airbyte-db:lib')
 
-    implementation 'org.testcontainers:postgresql:1.15.1'
+    implementation 'org.testcontainers:postgresql:1.15.3'
     implementation "org.testcontainers:cockroachdb:1.15.3"
 }

airbyte-tests/build.gradle

@@ -49,7 +49,7 @@ dependencies {
     acceptanceTestsImplementation 'com.fasterxml.jackson.core:jackson-databind'
     acceptanceTestsImplementation 'io.github.cdimascio:java-dotenv:3.0.0'
     acceptanceTestsImplementation 'org.apache.commons:commons-csv:1.4'
-    acceptanceTestsImplementation 'org.testcontainers:postgresql:1.15.1'
+    acceptanceTestsImplementation 'org.testcontainers:postgresql:1.15.3'
     acceptanceTestsImplementation 'org.postgresql:postgresql:42.2.18'
 
     automaticMigrationAcceptanceTestImplementation project(':airbyte-api')

airbyte-webapp/build.gradle

@@ -3,9 +3,11 @@ plugins {
     id "com.github.node-gradle.node" version "3.1.1"
 }
 
+def nodeVersion = System.getenv('NODE_VERSION') ?: '14.11.0'
+
 node {
     download = true
-    version = "14.11.0"
+    version = nodeVersion
 }
 
 npm_run_build {

airbyte-workers/Dockerfile

@@ -1,4 +1,7 @@
-FROM openjdk:14.0.2-slim AS worker
+ARG JDK_VERSION=14.0.2
+FROM openjdk:${JDK_VERSION}-slim AS worker
+
+ARG ARCH=amd64
 
 # Install Docker to launch worker images. Eventually should be replaced with Docker-java.
 # See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java
@@ -10,7 +13,7 @@ RUN apt-get update && apt-get install -y \
                           software-properties-common
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
 RUN add-apt-repository \
-       "deb [arch=amd64] https://download.docker.com/linux/debian \
+       "deb [arch=${ARCH}] https://download.docker.com/linux/debian \
        $(lsb_release -cs) \
        stable"
 RUN apt-get update && apt-get install -y docker-ce-cli jq
@@ -20,7 +23,7 @@ ENV APPLICATION airbyte-workers
 WORKDIR /app
 
 # Install kubectl
-RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.17.14/bin/linux/amd64/kubectl
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.17.14/bin/linux/${ARCH}/kubectl
 RUN chmod +x ./kubectl
 RUN mv ./kubectl /usr/local/bin