Publish connector Docker images for multiple platform architectures (#…

…13004)

* Publish connector images for multiple platform architectures

* testing

* notes

* show path in log

* Build arch 1-by-1 and use manifest file

* build & test comment

* delete tmp docker tags

* share docker auth code

* remove manifests when done & fix trailing / for tag delete

tools/integrations/manage.sh

@@ -41,6 +41,8 @@ cmd_build() {
   local run_tests=$1; shift || run_tests=true
 
   echo "Building $path"
+  # Note that we are only building (and testing) once on this build machine's architecture
+  # Learn more @ https://github.com/airbytehq/airbyte/pull/13004
   ./gradlew --no-daemon "$(_to_gradle_path "$path" clean)"
   ./gradlew --no-daemon "$(_to_gradle_path "$path" build)"
 
@@ -198,7 +200,12 @@ cmd_publish() {
   local image_name; image_name=$(_get_docker_image_name "$path"/Dockerfile)
   local image_version; image_version=$(_get_docker_image_version "$path"/Dockerfile)
   local versioned_image=$image_name:$image_version
-  local latest_image=$image_name:latest
+  local latest_image="$image_name" # don't include ":latest", that's assumed here
+  local build_arch="linux/amd64,linux/arm64"
+
+  # log into docker
+  DOCKER_USERNAME=${DOCKER_USERNAME:-airbytebot}
+  DOCKER_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'${DOCKER_USERNAME}'", "password": "'${DOCKER_PASSWORD}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token)
 
   echo "image_name $image_name"
   echo "versioned_image $versioned_image"
@@ -217,26 +224,61 @@ cmd_publish() {
   if [[ "airbyte/normalization" == "${image_name}" ]]; then
     echo "Publishing normalization images (version: $versioned_image)"
     GIT_REVISION=$(git rev-parse HEAD)
-    VERSION=$image_version GIT_REVISION=$GIT_REVISION docker-compose -f airbyte-integrations/bases/base-normalization/docker-compose.build.yaml build
-    VERSION=$image_version GIT_REVISION=$GIT_REVISION docker-compose -f airbyte-integrations/bases/base-normalization/docker-compose.build.yaml push
-    VERSION=latest         GIT_REVISION=$GIT_REVISION docker-compose -f airbyte-integrations/bases/base-normalization/docker-compose.build.yaml build
-    VERSION=latest         GIT_REVISION=$GIT_REVISION docker-compose -f airbyte-integrations/bases/base-normalization/docker-compose.build.yaml push
+
+    # Note: "buildx bake" needs to be run within the directory
+    local original_pwd=$PWD
+    cd airbyte-integrations/bases/base-normalization
+
+    VERSION=$image_version GIT_REVISION=$GIT_REVISION docker buildx bake \
+      --set "*.platform=$build_arch"                                     \
+      -f docker-compose.build.yaml                                       \
+      --push
+
+    VERSION=latest GIT_REVISION=$GIT_REVISION docker buildx bake \
+      --set "*.platform=$build_arch"                             \
+      -f docker-compose.build.yaml                               \
+      --push
+
+    cd $original_pwd
   else
-    docker tag "$image_name:dev" "$versioned_image"
-    docker tag "$image_name:dev" "$latest_image"
+    # We have to go arch-by-arch locally (see https://github.com/docker/buildx/issues/59 for more info) due to our base images (e.g. airbyte-integrations/bases/base-java)
+    # Alternative local approach @ https://github.com/docker/buildx/issues/301#issuecomment-755164475
+
+    for arch in $(echo $build_arch | sed "s/,/ /g")
+    do
+      echo "building base images for $arch"
+      docker buildx build -t airbyte/integration-base:dev --platform $arch --load airbyte-integrations/bases/base
+      docker buildx build -t airbyte/integration-base-java:dev --platform $arch --load airbyte-integrations/bases/base-java
+
+      local arch_versioned_image=$image_name:`echo $arch | sed "s/\//-/g"`-$image_version
+      echo "Publishing new version ($arch_versioned_image) from $path"
+      docker buildx build -t $arch_versioned_image --platform $arch --push $path
+      docker manifest create $latest_image --amend $arch_versioned_image
+      docker manifest create $versioned_image --amend $arch_versioned_image
+    done
+
+    docker manifest push $latest_image
+    docker manifest push $versioned_image
+    docker manifest rm $latest_image
+    docker manifest rm $versioned_image
+
+    # delete the temporary image tags made with arch_versioned_image
+    sleep 5
+    for arch in $(echo $build_arch | sed "s/,/ /g")
+    do
+      local arch_versioned_tag=`echo $arch | sed "s/\//-/g"`-$image_version
+      echo "deleting temporary tag: ${image_name}/tags/${arch_versioned_tag}"
+      TAG_URL="https://hub.docker.com/v2/repositories/${image_name}/tags/${arch_versioned_tag}/" # trailing slash is needed!
+      curl -X DELETE -H "Authorization: JWT ${DOCKER_TOKEN}" "$TAG_URL"
+    done
 
-    echo "Publishing new version ($versioned_image)"
-    docker push "$versioned_image"
-    docker push "$latest_image"
   fi
 
   # Checking if the image was successfully registered on DockerHub
   # see the description of this PR to understand why this is needed https://github.com/airbytehq/airbyte/pull/11654/
   sleep 5
 
   # To work for private repos we need a token as well
-  DOCKER_USERNAME=${DOCKER_USERNAME:-airbytebot}
-  DOCKER_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'${DOCKER_USERNAME}'", "password": "'${DOCKER_PASSWORD}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token)
   TAG_URL="https://hub.docker.com/v2/repositories/${image_name}/tags/${image_version}"
   DOCKERHUB_RESPONSE_CODE=$(curl --silent --output /dev/null --write-out "%{http_code}" -H "Authorization: JWT ${DOCKER_TOKEN}" ${TAG_URL})
   if [[ "${DOCKERHUB_RESPONSE_CODE}" == "404" ]]; then