[BEAM-9541] Push all docker images from RC instead of hard-coding them.

This greatly simplifies the script. It also spares us from having to hard-code yet another list of supported Flink versions, which I'm trying to avoid. There is the minor downside that listing images requires them to be present locally from build_release_candidate.sh, so we can't clean them up in between builds. However, I don't think that was ever really necessary except to save disk space.
apache · Jan 23, 2021 · 34ce707 · 34ce707
1 parent da2f431
commit 34ce707
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 88 deletions.
diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
@@ -290,16 +290,6 @@ if [[ $confirmation = "y" ]]; then
   ./gradlew ":runners:spark:job-server:container:dockerPush" -Pdocker-tag="${RELEASE}_rc${RC_NUM}"
 
   rm -rf ~/${PYTHON_ARTIFACTS_DIR}
-
-  echo '-------------------Clean up images at local-----------------'
-  for ver in "${PYTHON_VER[@]}"; do
-     docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_rc${RC_NUM}
-  done
-  docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_rc${RC_NUM}
-  for ver in "${FLINK_VER[@]}"; do
-    docker rmi -f "${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}flink${ver}_job_server:${RELEASE}_rc${RC_NUM}"
-  done
-  docker rmi -f "${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}spark_job_server:${RELEASE}_rc${RC_NUM}"
 fi
 
 echo "[Current Step]: Update beam-site"

diff --git a/release/src/main/scripts/publish_docker_images.sh b/release/src/main/scripts/publish_docker_images.sh
@@ -27,10 +27,6 @@ set -e
 DOCKER_IMAGE_DEFAULT_REPO_ROOT=apache
 DOCKER_IMAGE_DEFAULT_REPO_PREFIX=beam_
 
-JAVA_VER=("java8" "java11")
-PYTHON_VER=("python3.6" "python3.7" "python3.8")
-FLINK_VER=("1.8" "1.9" "1.10")
-
 echo "Publish SDK docker images to Docker Hub."
 
 echo "================Setting Up Environment Variables==========="
@@ -47,84 +43,21 @@ echo "Do you want to proceed? [y|N]"
 read confirmation
 if [[ $confirmation = "y" ]]; then
 
-  echo '-------------------Tagging and Pushing Python images-----------------'
-  for ver in "${PYTHON_VER[@]}"; do
-    # Pull verified RC from dockerhub.
-    docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
-
-    # Tag with ${RELEASE} and push to dockerhub.
-    docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-    docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-
-    # Tag with latest and push to dockerhub.
-    docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-    docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-
-    # Cleanup images from local
-    docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
-    docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-    docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-  done
-
-  echo '-------------------Tagging and Pushing Java images-----------------'
-  for ver in "${JAVA_VER[@]}"; do
-    # Pull verified RC from dockerhub.
-    docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
-
-    # Tag with ${RELEASE} and push to dockerhub.
-    docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-    docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-
-    # Tag with latest and push to dockerhub.
-    docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-    docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-
-    # Cleanup images from local
-    docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
-    docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-    docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-  done
-
-  echo '-------------Tagging and Pushing Flink job server images-------------'
-  echo "Publishing images for the following Flink versions:" "${FLINK_VER[@]}"
-  echo "Make sure the versions are correct, then press any key to proceed."
+  IMAGES=$(docker images --filter "reference=apache/beam_*:${RELEASE}_${RC_VERSION}" --format "{{.Repository}}")
+  echo "Publishing the following images:" "${IMAGES[@]}"
+  echo "Make sure the images are correct, then press any key to proceed."
   read
-  for ver in "${FLINK_VER[@]}"; do
-    FLINK_IMAGE_NAME=${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}flink${ver}_job_server
-
+  for IMAGE in "${IMAGES[@]}"; do
     # Pull verified RC from dockerhub.
-    docker pull "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
+    docker pull ${IMAGE}:${RELEASE}_${RC_VERSION}
 
     # Tag with ${RELEASE} and push to dockerhub.
-    docker tag "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}" "${FLINK_IMAGE_NAME}:${RELEASE}"
-    docker push "${FLINK_IMAGE_NAME}:${RELEASE}"
+    docker tag ${IMAGE}:${RELEASE}_${RC_VERSION} ${IMAGE}:${RELEASE}
+    docker push ${IMAGE}:${RELEASE}
 
     # Tag with latest and push to dockerhub.
-    docker tag "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}" "${FLINK_IMAGE_NAME}:latest"
-    docker push "${FLINK_IMAGE_NAME}:latest"
-
-    # Cleanup images from local
-    docker rmi -f "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
-    docker rmi -f "${FLINK_IMAGE_NAME}:${RELEASE}"
-    docker rmi -f "${FLINK_IMAGE_NAME}:latest"
+    docker tag ${IMAGE}:${RELEASE}_${RC_VERSION} ${IMAGE}:latest
+    docker push ${IMAGE}:latest
   done
 
-  echo '-------------Tagging and Pushing Spark job server image-------------'
-  SPARK_IMAGE_NAME=${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}spark_job_server
-
-  # Pull verified RC from dockerhub.
-  docker pull "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
-
-  # Tag with ${RELEASE} and push to dockerhub.
-  docker tag "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}" "${SPARK_IMAGE_NAME}:${RELEASE}"
-  docker push "${SPARK_IMAGE_NAME}:${RELEASE}"
-
-  # Tag with latest and push to dockerhub.
-  docker tag "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}" "${SPARK_IMAGE_NAME}:latest"
-  docker push "${SPARK_IMAGE_NAME}:latest"
-
-  # Cleanup images from local
-  docker rmi -f "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
-  docker rmi -f "${SPARK_IMAGE_NAME}:${RELEASE}"
-  docker rmi -f "${SPARK_IMAGE_NAME}:latest"
-fi
+fi
diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md
@@ -1089,7 +1089,7 @@ please follow [the guide](https://help.github.com/articles/creating-a-personal-a
 All wheels should be published, in addition to the zip of the release source.
 (Signatures and hashes do _not_ need to be uploaded.)
 
-### Deploy SDK docker images to DockerHub
+### Deploy docker images to DockerHub
 * Script: [publish_docker_images.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/publish_docker_images.sh)
 * Usage
 ```
@@ -1100,6 +1100,8 @@ Verify that:
 * Images with *latest* tag are pointing to current release by confirming
   1. Digest of the image with *latest* tag is the same as the one with {RELEASE} tag.
 
+(Optional) Clean up any unneeded local images afterward to save disk space.
+
 ### Merge Website pull requests
 
 Merge all of the website pull requests