build: Parallelize the CI image builds (continued) (apache#26698)

sfirke · Mar 22, 2024 · 0924e7b · 0924e7b
1 parent ae8d9f8
commit 0924e7b
Show file tree

Hide file tree

Showing 5 changed files with 126 additions and 135 deletions.
diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
@@ -22,21 +22,29 @@ jobs:
     if: needs.config.outputs.has-secrets
     name: docker-release
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: ["dev", "lean", "lean310", "websocket", "dockerize"]
+        platform: ["linux/amd64", "linux/arm64"]
+      fail-fast: false
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           submodules: recursive
           ref: ${{ github.ref }}
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - shell: bash
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker Image
         env:
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}"
-          ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME"
+          ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME" ${{ matrix.target }} ${{ matrix.platform }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -24,27 +24,52 @@ jobs:
             echo "has-secrets=0" >> "$GITHUB_OUTPUT"
             echo "no secrets!"
           fi
-
   docker-build:
     needs: config
     if: needs.config.outputs.has-secrets
     name: docker-build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: ["dev", "lean", "lean310", "websocket", "dockerize"]
+        platform: ["linux/amd64", "linux/arm64"]
+      fail-fast: false
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - shell: bash
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker Image
+        shell: bash
         env:
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
-          ./scripts/docker_build_push.sh
+          ./scripts/docker_build_push.sh "" ${{ matrix.target }} ${{ matrix.platform }}
+
+  ephemeral-docker-build:
+    needs: config
+    if: needs.config.outputs.has-secrets
+    name: docker-build
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Build ephemeral env image
         if: github.event_name == 'pull_request'
@@ -54,7 +79,7 @@ jobs:
           echo ${{ github.event.pull_request.number }} > ./build/PR-NUM
           docker buildx build --target ci \
             --load \
-            --cache-from=type=local,src=/tmp/superset \
+            --cache-from=type=registry,ref=apache/superset:lean \
             -t ${{ github.sha }} \
             -t "pr-${{ github.event.pull_request.number }}" \
             --platform linux/amd64 \
@@ -64,7 +89,7 @@ jobs:
 
       - name: Upload build artifacts
         if: github.event_name == 'pull_request'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build
           path: build/
diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
@@ -18,9 +18,15 @@
 set -eo pipefail
 
 GITHUB_RELEASE_TAG_NAME="$1"
+TARGET="$2"
+BUILD_PLATFORM="$3" # should be either 'linux/amd64' or 'linux/arm64'
 
+# Common variables
 SHA=$(git rev-parse HEAD)
 REPO_NAME="apache/superset"
+DOCKER_ARGS="--load" # default args, change as needed
+DOCKER_CONTEXT="."
+
 
 if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
   REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
@@ -54,6 +60,45 @@ if [[ "${TEST_ENV}" == "true" ]]; then
   exit 0
 fi
 
+# for the dev image, it's ok to tag master as latest-dev
+# for production, we only want to tag the latest official release as latest
+if [ "${LATEST_TAG}" = "master" ]; then
+  DEV_TAG="${REPO_NAME}:latest-dev"
+else
+  DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
+fi
+
+BUILD_ARG="3.9-slim-bookworm"
+
+case "${TARGET}" in
+  "dev")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}"
+    BUILD_TARGET="dev"
+    ;;
+  "lean")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA} -t ${REPO_NAME}:${REFSPEC} -t ${REPO_NAME}:${LATEST_TAG}"
+    BUILD_TARGET="lean"
+    ;;
+  "lean310")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-py310 -t ${REPO_NAME}:${REFSPEC}-py310 -t ${REPO_NAME}:${LATEST_TAG}-py310"
+    BUILD_TARGET="lean"
+    BUILD_ARG="3.10-slim-bookworm"
+    ;;
+  "websocket")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket"
+    BUILD_TARGET=""
+	DOCKER_CONTEXT="superset-websocket"
+    ;;
+  "dockerize")
+    DOCKER_TAGS="-t ${REPO_NAME}:dockerize"
+    BUILD_TARGET=""
+	DOCKER_CONTEXT="-f dockerize.Dockerfile ."
+    ;;
+  *)
+    echo "Invalid TARGET: ${TARGET}"
+    exit 1
+    ;;
+esac
 
 cat<<EOF
   Rolling with tags:
@@ -67,123 +112,30 @@ if [ -z "${DOCKERHUB_TOKEN}" ]; then
   echo "Skipping Docker push"
   # By default load it back
   DOCKER_ARGS="--load"
-  ARCHITECTURE_FOR_BUILD="linux/amd64 linux/arm64"
 else
   # Login and push
   docker logout
   docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
   DOCKER_ARGS="--push"
-  ARCHITECTURE_FOR_BUILD="linux/amd64,linux/arm64"
 fi
 set -x
 
-# for the dev image, it's ok to tag master as latest-dev
-# for production, we only want to tag the latest official release as latest
-if [ "${LATEST_TAG}" = "master" ]; then
-  DEV_TAG="${REPO_NAME}:latest-dev"
-else
-  DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
+TARGET_ARGUMENT=""
+if [[ -n "${BUILD_TARGET}" ]]; then
+  TARGET_ARGUMENT="--target ${BUILD_TARGET}"
 fi
 
-for BUILD_PLATFORM in $ARCHITECTURE_FOR_BUILD; do
-#
-# Build the dev image
-#
-docker buildx build --target dev \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:master-dev \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-dev" \
-  -t "${REPO_NAME}:${REFSPEC}-dev" \
-  -t "${DEV_TAG}" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=dev" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}" \
-  -t "${REPO_NAME}:${REFSPEC}" \
-  -t "${REPO_NAME}:${LATEST_TAG}" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean310" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-py310" \
-  -t "${REPO_NAME}:${REFSPEC}-py310" \
-  -t "${REPO_NAME}:${LATEST_TAG}-py310" \
-  --platform ${BUILD_PLATFORM} \
-  --build-arg PY_VER="3.10-slim-bookworm"\
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean310" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean39" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-py39" \
-  -t "${REPO_NAME}:${REFSPEC}-py39" \
-  -t "${REPO_NAME}:${LATEST_TAG}-py39" \
-  --platform ${BUILD_PLATFORM} \
-  --build-arg PY_VER="3.9-slim-bullseye"\
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean39" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "websocket" image
-#
-docker buildx build \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:master-websocket \
-  -t "${REPO_NAME}:${SHA}-websocket" \
-  -t "${REPO_NAME}:${REFSPEC}-websocket" \
-  -t "${REPO_NAME}:${LATEST_TAG}-websocket" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=websocket" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  superset-websocket
-
-#
-# Build the dockerize image
-#
 docker buildx build \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:dockerize \
-  -t "${REPO_NAME}:dockerize" \
+  ${TARGET_ARGUMENT} \
+  ${DOCKER_ARGS} \
+  --cache-from=type=registry,ref=apache/superset:${TARGET} \
+  --cache-to=type=registry,mode=max,ref=apache/superset:${TARGET} \
+  ${DOCKER_TAGS} \
   --platform ${BUILD_PLATFORM} \
   --label "sha=${SHA}" \
   --label "built_at=$(date)" \
+  --label "target=${TARGET}" \
+  --label "base=${PY_VER}" \
   --label "build_actor=${GITHUB_ACTOR}" \
-  -f dockerize.Dockerfile \
-  .
-done
+  ${BUILD_ARG:+--build-arg PY_VER="${BUILD_ARG}"} \
+  ${DOCKER_CONTEXT}
diff --git a/tests/unit_tests/fixtures/bash_mock.py b/tests/unit_tests/fixtures/bash_mock.py
@@ -32,10 +32,10 @@ def tag_latest_release(tag):
         return result
 
     @staticmethod
-    def docker_build_push(tag, branch):
-        bash_command = f"./scripts/docker_build_push.sh {tag}"
+    def docker_build_push(tag, target, platform, branch):
+        cmd = f'./scripts/docker_build_push.sh "{tag}" "{target}" "{platform}"'
         result = subprocess.run(
-            bash_command,
+            cmd,
             shell=True,
             capture_output=True,
             text=True,

diff --git a/tests/unit_tests/scripts/docker_build_push_test.py b/tests/unit_tests/scripts/docker_build_push_test.py
@@ -15,30 +15,36 @@ def wrapped(*args, **kwargs):
 
 
 @pytest.mark.parametrize(
-    "tag, expected_output, branch",
+    "tag, target, platform, expected_output, branch",
     [
-        ("1.0.0", "LATEST_TAG is master", "master"),
-        ("2.1.0", "LATEST_TAG is master", "master"),
-        ("2.1.1", "LATEST_TAG is latest", "master"),
-        ("3.0.0", "LATEST_TAG is latest", "master"),
-        ("2.1.0rc1", "LATEST_TAG is 2.1.0", "2.1.0"),
-        ("", "LATEST_TAG is foo", "foo"),
-        ("2.1", "LATEST_TAG is 2.1", "2.1"),
-        ("does_not_exist", "LATEST_TAG is does-not-exist", "does_not_exist"),
+        ("1.0.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        ("2.1.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        ("2.1.1", "lean", "linux/amd64", "LATEST_TAG is latest", "master"),
+        ("3.0.0", "lean", "linux/amd64", "LATEST_TAG is latest", "master"),
+        ("2.1.0rc1", "lean", "linux/amd64", "LATEST_TAG is 2.1.0", "2.1.0"),
+        ("", "lean", "linux/amd64", "LATEST_TAG is foo", "foo"),
+        ("2.1", "lean", "linux/amd64", "LATEST_TAG is 2.1", "2.1"),
+        (
+            "does_not_exist",
+            "lean",
+            "linux/amd64",
+            "LATEST_TAG is does-not-exist",
+            "does_not_exist",
+        ),
     ],
 )
-def test_tag_latest_release(tag, expected_output, branch):
+def test_tag_latest_release(tag, target, platform, expected_output, branch):
     with mock.patch(
         "tests.unit_tests.fixtures.bash_mock.subprocess.run", wraps=wrapped
     ) as subprocess_mock:
-        result = BashMock.docker_build_push(tag, branch)
+        result = BashMock.docker_build_push(tag, target, platform, branch)
 
+        cmd = f'./scripts/docker_build_push.sh "{tag}" "{target}" "{platform}"'
         subprocess_mock.assert_called_once_with(
-            f"./scripts/docker_build_push.sh {tag}",
+            cmd,
             shell=True,
             capture_output=True,
             text=True,
             env={"TEST_ENV": "true", "GITHUB_REF": f"refs/heads/{branch}"},
         )
-
-        assert re.search(expected_output, result.stdout, re.MULTILINE)
+        assert re.search(expected_output, result.stdout.strip(), re.MULTILINE)