diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
index 827ddb93a..7fbfa66ef 100644
--- a/.github/workflows/build-docker.yml
+++ b/.github/workflows/build-docker.yml
@@ -93,15 +93,16 @@ jobs:
       - name: Build and Deploy Docker image
         uses: docker/build-push-action@v2
         env:
-          IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ matrix.tag }}${{ (matrix.tag != '' && '-') || '' }}latest
-          VERSIONED_IMAGE_TAG: |
+          IMAGE_TAG: |- # If it's a release, add the version, otherwise add the `latest`
             ${{ (
               github.event.release.tag_name != '' && (
                 matrix.tag != '' && (
                   format('{0}:{1}-{2}', env.IMAGE_NAME, matrix.tag, github.event.release.tag_name)
                 ) || format('{0}:{1}', env.IMAGE_NAME, github.event.release.tag_name)
               )
-            ) || '' }}
+            ) || (
+              format('{0}:{1}{2}latest', env.IMAGE_NAME, matrix.tag, (matrix.tag != '' && '-') || ''
+            ) }}
         with:
           context: .
           builder: ${{ steps.buildx.outputs.name }}
@@ -117,8 +118,6 @@ jobs:
             ONNXRUNTIME_URL=${{ matrix.onnxruntime_url }}
           target: ${{ matrix.target }}
           push: true
-          tags: |
-            ${{ env.IMAGE_TAG }}
-            ${{ env.VERSIONED_IMAGE_TAG }}
+          tags: ${{ env.IMAGE_TAG }}
           cache-from: type=registry,ref=${{ env.IMAGE_TAG }}-buildcache
           cache-to: type=registry,ref=${{ env.IMAGE_TAG }}-buildcache,mode=max