Docker build improvements (#8999)

* build once and push cache later * Separate cache job
mindsdb · Apr 5, 2024 · 1aed2ac · 1aed2ac
1 parent f758594
commit 1aed2ac
Show file tree

Hide file tree

Showing 6 changed files with 84 additions and 29 deletions.
diff --git a/.github/actions/docker-bake-cache/action.yml b/.github/actions/docker-bake-cache/action.yml
@@ -0,0 +1,26 @@
+# Builds our docker bake file without pushing to a repo, and pushes the layers to repo cache
+
+runs:
+  using: 'composite'
+  steps:
+    # Get clean environment variables via https://github.com/marketplace/actions/github-environment-variables-action
+    - uses: FranzDiebold/github-env-vars-action@v2
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Login to Amazon ECR
+      uses: aws-actions/amazon-ecr-login@v1
+    - name: Build and push
+      shell: bash
+      run: |
+        # Get a githash or tag name to use as image prefix
+        TAG_NAME=${{ github.event.release.tag_name }}
+        GIT_SHA=${{ env.CI_SHA }}
+        IMAGE_PREFIX=${TAG_NAME:-$GIT_SHA}
+
+        # Configure our buildkit builders
+        docker buildx create --name remote-buildkit-agent --node mdb_amd64 --platform linux/amd64 --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist)
+        docker buildx create --name=remote-buildkit-agent --node mdb_arm64 --platform linux/arm64 --append --driver=remote --use tcp://remote-buildkit-agent-arm.infrastructure.svc.cluster.local:80 || true # Same for ARM
+        
+        # Build each platform individually and don't push (bake file has logic to push cache when only one platform is built)
+        VERSION=$IMAGE_PREFIX BRANCH=${{ env.CI_ACTION_REF_NAME }} PLATFORMS=linux/amd64 docker buildx bake --progress plain -f docker/docker-bake.hcl
+        VERSION=$IMAGE_PREFIX BRANCH=${{ env.CI_ACTION_REF_NAME }} PLATFORMS=linux/arm64 docker buildx bake --progress plain -f docker/docker-bake.hcl
diff --git a/.github/actions/docker-bake/action.yml b/.github/actions/docker-bake/action.yml
@@ -26,12 +26,8 @@ runs:
         docker buildx create --name remote-buildkit-agent --node mdb_amd64 --platform linux/amd64 --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist)
         docker buildx create --name=remote-buildkit-agent --node mdb_arm64 --platform linux/arm64 --append --driver=remote --use tcp://remote-buildkit-agent-arm.infrastructure.svc.cluster.local:80 || true # Same for ARM
 
+        # Print what bake is going to do
         VERSION=$IMAGE_PREFIX BRANCH=${{ env.CI_ACTION_REF_NAME }} PUSH_TO_DOCKERHUB=${{ inputs.push-to-dockerhub }} PLATFORM="" docker buildx bake -f docker/docker-bake.hcl --print
 
-        # Build amd64 and arm64 images in parallel and wait for both to finish
-        VERSION=$IMAGE_PREFIX BRANCH=${{ env.CI_ACTION_REF_NAME }} PUSH_TO_DOCKERHUB=${{ inputs.push-to-dockerhub }} PLATFORM=linux/amd64 docker buildx bake --progress plain --push -f docker/docker-bake.hcl &
-        VERSION=$IMAGE_PREFIX BRANCH=${{ env.CI_ACTION_REF_NAME }} PUSH_TO_DOCKERHUB=${{ inputs.push-to-dockerhub }} PLATFORM=linux/arm64 docker buildx bake --progress plain --push -f docker/docker-bake.hcl &
-        wait
-
-        # Create multi-arch manifests
-        VERSION=$IMAGE_PREFIX BRANCH=${{ env.CI_ACTION_REF_NAME }} PUSH_TO_DOCKERHUB=${{ inputs.push-to-dockerhub }} PLATFORM="" docker buildx bake -f docker/docker-bake.hcl --print | jq '.target[] | .tags[]' | xargs -I'{}' docker buildx imagetools create -t '{}' '{}-amd64' '{}-arm64'
+        # Build amd64 and arm64 images and push to repos
+        VERSION=$IMAGE_PREFIX BRANCH=${{ env.CI_ACTION_REF_NAME }} PUSH_TO_DOCKERHUB=${{ inputs.push-to-dockerhub }} docker buildx bake --progress plain --push -f docker/docker-bake.hcl
diff --git a/.github/workflows/build_deploy_dev.yml b/.github/workflows/build_deploy_dev.yml
@@ -37,6 +37,16 @@ jobs:
       - uses: actions/checkout@v2
       # Build the bakefile and push
       - uses: ./.github/actions/docker-bake
+
+  build_cache:
+    # Build our docker images based on our bake file
+    # This job only pushes the layers to the cache repo
+    # It's done separately so other jobs can run without waiting for this one
+    runs-on: [self-hosted, dev]
+    needs: [build]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/docker-bake-cache
 
   trigger_deploy:
     # Trigger private repo to deploy to dev envs

diff --git a/.github/workflows/build_deploy_prod.yml b/.github/workflows/build_deploy_prod.yml
@@ -63,6 +63,15 @@ jobs:
         with:
           push-to-dockerhub: true
 
+  docker_build_cache:
+    # Build our docker images based on our bake file
+    # This job only pushes the layers to the cache repo
+    # It's done separately so other jobs can run without waiting for this one
+    runs-on: [self-hosted, dev]
+    needs: [docker_build]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/docker-bake-cache
 
   trigger_deploy:
     # Trigger private repo to deploy to prod env

diff --git a/.github/workflows/build_deploy_staging.yml b/.github/workflows/build_deploy_staging.yml
@@ -14,6 +14,16 @@ jobs:
       - uses: actions/checkout@v2
       # Build the bakefile and push
       - uses: ./.github/actions/docker-bake
+
+  build_cache:
+    # Build our docker images based on our bake file
+    # This job only pushes the layers to the cache repo
+    # It's done separately so other jobs can run without waiting for this one
+    runs-on: [self-hosted, dev]
+    needs: [build]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/docker-bake-cache
 
   trigger_deploy:
     # Trigger private repo to deploy to staging and dev related environments

diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl
@@ -13,31 +13,35 @@ variable "IMAGE" {
 variable "VERSION" {
   default = "unknown"
 }
-variable "PLATFORM" {
-  default = "linux/amd64"
+variable "PLATFORMS" {
+  default = "linux/amd64,linux/arm64"
+}
+variable PLATFORM_LIST {
+  default = split(",", PLATFORMS)
 }
 variable "BRANCH" {
   default = "stable"
 }
-
-function "get_platform_tag" {
-  params = []
-  result = replace("${equal(PLATFORM, "") ? "" : "-"}${PLATFORM}", "linux/", "")
+variable "ECR_REPO" {
+  default = "454861456664.dkr.ecr.us-east-2.amazonaws.com"
 }
 
 function "get_cache_to" {
-  params = [target]
-  result = [
-    "type=registry,image-manifest=true,oci-mediatypes=true,mode=max,ref=454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}-cache:${replace("${BRANCH}", "/", "-")}-${target}${get_platform_tag()}"
+  params = [image]
+  result = length(PLATFORM_LIST) > 1 ? [] : [
+    "type=registry,image-manifest=true,oci-mediatypes=true,mode=max,ref=${ECR_REPO}/${IMAGE}-cache:${replace("${BRANCH}", "/", "-")}-${image}-${replace("${PLATFORM_LIST[0]}", "linux/", "")}"
   ]
 }
 function "get_cache_from" {
-  params = [target]
-  result = [
-    "type=registry,ref=454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}-cache:${replace("${BRANCH}", "/", "-")}-${target}${get_platform_tag()}",
-    "type=registry,ref=454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}-cache:staging-${target}${get_platform_tag()}",
-    "type=registry,ref=454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}-cache:stable-${target}${get_platform_tag()}"
-  ]
+  params = [image]
+  result = flatten([for p in PLATFORM_LIST:
+    split("\n", <<EOT
+type=registry,ref=${ECR_REPO}/${IMAGE}-cache:${replace("${BRANCH}", "/", "-")}-${image}-${replace("${p}", "linux/", "")}
+type=registry,ref=${ECR_REPO}/${IMAGE}-cache:staging-${image}-${replace("${p}", "linux/", "")}
+type=registry,ref=${ECR_REPO}/${IMAGE}-cache:stable-${image}-${replace("${p}", "linux/", "")}
+EOT
+    )
+  ])
 }
 
 # Generate the list of tags for a given image.
@@ -46,12 +50,12 @@ function "get_cache_from" {
 # - "mindsdb:v1.2.3-cloud" - For this specific version
 # The same tags are pushed to dockerhub as well if the PUSH_TO_DOCKERHUB variable is set.
 function "get_tags" {
-  params = [target]
+  params = [image]
   result = [
-    "454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}:${VERSION}${notequal(target, "bare") ? "-${target}" : ""}${get_platform_tag()}",
-    "454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}:${notequal(target, "bare") ? target : "latest"}${get_platform_tag()}",
-    PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${VERSION}${notequal(target, "bare") ? "-${target}" : ""}${get_platform_tag()}" : "",
-    PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${notequal(target, "bare") ? target : "latest"}${get_platform_tag()}" : ""
+    "${ECR_REPO}/${IMAGE}:${VERSION}${notequal(image, "bare") ? "-${image}" : ""}",
+    "${ECR_REPO}/${IMAGE}:${notequal(image, "bare") ? image : "latest"}",
+    PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${VERSION}${notequal(image, "bare") ? "-${image}" : ""}" : "",
+    PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${notequal(image, "bare") ? image : "latest"}" : ""
   ]
 } 
 
@@ -61,8 +65,8 @@ function "get_tags" {
 
 target "images" {
   name = item.name
-  dockerfile = "docker/mindsdb.Dockerfile" # If you change this, also change it in target:builder
-  platforms = ["${PLATFORM}"]
+  dockerfile = "docker/mindsdb.Dockerfile"
+  platforms = PLATFORM_LIST
   matrix = {
     item = [
       {