feat(k8s): support mode=max for AWS ECR with cluster-buildkit build m…

…ode (#5758) **What this PR does / why we need it**: AWS ECR now supports `mode=max` together with an extra option `image-manifest=true`. See also https://aws.amazon.com/blogs/containers/announcing-remote-cache-support-in-amazon-ecr-for-buildkit-clients/ **Which issue(s) this PR fixes**: Fixes #5683 **Special notes for your reviewer**:
garden-io · Feb 20, 2024 · 6a94cec · 6a94cec
1 parent a145621
commit 6a94cec
Show file tree

Hide file tree

Showing 6 changed files with 105 additions and 50 deletions.
diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts
@@ -449,13 +449,14 @@ export const kubernetesConfigBase = () =>
 
             See the following table for details on our detection mechanism:
 
-            | Registry Name                   | Registry Domain         | Assumed \`mode=max\` support |
-            |---------------------------------|-------------------------|------------------------------|
-            | Google Cloud Artifact Registry  | \`pkg.dev\`             | Yes                          |
-            | Azure Container Registry        | \`azurecr.io\`          | Yes                          |
-            | GitHub Container Registry       | \`ghcr.io\`             | Yes                          |
-            | DockerHub                       | \`hub.docker.com\`     | Yes                          |
-            | Any other registry              |                         | No                           |
+            | Registry Name                   | Registry Domain                    | Assumed \`mode=max\` support |
+            |---------------------------------|------------------------------------|------------------------------|
+            | AWS Elastic Container Registry  | \`dkr.ecr.<region>.amazonaws.com\` | Yes (with \`image-manifest=true\`) |
+            | Google Cloud Artifact Registry  | \`pkg.dev\`                        | Yes                          |
+            | Azure Container Registry        | \`azurecr.io\`                     | Yes                          |
+            | GitHub Container Registry       | \`ghcr.io\`                        | Yes                          |
+            | DockerHub                       | \`hub.docker.com\`                 | Yes                          |
+            | Any other registry              |                                    | No                           |
 
             In case you need to override the defaults for your registry, you can do it like so:
 

diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts
@@ -48,6 +48,17 @@ import { stringifyResources } from "../util.js"
 import { styles } from "../../../../logger/styles.js"
 import type { ResolvedBuildAction } from "../../../../actions/build.js"
 
+const AWS_ECR_REGEX = /^([^\.]+\.)?dkr\.ecr\.([^\.]+\.)amazonaws\.com\//i // AWS Elastic Container Registry
+
+// NOTE: If you change this, please make sure to also change the table in our documentation in config.ts
+const MODE_MAX_ALLOWED_REGISTRIES = [
+  AWS_ECR_REGEX,
+  /^([^/]+\.)?pkg\.dev\//i, // Google Package Registry
+  /^([^/]+\.)?azurecr\.io\//i, // Azure Container registry
+  /^hub\.docker\.com\//i, // DockerHub
+  /^ghcr\.io\//i, // GitHub Container registry
+]
+
 const deployLock = new AsyncLock()
 
 export const getBuildkitBuildStatus: BuildStatusHandler = async (params) => {
@@ -355,9 +366,16 @@ export function getBuildkitImageFlags(
       continue
     }
 
+    // AWS ECR needs extra flag image-manifest=true with mode=max
+    // See also https://aws.amazon.com/blogs/containers/announcing-remote-cache-support-in-amazon-ecr-for-buildkit-clients/
+    let imageManifestFlag = ""
+    if (cacheMode === "max" && AWS_ECR_REGEX.test(cacheImageName)) {
+      imageManifestFlag = "image-manifest=true,"
+    }
+
     args.push(
       "--export-cache",
-      `type=registry,ref=${cacheImageName}:${cache.tag},mode=${cacheMode}${registryExtraSpec}`
+      `${imageManifestFlag}type=registry,ref=${cacheImageName}:${cache.tag},mode=${cacheMode}${registryExtraSpec}`
     )
   }
 
@@ -382,16 +400,8 @@ export const getSupportedCacheMode = (
     return cache.mode
   }
 
-  // NOTE: If you change this, please make sure to also change the table in our documentation in config.ts
-  const allowList = [
-    /^([^/]+\.)?pkg\.dev\//i, // Google Package Registry
-    /^([^/]+\.)?azurecr\.io\//i, // Azure Container registry
-    /^hub\.docker\.com\//i, // DockerHub
-    /^ghcr\.io\//i, // GitHub Container registry
-  ]
-
   // use mode=max for all registries that are known to support it
-  for (const allowed of allowList) {
+  for (const allowed of MODE_MAX_ALLOWED_REGISTRIES) {
     if (allowed.test(deploymentImageName)) {
       return "max"
     }

diff --git a/core/test/unit/src/plugins/kubernetes/container/build/buildkit.ts b/core/test/unit/src/plugins/kubernetes/container/build/buildkit.ts
@@ -277,8 +277,6 @@ describe("buildkit build", () => {
       // The following registries are actually known NOT to support mode=max
       "eu.gcr.io",
       "gcr.io",
-      "aws_account_id.dkr.ecr.region.amazonaws.com",
-      "keks.dkr.ecr.bla.amazonaws.com",
       // Most self-hosted registries actually support mode=max, but because
       // Harbor actually doesn't, we need to default to inline.
       "anyOtherRegistry",
@@ -306,6 +304,33 @@ describe("buildkit build", () => {
       })
     }
 
+    // AWS ECR supports mode=max with image-manifest=true option
+    const expectedMaxWithImageManifest = [
+      "aws_account_id.dkr.ecr.region.amazonaws.com",
+      "keks.dkr.ecr.bla.amazonaws.com",
+    ]
+    for (const registry of expectedMaxWithImageManifest) {
+      it(`returns mode=max cache flags with image-manifest=true for registry ${registry}`, async () => {
+        const moduleOutputs = {
+          "local-image-id": "name:v-xxxxxx",
+          "local-image-name": "name",
+          "deployment-image-id": `${registry}/namespace/name:v-xxxxxx`,
+          "deployment-image-name": `${registry}/namespace/name`,
+        }
+
+        const flags = getBuildkitImageFlags(defaultConfig, moduleOutputs, false)
+
+        expect(flags).to.eql([
+          "--output",
+          `type=image,\\"name=${registry}/namespace/name:v-xxxxxx\\",push=true`,
+          "--import-cache",
+          `type=registry,ref=${registry}/namespace/name:_buildcache`,
+          "--export-cache",
+          `image-manifest=true,type=registry,ref=${registry}/namespace/name:_buildcache,mode=max`,
+        ])
+      })
+    }
+
     // test autodetection for mode=max
     const expectedMax = [
       // The following registries are known to actually support mode=max
@@ -317,6 +342,7 @@ describe("buildkit build", () => {
       "azurecr.io",
       "some.subdomain.azurecr.io",
     ]
+
     for (const registry of expectedMax) {
       it(`returns mode=max cache flags with default config with registry ${registry}`, async () => {
         const moduleOutputs = {

diff --git a/docs/k8s-plugins/guides/in-cluster-building.md b/docs/k8s-plugins/guides/in-cluster-building.md
@@ -202,10 +202,11 @@ Please keep in mind that you should also configure a garbage collection policy i
 
 #### Multi-stage caching
 
-If your `Dockerfile` has multiple stages, you can benefit from `mode=max` caching. It is automatically enabled, if your registry is not in our list of unsupported registries.
-Currently, those are AWS ECR and Google GCR. If you are using GCR, you can switch to the Google Artifact Registry, which supports `mode=max`.
+If your `Dockerfile` has multiple stages, you can benefit from `mode=max` caching. It is automatically enabled, if your registry is in our list of supported registries.
 
-You can also configure a different cache registry for your images. That way you can keep using ECR or GCR, while having better cache hit rate with `mode=max`:
+You can find the list of supported registries in [Kubernetes provider configuration guide](../../reference/providers/kubernetes.md#providersclusterbuildkitcache).
+
+You can also configure a different cache registry for your images. That way you can use `mode=max` to achieve a better cache hit rate, even if your registry does not support `mode=max`.
 
 ```yaml
 clusterBuildkit:
@@ -218,6 +219,19 @@ clusterBuildkit:
 
 For this mode of operation you need secrets for all the registries configured in your `imagePullSecrets`.
 
+Please note that most registries do support `mode=max`. If you are using a self-hosted registry, we do not use `mode=max` by default out of caution. You can force to enable it to achieve a better cache-hit rate with self-hosted registries:
+
+```
+clusterBuildkit:
+  cache:
+      - type: registry
+        mode: max # Force mode=max as our self-hosted registry is not in the list of supported registries
+        registry:
+          hostname: company-registry.example.com
+          namespace: my-team-cache
+
+```
+
 ### Local Docker
 
 This is the default build mode. It is usually the least efficient one for remote clusters, but requires no additional services

diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md
@@ -65,13 +65,14 @@ providers:
       #
       # See the following table for details on our detection mechanism:
       #
-      # | Registry Name                   | Registry Domain         | Assumed `mode=max` support |
-      # |---------------------------------|-------------------------|------------------------------|
-      # | Google Cloud Artifact Registry  | `pkg.dev`             | Yes                          |
-      # | Azure Container Registry        | `azurecr.io`          | Yes                          |
-      # | GitHub Container Registry       | `ghcr.io`             | Yes                          |
-      # | DockerHub                       | `hub.docker.com`     | Yes                          |
-      # | Any other registry              |                         | No                           |
+      # | Registry Name                   | Registry Domain                    | Assumed `mode=max` support |
+      # |---------------------------------|------------------------------------|------------------------------|
+      # | AWS Elastic Container Registry  | `dkr.ecr.<region>.amazonaws.com` | Yes (with `image-manifest=true`) |
+      # | Google Cloud Artifact Registry  | `pkg.dev`                        | Yes                          |
+      # | Azure Container Registry        | `azurecr.io`                     | Yes                          |
+      # | GitHub Container Registry       | `ghcr.io`                        | Yes                          |
+      # | DockerHub                       | `hub.docker.com`                 | Yes                          |
+      # | Any other registry              |                                    | No                           |
       #
       # In case you need to override the defaults for your registry, you can do it like so:
       #
@@ -617,13 +618,14 @@ option.
 
 See the following table for details on our detection mechanism:
 
-| Registry Name                   | Registry Domain         | Assumed `mode=max` support |
-|---------------------------------|-------------------------|------------------------------|
-| Google Cloud Artifact Registry  | `pkg.dev`             | Yes                          |
-| Azure Container Registry        | `azurecr.io`          | Yes                          |
-| GitHub Container Registry       | `ghcr.io`             | Yes                          |
-| DockerHub                       | `hub.docker.com`     | Yes                          |
-| Any other registry              |                         | No                           |
+| Registry Name                   | Registry Domain                    | Assumed `mode=max` support |
+|---------------------------------|------------------------------------|------------------------------|
+| AWS Elastic Container Registry  | `dkr.ecr.<region>.amazonaws.com` | Yes (with `image-manifest=true`) |
+| Google Cloud Artifact Registry  | `pkg.dev`                        | Yes                          |
+| Azure Container Registry        | `azurecr.io`                     | Yes                          |
+| GitHub Container Registry       | `ghcr.io`                        | Yes                          |
+| DockerHub                       | `hub.docker.com`                 | Yes                          |
+| Any other registry              |                                    | No                           |
 
 In case you need to override the defaults for your registry, you can do it like so:
 

diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md
@@ -58,13 +58,14 @@ providers:
       #
       # See the following table for details on our detection mechanism:
       #
-      # | Registry Name                   | Registry Domain         | Assumed `mode=max` support |
-      # |---------------------------------|-------------------------|------------------------------|
-      # | Google Cloud Artifact Registry  | `pkg.dev`             | Yes                          |
-      # | Azure Container Registry        | `azurecr.io`          | Yes                          |
-      # | GitHub Container Registry       | `ghcr.io`             | Yes                          |
-      # | DockerHub                       | `hub.docker.com`     | Yes                          |
-      # | Any other registry              |                         | No                           |
+      # | Registry Name                   | Registry Domain                    | Assumed `mode=max` support |
+      # |---------------------------------|------------------------------------|------------------------------|
+      # | AWS Elastic Container Registry  | `dkr.ecr.<region>.amazonaws.com` | Yes (with `image-manifest=true`) |
+      # | Google Cloud Artifact Registry  | `pkg.dev`                        | Yes                          |
+      # | Azure Container Registry        | `azurecr.io`                     | Yes                          |
+      # | GitHub Container Registry       | `ghcr.io`                        | Yes                          |
+      # | DockerHub                       | `hub.docker.com`                 | Yes                          |
+      # | Any other registry              |                                    | No                           |
       #
       # In case you need to override the defaults for your registry, you can do it like so:
       #
@@ -562,13 +563,14 @@ option.
 
 See the following table for details on our detection mechanism:
 
-| Registry Name                   | Registry Domain         | Assumed `mode=max` support |
-|---------------------------------|-------------------------|------------------------------|
-| Google Cloud Artifact Registry  | `pkg.dev`             | Yes                          |
-| Azure Container Registry        | `azurecr.io`          | Yes                          |
-| GitHub Container Registry       | `ghcr.io`             | Yes                          |
-| DockerHub                       | `hub.docker.com`     | Yes                          |
-| Any other registry              |                         | No                           |
+| Registry Name                   | Registry Domain                    | Assumed `mode=max` support |
+|---------------------------------|------------------------------------|------------------------------|
+| AWS Elastic Container Registry  | `dkr.ecr.<region>.amazonaws.com` | Yes (with `image-manifest=true`) |
+| Google Cloud Artifact Registry  | `pkg.dev`                        | Yes                          |
+| Azure Container Registry        | `azurecr.io`                     | Yes                          |
+| GitHub Container Registry       | `ghcr.io`                        | Yes                          |
+| DockerHub                       | `hub.docker.com`                 | Yes                          |
+| Any other registry              |                                    | No                           |
 
 In case you need to override the defaults for your registry, you can do it like so: