feat(k8s): add clusterBuildkit.nodeSelector config option

This allows users to set which nodes are used for the buildkit daemons.
I felt it made more sense to make this a separate option, since the
existing `systemNodeSelector` option was meant for the `garden-system`
services, and it's anyway likely good to be fairly granular with this.

core/src/plugins/kubernetes/config.ts

@@ -107,6 +107,7 @@ export interface KubernetesConfig extends GenericProviderConfig {
   buildMode: ContainerBuildMode
   clusterBuildkit?: {
     rootless?: boolean
+    nodeSelector?: StringMap
   }
   clusterDocker?: {
     enableBuildKit?: boolean
@@ -329,6 +330,16 @@ export const kubernetesConfigBase = () =>
             Please see [the buildkit docs](https://github.com/moby/buildkit/blob/master/docs/rootless.md) for caveats when using this mode.
             `
           ),
+        nodeSelector: joiStringMap(joi.string())
+          .description(
+            dedent`
+            Exposes the \`nodeSelector\` field on the PodSpec of the BuildKit deployment. This allows you to constrain the BuildKit daemon to only run on particular nodes.
+
+            [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+            `
+          )
+          .example({ disktype: "ssd" })
+          .default(() => ({})),
       })
       .default(() => {})
       .description("Configuration options for the `cluster-buildkit` build mode."),
@@ -523,9 +534,10 @@ export const kubernetesConfigBase = () =>
     systemNodeSelector: joiStringMap(joi.string())
       .description(
         dedent`
-      Exposes the \`nodeSelector\` field on the PodSpec of system services. This allows you to constrain
-      the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
-    `
+        Exposes the \`nodeSelector\` field on the PodSpec of system services. This allows you to constrain the system services to only run on particular nodes.
+
+        [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+        `
       )
       .example({ disktype: "ssd" })
       .default(() => ({})),

core/src/plugins/kubernetes/container/build/buildkit.ts

@@ -252,7 +252,7 @@ function getDockerBuildFlags(module: ContainerModule) {
   return args
 }
 
-function getBuildkitDeployment(provider: KubernetesProvider) {
+export function getBuildkitDeployment(provider: KubernetesProvider) {
   const deployment = cloneDeep(baseBuildkitDeployment)
   const buildkitContainer = deployment.spec!.template.spec!.containers[0]
 
@@ -289,6 +289,11 @@ function getBuildkitDeployment(provider: KubernetesProvider) {
   const registryHostname = getRegistryHostname(provider.config)
   deployment.spec!.template.spec!.containers.push(getSocatContainer(registryHostname))
 
+  // Set the configured nodeSelector, if any
+  if (provider.config.clusterBuildkit?.nodeSelector) {
+    deployment.spec!.template.spec!.nodeSelector = provider.config.clusterBuildkit?.nodeSelector
+  }
+
   return deployment
 }
 

core/test/integ/src/plugins/kubernetes/container/build/buildkit.ts

@@ -67,6 +67,28 @@ describe("ensureBuildkit", () => {
       expect(deployed).to.be.true
     })
 
+    it("deploys buildkit with the configured nodeSelector", async () => {
+      try {
+        await api.apps.deleteNamespacedDeployment(buildkitDeploymentName, namespace)
+      } catch {}
+
+      const nodeSelector = { "kubernetes.io/os": "linux" }
+
+      provider.config.clusterBuildkit = { nodeSelector }
+
+      await ensureBuildkit({
+        ctx,
+        provider,
+        log: garden.log,
+        api,
+        namespace,
+      })
+
+      const deployment = await api.apps.readNamespacedDeployment(buildkitDeploymentName, namespace)
+
+      expect(deployment.spec.template.spec?.nodeSelector).to.eql(nodeSelector)
+    })
+
     it("creates a docker auth secret from configured imagePullSecrets", async () => {
       await ensureBuildkit({
         ctx,

docs/reference/providers/kubernetes.md

@@ -50,6 +50,13 @@ providers:
       # using this mode.
       rootless: false
 
+      # Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the
+      # BuildKit daemon to only run on particular nodes.
+      #
+      # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes
+      # guide to assigning Pods to nodes.
+      nodeSelector: {}
+
     # Configuration options for the `cluster-docker` build mode.
     clusterDocker:
       # Enable [BuildKit](https://github.com/moby/buildkit) support. This should in most cases work well and be more
@@ -250,10 +257,11 @@ providers:
       # for now).
       acmeChallengeType: HTTP-01
 
-    # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
-    # the system services to only run on particular nodes. [See
-    # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to
-    # assigning Pods to nodes.
+    # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system
+    # services to only run on particular nodes.
+    #
+    # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide
+    # to assigning Pods to nodes.
     systemNodeSelector: {}
 
     # For setting tolerations on the registry-proxy when using in-cluster building.
@@ -422,6 +430,28 @@ Please see [the buildkit docs](https://github.com/moby/buildkit/blob/master/docs
 | --------- | ------- | -------- |
 | `boolean` | `false` | No       |
 
+### `providers[].clusterBuildkit.nodeSelector`
+
+[providers](#providers) > [clusterBuildkit](#providersclusterbuildkit) > nodeSelector
+
+Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the BuildKit daemon to only run on particular nodes.
+
+[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+
+| Type     | Default | Required |
+| -------- | ------- | -------- |
+| `object` | `{}`    | No       |
+
+Example:
+
+```yaml
+providers:
+  - clusterBuildkit:
+      ...
+      nodeSelector:
+          disktype: ssd
+```
+
 ### `providers[].clusterDocker`
 
 [providers](#providers) > clusterDocker
@@ -1288,8 +1318,9 @@ providers:
 
 [providers](#providers) > systemNodeSelector
 
-Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
-the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system services to only run on particular nodes.
+
+[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
 
 | Type     | Default | Required |
 | -------- | ------- | -------- |

docs/reference/providers/local-kubernetes.md

@@ -46,6 +46,13 @@ providers:
       # using this mode.
       rootless: false
 
+      # Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the
+      # BuildKit daemon to only run on particular nodes.
+      #
+      # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes
+      # guide to assigning Pods to nodes.
+      nodeSelector: {}
+
     # Configuration options for the `cluster-docker` build mode.
     clusterDocker:
       # Enable [BuildKit](https://github.com/moby/buildkit) support. This should in most cases work well and be more
@@ -246,10 +253,11 @@ providers:
       # for now).
       acmeChallengeType: HTTP-01
 
-    # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
-    # the system services to only run on particular nodes. [See
-    # here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to
-    # assigning Pods to nodes.
+    # Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system
+    # services to only run on particular nodes.
+    #
+    # [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide
+    # to assigning Pods to nodes.
     systemNodeSelector: {}
 
     # For setting tolerations on the registry-proxy when using in-cluster building.
@@ -384,6 +392,28 @@ Please see [the buildkit docs](https://github.com/moby/buildkit/blob/master/docs
 | --------- | ------- | -------- |
 | `boolean` | `false` | No       |
 
+### `providers[].clusterBuildkit.nodeSelector`
+
+[providers](#providers) > [clusterBuildkit](#providersclusterbuildkit) > nodeSelector
+
+Exposes the `nodeSelector` field on the PodSpec of the BuildKit deployment. This allows you to constrain the BuildKit daemon to only run on particular nodes.
+
+[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+
+| Type     | Default | Required |
+| -------- | ------- | -------- |
+| `object` | `{}`    | No       |
+
+Example:
+
+```yaml
+providers:
+  - clusterBuildkit:
+      ...
+      nodeSelector:
+          disktype: ssd
+```
+
 ### `providers[].clusterDocker`
 
 [providers](#providers) > clusterDocker
@@ -1250,8 +1280,9 @@ providers:
 
 [providers](#providers) > systemNodeSelector
 
-Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain
-the system services to only run on particular nodes. [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
+Exposes the `nodeSelector` field on the PodSpec of system services. This allows you to constrain the system services to only run on particular nodes.
+
+[See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
 
 | Type     | Default | Required |
 | -------- | ------- | -------- |