feat: allow passing additional tolerations to kaniko pods (#2540)

* feat: allow passing additional tolerations to kaniko pods

* chore: add documentation for the additional kaniko tolerations

* chore: add description for kaniko tolerations docs

core/src/plugins/kubernetes/config.ts

@@ -120,6 +120,7 @@ export interface KubernetesConfig extends BaseProviderConfig {
     extraFlags?: string[]
     namespace?: string | null
     nodeSelector?: StringMap
+    tolerations?: V1Toleration[]
   }
   context: string
   defaultHostname?: string
@@ -420,6 +421,9 @@ export const kubernetesConfigBase = () =>
             [See here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for the official Kubernetes guide to assigning Pods to nodes.
           `
         ),
+        tolerations: joiSparseArray(tolerationSchema()).description(
+          "Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds."
+        ),
       })
       .default(() => {})
       .description("Configuration options for the `kaniko` build mode."),
@@ -615,39 +619,40 @@ export const kubernetesConfigBase = () =>
       )
       .example({ disktype: "ssd" })
       .default(() => ({})),
-    registryProxyTolerations: joiSparseArray(
-      joi.object().keys({
-        effect: joi.string().allow("NoSchedule", "PreferNoSchedule", "NoExecute").description(dedent`
+    registryProxyTolerations: joiSparseArray(tolerationSchema()).description(dedent`
+        For setting tolerations on the registry-proxy when using in-cluster building.
+        The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node.
+
+        Use this only if you're doing in-cluster building and the nodes in your cluster
+        have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/).
+      `),
+  })
+
+export const tolerationSchema = () =>
+  joi.object().keys({
+    effect: joi.string().allow("NoSchedule", "PreferNoSchedule", "NoExecute").description(dedent`
           "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
           allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
         `),
-        key: joi.string().description(dedent`
+    key: joi.string().description(dedent`
           "Key" is the taint key that the toleration applies to. Empty means match all taint keys.
           If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
         `),
-        operator: joi.string().allow("Exists", "Equal").default("Equal").description(dedent`
+    operator: joi.string().allow("Exists", "Equal").default("Equal").description(dedent`
           "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to
           "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
           particular category.
         `),
-        tolerationSeconds: joi.string().description(dedent`
+    tolerationSeconds: joi.string().description(dedent`
           "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
           otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
           the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
           by the system.
         `),
-        value: joi.string().description(dedent`
+    value: joi.string().description(dedent`
           "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty,
           otherwise just a regular string.
         `),
-      })
-    ).description(dedent`
-        For setting tolerations on the registry-proxy when using in-cluster building.
-        The registry-proxy is a DaemonSet that proxies connections to the docker registry service on each node.
-
-        Use this only if you're doing in-cluster building and the nodes in your cluster
-        have [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/).
-      `),
   })
 
 export const namespaceSchema = () =>

core/src/plugins/kubernetes/container/build/kaniko.ts

@@ -347,6 +347,7 @@ async function runKaniko({
   }
 
   const kanikoImage = provider.config.kaniko?.image || DEFAULT_KANIKO_IMAGE
+  const kanikoTolerations = [...(provider.config.kaniko?.tolerations || []), builderToleration]
   const utilHostname = `${utilDeploymentName}.${utilNamespace}.svc.cluster.local`
   const sourceUrl = `rsync://${utilHostname}:${utilRsyncPort}/volume/${ctx.workingCopyId}/${module.name}/`
 
@@ -433,7 +434,7 @@ async function runKaniko({
         },
       },
     ],
-    tolerations: [builderToleration],
+    tolerations: kanikoTolerations,
   }
 
   if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) {
@@ -516,6 +517,7 @@ async function runKaniko({
 }
 
 export function getUtilManifests(provider: KubernetesProvider, authSecretName: string) {
+  const kanikoTolerations = [...(provider.config.kaniko?.tolerations || []), builderToleration]
   const deployment: KubernetesDeployment = {
     apiVersion: "apps/v1",
     kind: "Deployment",
@@ -558,7 +560,7 @@ export function getUtilManifests(provider: KubernetesProvider, authSecretName: s
               emptyDir: {},
             },
           ],
-          tolerations: [builderToleration],
+          tolerations: kanikoTolerations,
         },
       },
     },

docs/reference/providers/kubernetes.md

@@ -81,6 +81,33 @@ providers:
       # guide to assigning Pods to nodes.
       nodeSelector:
 
+      # Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds.
+      tolerations:
+        - # "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
+          # allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
+          effect:
+
+          # "Key" is the taint key that the toleration applies to. Empty means match all taint keys.
+          # If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
+          key:
+
+          # "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal".
+          # Defaults to
+          # "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
+          # particular category.
+          operator: Equal
+
+          # "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
+          # otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
+          # the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
+          # by the system.
+          tolerationSeconds:
+
+          # "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be
+          # empty,
+          # otherwise just a regular string.
+          value:
+
     # A default hostname to use when no hostname is explicitly configured for a service.
     defaultHostname:
 
@@ -507,6 +534,74 @@ Exposes the `nodeSelector` field on the PodSpec of the Kaniko pods. This allows
 | -------- | -------- |
 | `object` | No       |
 
+### `providers[].kaniko.tolerations[]`
+
+[providers](#providers) > [kaniko](#providerskaniko) > tolerations
+
+Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds.
+
+| Type            | Default | Required |
+| --------------- | ------- | -------- |
+| `array[object]` | `[]`    | No       |
+
+### `providers[].kaniko.tolerations[].effect`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > effect
+
+"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
+allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].kaniko.tolerations[].key`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > key
+
+"Key" is the taint key that the toleration applies to. Empty means match all taint keys.
+If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].kaniko.tolerations[].operator`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > operator
+
+"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to
+"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
+particular category.
+
+| Type     | Default   | Required |
+| -------- | --------- | -------- |
+| `string` | `"Equal"` | No       |
+
+### `providers[].kaniko.tolerations[].tolerationSeconds`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > tolerationSeconds
+
+"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
+otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
+the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
+by the system.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].kaniko.tolerations[].value`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > value
+
+"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty,
+otherwise just a regular string.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
 ### `providers[].defaultHostname`
 
 [providers](#providers) > defaultHostname

docs/reference/providers/local-kubernetes.md

@@ -77,6 +77,33 @@ providers:
       # guide to assigning Pods to nodes.
       nodeSelector:
 
+      # Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds.
+      tolerations:
+        - # "Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
+          # allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
+          effect:
+
+          # "Key" is the taint key that the toleration applies to. Empty means match all taint keys.
+          # If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
+          key:
+
+          # "Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal".
+          # Defaults to
+          # "Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
+          # particular category.
+          operator: Equal
+
+          # "TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
+          # otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
+          # the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
+          # by the system.
+          tolerationSeconds:
+
+          # "Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be
+          # empty,
+          # otherwise just a regular string.
+          value:
+
     # A default hostname to use when no hostname is explicitly configured for a service.
     defaultHostname:
 
@@ -469,6 +496,74 @@ Exposes the `nodeSelector` field on the PodSpec of the Kaniko pods. This allows
 | -------- | -------- |
 | `object` | No       |
 
+### `providers[].kaniko.tolerations[]`
+
+[providers](#providers) > [kaniko](#providerskaniko) > tolerations
+
+Specify tolerations to apply to each Kaniko Pod. Useful to control which nodes in a cluster can run builds.
+
+| Type            | Default | Required |
+| --------------- | ------- | -------- |
+| `array[object]` | `[]`    | No       |
+
+### `providers[].kaniko.tolerations[].effect`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > effect
+
+"Effect" indicates the taint effect to match. Empty means match all taint effects. When specified,
+allowed values are "NoSchedule", "PreferNoSchedule" and "NoExecute".
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].kaniko.tolerations[].key`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > key
+
+"Key" is the taint key that the toleration applies to. Empty means match all taint keys.
+If the key is empty, operator must be "Exists"; this combination means to match all values and all keys.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].kaniko.tolerations[].operator`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > operator
+
+"Operator" represents a key's relationship to the value. Valid operators are "Exists" and "Equal". Defaults to
+"Equal". "Exists" is equivalent to wildcard for value, so that a pod can tolerate all taints of a
+particular category.
+
+| Type     | Default   | Required |
+| -------- | --------- | -------- |
+| `string` | `"Equal"` | No       |
+
+### `providers[].kaniko.tolerations[].tolerationSeconds`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > tolerationSeconds
+
+"TolerationSeconds" represents the period of time the toleration (which must be of effect "NoExecute",
+otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate
+the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately)
+by the system.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
+### `providers[].kaniko.tolerations[].value`
+
+[providers](#providers) > [kaniko](#providerskaniko) > [tolerations](#providerskanikotolerations) > value
+
+"Value" is the taint value the toleration matches to. If the operator is "Exists", the value should be empty,
+otherwise just a regular string.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | No       |
+
 ### `providers[].defaultHostname`
 
 [providers](#providers) > defaultHostname