feat(k8s): add service account and irsa support for in-cluster-builder (

#3384)

* feat(k8s): add service account and irsa support for in-cluster-builder

This commit introduces a new service account, and makes the annotations
configurable in kaniko and buildkit in-cluster-builders.

This change enables the use of IRSA for in-cluter-building which makes
it more secure.

Fixes #2931

* docs(irsa): add docs for using in-cluster-building with IRSA

I decided to replace the existing docs, because the existing approach
doesn't provide defense-in-depth. Even in dev clusters, the access to
the Docker registries should be restricted to as few places as
possible, to make it harder for attackers to push bad images.

* chore: fix test

* Update core/src/plugins/kubernetes/container/build/common.ts

* chore: resolve conflicts on rebase

* chore: resolve more conflicts on rebase

* revert in-cluster-building.md

* docs: add irsa separately

* chore: add gcr credential helper to builder and util images

* chore: fix serviceAccount for kaniko and support  using different kaniko namespace

* chore: fix configschema

* chore: update reference docs

* chore: fix typo in kubernetes config.ts

* chore: remove var used for debugging

* chore: fix test-framework

* chore: add namespace to serviceAccount manifest

* chore: add integ tests

* chore: remove namespace from util manifest since it is always in project namepsace

* chore: use kaniko and buildkit envs for integ tests

* chore: add docs for google workload identity

* chore: move tests to their own suite

* chore: fix docs

* chore: remove unused kube api in test

* chore: add assertions and use isEqual

* Update core/test/integ/src/plugins/kubernetes/container/build/build.ts

Co-authored-by: Steffen Neubauer <steffen@garden.io>

* chore: use constant

* chore: compare resources in test for better output

---------

Co-authored-by: Anna Mager <anna@garden.io>
Co-authored-by: Anna Mager <78752267+twelvemo@users.noreply.github.com>

core/src/plugins/kubernetes/api.ts

@@ -31,6 +31,7 @@ import {
   V1Deployment,
   V1Secret,
   V1Service,
+  V1ServiceAccount,
   ServerConfiguration,
   createConfiguration,
 } from "@kubernetes/client-node"
@@ -129,6 +130,15 @@ const crudMap = {
     delete: "deleteNamespacedService",
     patch: "patchNamespacedService",
   },
+  ServiceAccount: {
+    cls: new V1ServiceAccount(),
+    group: "core",
+    read: "readNamespacedServiceAccount",
+    create: "createNamespacedServiceAccount",
+    replace: "replaceNamespacedServiceAccount",
+    delete: "deleteNamespacedServiceAccount",
+    patch: "patchNamespacedServiceAccount",
+  },
 }
 
 type CrudMap = typeof crudMap

core/src/plugins/kubernetes/config.ts

@@ -132,6 +132,7 @@ export interface KubernetesConfig extends BaseProviderConfig {
     nodeSelector?: StringMap
     tolerations?: V1Toleration[]
     annotations?: StringMap
+    serviceAccountAnnotations?: StringMap
   }
   jib?: {
     pushViaCluster?: boolean
@@ -143,6 +144,7 @@ export interface KubernetesConfig extends BaseProviderConfig {
     nodeSelector?: StringMap
     tolerations?: V1Toleration[]
     annotations?: StringMap
+    serviceAccountAnnotations?: StringMap
     util?: {
       tolerations?: V1Toleration[]
       annotations?: StringMap
@@ -517,6 +519,9 @@ export const kubernetesConfigBase = () =>
           annotations: annotationsSchema().description(
             "Specify annotations to apply to both the Pod and Deployment resources associated with cluster-buildkit. Annotations may have an effect on the behaviour of certain components, for example autoscalers."
           ),
+          serviceAccountAnnotations: serviceAccountAnnotationsSchema().description(
+            "Specify annotations to apply to the Kubernetes service account used by cluster-buildkit. This can be useful to set up IRSA with in-cluster building."
+          ),
         })
         .default(() => ({}))
         .description("Configuration options for the `cluster-buildkit` build mode."),
@@ -567,6 +572,9 @@ export const kubernetesConfigBase = () =>
             deline`Specify annotations to apply to each Kaniko builder pod. Annotations may have an effect on the behaviour of certain components, for example autoscalers.
           The same annotations will be used for each util pod unless they are specifically set under \`util.annotations\``
           ),
+          serviceAccountAnnotations: serviceAccountAnnotationsSchema().description(
+            "Specify annotations to apply to the Kubernetes service account used by kaniko. This can be useful to set up IRSA with in-cluster building."
+          ),
           util: joi.object().keys({
             tolerations: joiSparseArray(tolerationSchema()).description(
               "Specify tolerations to apply to each garden-util pod."
@@ -681,6 +689,13 @@ const annotationsSchema = () =>
     })
     .optional()
 
+const serviceAccountAnnotationsSchema = () =>
+  joiStringMap(joi.string())
+    .example({
+      "eks.amazonaws.com/role-arn": "arn:aws:iam::111122223333:role/my-role",
+    })
+    .optional()
+
 export const namespaceSchema = () =>
   joi.alternatives(
     joi.object().keys({

core/src/plugins/kubernetes/constants.ts

@@ -31,15 +31,15 @@ export const defaultIngressClass = "nginx"
 
 // Docker images that Garden ships with
 export const k8sUtilImageName: DockerImageWithDigest =
-  "gardendev/k8s-util:0.5.6@sha256:dce403dc7951e3f714fbb0157aaa08d010601049ea939517957e46ac332073ad"
+  "gardendev/k8s-util:0.5.7@sha256:522da245a5e6ae7c711aa94f84fc83f82a8fdffbf6d8bc48f4d80fee0e0e631b"
 export const k8sSyncUtilImageName: DockerImageWithDigest =
   "gardendev/k8s-sync:0.1.5@sha256:28263cee5ac41acebb8c08f852c4496b15e18c0c94797d7a949a4453b5f91578"
 export const k8sReverseProxyImageName: DockerImageWithDigest =
   "gardendev/k8s-reverse-proxy:0.1.0@sha256:df2976dc67c237114bd9c70e32bfe4d7131af98e140adf6dac29b47b85e07232"
 export const buildkitImageName: DockerImageWithDigest =
-  "gardendev/buildkit:v0.12.2@sha256:2e40f645994b55e03b75b07fbb574dac3d08463a7dda31a958a8619ed011aed6"
+  "gardendev/buildkit:v0.12.2-1@sha256:5b30f6fa46e1fdb89b2255b4290dd3f9072b8f91fd6927b8d428e92498fbf8d0"
 export const buildkitRootlessImageName: DockerImageWithDigest =
-  "gardendev/buildkit:v0.12.2-rootless@sha256:e30b7830078d51e66f1a861024dcc91f2ae5cb1108789c74d0e43ffe0d065b20"
+  "gardendev/buildkit:v0.12.2-1-rootless@sha256:d60e79c66832a95b89f67b1dbee255a561b20105f4d3ec9903dcc7dc4c40f19b"
 export const defaultKanikoImageName: DockerImageWithDigest =
   "gcr.io/kaniko-project/executor:v1.11.0-debug@sha256:32ba2214921892c2fa7b5f9c4ae6f8f026538ce6b2105a93a36a8b5ee50fe517"
 export const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest =

core/src/plugins/kubernetes/container/build/buildkit.ts

@@ -30,6 +30,9 @@ import {
   getUtilContainer,
   ensureBuilderSecret,
   builderToleration,
+  inClusterBuilderServiceAccount,
+  ensureServiceAccount,
+  cycleDeployment,
 } from "./common.js"
 import { getNamespaceStatus } from "../../namespace.js"
 import { sleep } from "../../../../util/util.js"
@@ -187,6 +190,14 @@ export async function ensureBuildkit({
   api: KubeApi
   namespace: string
 }) {
+  const serviceAccountChanged = await ensureServiceAccount({
+    ctx,
+    log,
+    api,
+    namespace,
+    annotations: provider.config.clusterBuildkit?.serviceAccountAnnotations,
+  })
+
   return deployLock.acquire(namespace, async () => {
     const deployLog = log.createLog()
 
@@ -210,12 +221,18 @@ export async function ensureBuildkit({
       log: deployLog,
     })
 
+    // if the service account changed, all pods part of the deployment must be restarted
+    // so that they receive new credentials (e.g. for IRSA)
+    if (status.remoteResources.length > 0 && serviceAccountChanged) {
+      await cycleDeployment({ ctx, provider, deployment: manifest, api, namespace, deployLog })
+    }
+
     if (status.state === "ready") {
       // Need to wait a little to ensure the secret is updated in the deployment
       if (secretUpdated) {
         await sleep(1000)
       }
-      return { authSecret, updated: false }
+      return { authSecret, updated: serviceAccountChanged }
     }
 
     // Deploy the buildkit daemon
@@ -397,6 +414,7 @@ export function getBuildkitDeployment(
           annotations: provider.config.clusterBuildkit?.annotations,
         },
         spec: {
+          serviceAccountName: inClusterBuilderServiceAccount,
           containers: [
             {
               name: buildkitContainerName,

core/src/plugins/kubernetes/container/build/common.ts

@@ -20,9 +20,9 @@ import { prepareSecrets } from "../../secrets.js"
 import { Mutagen } from "../../../../mutagen.js"
 import { randomString } from "../../../../util/string.js"
 import type { V1Container, V1Service } from "@kubernetes/client-node"
-import { cloneDeep, isEmpty } from "lodash-es"
+import { cloneDeep, isEmpty, isEqual } from "lodash-es"
 import { compareDeployedResources, waitForResources } from "../../status/status.js"
-import type { KubernetesDeployment, KubernetesResource } from "../../types.js"
+import type { KubernetesDeployment, KubernetesResource, KubernetesServiceAccount } from "../../types.js"
 import type { BuildActionHandler, BuildActionResults } from "../../../../plugin/action-types.js"
 import { k8sGetContainerBuildActionOutputs } from "../handlers.js"
 import type { Resolved } from "../../../../actions/types.js"
@@ -31,7 +31,10 @@ import { getKubectlExecDestination } from "../../sync.js"
 import { getRunningDeploymentPod } from "../../util.js"
 import { buildSyncVolumeName, dockerAuthSecretKey, k8sUtilImageName, rsyncPortName } from "../../constants.js"
 import { styles } from "../../../../logger/styles.js"
+import type { StringMap } from "../../../../config/common.js"
 
+export const inClusterBuilderServiceAccount = "garden-in-cluster-builder"
+export const sharedBuildSyncDeploymentName = "garden-build-sync"
 export const utilContainerName = "util"
 export const utilRsyncPort = 8730
 export const utilDeploymentName = "garden-util"
@@ -189,7 +192,7 @@ export async function skopeoBuildStatus({
   const outputs = k8sGetContainerBuildActionOutputs({ action, provider })
 
   const remoteId = outputs.deploymentImageId
-  const skopeoCommand = ["skopeo", "--command-timeout=30s", "inspect", "--raw", "--authfile", "/.docker/config.json"]
+  const skopeoCommand = ["skopeo", "--command-timeout=30s", "inspect", "--raw", "--authfile", "~/.docker/config.json"]
 
   if (deploymentRegistry?.insecure === true) {
     skopeoCommand.push("--tls-verify=false")
@@ -262,6 +265,57 @@ export function skopeoManifestUnknown(errMsg: string | null | undefined): boolea
   )
 }
 
+export async function ensureServiceAccount({
+  ctx,
+  log,
+  api,
+  namespace,
+  annotations,
+}: {
+  ctx: PluginContext
+  log: Log
+  api: KubeApi
+  namespace: string
+  annotations?: StringMap
+}): Promise<boolean> {
+  return deployLock.acquire(namespace, async () => {
+    const serviceAccount = getBuilderServiceAccountSpec(namespace, annotations)
+
+    const status = await compareDeployedResources({
+      ctx: ctx as KubernetesPluginContext,
+      api,
+      namespace,
+      manifests: [serviceAccount],
+      log,
+    })
+
+    // NOTE: This is here to make sure that we remove annotations in case they are removed in the garden config.
+    // `compareDeployedResources` as of today only checks whether the manifest is a subset of the deployed manifest.
+    // The manifest is still a subset of the deployed manifest, if an annotation has been removed. But we want the
+    // annotation to be actually removed.
+    // NOTE(steffen): I tried to change the behaviour of `compareDeployedResources` to return "outdated" when the
+    // annotations have changed. But a lot of code actually depends on the behaviour of it with missing annotations.
+    const annotationsNeedUpdate =
+      status.remoteResources.length > 0 && !isEqualAnnotations(serviceAccount, status.remoteResources[0])
+
+    const needUpsert = status.state !== "ready" || annotationsNeedUpdate
+
+    if (needUpsert) {
+      await api.upsert({ kind: "ServiceAccount", namespace, log, obj: serviceAccount })
+      return true
+    }
+
+    return false
+  })
+}
+
+export function isEqualAnnotations(r1: KubernetesResource, r2: KubernetesResource): boolean {
+  // normalize annotations before comparison
+  const a1 = r1.metadata.annotations !== undefined ? r1.metadata.annotations : {}
+  const a2 = r2.metadata.annotations !== undefined ? r2.metadata.annotations : {}
+  return isEqual(a1, a2)
+}
+
 /**
  * Ensures that a garden-util deployment exists in the specified namespace.
  * Returns the docker auth secret that's generated and mounted in the deployment.
@@ -279,6 +333,14 @@ export async function ensureUtilDeployment({
   api: KubeApi
   namespace: string
 }) {
+  const serviceAccountChanged = await ensureServiceAccount({
+    ctx,
+    log,
+    api,
+    namespace,
+    annotations: provider.config.kaniko?.serviceAccountAnnotations,
+  })
+
   return deployLock.acquire(namespace, async () => {
     const deployLog = log.createLog()
 
@@ -301,12 +363,18 @@ export async function ensureUtilDeployment({
       log: deployLog,
     })
 
+    // if the service account changed, all pods part of the deployment must be restarted
+    // so that they receive new credentials (e.g. for IRSA)
+    if (status.remoteResources.length > 0 && serviceAccountChanged) {
+      await cycleDeployment({ ctx, provider, deployment, api, namespace, deployLog })
+    }
+
     if (status.state === "ready") {
       // Need to wait a little to ensure the secret is updated in the deployment
       if (secretUpdated) {
         await sleep(1000)
       }
-      return { authSecret, updated: false }
+      return { authSecret, updated: serviceAccountChanged }
     }
 
     // Deploy the service
@@ -333,6 +401,46 @@ export async function ensureUtilDeployment({
   })
 }
 
+export async function cycleDeployment({
+  ctx,
+  provider,
+  deployment,
+  api,
+  namespace,
+  deployLog,
+}: {
+  ctx: PluginContext
+  provider: KubernetesProvider
+  deployment: KubernetesDeployment
+  api: KubeApi
+  namespace: string
+  deployLog: Log
+}) {
+  const originalReplicas = deployment.spec.replicas
+
+  deployment.spec.replicas = 0
+  await api.upsert({ kind: "Deployment", namespace, log: deployLog, obj: deployment })
+  await waitForResources({
+    namespace,
+    ctx,
+    provider,
+    resources: [deployment],
+    log: deployLog,
+    timeoutSec: 600,
+  })
+
+  deployment.spec.replicas = originalReplicas
+  await api.upsert({ kind: "Deployment", namespace, log: deployLog, obj: deployment })
+  await waitForResources({
+    namespace,
+    ctx,
+    provider,
+    resources: [deployment],
+    log: deployLog,
+    timeoutSec: 600,
+  })
+}
+
 export async function getManifestInspectArgs(remoteId: string, deploymentRegistry: ContainerRegistryConfig) {
   const dockerArgs = ["manifest", "inspect", remoteId]
   const { hostname } = deploymentRegistry
@@ -386,6 +494,21 @@ export async function ensureBuilderSecret({
   return { authSecret, updated }
 }
 
+export function getBuilderServiceAccountSpec(namespace: string, annotations?: StringMap) {
+  const serviceAccount: KubernetesServiceAccount = {
+    apiVersion: "v1",
+    kind: "ServiceAccount",
+    metadata: {
+      name: inClusterBuilderServiceAccount,
+      // ensure we clear old annotations if config flags are removed
+      annotations: annotations || {},
+      namespace,
+    },
+  }
+
+  return serviceAccount
+}
+
 export function getUtilContainer(authSecretName: string, provider: KubernetesProvider): V1Container {
   return {
     name: utilContainerName,
@@ -491,6 +614,7 @@ export function getUtilManifests(
           annotations: kanikoAnnotations,
         },
         spec: {
+          serviceAccountName: inClusterBuilderServiceAccount,
           containers: [utilContainer],
           imagePullSecrets,
           volumes: [

core/src/plugins/kubernetes/container/build/kaniko.ts

@@ -34,6 +34,8 @@ import {
   builderToleration,
   ensureUtilDeployment,
   utilDeploymentName,
+  inClusterBuilderServiceAccount,
+  ensureServiceAccount,
 } from "./common.js"
 import { differenceBy, isEmpty } from "lodash-es"
 import { getDockerBuildFlags } from "../../../container/build.js"
@@ -118,6 +120,8 @@ export const kanikoBuild: BuildHandler = async (params) => {
     kanikoNamespace = await getSystemNamespace(k8sCtx, provider, log)
   }
 
+  await ensureNamespace(api, k8sCtx, { name: kanikoNamespace }, log)
+
   if (kanikoNamespace !== projectNamespace) {
     // Make sure the Kaniko Pod namespace has the auth secret ready
     const secretRes = await ensureBuilderSecret({
@@ -128,9 +132,16 @@ export const kanikoBuild: BuildHandler = async (params) => {
     })
 
     authSecret = secretRes.authSecret
-  }
 
-  await ensureNamespace(api, k8sCtx, { name: kanikoNamespace }, log)
+    // Make sure the Kaniko Pod namespace has the garden-in-cluster-builder service account
+    await ensureServiceAccount({
+      ctx,
+      log,
+      api,
+      namespace: kanikoNamespace,
+      annotations: provider.config.kaniko?.serviceAccountAnnotations,
+    })
+  }
 
   // Execute the build
   const args = [
@@ -323,6 +334,7 @@ export function getKanikoBuilderPodManifest({
       },
     ],
     tolerations: kanikoTolerations,
+    serviceAccountName: inClusterBuilderServiceAccount,
   }
 
   const pod: KubernetesPod = {