diff --git a/core/src/plugins/kubernetes/api.ts b/core/src/plugins/kubernetes/api.ts index 4396ee5239..4d605df9b4 100644 --- a/core/src/plugins/kubernetes/api.ts +++ b/core/src/plugins/kubernetes/api.ts @@ -31,6 +31,7 @@ import { V1Deployment, V1Secret, V1Service, + V1ServiceAccount, ServerConfiguration, createConfiguration, } from "@kubernetes/client-node" @@ -129,6 +130,15 @@ const crudMap = { delete: "deleteNamespacedService", patch: "patchNamespacedService", }, + ServiceAccount: { + cls: new V1ServiceAccount(), + group: "core", + read: "readNamespacedServiceAccount", + create: "createNamespacedServiceAccount", + replace: "replaceNamespacedServiceAccount", + delete: "deleteNamespacedServiceAccount", + patch: "patchNamespacedServiceAccount", + }, } type CrudMap = typeof crudMap diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts index a880891c76..033e28ff26 100644 --- a/core/src/plugins/kubernetes/config.ts +++ b/core/src/plugins/kubernetes/config.ts @@ -132,6 +132,7 @@ export interface KubernetesConfig extends BaseProviderConfig { nodeSelector?: StringMap tolerations?: V1Toleration[] annotations?: StringMap + serviceAccountAnnotations?: StringMap } jib?: { pushViaCluster?: boolean @@ -143,6 +144,7 @@ export interface KubernetesConfig extends BaseProviderConfig { nodeSelector?: StringMap tolerations?: V1Toleration[] annotations?: StringMap + serviceAccountAnnotations?: StringMap util?: { tolerations?: V1Toleration[] annotations?: StringMap @@ -517,6 +519,9 @@ export const kubernetesConfigBase = () => annotations: annotationsSchema().description( "Specify annotations to apply to both the Pod and Deployment resources associated with cluster-buildkit. Annotations may have an effect on the behaviour of certain components, for example autoscalers." ), + serviceAccountAnnotations: serviceAccountAnnotationsSchema().description( + "Specify annotations to apply to the Kubernetes service account used by cluster-buildkit. This can be useful to set up IRSA with in-cluster building." + ), }) .default(() => ({})) .description("Configuration options for the `cluster-buildkit` build mode."), @@ -567,6 +572,9 @@ export const kubernetesConfigBase = () => deline`Specify annotations to apply to each Kaniko builder pod. Annotations may have an effect on the behaviour of certain components, for example autoscalers. The same annotations will be used for each util pod unless they are specifically set under \`util.annotations\`` ), + serviceAccountAnnotations: serviceAccountAnnotationsSchema().description( + "Specify annotations to apply to the Kubernetes service account used by kaniko. This can be useful to set up IRSA with in-cluster building." + ), util: joi.object().keys({ tolerations: joiSparseArray(tolerationSchema()).description( "Specify tolerations to apply to each garden-util pod." @@ -681,6 +689,13 @@ const annotationsSchema = () => }) .optional() +const serviceAccountAnnotationsSchema = () => + joiStringMap(joi.string()) + .example({ + "eks.amazonaws.com/role-arn": "arn:aws:iam::111122223333:role/my-role", + }) + .optional() + export const namespaceSchema = () => joi.alternatives( joi.object().keys({ diff --git a/core/src/plugins/kubernetes/constants.ts b/core/src/plugins/kubernetes/constants.ts index 43fd4af5bb..2d2d900c0d 100644 --- a/core/src/plugins/kubernetes/constants.ts +++ b/core/src/plugins/kubernetes/constants.ts @@ -31,15 +31,15 @@ export const defaultIngressClass = "nginx" // Docker images that Garden ships with export const k8sUtilImageName: DockerImageWithDigest = - "gardendev/k8s-util:0.5.6@sha256:dce403dc7951e3f714fbb0157aaa08d010601049ea939517957e46ac332073ad" + "gardendev/k8s-util:0.5.7@sha256:522da245a5e6ae7c711aa94f84fc83f82a8fdffbf6d8bc48f4d80fee0e0e631b" export const k8sSyncUtilImageName: DockerImageWithDigest = "gardendev/k8s-sync:0.1.5@sha256:28263cee5ac41acebb8c08f852c4496b15e18c0c94797d7a949a4453b5f91578" export const k8sReverseProxyImageName: DockerImageWithDigest = "gardendev/k8s-reverse-proxy:0.1.0@sha256:df2976dc67c237114bd9c70e32bfe4d7131af98e140adf6dac29b47b85e07232" export const buildkitImageName: DockerImageWithDigest = - "gardendev/buildkit:v0.12.2@sha256:2e40f645994b55e03b75b07fbb574dac3d08463a7dda31a958a8619ed011aed6" + "gardendev/buildkit:v0.12.2-1@sha256:5b30f6fa46e1fdb89b2255b4290dd3f9072b8f91fd6927b8d428e92498fbf8d0" export const buildkitRootlessImageName: DockerImageWithDigest = - "gardendev/buildkit:v0.12.2-rootless@sha256:e30b7830078d51e66f1a861024dcc91f2ae5cb1108789c74d0e43ffe0d065b20" + "gardendev/buildkit:v0.12.2-1-rootless@sha256:d60e79c66832a95b89f67b1dbee255a561b20105f4d3ec9903dcc7dc4c40f19b" export const defaultKanikoImageName: DockerImageWithDigest = "gcr.io/kaniko-project/executor:v1.11.0-debug@sha256:32ba2214921892c2fa7b5f9c4ae6f8f026538ce6b2105a93a36a8b5ee50fe517" export const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest = diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts index 4b68d2fc7d..c07de229fb 100644 --- a/core/src/plugins/kubernetes/container/build/buildkit.ts +++ b/core/src/plugins/kubernetes/container/build/buildkit.ts @@ -30,6 +30,9 @@ import { getUtilContainer, ensureBuilderSecret, builderToleration, + inClusterBuilderServiceAccount, + ensureServiceAccount, + cycleDeployment, } from "./common.js" import { getNamespaceStatus } from "../../namespace.js" import { sleep } from "../../../../util/util.js" @@ -187,6 +190,14 @@ export async function ensureBuildkit({ api: KubeApi namespace: string }) { + const serviceAccountChanged = await ensureServiceAccount({ + ctx, + log, + api, + namespace, + annotations: provider.config.clusterBuildkit?.serviceAccountAnnotations, + }) + return deployLock.acquire(namespace, async () => { const deployLog = log.createLog() @@ -210,12 +221,18 @@ export async function ensureBuildkit({ log: deployLog, }) + // if the service account changed, all pods part of the deployment must be restarted + // so that they receive new credentials (e.g. for IRSA) + if (status.remoteResources.length > 0 && serviceAccountChanged) { + await cycleDeployment({ ctx, provider, deployment: manifest, api, namespace, deployLog }) + } + if (status.state === "ready") { // Need to wait a little to ensure the secret is updated in the deployment if (secretUpdated) { await sleep(1000) } - return { authSecret, updated: false } + return { authSecret, updated: serviceAccountChanged } } // Deploy the buildkit daemon @@ -397,6 +414,7 @@ export function getBuildkitDeployment( annotations: provider.config.clusterBuildkit?.annotations, }, spec: { + serviceAccountName: inClusterBuilderServiceAccount, containers: [ { name: buildkitContainerName, diff --git a/core/src/plugins/kubernetes/container/build/common.ts b/core/src/plugins/kubernetes/container/build/common.ts index ebd523a812..eea6254b0d 100644 --- a/core/src/plugins/kubernetes/container/build/common.ts +++ b/core/src/plugins/kubernetes/container/build/common.ts @@ -20,9 +20,9 @@ import { prepareSecrets } from "../../secrets.js" import { Mutagen } from "../../../../mutagen.js" import { randomString } from "../../../../util/string.js" import type { V1Container, V1Service } from "@kubernetes/client-node" -import { cloneDeep, isEmpty } from "lodash-es" +import { cloneDeep, isEmpty, isEqual } from "lodash-es" import { compareDeployedResources, waitForResources } from "../../status/status.js" -import type { KubernetesDeployment, KubernetesResource } from "../../types.js" +import type { KubernetesDeployment, KubernetesResource, KubernetesServiceAccount } from "../../types.js" import type { BuildActionHandler, BuildActionResults } from "../../../../plugin/action-types.js" import { k8sGetContainerBuildActionOutputs } from "../handlers.js" import type { Resolved } from "../../../../actions/types.js" @@ -31,7 +31,10 @@ import { getKubectlExecDestination } from "../../sync.js" import { getRunningDeploymentPod } from "../../util.js" import { buildSyncVolumeName, dockerAuthSecretKey, k8sUtilImageName, rsyncPortName } from "../../constants.js" import { styles } from "../../../../logger/styles.js" +import type { StringMap } from "../../../../config/common.js" +export const inClusterBuilderServiceAccount = "garden-in-cluster-builder" +export const sharedBuildSyncDeploymentName = "garden-build-sync" export const utilContainerName = "util" export const utilRsyncPort = 8730 export const utilDeploymentName = "garden-util" @@ -189,7 +192,7 @@ export async function skopeoBuildStatus({ const outputs = k8sGetContainerBuildActionOutputs({ action, provider }) const remoteId = outputs.deploymentImageId - const skopeoCommand = ["skopeo", "--command-timeout=30s", "inspect", "--raw", "--authfile", "/.docker/config.json"] + const skopeoCommand = ["skopeo", "--command-timeout=30s", "inspect", "--raw", "--authfile", "~/.docker/config.json"] if (deploymentRegistry?.insecure === true) { skopeoCommand.push("--tls-verify=false") @@ -262,6 +265,57 @@ export function skopeoManifestUnknown(errMsg: string | null | undefined): boolea ) } +export async function ensureServiceAccount({ + ctx, + log, + api, + namespace, + annotations, +}: { + ctx: PluginContext + log: Log + api: KubeApi + namespace: string + annotations?: StringMap +}): Promise { + return deployLock.acquire(namespace, async () => { + const serviceAccount = getBuilderServiceAccountSpec(namespace, annotations) + + const status = await compareDeployedResources({ + ctx: ctx as KubernetesPluginContext, + api, + namespace, + manifests: [serviceAccount], + log, + }) + + // NOTE: This is here to make sure that we remove annotations in case they are removed in the garden config. + // `compareDeployedResources` as of today only checks whether the manifest is a subset of the deployed manifest. + // The manifest is still a subset of the deployed manifest, if an annotation has been removed. But we want the + // annotation to be actually removed. + // NOTE(steffen): I tried to change the behaviour of `compareDeployedResources` to return "outdated" when the + // annotations have changed. But a lot of code actually depends on the behaviour of it with missing annotations. + const annotationsNeedUpdate = + status.remoteResources.length > 0 && !isEqualAnnotations(serviceAccount, status.remoteResources[0]) + + const needUpsert = status.state !== "ready" || annotationsNeedUpdate + + if (needUpsert) { + await api.upsert({ kind: "ServiceAccount", namespace, log, obj: serviceAccount }) + return true + } + + return false + }) +} + +export function isEqualAnnotations(r1: KubernetesResource, r2: KubernetesResource): boolean { + // normalize annotations before comparison + const a1 = r1.metadata.annotations !== undefined ? r1.metadata.annotations : {} + const a2 = r2.metadata.annotations !== undefined ? r2.metadata.annotations : {} + return isEqual(a1, a2) +} + /** * Ensures that a garden-util deployment exists in the specified namespace. * Returns the docker auth secret that's generated and mounted in the deployment. @@ -279,6 +333,14 @@ export async function ensureUtilDeployment({ api: KubeApi namespace: string }) { + const serviceAccountChanged = await ensureServiceAccount({ + ctx, + log, + api, + namespace, + annotations: provider.config.kaniko?.serviceAccountAnnotations, + }) + return deployLock.acquire(namespace, async () => { const deployLog = log.createLog() @@ -301,12 +363,18 @@ export async function ensureUtilDeployment({ log: deployLog, }) + // if the service account changed, all pods part of the deployment must be restarted + // so that they receive new credentials (e.g. for IRSA) + if (status.remoteResources.length > 0 && serviceAccountChanged) { + await cycleDeployment({ ctx, provider, deployment, api, namespace, deployLog }) + } + if (status.state === "ready") { // Need to wait a little to ensure the secret is updated in the deployment if (secretUpdated) { await sleep(1000) } - return { authSecret, updated: false } + return { authSecret, updated: serviceAccountChanged } } // Deploy the service @@ -333,6 +401,46 @@ export async function ensureUtilDeployment({ }) } +export async function cycleDeployment({ + ctx, + provider, + deployment, + api, + namespace, + deployLog, +}: { + ctx: PluginContext + provider: KubernetesProvider + deployment: KubernetesDeployment + api: KubeApi + namespace: string + deployLog: Log +}) { + const originalReplicas = deployment.spec.replicas + + deployment.spec.replicas = 0 + await api.upsert({ kind: "Deployment", namespace, log: deployLog, obj: deployment }) + await waitForResources({ + namespace, + ctx, + provider, + resources: [deployment], + log: deployLog, + timeoutSec: 600, + }) + + deployment.spec.replicas = originalReplicas + await api.upsert({ kind: "Deployment", namespace, log: deployLog, obj: deployment }) + await waitForResources({ + namespace, + ctx, + provider, + resources: [deployment], + log: deployLog, + timeoutSec: 600, + }) +} + export async function getManifestInspectArgs(remoteId: string, deploymentRegistry: ContainerRegistryConfig) { const dockerArgs = ["manifest", "inspect", remoteId] const { hostname } = deploymentRegistry @@ -386,6 +494,21 @@ export async function ensureBuilderSecret({ return { authSecret, updated } } +export function getBuilderServiceAccountSpec(namespace: string, annotations?: StringMap) { + const serviceAccount: KubernetesServiceAccount = { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: inClusterBuilderServiceAccount, + // ensure we clear old annotations if config flags are removed + annotations: annotations || {}, + namespace, + }, + } + + return serviceAccount +} + export function getUtilContainer(authSecretName: string, provider: KubernetesProvider): V1Container { return { name: utilContainerName, @@ -491,6 +614,7 @@ export function getUtilManifests( annotations: kanikoAnnotations, }, spec: { + serviceAccountName: inClusterBuilderServiceAccount, containers: [utilContainer], imagePullSecrets, volumes: [ diff --git a/core/src/plugins/kubernetes/container/build/kaniko.ts b/core/src/plugins/kubernetes/container/build/kaniko.ts index f69107c4ed..2dddfda7c4 100644 --- a/core/src/plugins/kubernetes/container/build/kaniko.ts +++ b/core/src/plugins/kubernetes/container/build/kaniko.ts @@ -34,6 +34,8 @@ import { builderToleration, ensureUtilDeployment, utilDeploymentName, + inClusterBuilderServiceAccount, + ensureServiceAccount, } from "./common.js" import { differenceBy, isEmpty } from "lodash-es" import { getDockerBuildFlags } from "../../../container/build.js" @@ -118,6 +120,8 @@ export const kanikoBuild: BuildHandler = async (params) => { kanikoNamespace = await getSystemNamespace(k8sCtx, provider, log) } + await ensureNamespace(api, k8sCtx, { name: kanikoNamespace }, log) + if (kanikoNamespace !== projectNamespace) { // Make sure the Kaniko Pod namespace has the auth secret ready const secretRes = await ensureBuilderSecret({ @@ -128,9 +132,16 @@ export const kanikoBuild: BuildHandler = async (params) => { }) authSecret = secretRes.authSecret - } - await ensureNamespace(api, k8sCtx, { name: kanikoNamespace }, log) + // Make sure the Kaniko Pod namespace has the garden-in-cluster-builder service account + await ensureServiceAccount({ + ctx, + log, + api, + namespace: kanikoNamespace, + annotations: provider.config.kaniko?.serviceAccountAnnotations, + }) + } // Execute the build const args = [ @@ -323,6 +334,7 @@ export function getKanikoBuilderPodManifest({ }, ], tolerations: kanikoTolerations, + serviceAccountName: inClusterBuilderServiceAccount, } const pod: KubernetesPod = { diff --git a/core/src/plugins/kubernetes/jib-container.ts b/core/src/plugins/kubernetes/jib-container.ts index bf0015d2a5..567b22b6cc 100644 --- a/core/src/plugins/kubernetes/jib-container.ts +++ b/core/src/plugins/kubernetes/jib-container.ts @@ -110,7 +110,6 @@ async function buildAndPushViaRemote(params: BuildActionParams<"build", Containe // Make sure the sync target is up if (buildMode === "kaniko") { - // Make sure the garden-util deployment is up await ensureUtilDeployment({ ctx, provider, @@ -120,7 +119,6 @@ async function buildAndPushViaRemote(params: BuildActionParams<"build", Containe }) deploymentName = utilDeploymentName } else if (buildMode === "cluster-buildkit") { - // Make sure the buildkit deployment is up await ensureBuildkit({ ctx, provider, diff --git a/core/src/plugins/kubernetes/types.ts b/core/src/plugins/kubernetes/types.ts index 4fe6453e45..5507a956ce 100644 --- a/core/src/plugins/kubernetes/types.ts +++ b/core/src/plugins/kubernetes/types.ts @@ -10,6 +10,7 @@ import type { KubernetesObject, V1DaemonSet, V1Deployment, + V1ServiceAccount, V1ObjectMeta, V1ReplicaSet, V1StatefulSet, @@ -92,6 +93,7 @@ export type KubernetesReplicaSet = KubernetesResource export type KubernetesStatefulSet = KubernetesResource export type KubernetesPod = KubernetesResource export type KubernetesService = KubernetesResource +export type KubernetesServiceAccount = KubernetesResource export type KubernetesWorkload = KubernetesResource export type KubernetesIngress = KubernetesResource diff --git a/core/test/integ/src/plugins/kubernetes/container/build/build.ts b/core/test/integ/src/plugins/kubernetes/container/build/build.ts index 57bf32f4db..c6fcc6ebc2 100644 --- a/core/test/integ/src/plugins/kubernetes/container/build/build.ts +++ b/core/test/integ/src/plugins/kubernetes/container/build/build.ts @@ -10,7 +10,11 @@ import { expectError, grouped } from "../../../../../../helpers.js" import type { Garden } from "../../../../../../../src/garden.js" import type { ConfigGraph } from "../../../../../../../src/graph/config-graph.js" import type { PluginContext } from "../../../../../../../src/plugin-context.js" -import type { KubernetesProvider } from "../../../../../../../src/plugins/kubernetes/config.js" +import type { + ClusterBuildkitCacheConfig, + KubernetesPluginContext, + KubernetesProvider, +} from "../../../../../../../src/plugins/kubernetes/config.js" import { expect } from "chai" import { getContainerTestGarden } from "../container.js" import { containerHelpers } from "../../../../../../../src/plugins/container/helpers.js" @@ -21,6 +25,14 @@ import type { ContainerBuildAction } from "../../../../../../../src/plugins/cont import { BuildTask } from "../../../../../../../src/tasks/build.js" import { k8sContainerBuildExtension } from "../../../../../../../src/plugins/kubernetes/container/extensions.js" import { deleteGoogleArtifactImage, listGoogleArtifactImageTags } from "../../../../../helpers.js" +import { + ensureServiceAccount, + ensureUtilDeployment, + getBuilderServiceAccountSpec, +} from "../../../../../../../src/plugins/kubernetes/container/build/common.js" +import { compareDeployedResources } from "../../../../../../../src/plugins/kubernetes/status/status.js" +import { KubeApi } from "../../../../../../../src/plugins/kubernetes/api.js" +import { ensureBuildkit } from "../../../../../../../src/plugins/kubernetes/container/build/buildkit.js" describe.skip("Kubernetes Container Build Extension", () => { const builder = k8sContainerBuildExtension() @@ -566,3 +578,170 @@ describe.skip("Kubernetes Container Build Extension", () => { }) }) }) + +describe("Ensure serviceAccount annotations for in-cluster building", () => { + let garden: Garden + let cleanup: (() => void) | undefined + let log: ActionLog + let provider: KubernetesProvider + let ctx: PluginContext + let api: KubeApi + after(async () => { + if (garden) { + garden.close() + } + }) + + const init = async (environmentName: string, remoteContainerAuth = false) => { + ;({ garden, cleanup } = await getContainerTestGarden(environmentName, { remoteContainerAuth })) + log = createActionLog({ log: garden.log, actionName: "", actionKind: "" }) + provider = await garden.resolveProvider(garden.log, "local-kubernetes") + ctx = await garden.getPluginContext({ provider, templateContext: undefined, events: undefined }) + api = await KubeApi.factory(log, ctx, provider) + } + grouped("kaniko").context("kaniko service account annotations", () => { + beforeEach(async () => { + await init("kaniko") + }) + it("should deploy a garden builder serviceAccount with specified annotations in the project namespace", async () => { + const annotations = { + "iam.gke.io/gcp-service-account": "workload-identity-gar@garden-ci.iam.gserviceaccount.com", + } + const projectNamespace = ctx.namespace + provider.config.kaniko = { serviceAccountAnnotations: annotations } + const serviceAccount = getBuilderServiceAccountSpec(projectNamespace, annotations) + + await ensureServiceAccount({ ctx, log, api, namespace: projectNamespace, annotations }) + + const status = await compareDeployedResources({ + ctx: ctx as KubernetesPluginContext, + api, + namespace: projectNamespace, + manifests: [serviceAccount], + log, + }) + expect(status.state).to.equal("ready") + }) + it("should remove annotations from the garden builder serviceAccount", async () => { + const projectNamespace = ctx.namespace + const originalAnnotations = { + "iam.gke.io/gcp-service-account": "workload-identity-gar@garden-ci.iam.gserviceaccount.com", + "foo": "bar", + } + provider.config.kaniko = { serviceAccountAnnotations: originalAnnotations } + const originalServiceAccount = getBuilderServiceAccountSpec(projectNamespace, originalAnnotations) + + await ensureServiceAccount({ ctx, log, api, namespace: projectNamespace, annotations: originalAnnotations }) + + const status = await compareDeployedResources({ + ctx: ctx as KubernetesPluginContext, + api, + namespace: projectNamespace, + manifests: [originalServiceAccount], + log, + }) + // Both annotations should be present + expect(originalServiceAccount.metadata.annotations).to.deep.equal(status.remoteResources[0].metadata.annotations) + + const reducedAnnotations = { + "iam.gke.io/gcp-service-account": "workload-identity-gar@garden-ci.iam.gserviceaccount.com", + } + provider.config.kaniko = { serviceAccountAnnotations: reducedAnnotations } + const updatedServiceAccount = getBuilderServiceAccountSpec(projectNamespace, reducedAnnotations) + + await ensureServiceAccount({ ctx, log, api, namespace: projectNamespace, annotations: reducedAnnotations }) + + const updatedStatus = await compareDeployedResources({ + ctx: ctx as KubernetesPluginContext, + api, + namespace: projectNamespace, + manifests: [updatedServiceAccount], + log, + }) + // Only reduced annotations should be present + expect(updatedServiceAccount.metadata.annotations).to.deep.equal( + updatedStatus.remoteResources[0].metadata.annotations + ) + }) + it("should cycle the util deployment when the serviceAccount annotations changed", async () => { + const originalAnnotations = { + "iam.gke.io/gcp-service-account": "workload-identity-gar@garden-ci.iam.gserviceaccount.com", + } + const projectNamespace = ctx.namespace + provider.config.buildMode = "kaniko" + provider.config.kaniko = { serviceAccountAnnotations: originalAnnotations } + + await ensureUtilDeployment({ ctx, provider, log, api, namespace: projectNamespace }) + + const updatedAnnotations = { + "iam.gke.io/gcp-service-account": "a-different-service-account@garden-ci.iam.gserviceaccount.com", + } + provider.config.kaniko = { serviceAccountAnnotations: updatedAnnotations } + const { updated } = await ensureUtilDeployment({ ctx, provider, log, api, namespace: projectNamespace }) + + expect(updated).to.be.true + }) + }) + + grouped("cluster-buildkit").context("cluster-buildkit service account annotations", () => { + beforeEach(async () => { + await init("cluster-buildkit") + }) + + afterEach(async () => { + if (cleanup) { + cleanup() + } + }) + + const defaultCacheConfig: ClusterBuildkitCacheConfig[] = [ + { + type: "registry", + mode: "auto", + tag: "_buildcache", + export: true, + }, + ] + + it("should deploy a garden builder serviceAccount with specified annotations in the project namespace", async () => { + const annotations = { + "iam.gke.io/gcp-service-account": "workload-identity-gar@garden-ci.iam.gserviceaccount.com", + } + const projectNamespace = ctx.namespace + + provider.config.clusterBuildkit = { serviceAccountAnnotations: annotations, cache: defaultCacheConfig } + const serviceAccount = getBuilderServiceAccountSpec(projectNamespace, annotations) + + await ensureServiceAccount({ ctx, log, api, namespace: projectNamespace, annotations }) + + const status = await compareDeployedResources({ + ctx: ctx as KubernetesPluginContext, + api, + namespace: projectNamespace, + manifests: [serviceAccount], + log: garden.log, + }) + + expect(status.state).to.equal("ready") + }) + + it("should cycle the buildkit deployment when the serviceAccount annotations changed", async () => { + const originalAnnotations = { + "iam.gke.io/gcp-service-account": "workload-identity-gar@garden-ci.iam.gserviceaccount.com", + } + const projectNamespace = ctx.namespace + provider.config.buildMode = "cluster-buildkit" + provider.config.clusterBuildkit = { serviceAccountAnnotations: originalAnnotations, cache: defaultCacheConfig } + + await ensureBuildkit({ ctx, provider, log, api, namespace: projectNamespace }) + + const updatedAnnotations = { + "iam.gke.io/gcp-service-account": "a-different-service-account@garden-ci.iam.gserviceaccount.com", + } + provider.config.clusterBuildkit = { serviceAccountAnnotations: updatedAnnotations, cache: defaultCacheConfig } + const { updated } = await ensureBuildkit({ ctx, provider, log: garden.log, api, namespace: projectNamespace }) + + expect(updated).to.be.true + }) + }) +}) diff --git a/core/test/unit/src/plugins/kubernetes/container/build/common.ts b/core/test/unit/src/plugins/kubernetes/container/build/common.ts index e495ba69dd..9458a686bd 100644 --- a/core/test/unit/src/plugins/kubernetes/container/build/common.ts +++ b/core/test/unit/src/plugins/kubernetes/container/build/common.ts @@ -7,7 +7,9 @@ */ import { + getBuilderServiceAccountSpec, getUtilManifests, + inClusterBuilderServiceAccount, skopeoManifestUnknown, } from "../../../../../../../src/plugins/kubernetes/container/build/common.js" import { expect } from "chai" @@ -46,6 +48,27 @@ describe("common build", () => { }) }) + describe("getBuilderServiceAccountSpec", () => { + it("should return the manifest", () => { + const annotation = { "some-annotation": "annotation-value" } + const result = getBuilderServiceAccountSpec("random-namespace", annotation) + expect(result).eql({ + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: inClusterBuilderServiceAccount, + annotations: annotation, + namespace: "random-namespace", + }, + }) + }) + + it("should return empty annotations when no annotations are provided", () => { + const result = getBuilderServiceAccountSpec("random-namespace") + expect(result.metadata.annotations).eql({}) + }) + }) + describe("getUtilManifests", () => { const _provider: DeepPartial = { config: { @@ -75,6 +98,7 @@ describe("common build", () => { template: { metadata: { labels: { app: "garden-util" }, annotations: undefined }, spec: { + serviceAccountName: inClusterBuilderServiceAccount, containers: [ { name: "util", diff --git a/core/test/unit/src/plugins/kubernetes/container/build/kaniko.ts b/core/test/unit/src/plugins/kubernetes/container/build/kaniko.ts index b72cdca029..962ea546bb 100644 --- a/core/test/unit/src/plugins/kubernetes/container/build/kaniko.ts +++ b/core/test/unit/src/plugins/kubernetes/container/build/kaniko.ts @@ -17,6 +17,7 @@ import type { KubernetesProvider } from "../../../../../../../src/plugins/kubern import { defaultResources } from "../../../../../../../src/plugins/kubernetes/config.js" import { defaultKanikoImageName, k8sUtilImageName } from "../../../../../../../src/plugins/kubernetes/constants.js" import type { DeepPartial } from "utility-types" +import { inClusterBuilderServiceAccount } from "../../../../../../../src/plugins/kubernetes/container/build/common.js" describe("kaniko build", () => { it("should return as successful when immutable tag already exists in destination", () => { @@ -139,6 +140,7 @@ describe("kaniko build", () => { }, ], shareProcessNamespace: true, + serviceAccountName: inClusterBuilderServiceAccount, tolerations: [ { effect: "NoSchedule", diff --git a/docs/k8s-plugins/advanced/in-cluster-building.md b/docs/k8s-plugins/advanced/in-cluster-building.md index 2d6bc25081..2c25090333 100644 --- a/docs/k8s-plugins/advanced/in-cluster-building.md +++ b/docs/k8s-plugins/advanced/in-cluster-building.md @@ -335,7 +335,7 @@ providers: #### Configuring Access -To grant your service account the right permission to push to ECR, add this policy to each of the repositories in the container registry that you want to use with in-cluster building: +If your Kubernetes cluster and ECR repositories are only used for development, an easy way to configure access is to allow push access to all the workers (and subsequently all pods): ```json { @@ -364,6 +364,130 @@ To grant your service account the right permission to push to ECR, add this poli To grant developers permission to push and pull directly from a repository, see [the AWS documentation](https://docs.aws.amazon.com/AmazonECR/latest/userguide/security_iam_id-based-policy-examples.html). +If you need more fine grained control, please use IRSA (see the next section). + +#### Using in-cluster-building with IRSA (IAM Roles for Service Accounts) + +Using IRSA we can reduce the ECR access from the worker nodes (and subsequently all pods running on these worker nodes) to readonly, and only provide push access to the in-cluster builder Pods. + +Depending on how you deployed your EKS cluster you already might have a policy attached to your worker nodes by default that allows read access to all ECR repositories. For more info please check [the ECR on EKS user guide of the AWS docs](https://docs.aws.amazon.com/AmazonECR/latest/userguide/ECR_on_EKS.html). + +If it does not exist yet, first create an IAM policy to allow the Kubernetes nodes to pull images from your ECR repositories: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "GardenAllowPull", + "Effect": "Allow", + "Principal": { + "AWS": ["arn:aws:iam:::role/"] + }, + "Action": [ + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + ] + "Resource": "arn:aws:ecr:::repository/" + }, + { + "Sid": "GetAuthorizationToken", + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken" + ], + "Resource": "*" + } + ] +} +``` + +Create a [web identity role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html) to allow pushing images from the in-cluster builder Pods, with the following trust relationship: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks..amazonaws.com/id/:sub": "system:serviceaccount:*:garden-in-cluster-builder", + "oidc.eks..amazonaws.com/id/:aud": "sts.amazonaws.com" + } + } + } + ] +} +``` + +Note that this trust relationship allows the Pods associated with the `garden-in-cluster-builder` serviceaccount in all namespaces (`*`) to push images. + +Configure the following IAM policy with the web identity role: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "GardenAllowPushPull", + "Effect": "Allow", + "Action": [ + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:CompleteLayerUpload", + "ecr:GetDownloadUrlForLayer", + "ecr:InitiateLayerUpload", + "ecr:PutImage", + "ecr:UploadLayerPart" + ], + "Resource": "arn:aws:ecr:::repository/" + }, + { + "Sid": "GetAuthorizationToken", + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken" + ], + "Resource": "*" + } + ] +} +``` + +NOTE: You need to replace the following placeholders: +- `` is your AWS Account ID +- `` is the Node IAM role name (You can find it in your EKS node group) +- `` AWS region +- `` name of the ECR repositories ([matching multiple names using wildcards](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_resource.html#reference_policies_elements_resource_wildcards) is allowed) +- `` Part of the OpenID Connect provider URL + +Add the IRSA `serviceAccountAnnotations` to your `project.garden.yml`: + +```yaml +kind: Project +name: my-project +... +providers: + - name: kubernetes + ... + # If you use the kaniko build mode + buildMode: kaniko + kaniko: + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/ + # If you use the buildkit build mode + buildMode: buildkit + clusterBuildkit: + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/ +``` + ### Using in-cluster building with GCR To use in-cluster building with GCR (Google Container Registry) you need to set up authentication, with the following steps: @@ -494,6 +618,59 @@ providers: namespace: default ``` +#### Using in-cluster building with Google Workload identity + +Workload identity for GKE clusters allows service acccounts in your cluster to impersonate IAM service accounts. Using this method for in-cluster building allows you to avoid storing IAM service account credentials as secrets in your cluster. + +Make sure that [workload identity is enabled on your cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#enable). + +First create an IAM service account: + +```sh +gcloud iam service-accounts create gar-access \ + --project=${PROJECT_ID} +``` + +Then attach the roles required to push and pull to Google Artifact Registry: + +```sh +gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member=serviceAccount:gar-access@${PROJECT_ID}.iam.gserviceaccount.com \ + --role=roles/artifactregistry.writer +``` + +Note that you can also use this method with Google Container Registry, for the required roles [check the section about GCR above](#using-in-cluster-building-with-gcr). + +Now you need to add an IAM policy binding to allow the Kubernetes service account to impersonate the IAM service account. Note that GCP workload identity for Kubernetes does not allow wildcards for in the member section. This means that every Kubernetes service account in each namespace must be registered as a member. Garden's build services always use a service account with the name `garden-in-cluster-builder`. + +```sh +gcloud iam service-accounts add-iam-policy-binding gar-access@${PROJECT_ID}.iam.gserviceaccount.com \ + --role roles/iam.workloadIdentityUser \ + --member "serviceAccount:${PROJECT_ID}.svc.id.goog[${K8S_NAMESPACE}/garden-in-cluster-builder]" +``` + +And finally add the annotation with your IAM service account to the garden project configuration. Garden will make sure to annotate the in cluster builder service account with this annotation. + +```yaml +kind: Project +name: my-project +... +providers: + - name: kubernetes + ... + # If you use the kaniko build mode + buildMode: kaniko + kaniko: + serviceAccountAnnotations: + iam.gke.io/gcp-service-account: gar-access@${PROJECT_ID}.iam.gserviceaccount.com + + # If you use the buildkit build mode + buildMode: buildkit + clusterBuildkit: + serviceAccountAnnotations: + iam.gke.io/gcp-service-account: gar-access@${PROJECT_ID}.iam.gserviceaccount.com +``` + ## Publishing images You can publish images that have been built in your cluster, using the `garden publish` command. See the [Publishing images](../../other-plugins/container.md#publishing-images) section in the [Container Action guide](../../other-plugins/container.md) for details. diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index 61ec8fa98b..ae76b92310 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -202,6 +202,10 @@ providers: # Annotations may have an effect on the behaviour of certain components, for example autoscalers. annotations: + # Specify annotations to apply to the Kubernetes service account used by cluster-buildkit. This can be useful to + # set up IRSA with in-cluster building. + serviceAccountAnnotations: + # Setting related to Jib image builds. jib: # In some cases you may need to push images built with Jib to the remote registry via Kubernetes cluster, e.g. @@ -265,6 +269,10 @@ providers: # are specifically set under `util.annotations` annotations: + # Specify annotations to apply to the Kubernetes service account used by kaniko. This can be useful to set up + # IRSA with in-cluster building. + serviceAccountAnnotations: + util: # Specify tolerations to apply to each garden-util pod. tolerations: @@ -908,6 +916,26 @@ providers: cluster-autoscaler.kubernetes.io/safe-to-evict: 'false' ``` +### `providers[].clusterBuildkit.serviceAccountAnnotations` + +[providers](#providers) > [clusterBuildkit](#providersclusterbuildkit) > serviceAccountAnnotations + +Specify annotations to apply to the Kubernetes service account used by cluster-buildkit. This can be useful to set up IRSA with in-cluster building. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +Example: + +```yaml +providers: + - clusterBuildkit: + ... + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/my-role +``` + ### `providers[].jib` [providers](#providers) > jib @@ -1068,6 +1096,26 @@ providers: cluster-autoscaler.kubernetes.io/safe-to-evict: 'false' ``` +### `providers[].kaniko.serviceAccountAnnotations` + +[providers](#providers) > [kaniko](#providerskaniko) > serviceAccountAnnotations + +Specify annotations to apply to the Kubernetes service account used by kaniko. This can be useful to set up IRSA with in-cluster building. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +Example: + +```yaml +providers: + - kaniko: + ... + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/my-role +``` + ### `providers[].kaniko.util` [providers](#providers) > [kaniko](#providerskaniko) > util diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index f8bbfd7faa..0b89228a60 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -198,6 +198,10 @@ providers: # Annotations may have an effect on the behaviour of certain components, for example autoscalers. annotations: + # Specify annotations to apply to the Kubernetes service account used by cluster-buildkit. This can be useful to + # set up IRSA with in-cluster building. + serviceAccountAnnotations: + # Setting related to Jib image builds. jib: # In some cases you may need to push images built with Jib to the remote registry via Kubernetes cluster, e.g. @@ -261,6 +265,10 @@ providers: # are specifically set under `util.annotations` annotations: + # Specify annotations to apply to the Kubernetes service account used by kaniko. This can be useful to set up + # IRSA with in-cluster building. + serviceAccountAnnotations: + util: # Specify tolerations to apply to each garden-util pod. tolerations: @@ -856,6 +864,26 @@ providers: cluster-autoscaler.kubernetes.io/safe-to-evict: 'false' ``` +### `providers[].clusterBuildkit.serviceAccountAnnotations` + +[providers](#providers) > [clusterBuildkit](#providersclusterbuildkit) > serviceAccountAnnotations + +Specify annotations to apply to the Kubernetes service account used by cluster-buildkit. This can be useful to set up IRSA with in-cluster building. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +Example: + +```yaml +providers: + - clusterBuildkit: + ... + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/my-role +``` + ### `providers[].jib` [providers](#providers) > jib @@ -1016,6 +1044,26 @@ providers: cluster-autoscaler.kubernetes.io/safe-to-evict: 'false' ``` +### `providers[].kaniko.serviceAccountAnnotations` + +[providers](#providers) > [kaniko](#providerskaniko) > serviceAccountAnnotations + +Specify annotations to apply to the Kubernetes service account used by kaniko. This can be useful to set up IRSA with in-cluster building. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +Example: + +```yaml +providers: + - kaniko: + ... + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/my-role +``` + ### `providers[].kaniko.util` [providers](#providers) > [kaniko](#providerskaniko) > util diff --git a/images/buildkit/Dockerfile b/images/buildkit/Dockerfile index 4685e25f0e..c21ca66aeb 100644 --- a/images/buildkit/Dockerfile +++ b/images/buildkit/Dockerfile @@ -9,12 +9,12 @@ RUN cd /usr/local/bin && \ chmod +x docker-credential-ecr-login # GCR credential helper -RUN wget "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz" && \ - echo "90837d1d9cf16809a60d5c7891d7d0b8445b1978ad43187032a0ca93bda49ed5 docker-credential-gcr_linux_amd64-2.0.1.tar.gz" | sha256sum -c && \ - tar xzf docker-credential-gcr_linux_amd64-2.0.1.tar.gz --to-stdout ./docker-credential-gcr \ - > /usr/local/bin/docker-credential-gcr && \ - chmod +x /usr/local/bin/docker-credential-gcr && \ - rm docker-credential-gcr_linux_amd64-2.0.1.tar.gz +RUN cd /usr/local/bin && \ + wget "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.1.14/docker-credential-gcr_linux_amd64-2.1.14.tar.gz" && \ + echo "81f2d215466ab5bf6a350aadab42b42ad29590d16eab39f28014e4a6563c848a docker-credential-gcr_linux_amd64-2.1.14.tar.gz" | sha256sum -c && \ + tar xzf docker-credential-gcr_linux_amd64-2.1.14.tar.gz && \ + rm docker-credential-gcr_linux_amd64-2.1.14.tar.gz && \ + chmod +x docker-credential-gcr FROM moby/buildkit:v0.12.2-rootless@sha256:0919807170af622451887366c17408dc9a946d04c6fe4fcca3071f9637f8598f as buildkit-rootless diff --git a/images/buildkit/garden.yml b/images/buildkit/garden.yml index 57a6ce93d3..388be92dfd 100644 --- a/images/buildkit/garden.yml +++ b/images/buildkit/garden.yml @@ -2,7 +2,7 @@ kind: Module type: container name: buildkit description: Used for the cluster-buildkit build mode in the kubernetes provider -image: gardendev/buildkit:v0.12.2 +image: gardendev/buildkit:v0.12.2-1 dockerfile: Dockerfile build: targetImage: buildkit @@ -14,7 +14,7 @@ kind: Module type: container name: buildkit-rootless description: Used for the cluster-buildkit build mode in the kubernetes provider, rootless variant -image: gardendev/buildkit:v0.12.2-rootless +image: gardendev/buildkit:v0.12.2-1-rootless dockerfile: Dockerfile build: dependencies: diff --git a/images/k8s-util/Dockerfile b/images/k8s-util/Dockerfile index 2a4e125170..290d1d5e54 100644 --- a/images/k8s-util/Dockerfile +++ b/images/k8s-util/Dockerfile @@ -7,6 +7,13 @@ RUN cd /usr/local/bin && \ echo "af805202cb5d627dde2e6d4be1f519b195fd5a3a35ddc88d5010b4a4e5a98dd8 docker-credential-ecr-login" | sha256sum -c && \ chmod +x docker-credential-ecr-login +RUN cd /usr/local/bin && \ + wget "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.1.14/docker-credential-gcr_linux_amd64-2.1.14.tar.gz" && \ + echo "81f2d215466ab5bf6a350aadab42b42ad29590d16eab39f28014e4a6563c848a docker-credential-gcr_linux_amd64-2.1.14.tar.gz" | sha256sum -c && \ + tar xzf docker-credential-gcr_linux_amd64-2.1.14.tar.gz && \ + rm docker-credential-gcr_linux_amd64-2.1.14.tar.gz && \ + chmod +x docker-credential-gcr + RUN adduser -g 1000 -D user && \ mkdir -p /data && \ chown -R user:user /data diff --git a/images/k8s-util/garden.yml b/images/k8s-util/garden.yml index 852865ec00..417cd401a5 100644 --- a/images/k8s-util/garden.yml +++ b/images/k8s-util/garden.yml @@ -2,7 +2,7 @@ kind: Module type: container name: k8s-util description: Used by the kubernetes provider for build-related activities -image: gardendev/k8s-util:0.5.6 +image: gardendev/k8s-util:0.5.7 dockerfile: Dockerfile build: dependencies: [k8s-sync] diff --git a/images/skopeo/Dockerfile b/images/skopeo/Dockerfile index 4d62936c9a..ad1fe3c0c3 100644 --- a/images/skopeo/Dockerfile +++ b/images/skopeo/Dockerfile @@ -6,9 +6,9 @@ RUN cd /usr/local/bin && \ echo "af805202cb5d627dde2e6d4be1f519b195fd5a3a35ddc88d5010b4a4e5a98dd8 docker-credential-ecr-login" | sha256sum -c && \ chmod +x docker-credential-ecr-login -RUN wget "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz" && \ - echo "90837d1d9cf16809a60d5c7891d7d0b8445b1978ad43187032a0ca93bda49ed5 docker-credential-gcr_linux_amd64-2.0.1.tar.gz" | sha256sum -c && \ - tar xzf docker-credential-gcr_linux_amd64-2.0.1.tar.gz --to-stdout ./docker-credential-gcr \ - > /usr/local/bin/docker-credential-gcr && \ - chmod +x /usr/local/bin/docker-credential-gcr && \ - rm docker-credential-gcr_linux_amd64-2.0.1.tar.gz +RUN cd /usr/local/bin && \ + wget "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.1.14/docker-credential-gcr_linux_amd64-2.1.14.tar.gz" && \ + echo "81f2d215466ab5bf6a350aadab42b42ad29590d16eab39f28014e4a6563c848a docker-credential-gcr_linux_amd64-2.1.14.tar.gz" | sha256sum -c && \ + tar xzf docker-credential-gcr_linux_amd64-2.1.14.tar.gz && \ + rm docker-credential-gcr_linux_amd64-2.1.14.tar.gz && \ + chmod +x docker-credential-gcr diff --git a/images/skopeo/garden.yml b/images/skopeo/garden.yml index f0c5461620..9a4e077486 100644 --- a/images/skopeo/garden.yml +++ b/images/skopeo/garden.yml @@ -2,6 +2,6 @@ kind: Module type: container name: skopeo description: Used by the kubernetes provider for interacting with container registries within a cluster -image: gardendev/skopeo:1.41.0-3 +image: gardendev/skopeo:1.41.0-4 dockerfile: Dockerfile extraFlags: [ "--platform", "linux/amd64" ]