diff --git a/core/src/plugins/kubernetes/commands/cluster-init.ts b/core/src/plugins/kubernetes/commands/cluster-init.ts index e325a53805..5c575ec669 100644 --- a/core/src/plugins/kubernetes/commands/cluster-init.ts +++ b/core/src/plugins/kubernetes/commands/cluster-init.ts @@ -7,30 +7,33 @@ */ import type { PluginCommand } from "../../../plugin/command.js" -import { prepareSystem, getEnvironmentStatus } from "../init.js" +import { prepareEnvironment, getEnvironmentStatus } from "../init.js" import chalk from "chalk" +import { emitNonRepeatableWarning } from "../../../warnings.js" +// TODO: remove in 0.14 export const clusterInit: PluginCommand = { name: "cluster-init", - description: "Initialize or update cluster-wide Garden services.", + description: "[DEPRECATED] Initialize or update cluster-wide Garden services.", title: ({ environmentName }) => { return `Initializing/updating cluster-wide services for ${chalk.white(environmentName)} environment` }, handler: async ({ ctx, log }) => { + emitNonRepeatableWarning(log, "This command is now deprecated and will be removed in Garden 0.14.") + const status = await getEnvironmentStatus({ ctx, log }) let result = {} if (status.ready) { log.info("All services already initialized!") } else { - result = await prepareSystem({ + result = await prepareEnvironment({ ctx, log, force: true, status, - clusterInit: true, }) } diff --git a/core/src/plugins/kubernetes/commands/uninstall-garden-services.ts b/core/src/plugins/kubernetes/commands/uninstall-garden-services.ts index 45a42133e4..99539e52d5 100644 --- a/core/src/plugins/kubernetes/commands/uninstall-garden-services.ts +++ b/core/src/plugins/kubernetes/commands/uninstall-garden-services.ts @@ -8,9 +8,8 @@ import chalk from "chalk" import type { PluginCommand } from "../../../plugin/command.js" -import { getKubernetesSystemVariables } from "../init.js" import type { KubernetesPluginContext } from "../config.js" -import { getSystemGarden } from "../system.js" +import { ingressControllerUninstall } from "../nginx/ingress-controller.js" export const uninstallGardenServices: PluginCommand = { name: "uninstall-garden-services", @@ -22,25 +21,13 @@ export const uninstallGardenServices: PluginCommand = { handler: async ({ ctx, log }) => { const k8sCtx = ctx - const variables = getKubernetesSystemVariables(k8sCtx.provider.config) - const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) - const actions = await sysGarden.getActionRouter() - - const graph = await sysGarden.getConfigGraph({ log, emit: false }) - const deploys = graph.getDeploys() - - log.info("") - - const deployNames = deploys.map((s) => s.name) - const statuses = await actions.deleteDeploys({ graph, log, names: deployNames }) - - log.info("") - - const environmentStatuses = await actions.provider.cleanupAll(log) + if (k8sCtx.provider.config.setupIngressController === "nginx") { + await ingressControllerUninstall(k8sCtx, log) + } log.info(chalk.green("\nDone!")) - return { result: { serviceStatuses: statuses, environmentStatuses } } + return { result: {} } }, } diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts index 9419723512..a880891c76 100644 --- a/core/src/plugins/kubernetes/config.ts +++ b/core/src/plugins/kubernetes/config.ts @@ -9,7 +9,6 @@ import type { StringMap } from "../../config/common.js" import { joi, - joiArray, joiIdentifier, joiIdentifierDescription, joiProviderName, @@ -28,7 +27,6 @@ import { } from "../container/moduleConfig.js" import type { PluginContext } from "../../plugin-context.js" import { dedent, deline } from "../../util/string.js" -import { defaultSystemNamespace } from "./system.js" import type { SyncableKind } from "./types.js" import { syncableKinds } from "./types.js" import type { BaseTaskSpec } from "../../config/task.js" @@ -42,7 +40,9 @@ import type { SyncDefaults } from "./sync.js" import { syncDefaultsSchema } from "./sync.js" import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl.js" import { DOCS_BASE_URL } from "../../constants.js" -import { defaultKanikoImageName } from "./constants.js" +import { defaultKanikoImageName, defaultSystemNamespace } from "./constants.js" +import type { LocalKubernetesClusterType } from "./local/config.js" +import type { EphemeralKubernetesClusterType } from "./ephemeral/config.js" export interface ProviderSecretRef { name: string @@ -122,6 +122,8 @@ export interface ClusterBuildkitCacheConfig { registry?: ContainerRegistryConfig } +export type KubernetesClusterType = LocalKubernetesClusterType | EphemeralKubernetesClusterType + export interface KubernetesConfig extends BaseProviderConfig { buildMode: ContainerBuildMode clusterBuildkit?: { @@ -173,8 +175,7 @@ export interface KubernetesConfig extends BaseProviderConfig { gardenSystemNamespace: string tlsCertificates: IngressTlsCertificate[] certManager?: CertManagerConfig - clusterType?: "kind" | "minikube" | "microk8s" | "k3s" - _systemServices: string[] + clusterType?: KubernetesClusterType } export type KubernetesProvider = Provider @@ -633,7 +634,6 @@ export const kubernetesConfigBase = () => tlsCertificates: joiSparseArray(tlsCertificateSchema()) .unique("name") .description("One or more certificates to use for ingress."), - _systemServices: joiArray(joiIdentifier()).meta({ internal: true }), systemNodeSelector: joiStringMap(joi.string()) .description( dedent` diff --git a/core/src/plugins/kubernetes/constants.ts b/core/src/plugins/kubernetes/constants.ts index 22414c5f41..43fd4af5bb 100644 --- a/core/src/plugins/kubernetes/constants.ts +++ b/core/src/plugins/kubernetes/constants.ts @@ -42,6 +42,13 @@ export const buildkitRootlessImageName: DockerImageWithDigest = "gardendev/buildkit:v0.12.2-rootless@sha256:e30b7830078d51e66f1a861024dcc91f2ae5cb1108789c74d0e43ffe0d065b20" export const defaultKanikoImageName: DockerImageWithDigest = "gcr.io/kaniko-project/executor:v1.11.0-debug@sha256:32ba2214921892c2fa7b5f9c4ae6f8f026538ce6b2105a93a36a8b5ee50fe517" +export const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest = + "gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf" +export const defaultGardenIngressControllerImage: DockerImageWithDigest = + "k8s.gcr.io/ingress-nginx/controller:v1.1.3@sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2" +export const defaultGardenIngressControllerKubeWebhookCertGenImage: DockerImageWithDigest = + "k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660" export const buildkitDeploymentName = "garden-buildkit" export const buildkitContainerName = "buildkitd" +export const defaultSystemNamespace = "garden-system" diff --git a/core/src/plugins/kubernetes/container/build/local.ts b/core/src/plugins/kubernetes/container/build/local.ts index c5c618883f..a41a736ce1 100644 --- a/core/src/plugins/kubernetes/container/build/local.ts +++ b/core/src/plugins/kubernetes/container/build/local.ts @@ -93,7 +93,7 @@ export const localBuild: BuildHandler = async (params) => { } /** - * Loads a built local image to a local Kubernetes instance + * Loads a built local image to a local Kubernetes instance. */ export async function loadToLocalK8s(params: BuildActionParams<"build", ContainerBuildAction>) { const { ctx, log, action } = params diff --git a/core/src/plugins/kubernetes/ephemeral/config.ts b/core/src/plugins/kubernetes/ephemeral/config.ts index 883e7022f2..a0f0891914 100644 --- a/core/src/plugins/kubernetes/ephemeral/config.ts +++ b/core/src/plugins/kubernetes/ephemeral/config.ts @@ -10,7 +10,6 @@ import chalk from "chalk" import fsExtra from "fs-extra" const { mkdirp, writeFile } = fsExtra import { load } from "js-yaml" -import { remove } from "lodash-es" import moment from "moment" import { join } from "path" import { joi, joiProviderName } from "../../../config/common.js" @@ -19,9 +18,13 @@ import { ConfigurationError } from "../../../exceptions.js" import type { ConfigureProviderParams } from "../../../plugin/handlers/Provider/configureProvider.js" import { dedent } from "../../../util/string.js" import type { KubernetesConfig } from "../config.js" +import { defaultResources } from "../config.js" import { namespaceSchema } from "../config.js" import { EPHEMERAL_KUBERNETES_PROVIDER_NAME } from "./ephemeral.js" import { DEFAULT_GARDEN_CLOUD_DOMAIN } from "../../../constants.js" +import { defaultSystemNamespace } from "../constants.js" + +export type EphemeralKubernetesClusterType = "ephemeral" export const configSchema = () => providerConfigBaseSchema() @@ -43,12 +46,7 @@ export const configSchema = () => export async function configureProvider(params: ConfigureProviderParams) { const { base, log, projectName, ctx, config: baseConfig } = params - if (projectName === "garden-system") { - // avoid configuring ephemeral-kubernetes provider and creating ephemeral-cluster for garden-system project - return { - config: baseConfig, - } - } + log.info(`Configuring ${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider for project ${projectName}`) if (!ctx.cloudApi) { throw new ConfigurationError({ @@ -60,13 +58,16 @@ export async function configureProvider(params: ConfigureProviderParams nginxServices.includes(s)) - _systemServices.push("nginx-ephemeral") - updatedConfig.setupIngressController = "nginx" - // set default hostname - updatedConfig.defaultHostname = createEphemeralClusterResponse.ingressesHostname - } - return { config: updatedConfig, } diff --git a/core/src/plugins/kubernetes/helm/status.ts b/core/src/plugins/kubernetes/helm/status.ts index ef099a4ce9..4dfa3f1ba5 100644 --- a/core/src/plugins/kubernetes/helm/status.ts +++ b/core/src/plugins/kubernetes/helm/status.ts @@ -28,7 +28,7 @@ import { gardenAnnotationKey } from "../../../util/string.js" export const gardenCloudAECPauseAnnotation = gardenAnnotationKey("aec-status") -const helmStatusMap: { [status: string]: DeployState } = { +export const helmStatusMap: { [status: string]: DeployState } = { unknown: "unknown", deployed: "ready", deleted: "missing", diff --git a/core/src/plugins/kubernetes/init.ts b/core/src/plugins/kubernetes/init.ts index 519980ae09..3a241e955d 100644 --- a/core/src/plugins/kubernetes/init.ts +++ b/core/src/plugins/kubernetes/init.ts @@ -11,12 +11,11 @@ import { getAppNamespace, prepareNamespaces, deleteNamespaces, - getSystemNamespace, getNamespaceStatus, clearNamespaceCache, + getSystemNamespace, } from "./namespace.js" import type { KubernetesPluginContext, KubernetesConfig, KubernetesProvider, ProviderSecretRef } from "./config.js" -import { prepareSystemServices, getSystemServiceStatus, getSystemGarden } from "./system.js" import type { GetEnvironmentStatusParams, EnvironmentStatus, @@ -30,22 +29,17 @@ import type { CleanupEnvironmentResult, } from "../../plugin/handlers/Provider/cleanupEnvironment.js" import { millicpuToString, megabytesToString } from "./util.js" -import chalk from "chalk" import { deline, dedent, gardenAnnotationKey } from "../../util/string.js" -import type { DeployState } from "../../types/service.js" -import { combineStates } from "../../types/service.js" import { ConfigurationError } from "../../exceptions.js" import { readSecret } from "./secrets.js" import { systemDockerAuthSecretName, dockerAuthSecretKey } from "./constants.js" import type { V1IngressClass, V1Secret, V1Toleration } from "@kubernetes/client-node" import type { KubernetesResource } from "./types.js" -import { compareDeployedResources } from "./status/status.js" import type { PrimitiveMap } from "../../config/common.js" -import { mapValues, omit } from "lodash-es" +import { mapValues } from "lodash-es" import { getIngressApiVersion, supportedIngressApiVersions } from "./container/ingress.js" import type { Log } from "../../logger/log-entry.js" -import type { DeployStatusMap } from "../../plugin/handlers/Deploy/get-status.js" -import { isProviderEphemeralKubernetes } from "./ephemeral/ephemeral.js" +import { ingressControllerInstall, ingressControllerReady } from "./nginx/ingress-controller.js" const dockerAuthSecretType = "kubernetes.io/dockerconfigjson" const dockerAuthDocsLink = ` @@ -59,14 +53,12 @@ interface KubernetesProviderOutputs extends PrimitiveMap { } interface KubernetesEnvironmentDetail { - deployStatuses: DeployStatusMap systemReady: boolean - systemServiceState: DeployState systemCertManagerReady: boolean systemManagedCertificatesReady: boolean } -type KubernetesEnvironmentStatus = EnvironmentStatus +export type KubernetesEnvironmentStatus = EnvironmentStatus /** * Checks system service statuses (if provider has system services) @@ -82,26 +74,13 @@ export async function getEnvironmentStatus({ const api = await KubeApi.factory(log, ctx, provider) const namespaces = await prepareNamespaces({ ctx, log }) - const systemServiceNames = k8sCtx.provider.config._systemServices - const systemNamespace = await getSystemNamespace(k8sCtx, k8sCtx.provider, log) const detail: KubernetesEnvironmentDetail = { - deployStatuses: {}, systemReady: true, - systemServiceState: "unknown", systemCertManagerReady: true, systemManagedCertificatesReady: true, } - const ingressApiVersion = await getIngressApiVersion(log, api, supportedIngressApiVersions) - const ingressWarnings = await getIngressMisconfigurationWarnings( - provider.config.ingressClass, - ingressApiVersion, - log, - api - ) - ingressWarnings.forEach((w) => log.warn(w)) - const namespaceNames = mapValues(namespaces, (s) => s.namespaceName) const result: KubernetesEnvironmentStatus = { ready: true, @@ -112,52 +91,21 @@ export async function getEnvironmentStatus({ }, } - if ( - // No need to continue if we don't need any system services - systemServiceNames.length === 0 || - // Make sure we don't recurse infinitely - provider.config.namespace?.name === systemNamespace - ) { - return result - } - - const variables = getKubernetesSystemVariables(provider.config) - const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) - - // Check if builder auth secret is up-to-date - let secretsUpToDate = true - - if (provider.config.buildMode !== "local-docker") { - const authSecret = await prepareDockerAuth(api, provider, systemNamespace) - const comparison = await compareDeployedResources({ - ctx: k8sCtx, - api, - namespace: systemNamespace, - manifests: [authSecret], + if (provider.config.setupIngressController === "nginx") { + const ingressControllerReadiness = await ingressControllerReady(ctx, log) + result.ready = ingressControllerReadiness + detail.systemReady = ingressControllerReadiness + } else { + // We only need to warn about missing ingress classes if we're not using garden installed nginx + const ingressApiVersion = await getIngressApiVersion(log, api, supportedIngressApiVersions) + const ingressWarnings = await getIngressMisconfigurationWarnings( + provider.config.ingressClass, + ingressApiVersion, log, - }) - secretsUpToDate = comparison.state === "ready" - } - - // Get system service statuses - const systemServiceStatus = await getSystemServiceStatus({ - ctx: k8sCtx, - log, - sysGarden, - namespace: systemNamespace, - names: systemServiceNames, - }) - - if (!secretsUpToDate || systemServiceStatus.state !== "ready") { - result.ready = false - detail.systemReady = false + api + ) + ingressWarnings.forEach((w) => log.warn(w)) } - - detail.deployStatuses = mapValues(systemServiceStatus.serviceStatuses, (s) => omit(s, "executedAction")) - detail.systemServiceState = systemServiceStatus.state - - sysGarden.log.success("Done") - return result } @@ -175,14 +123,14 @@ export async function getIngressMisconfigurationWarnings( if (ingressApiVersion === "networking.k8s.io/v1") { // Note: We do not create the IngressClass resource automatically here so add a warning if it's not there! - const ingressclasses = await api.listResources>({ + const ingressClasses = await api.listResources>({ apiVersion: ingressApiVersion, kind: "IngressClass", log, namespace: "all", }) - const ingressclassWithCorrectName = ingressclasses.items.find((ic) => ic.metadata.name === customIngressClassName) - if (!ingressclassWithCorrectName) { + const ingressClassWithCorrectName = ingressClasses.items.find((ic) => ic.metadata.name === customIngressClassName) + if (!ingressClassWithCorrectName) { warnings.push(deline`An ingressClass “${customIngressClassName}" was set in the provider config for the Kubernetes provider but no matching IngressClass resource was found in the cluster. IngressClass resources are typically created by your Ingress Controller so this may suggest that it has not been properly set up.`) @@ -193,114 +141,28 @@ export async function getIngressMisconfigurationWarnings( } /** - * Deploys system services (if any) + * Deploys system services (if any). */ export async function prepareEnvironment( params: PrepareEnvironmentParams ): Promise { const { ctx, log, status } = params const k8sCtx = ctx - - // Prepare system services - await prepareSystem({ ...params, clusterInit: false }) - const nsStatus = await getNamespaceStatus({ ctx: k8sCtx, log, provider: k8sCtx.provider }) - ctx.events.emit("namespaceStatus", nsStatus) - return { status: { ready: true, outputs: status.outputs } } -} - -export async function prepareSystem({ - ctx, - log, - force, - status, - clusterInit, -}: PrepareEnvironmentParams & { clusterInit: boolean }) { - const k8sCtx = ctx const provider = k8sCtx.provider - const variables = getKubernetesSystemVariables(provider.config) - - const systemReady = status.detail && !!status.detail.systemReady && !force - const systemServiceNames = k8sCtx.provider.config._systemServices - - if (systemServiceNames.length === 0 || systemReady) { - return {} - } - - const deployStatuses: DeployStatusMap = (status.detail && status.detail.deployStatuses) || {} - const serviceStates = Object.values(deployStatuses).map((s) => s.detail?.state || "unknown") - const combinedState = combineStates(serviceStates) - - const remoteCluster = provider.name !== "local-kubernetes" - - // Don't attempt to prepare environment automatically when running the init or uninstall commands - if ( - !clusterInit && - ctx.command?.name === "plugins" && - (ctx.command?.args.command === "uninstall-garden-services" || ctx.command?.args.command === "cluster-init") - ) { - return {} - } - - // We require manual init if we're installing any system services to remote clusters unless the remote cluster - // is an ephemeral cluster, to avoid conflicts between users or unnecessary work. - if (!clusterInit && remoteCluster && !isProviderEphemeralKubernetes(provider)) { - const initCommand = chalk.white.bold(`garden --env=${ctx.environmentName} plugins kubernetes cluster-init`) - - if (combinedState === "ready") { - return {} - } else if ( - combinedState === "deploying" || - combinedState === "unhealthy" || - serviceStates.includes("missing") || - serviceStates.includes("unknown") - ) { - // If any of the services are not ready or missing, we throw, since builds and deployments are likely to fail. - throw new KubernetesError({ - message: deline` - One or more cluster-wide system services are missing or not ready. You need to run ${initCommand} - to initialize them, or contact a cluster admin to do so, before deploying services to this cluster. - `, - }) - } else { - // If system services are outdated but none are *missing*, we warn instead of flagging as not ready here. - // This avoids blocking users where there's variance in configuration between users of the same cluster, - // that often doesn't affect usage. - log.warn(deline` - One or more cluster-wide system services are outdated or their configuration does not match your current - configuration. You may want to run ${initCommand} to update them, or contact a cluster admin to do so. - `) - - return {} - } - } + const config = provider.config - const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) - const sysProvider = await sysGarden.resolveProvider(log, provider.name) - const systemNamespace = await getSystemNamespace(k8sCtx, sysProvider, log) - const sysApi = await KubeApi.factory(log, ctx, sysProvider) + // make sure that the system namespace exists + await getSystemNamespace(ctx, ctx.provider, log) - await sysGarden.clearBuilds() - - // Set auth secret for in-cluster builder - if (provider.config.buildMode !== "local-docker") { - log.info("Updating builder auth secret") - const authSecret = await prepareDockerAuth(sysApi, sysProvider, systemNamespace) - await sysApi.upsert({ kind: "Secret", namespace: systemNamespace, obj: authSecret, log }) + // TODO-0.13/TODO-0.14: remove this option for remote kubernetes clusters? + if (config.setupIngressController === "nginx") { + // Install nginx ingress controller + await ingressControllerInstall(k8sCtx, log) } - // Install system services - await prepareSystemServices({ - log, - sysGarden, - namespace: systemNamespace, - force, - ctx: k8sCtx, - names: systemServiceNames, - }) - - sysGarden.log.success("Done") - - return {} + const nsStatus = await getNamespaceStatus({ ctx: k8sCtx, log, provider }) + ctx.events.emit("namespaceStatus", nsStatus) + return { status: { ready: true, outputs: status.outputs } } } export async function cleanupEnvironment({ @@ -366,7 +228,7 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) { const systemNamespace = config.gardenSystemNamespace const systemTolerations: V1Toleration[] = [ { - key: "garden-system", + key: systemNamespace, operator: "Equal", value: "true", effect: "NoSchedule", @@ -391,11 +253,14 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) { } } +export type SystemVars = ReturnType + interface DockerConfigJson { experimental: string auths: { [registry: string]: { [key: string]: string } } credHelpers: { [registry: string]: any } } + export async function buildDockerAuthConfig( imagePullSecrets: ProviderSecretRef[], api: KubeApi diff --git a/core/src/plugins/kubernetes/kubectl.ts b/core/src/plugins/kubernetes/kubectl.ts index a5fbe3694e..4d9674c955 100644 --- a/core/src/plugins/kubernetes/kubectl.ts +++ b/core/src/plugins/kubernetes/kubectl.ts @@ -10,13 +10,13 @@ import { encodeYamlMulti } from "../../util/serialization.js" import type { ExecParams } from "../../util/ext-tools.js" import { PluginTool } from "../../util/ext-tools.js" import type { Log } from "../../logger/log-entry.js" -import type { KubernetesProvider } from "./config.js" +import type { KubernetesPluginContext, KubernetesProvider } from "./config.js" import type { KubernetesResource } from "./types.js" import { dedent } from "../../util/string.js" import { getResourceKey, hashManifest } from "./util.js" import type { PluginToolSpec } from "../../plugin/tools.js" import type { PluginContext } from "../../plugin-context.js" -import type { KubeApi } from "./api.js" +import { KubeApi } from "./api.js" import { KUBECTL_RETRY_OPTS, KubernetesError } from "./api.js" import fsExtra from "fs-extra" const { pathExists } = fsExtra @@ -24,6 +24,9 @@ import { ChildProcessError, ConfigurationError } from "../../exceptions.js" import type { RetryOpts } from "./retry.js" import { requestWithRetry } from "./retry.js" import { k8sManifestHashAnnotationKey } from "./status/status.js" +import { loadAll } from "js-yaml" +import { isTruthy } from "../../util/util.js" +import { readFile } from "fs/promises" // Corresponds to the default prune whitelist in `kubectl`. // See: https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/apply/prune.go#L176-L192 @@ -168,6 +171,14 @@ export async function apply({ } } +export async function applyYamlFromFile(ctx: KubernetesPluginContext, log: Log, path: string) { + const api = await KubeApi.factory(log, ctx, ctx.provider) + const manifests = loadAll((await readFile(path)).toString()) + .filter(isTruthy) + .map((m) => m as KubernetesResource) + await apply({ log, ctx, api, provider: ctx.provider, manifests, validate: false }) +} + export async function deleteResources(params: { log: Log ctx: PluginContext diff --git a/core/src/plugins/kubernetes/kubernetes.ts b/core/src/plugins/kubernetes/kubernetes.ts index 3eb345ded5..e94961b3de 100644 --- a/core/src/plugins/kubernetes/kubernetes.ts +++ b/core/src/plugins/kubernetes/kubernetes.ts @@ -27,7 +27,6 @@ import { resolve } from "path" import { dedent } from "../../util/string.js" import { kubernetesModuleSpecSchema } from "./kubernetes-type/module-config.js" import { helmModuleSpecSchema, helmModuleOutputsSchema } from "./helm/module-config.js" -import { getSystemMetadataNamespaceName } from "./system.js" import { defaultIngressClass } from "./constants.js" import { pvcModuleDefinition, persistentvolumeclaimDeployDefinition } from "./volumes/persistentvolumeclaim.js" import { helm3Spec } from "./helm/helm-cli.js" @@ -57,8 +56,6 @@ export async function configureProvider({ projectRoot, config, }: ConfigureProviderParams) { - config._systemServices = [] - // Convert string shorthand to canonical format if (isString(config.namespace)) { config.namespace = { name: config.namespace } @@ -69,8 +66,6 @@ export async function configureProvider({ } if (config.setupIngressController === "nginx") { - config._systemServices.push("ingress-controller", "default-backend") - if (!config.ingressClass) { config.ingressClass = defaultIngressClass } @@ -101,9 +96,8 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams .info("collecting provider configuration") const systemNamespace = await getSystemNamespace(k8sCtx, provider, log) - const systemMetadataNamespace = getSystemMetadataNamespaceName(provider.config) - const namespacesList = [systemNamespace, systemMetadataNamespace] + const namespacesList = [systemNamespace] if (includeProject) { const appNamespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider) namespacesList.push(appNamespace) diff --git a/core/src/plugins/kubernetes/local/config.ts b/core/src/plugins/kubernetes/local/config.ts index 32a72c8ae8..9b15b160cb 100644 --- a/core/src/plugins/kubernetes/local/config.ts +++ b/core/src/plugins/kubernetes/local/config.ts @@ -11,23 +11,14 @@ import { kubernetesConfigBase, k8sContextSchema, namespaceSchema } from "../conf import type { ConfigureProviderParams } from "../../../plugin/handlers/Provider/configureProvider.js" import { joiProviderName, joi } from "../../../config/common.js" import { getKubeConfig } from "../api.js" -import { configureMicrok8sAddons } from "./microk8s.js" -import { setMinikubeDockerEnv } from "./minikube.js" import { exec } from "../../../util/util.js" -import { remove } from "lodash-es" -import chalk from "chalk" -import { isKindCluster } from "./kind.js" -import { isK3sFamilyCluster } from "./k3s.js" -import type { K8sClientServerVersions } from "../util.js" -import { getK8sClientServerVersions } from "../util.js" -import { ChildProcessError } from "../../../exceptions.js" // TODO: split this into separate plugins to handle Docker for Mac and Minikube // note: this is in order of preference, in case neither is set as the current kubectl context // and none is explicitly configured in the garden.yml const supportedContexts = [ - "docker-for-desktop", "docker-desktop", + "docker-for-desktop", "microk8s", "minikube", "kind-kind", @@ -36,15 +27,14 @@ const supportedContexts = [ "k3d-k3s-default", "orbstack", ] -const nginxServices = ["ingress-controller", "default-backend"] + +export type LocalKubernetesClusterType = "kind" | "minikube" | "microk8s" | "k3s" | "generic" function isSupportedContext(context: string) { return supportedContexts.includes(context) || context.startsWith("kind-") } -export interface LocalKubernetesConfig extends KubernetesConfig { - setupIngressController: string | null -} +export type LocalKubernetesConfig = KubernetesConfig export const configSchema = () => kubernetesConfigBase() @@ -71,7 +61,6 @@ export async function configureProvider(params: ConfigureProviderParams nginxServices.includes(s)) - _systemServices.push("nginx-k3s") - } - } - - if (await isKindCluster(ctx, provider, providerLog)) { - config.clusterType = "kind" - - if (config.setupIngressController === "nginx") { - providerLog.debug("Using nginx-kind service for ingress") - remove(_systemServices, (s) => nginxServices.includes(s)) - let versions: K8sClientServerVersions | undefined - try { - versions = await getK8sClientServerVersions(config.context) - } catch (err) { - providerLog.debug("failed to get k8s version with error: " + err) - } - // TODO: remove this once we no longer support k8s v1.20 - if (versions && versions.serverVersion.minor >= 21) { - _systemServices.push("nginx-kind-new") - } else { - _systemServices.push("nginx-kind-old") - } - } - } else if (config.context === "minikube") { + if (config.context === "minikube") { await exec("minikube", ["config", "set", "WantUpdateNotification", "false"]) config.clusterType = "minikube" @@ -145,34 +106,6 @@ export async function configureProvider(params: ConfigureProviderParams nginxServices.includes(s)) - } - - await setMinikubeDockerEnv() - } else if (config.context === "microk8s") { - const addons = ["dns", "registry", "storage"] - - config.clusterType = "microk8s" - - if (config.setupIngressController === "nginx") { - providerLog.debug("Using microk8s's ingress addon") - addons.push("ingress") - remove(_systemServices, (s) => nginxServices.includes(s)) - _systemServices.push("nginx-ingress-class") - } - - await configureMicrok8sAddons(providerLog, addons) } if (!config.defaultHostname) { diff --git a/core/src/plugins/kubernetes/local/kind.ts b/core/src/plugins/kubernetes/local/kind.ts index 245371e752..b885a95187 100644 --- a/core/src/plugins/kubernetes/local/kind.ts +++ b/core/src/plugins/kubernetes/local/kind.ts @@ -131,15 +131,16 @@ async function getKindClusters(): Promise> { } async function getClusterForContext(context: string) { - for (const cluster of await getKindClusters()) { - if (await isContextAMatch(cluster, context)) { + const clusters = await getKindClusters() + for (const cluster of clusters) { + if (await contextMatches(cluster, context)) { return cluster } } return null } -async function isContextAMatch(cluster: string, context: string): Promise { +async function contextMatches(cluster: string, context: string): Promise { try { const kubeConfigString = (await exec("kind", ["get", "kubeconfig", `--name=${cluster}`])).stdout const kubeConfig = load(kubeConfigString)! diff --git a/core/src/plugins/kubernetes/local/local.ts b/core/src/plugins/kubernetes/local/local.ts index c57b5c75f5..74c1501906 100644 --- a/core/src/plugins/kubernetes/local/local.ts +++ b/core/src/plugins/kubernetes/local/local.ts @@ -6,10 +6,22 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import type { LocalKubernetesClusterType, LocalKubernetesConfig } from "./config.js" import { configureProvider, configSchema } from "./config.js" import { createGardenPlugin } from "../../../plugin/plugin.js" import { dedent } from "../../../util/string.js" import { DOCS_BASE_URL } from "../../../constants.js" +import type { + PrepareEnvironmentParams, + PrepareEnvironmentResult, +} from "../../../plugin/handlers/Provider/prepareEnvironment.js" +import type { KubernetesPluginContext } from "../config.js" +import { prepareEnvironment as _prepareEnvironmentBase } from "../init.js" +import type { Log } from "../../../logger/log-entry.js" +import { setMinikubeDockerEnv } from "./minikube.js" +import { isKindCluster } from "./kind.js" +import { configureMicrok8sAddons } from "./microk8s.js" +import { isK3sFamilyCluster } from "./k3s.js" const providerUrl = "./kubernetes.md" @@ -27,5 +39,47 @@ export const gardenPlugin = () => configSchema: configSchema(), handlers: { configureProvider, + prepareEnvironment, }, }) + +async function prepareEnvironment( + params: PrepareEnvironmentParams +): Promise { + const { ctx, log } = params + const provider = ctx.provider + + let clusterType = provider.config.clusterType + if (!clusterType) { + clusterType = await getClusterType(ctx, log) + provider.config.clusterType = clusterType + } + + const result = await _prepareEnvironmentBase(params) + + if (clusterType === "minikube") { + await setMinikubeDockerEnv() + } else if (clusterType === "microk8s") { + const microk8sAddons = ["dns", "registry", "storage"] + await configureMicrok8sAddons(log, microk8sAddons) + } + + return result +} + +async function getClusterType(ctx: KubernetesPluginContext, log: Log): Promise { + const provider = ctx.provider + const config = provider.config + + if (await isKindCluster(ctx, provider, log)) { + return "kind" + } else if (await isK3sFamilyCluster(ctx, provider, log)) { + return "k3s" + } else if (config.context === "minikube") { + return "minikube" + } else if (config.context === "microk8s") { + return "microk8s" + } else { + return "generic" + } +} diff --git a/core/src/plugins/kubernetes/nginx/default-backend.ts b/core/src/plugins/kubernetes/nginx/default-backend.ts new file mode 100644 index 0000000000..a3aca3935a --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/default-backend.ts @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { KubernetesPluginContext } from "../config.js" +import type { Log } from "../../../logger/log-entry.js" +import type { DeployState } from "../../../types/service.js" +import { KubeApi } from "../api.js" +import { checkResourceStatus, waitForResources } from "../status/status.js" +import chalk from "chalk" +import type { KubernetesDeployment, KubernetesService } from "../types.js" +import { defaultGardenIngressControllerDefaultBackendImage } from "../constants.js" +import { GardenIngressComponent } from "./ingress-controller-base.js" + +export class GardenDefaultBackend extends GardenIngressComponent { + override async install(ctx: KubernetesPluginContext, log: Log): Promise { + const { deployment, service } = defaultBackendGetManifests(ctx) + const status = await this.getStatus(ctx, log) + if (status === "ready") { + return + } + + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + const api = await KubeApi.factory(log, ctx, provider) + await api.upsert({ kind: "Service", namespace, log, obj: service }) + await api.upsert({ kind: "Deployment", namespace, log, obj: deployment }) + await waitForResources({ + // this is necessary to display the logs in provider-section + // because the function waitForResources uses actionName as a new Log name + actionName: "providers", + namespace, + ctx, + provider, + resources: [deployment], + log, + timeoutSec: 20, + }) + } + + override async getStatus(ctx: KubernetesPluginContext, log: Log): Promise { + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + const api = await KubeApi.factory(log, ctx, provider) + const { deployment } = defaultBackendGetManifests(ctx) + + const deploymentStatus = await checkResourceStatus({ api, namespace, manifest: deployment, log }) + log.debug(chalk.yellow(`Status of ingress controller default-backend: ${deploymentStatus}`)) + return deploymentStatus.state + } + + override async uninstall(ctx: KubernetesPluginContext, log: Log): Promise { + const { deployment, service } = defaultBackendGetManifests(ctx) + const status = await this.getStatus(ctx, log) + if (status === "missing") { + return + } + + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + const api = await KubeApi.factory(log, ctx, provider) + await api.deleteBySpec({ namespace, manifest: service, log }) + await api.deleteBySpec({ namespace, manifest: deployment, log }) + } +} + +function defaultBackendGetManifests(ctx: KubernetesPluginContext): { + deployment: KubernetesDeployment + service: KubernetesService +} { + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + + const deployment: KubernetesDeployment = { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + labels: { + app: "default-backend", + }, + name: "default-backend", + namespace, + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: "default-backend", + }, + }, + strategy: { + rollingUpdate: { + maxSurge: 1, + maxUnavailable: 1, + }, + type: "RollingUpdate", + }, + template: { + metadata: { + labels: { + app: "default-backend", + }, + }, + spec: { + containers: [ + { + image: defaultGardenIngressControllerDefaultBackendImage, + imagePullPolicy: "IfNotPresent", + name: "default-backend", + ports: [ + { + containerPort: 80, + name: "http", + protocol: "TCP", + }, + ], + resources: { + limits: { + cpu: "100m", + memory: "200Mi", + }, + requests: { + cpu: "10m", + memory: "90Mi", + }, + }, + securityContext: { + allowPrivilegeEscalation: false, + }, + }, + ], + }, + }, + }, + } + + const service: KubernetesService = { + apiVersion: "v1", + kind: "Service", + metadata: { + labels: { + app: "default-backend", + }, + name: "default-backend", + namespace, + }, + spec: { + type: "ClusterIP", + ports: [ + { + name: "http", + port: 80, + protocol: "TCP", + targetPort: 80, + }, + ], + selector: { + app: "default-backend", + }, + }, + } + return { deployment, service } +} diff --git a/core/src/plugins/kubernetes/nginx/ingress-controller-base.ts b/core/src/plugins/kubernetes/nginx/ingress-controller-base.ts new file mode 100644 index 0000000000..7cf3b26cc9 --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/ingress-controller-base.ts @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { Log } from "../../../logger/log-entry.js" +import type { KubernetesPluginContext } from "../config.js" +import type { DeployState } from "../../../types/service.js" + +export abstract class GardenIngressComponent { + abstract install(ctx: KubernetesPluginContext, log: Log): Promise + + abstract getStatus(ctx: KubernetesPluginContext, log: Log): Promise + + async ready(ctx: KubernetesPluginContext, log: Log): Promise { + return (await this.getStatus(ctx, log)) === "ready" + } + + abstract uninstall(ctx: KubernetesPluginContext, log: Log): Promise +} diff --git a/core/src/plugins/kubernetes/nginx/ingress-controller.ts b/core/src/plugins/kubernetes/nginx/ingress-controller.ts new file mode 100644 index 0000000000..8dfcfeb649 --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/ingress-controller.ts @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { Log } from "../../../logger/log-entry.js" +import type { KubernetesPluginContext } from "../config.js" +import { GenericHelmGardenIngressController } from "./nginx-helm-generic.js" +import { K3sHelmGardenIngressController } from "./nginx-helm-k3s.js" +import { Microk8sGardenIngressController } from "./nginx-microk8s.js" +import { MinikubeGardenIngressController } from "./nginx-minikube.js" +import { KindGardenIngressController } from "./nginx-kind.js" +import { EphemeralHelmGardenIngressController } from "./nginx-helm-ephemeral.js" +import { GardenIngressComponent } from "./ingress-controller-base.js" +import type { DeployState } from "../../../types/service.js" + +class NoOpGardenIngressController extends GardenIngressComponent { + override install(_ctx: KubernetesPluginContext, _log: Log): Promise { + return Promise.resolve(undefined) + } + + override getStatus(_ctx: KubernetesPluginContext, _log: Log): Promise { + return Promise.resolve("missing") + } + + override async ready(_ctx: KubernetesPluginContext, _log: Log): Promise { + return false + } + + override uninstall(_ctx: KubernetesPluginContext, _log: Log): Promise { + return Promise.resolve(undefined) + } +} + +export function getGardenIngressController(ctx: KubernetesPluginContext): GardenIngressComponent { + const clusterType = ctx.provider.config.clusterType + if (clusterType === undefined) { + return new NoOpGardenIngressController() + } + + if (clusterType === "kind") { + return new KindGardenIngressController() + } else if (clusterType === "microk8s") { + return new Microk8sGardenIngressController() + } else if (clusterType === "minikube") { + return new MinikubeGardenIngressController() + } else if (clusterType === "k3s") { + return new K3sHelmGardenIngressController() + } else if (clusterType === "generic") { + return new GenericHelmGardenIngressController() + } else if (clusterType === "ephemeral") { + return new EphemeralHelmGardenIngressController() + } else { + return clusterType satisfies never + } +} + +export async function ingressControllerReady(ctx: KubernetesPluginContext, log: Log): Promise { + return await getGardenIngressController(ctx).ready(ctx, log) +} + +export async function ingressControllerInstall(ctx: KubernetesPluginContext, log: Log) { + await getGardenIngressController(ctx).install(ctx, log) +} + +export async function ingressControllerUninstall(ctx: KubernetesPluginContext, log: Log) { + return await getGardenIngressController(ctx).uninstall(ctx, log) +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-helm-ephemeral.ts b/core/src/plugins/kubernetes/nginx/nginx-helm-ephemeral.ts new file mode 100644 index 0000000000..a2d5310a40 --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-helm-ephemeral.ts @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { SystemVars } from "../init.js" +import type { NginxHelmValues } from "./nginx-helm.js" +import { HelmGardenIngressController } from "./nginx-helm.js" + +export class EphemeralHelmGardenIngressController extends HelmGardenIngressController { + override getNginxHelmValues(systemVars: SystemVars): NginxHelmValues { + return { + name: "ingress-controller", + controller: { + kind: "Deployment", + updateStrategy: { + type: "RollingUpdate", + rollingUpdate: { + maxUnavailable: 1, + }, + }, + extraArgs: { + "default-backend-service": `${systemVars.namespace}/default-backend`, + }, + minReadySeconds: 1, + tolerations: systemVars["system-tolerations"], + nodeSelector: systemVars["system-node-selector"], + admissionWebhooks: { + enabled: false, + }, + ingressClassResource: { + name: "nginx", + enabled: true, + default: true, + }, + replicaCount: 1, + service: { + annotations: { + "kubernetes.namespace.so/expose": "true", + "kubernetes.namespace.so/exposed-port-80": "wildcard", + "kubernetes.namespace.so/exposed-port-443": "wildcard", + }, + type: "LoadBalancer", + }, + }, + defaultBackend: { + enabled: false, + }, + } + } +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-helm-generic.ts b/core/src/plugins/kubernetes/nginx/nginx-helm-generic.ts new file mode 100644 index 0000000000..0d46bbc507 --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-helm-generic.ts @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { SystemVars } from "../init.js" +import type { NginxHelmValues } from "./nginx-helm.js" +import { HelmGardenIngressController } from "./nginx-helm.js" + +export class GenericHelmGardenIngressController extends HelmGardenIngressController { + override getNginxHelmValues(systemVars: SystemVars): NginxHelmValues { + return { + name: "ingress-controller", + controller: { + kind: "DaemonSet", + updateStrategy: { + type: "RollingUpdate", + rollingUpdate: { + maxUnavailable: 1, + }, + }, + extraArgs: { + "default-backend-service": `${systemVars.namespace}/default-backend`, + }, + minReadySeconds: 1, + tolerations: systemVars["system-tolerations"], + nodeSelector: systemVars["system-node-selector"], + admissionWebhooks: { + enabled: false, + }, + ingressClassResource: { + name: "nginx", + enabled: true, + default: true, + }, + hostPort: { + enabled: true, + ports: { + http: systemVars["ingress-http-port"], + https: systemVars["ingress-https-port"], + }, + }, + }, + defaultBackend: { + enabled: false, + }, + } + } +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-helm-k3s.ts b/core/src/plugins/kubernetes/nginx/nginx-helm-k3s.ts new file mode 100644 index 0000000000..38d76d39a6 --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-helm-k3s.ts @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { SystemVars } from "../init.js" +import type { NginxHelmValues } from "./nginx-helm.js" +import { HelmGardenIngressController } from "./nginx-helm.js" + +export class K3sHelmGardenIngressController extends HelmGardenIngressController { + override getNginxHelmValues(systemVars: SystemVars): NginxHelmValues { + return { + name: "ingress-controller", + controller: { + kind: "Deployment", + updateStrategy: { + type: "RollingUpdate", + rollingUpdate: { + maxUnavailable: 1, + }, + }, + extraArgs: { + "default-backend-service": `${systemVars.namespace}/default-backend`, + }, + minReadySeconds: 1, + tolerations: systemVars["system-tolerations"], + nodeSelector: systemVars["system-node-selector"], + admissionWebhooks: { + enabled: false, + }, + ingressClassResource: { + name: "nginx", + enabled: true, + default: true, + }, + replicaCount: 1, + }, + defaultBackend: { + enabled: false, + }, + } + } +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-helm.ts b/core/src/plugins/kubernetes/nginx/nginx-helm.ts new file mode 100644 index 0000000000..5e3e6bb7eb --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-helm.ts @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import chalk from "chalk" +import type { Log } from "../../../logger/log-entry.js" +import type { DeployState } from "../../../types/service.js" +import type { KubernetesPluginContext } from "../config.js" +import { helm } from "../helm/helm-cli.js" +import { helmStatusMap } from "../helm/status.js" +import { getKubernetesSystemVariables } from "../init.js" +import type { SystemVars } from "../init.js" +import { GardenDefaultBackend } from "./default-backend.js" +import { checkResourceStatus, waitForResources } from "../status/status.js" +import { KubeApi } from "../api.js" + +import { GardenIngressComponent } from "./ingress-controller-base.js" + +const HELM_INGRESS_NGINX_REPO = "https://kubernetes.github.io/ingress-nginx" +const HELM_INGRESS_NGINX_VERSION = "4.0.13" +const HELM_INGRESS_NGINX_CHART = "ingress-nginx" +const HELM_INGRESS_NGINX_RELEASE_NAME = "garden-nginx" +const HELM_INGRESS_NGINX_DEPLOYMENT_TIMEOUT = "300s" + +type _HelmValue = number | string | boolean | object | null | undefined + +export abstract class HelmGardenIngressController extends GardenIngressComponent { + private readonly defaultBackend = new GardenDefaultBackend() + + override async install(ctx: KubernetesPluginContext, log: Log): Promise { + const ingressControllerReady = await this.ready(ctx, log) + if (ingressControllerReady) { + return + } + + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + const systemVars: SystemVars = getKubernetesSystemVariables(config) + const values = this.getNginxHelmValues(systemVars) + + const valueArgs: string[] = [] + for (const key in values) { + if (values.hasOwnProperty(key)) { + valueArgs.push(`${key}=${JSON.stringify(values[key])}`) + } + } + + const args = [ + "upgrade", + "--install", + HELM_INGRESS_NGINX_RELEASE_NAME, + HELM_INGRESS_NGINX_CHART, + "--version", + HELM_INGRESS_NGINX_VERSION, + "--repo", + HELM_INGRESS_NGINX_REPO, + "--timeout", + HELM_INGRESS_NGINX_DEPLOYMENT_TIMEOUT, + "--set-json", + `${valueArgs.join(",")}`, + ] + + log.info(`Installing nginx in ${namespace} namespace...`) + await this.defaultBackend.install(ctx, log) + await helm({ ctx, namespace, log, args, emitLogEvents: false }) + + const nginxHelmMainResource = getNginxHelmMainResource(values) + await waitForResources({ + // setting the action name to providers is necessary to display the logs in provider-section + actionName: "providers", + namespace, + ctx, + provider, + resources: [nginxHelmMainResource], + log, + timeoutSec: 60, + }) + + log.success(`nginx successfully installed in ${namespace} namespace`) + } + + override async getStatus(ctx: KubernetesPluginContext, log: Log): Promise { + const provider = ctx.provider + const config = provider.config + const api = await KubeApi.factory(log, ctx, provider) + const systemVars: SystemVars = getKubernetesSystemVariables(config) + const values = this.getNginxHelmValues(systemVars) + + const namespace = config.gardenSystemNamespace + try { + const statusRes = JSON.parse( + await helm({ + ctx, + log, + namespace, + args: ["status", HELM_INGRESS_NGINX_RELEASE_NAME, "--output", "json"], + // do not send JSON output to Garden Cloud or CLI verbose log + emitLogEvents: false, + }) + ) + const releaseStatus = statusRes.info?.status || "unknown" + + if (releaseStatus !== "deployed") { + log.debug(chalk.yellow(`Helm release status for ${HELM_INGRESS_NGINX_RELEASE_NAME}: ${releaseStatus}`)) + return helmStatusMap[releaseStatus] || "unknown" + } + + // we check that the deployment or daemonset is ready because the status of the helm release + // can be "deployed" even if the deployed resource is not ready. + const nginxHelmMainResource = getNginxHelmMainResource(values) + const deploymentStatus = await checkResourceStatus({ api, namespace, manifest: nginxHelmMainResource, log }) + return deploymentStatus.state + } catch (error) { + log.debug(chalk.yellow(`Helm release ${HELM_INGRESS_NGINX_RELEASE_NAME} missing.`)) + return "missing" + } + } + + override async ready(ctx: KubernetesPluginContext, log: Log): Promise { + const nginxStatus = await this.getStatus(ctx, log) + const backendStatus = await this.defaultBackend.getStatus(ctx, log) + + return nginxStatus === "ready" && backendStatus === "ready" + } + + override async uninstall(ctx: KubernetesPluginContext, log: Log): Promise { + const status = await this.getStatus(ctx, log) + if (status === "missing") { + return + } + + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + + await helm({ ctx, namespace, log, args: ["uninstall", HELM_INGRESS_NGINX_RELEASE_NAME], emitLogEvents: false }) + await this.defaultBackend.uninstall(ctx, log) + } + + abstract getNginxHelmValues(systemVars: SystemVars): NginxHelmValues +} + +export interface NginxHelmValues { + name: string + controller: { + kind: string + // change this if necessary to support more update strategies + updateStrategy: { + type: "RollingUpdate" + rollingUpdate: { + maxUnavailable: number + } + } + extraArgs: { + [key: string]: string + } + minReadySeconds: number + tolerations: SystemVars["system-tolerations"] + nodeSelector: SystemVars["system-node-selector"] + admissionWebhooks: { + enabled: boolean + } + ingressClassResource: { + name: string + enabled: boolean + default: boolean + } + replicaCount?: number + [key: string]: _HelmValue + } + defaultBackend: { + enabled: boolean + } + + [key: string]: _HelmValue +} + +function getNginxHelmMainResource(values: NginxHelmValues) { + return { + apiVersion: "apps/v1", + kind: values.controller.kind, + metadata: { + name: "garden-nginx-ingress-nginx-controller", + }, + } +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-kind-manifests.ts b/core/src/plugins/kubernetes/nginx/nginx-kind-manifests.ts new file mode 100644 index 0000000000..38da7ae61b --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-kind-manifests.ts @@ -0,0 +1,780 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { KubernetesResource } from "../types.js" +import { + defaultGardenIngressControllerImage, + defaultGardenIngressControllerKubeWebhookCertGenImage, +} from "../constants.js" + +export function kindNginxGetManifests(namespace: string): KubernetesResource[] { + return [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-controller", + namespace, + }, + spec: { + ports: [ + { + appProtocol: "http", + name: "http", + port: 80, + protocol: "TCP", + targetPort: "http", + }, + { + appProtocol: "https", + name: "https", + port: 443, + protocol: "TCP", + targetPort: "https", + }, + ], + selector: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + }, + type: "NodePort", + }, + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-controller-admission", + namespace, + }, + spec: { + ports: [ + { + appProtocol: "https", + name: "https-webhook", + port: 443, + targetPort: "webhook", + }, + ], + selector: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + }, + type: "ClusterIP", + }, + }, + { + apiVersion: "v1", + kind: "Namespace", + metadata: { + labels: { + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + }, + name: namespace, + }, + }, + { + apiVersion: "v1", + data: { + "allow-snippet-annotations": "true", + }, + kind: "ConfigMap", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-controller", + namespace, + }, + }, + { + apiVersion: "v1", + automountServiceAccountToken: true, + kind: "ServiceAccount", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx", + namespace, + }, + }, + { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission", + namespace, + }, + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRole", + metadata: { + labels: { + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx", + }, + rules: [ + { + apiGroups: [""], + resources: ["configmaps", "endpoints", "nodes", "pods", "secrets", "namespaces"], + verbs: ["list", "watch"], + }, + { + apiGroups: [""], + resources: ["nodes"], + verbs: ["get"], + }, + { + apiGroups: [""], + resources: ["services"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: ["networking.k8s.io"], + resources: ["ingresses"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: [""], + resources: ["events"], + verbs: ["create", "patch"], + }, + { + apiGroups: ["networking.k8s.io"], + resources: ["ingresses/status"], + verbs: ["update"], + }, + { + apiGroups: ["networking.k8s.io"], + resources: ["ingressclasses"], + verbs: ["get", "list", "watch"], + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRole", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission", + }, + rules: [ + { + apiGroups: ["admissionregistration.k8s.io"], + resources: ["validatingwebhookconfigurations"], + verbs: ["get", "update"], + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx", + namespace, + }, + rules: [ + { + apiGroups: [""], + resources: ["namespaces"], + verbs: ["get"], + }, + { + apiGroups: [""], + resources: ["configmaps", "pods", "secrets", "endpoints"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: [""], + resources: ["services"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: ["networking.k8s.io"], + resources: ["ingresses"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: ["networking.k8s.io"], + resources: ["ingresses/status"], + verbs: ["update"], + }, + { + apiGroups: ["networking.k8s.io"], + resources: ["ingressclasses"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: [""], + resourceNames: ["ingress-controller-leader"], + resources: ["configmaps"], + verbs: ["get", "update"], + }, + { + apiGroups: [""], + resources: ["configmaps"], + verbs: ["create"], + }, + { + apiGroups: [""], + resources: ["events"], + verbs: ["create", "patch"], + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission", + namespace, + }, + rules: [ + { + apiGroups: [""], + resources: ["secrets"], + verbs: ["get", "create"], + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx", + namespace, + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: "ingress-nginx", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ingress-nginx", + namespace, + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission", + namespace, + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: "ingress-nginx-admission", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ingress-nginx-admission", + namespace, + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRoleBinding", + metadata: { + labels: { + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx", + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + name: "ingress-nginx", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ingress-nginx", + namespace, + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRoleBinding", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission", + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + name: "ingress-nginx-admission", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ingress-nginx-admission", + namespace, + }, + ], + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-controller", + namespace, + }, + spec: { + minReadySeconds: 0, + revisionHistoryLimit: 10, + selector: { + matchLabels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + }, + }, + strategy: { + rollingUpdate: { + maxUnavailable: 1, + }, + type: "RollingUpdate", + }, + template: { + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + }, + }, + spec: { + containers: [ + { + args: [ + "/nginx-ingress-controller", + "--election-id=ingress-controller-leader", + "--controller-class=k8s.io/ingress-nginx", + "--ingress-class=nginx", + "--configmap=$(POD_NAMESPACE)/ingress-nginx-controller", + "--validating-webhook=:8443", + "--validating-webhook-certificate=/usr/local/certificates/cert", + "--validating-webhook-key=/usr/local/certificates/key", + "--watch-ingress-without-class=true", + "--publish-status-address=localhost", + ], + env: [ + { + name: "POD_NAME", + valueFrom: { + fieldRef: { + fieldPath: "metadata.name", + }, + }, + }, + { + name: "POD_NAMESPACE", + valueFrom: { + fieldRef: { + fieldPath: "metadata.namespace", + }, + }, + }, + { + name: "LD_PRELOAD", + value: "/usr/local/lib/libmimalloc.so", + }, + ], + image: defaultGardenIngressControllerImage, + imagePullPolicy: "IfNotPresent", + lifecycle: { + preStop: { + exec: { + command: ["/wait-shutdown"], + }, + }, + }, + livenessProbe: { + failureThreshold: 5, + httpGet: { + path: "/healthz", + port: 10254, + scheme: "HTTP", + }, + initialDelaySeconds: 10, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 1, + }, + name: "controller", + ports: [ + { + containerPort: 80, + hostPort: 80, + name: "http", + protocol: "TCP", + }, + { + containerPort: 443, + hostPort: 443, + name: "https", + protocol: "TCP", + }, + { + containerPort: 8443, + name: "webhook", + protocol: "TCP", + }, + ], + readinessProbe: { + failureThreshold: 3, + httpGet: { + path: "/healthz", + port: 10254, + scheme: "HTTP", + }, + initialDelaySeconds: 10, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 1, + }, + resources: { + requests: { + cpu: "100m", + memory: "90Mi", + }, + }, + securityContext: { + allowPrivilegeEscalation: true, + capabilities: { + add: ["NET_BIND_SERVICE"], + drop: ["ALL"], + }, + runAsUser: 101, + }, + volumeMounts: [ + { + mountPath: "/usr/local/certificates/", + name: "webhook-cert", + readOnly: true, + }, + ], + }, + ], + dnsPolicy: "ClusterFirst", + nodeSelector: { + "ingress-ready": "true", + "kubernetes.io/os": "linux", + }, + serviceAccountName: "ingress-nginx", + terminationGracePeriodSeconds: 0, + tolerations: [ + { + effect: "NoSchedule", + key: "node-role.kubernetes.io/master", + operator: "Equal", + }, + ], + volumes: [ + { + name: "webhook-cert", + secret: { + secretName: "ingress-nginx-admission", + }, + }, + ], + }, + }, + }, + }, + { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission-create", + namespace, + }, + spec: { + template: { + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission-create", + }, + spec: { + containers: [ + { + args: [ + "create", + "--host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc", + "--namespace=$(POD_NAMESPACE)", + "--secret-name=ingress-nginx-admission", + ], + env: [ + { + name: "POD_NAMESPACE", + valueFrom: { + fieldRef: { + fieldPath: "metadata.namespace", + }, + }, + }, + ], + image: defaultGardenIngressControllerKubeWebhookCertGenImage, + imagePullPolicy: "IfNotPresent", + name: "create", + securityContext: { + allowPrivilegeEscalation: false, + }, + }, + ], + nodeSelector: { + "kubernetes.io/os": "linux", + }, + restartPolicy: "OnFailure", + securityContext: { + fsGroup: 2000, + runAsNonRoot: true, + runAsUser: 2000, + }, + serviceAccountName: "ingress-nginx-admission", + }, + }, + }, + }, + { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission-patch", + namespace, + }, + spec: { + template: { + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission-patch", + }, + spec: { + containers: [ + { + args: [ + "patch", + "--webhook-name=ingress-nginx-admission", + "--namespace=$(POD_NAMESPACE)", + "--patch-mutating=false", + "--secret-name=ingress-nginx-admission", + "--patch-failure-policy=Fail", + ], + env: [ + { + name: "POD_NAMESPACE", + valueFrom: { + fieldRef: { + fieldPath: "metadata.namespace", + }, + }, + }, + ], + image: defaultGardenIngressControllerKubeWebhookCertGenImage, + imagePullPolicy: "IfNotPresent", + name: "patch", + securityContext: { + allowPrivilegeEscalation: false, + }, + }, + ], + nodeSelector: { + "kubernetes.io/os": "linux", + }, + restartPolicy: "OnFailure", + securityContext: { + fsGroup: 2000, + runAsNonRoot: true, + runAsUser: 2000, + }, + serviceAccountName: "ingress-nginx-admission", + }, + }, + }, + }, + { + apiVersion: "networking.k8s.io/v1", + kind: "IngressClass", + metadata: { + labels: { + "app.kubernetes.io/component": "controller", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + annotations: { + "ingressclass.kubernetes.io/is-default-class": "true", + }, + name: "nginx", + }, + spec: { + controller: "k8s.io/ingress-nginx", + }, + }, + { + apiVersion: "admissionregistration.k8s.io/v1", + kind: "ValidatingWebhookConfiguration", + metadata: { + labels: { + "app.kubernetes.io/component": "admission-webhook", + "app.kubernetes.io/instance": "ingress-nginx", + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + "app.kubernetes.io/version": "1.1.3", + }, + name: "ingress-nginx-admission", + }, + webhooks: [ + { + admissionReviewVersions: ["v1"], + clientConfig: { + service: { + name: "ingress-nginx-controller-admission", + namespace, + path: "/networking/v1/ingresses", + }, + }, + failurePolicy: "Fail", + matchPolicy: "Equivalent", + name: "validate.nginx.ingress.kubernetes.io", + rules: [ + { + apiGroups: ["networking.k8s.io"], + apiVersions: ["v1"], + operations: ["CREATE", "UPDATE"], + resources: ["ingresses"], + }, + ], + sideEffects: "None", + }, + ], + }, + ] +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-kind.ts b/core/src/plugins/kubernetes/nginx/nginx-kind.ts new file mode 100644 index 0000000000..c14c782ede --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-kind.ts @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { Log } from "../../../logger/log-entry.js" +import type { KubernetesPluginContext } from "../config.js" +import { KubeApi } from "../api.js" +import { checkResourceStatus, waitForResources } from "../status/status.js" +import chalk from "chalk" +import { apply, deleteResources } from "../kubectl.js" +import type { DeployState } from "../../../types/service.js" +import { kindNginxGetManifests } from "./nginx-kind-manifests.js" +import { GardenIngressComponent } from "./ingress-controller-base.js" + +const nginxKindMainResource = { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: "ingress-nginx-controller", + }, +} + +export class KindGardenIngressController extends GardenIngressComponent { + override async install(ctx: KubernetesPluginContext, log: Log): Promise { + const status = await this.getStatus(ctx, log) + if (status === "ready") { + return + } + + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + const api = await KubeApi.factory(log, ctx, provider) + + const manifests = kindNginxGetManifests(namespace) + + log.info("Installing ingress controller for kind cluster") + await apply({ log, ctx, api, provider, manifests, namespace }) + + await waitForResources({ + // setting the action name to providers is necessary to display the logs in provider-section + actionName: "providers", + namespace, + ctx, + provider, + resources: [nginxKindMainResource], + log, + timeoutSec: 60, + }) + } + + override async getStatus(ctx: KubernetesPluginContext, log: Log): Promise { + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + const api = await KubeApi.factory(log, ctx, provider) + + const deploymentStatus = await checkResourceStatus({ api, namespace, manifest: nginxKindMainResource, log }) + + log.debug(chalk.yellow(`Status of ingress controller: ${deploymentStatus.state}`)) + return deploymentStatus.state + } + + override async uninstall(ctx: KubernetesPluginContext, log: Log): Promise { + const status = await this.getStatus(ctx, log) + if (status === "missing") { + return + } + + const provider = ctx.provider + const config = provider.config + const namespace = config.gardenSystemNamespace + + const manifests = kindNginxGetManifests(namespace) + + log.info("Uninstalling ingress controller for kind cluster") + await deleteResources({ log, ctx, provider, namespace, resources: manifests }) + } +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-microk8s.ts b/core/src/plugins/kubernetes/nginx/nginx-microk8s.ts new file mode 100644 index 0000000000..d9b73f30a2 --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-microk8s.ts @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { Log } from "../../../logger/log-entry.js" +import { exec } from "../../../util/util.js" +import chalk from "chalk" +import type { KubernetesPluginContext } from "../config.js" +import { type DeployState } from "../../../types/service.js" +import { configureMicrok8sAddons } from "../local/microk8s.js" +import { waitForResources } from "../status/status.js" +import { GardenIngressComponent } from "./ingress-controller-base.js" + +export class Microk8sGardenIngressController extends GardenIngressComponent { + override async install(ctx: KubernetesPluginContext, log: Log): Promise { + const provider = ctx.provider + + const status = await this.getStatus(ctx, log) + if (status === "ready") { + return + } + log.info("Enabling microk8s ingress controller addon") + await configureMicrok8sAddons(log, ["ingress"]) + const nginxMainResource = { + apiVersion: "apps/v1", + kind: "DaemonSet", + metadata: { + name: "nginx-ingress-microk8s-controller", + }, + } + await waitForResources({ + // setting the action name to providers is necessary to display the logs in provider-section + actionName: "providers", + namespace: "ingress", + ctx, + provider, + resources: [nginxMainResource], + log, + timeoutSec: 60, + }) + } + + override async getStatus(_ctx: KubernetesPluginContext, log: Log): Promise { + // The microk8s addons implement healthchecks and auto-corrects the addon status + // in case the deployment becomes unhealthy so we can just check if the addon is enabled + const statusCommandResult = await exec("microk8s", ["status", "--format", "short"]) + const status = statusCommandResult.stdout + const addonEnabled = status.includes("core/ingress: enabled") + log.debug(chalk.yellow(`Status of microk8s ingress controller addon: ${addonEnabled ? "enabled" : "disabled"}`)) + return addonEnabled ? "ready" : "missing" + } + + override async uninstall(ctx: KubernetesPluginContext, log: Log): Promise { + const status = await this.getStatus(ctx, log) + if (status === "missing") { + return + } + log.info("Disabling microk8s ingress controller addon") + await exec("microk8s", ["disable", "ingress"]) + } +} diff --git a/core/src/plugins/kubernetes/nginx/nginx-minikube.ts b/core/src/plugins/kubernetes/nginx/nginx-minikube.ts new file mode 100644 index 0000000000..1411ba621d --- /dev/null +++ b/core/src/plugins/kubernetes/nginx/nginx-minikube.ts @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { Log } from "../../../logger/log-entry.js" +import type { DeployState } from "../../../types/service.js" +import { exec } from "../../../util/util.js" +import chalk from "chalk" +import type { KubernetesPluginContext } from "../config.js" +import { KubeApi } from "../api.js" +import { checkResourceStatus, waitForResources } from "../status/status.js" +import { GardenIngressComponent } from "./ingress-controller-base.js" + +export class MinikubeGardenIngressController extends GardenIngressComponent { + override async install(ctx: KubernetesPluginContext, log: Log): Promise { + const provider = ctx.provider + const status = await this.getStatus(ctx, log) + if (status === "ready") { + return + } + log.info("Enabling minikube ingress controller addon") + await exec("minikube", ["addons", "enable", "ingress"]) + await waitForResources({ + // setting the action name to providers is necessary to display the logs in provider-section + actionName: "providers", + namespace: "ingress-nginx", + ctx, + provider, + resources: [nginxKindMainResource], + log, + timeoutSec: 60, + }) + } + + override async getStatus(ctx: KubernetesPluginContext, log: Log): Promise { + // The minikube addons don't implement healthchecks, so we have to check the status of the addon and the deployment + const provider = ctx.provider + const api = await KubeApi.factory(log, ctx, provider) + const result = await exec("minikube", ["addons", "list", "-o=json"]) + const minikubeAddons = JSON.parse(result.stdout) as MinikubeAddons + const addonEnabled = minikubeAddons.ingress.Status === "enabled" + + if (!addonEnabled) { + log.debug(chalk.yellow("Status of minikube ingress controller addon: missing")) + return "missing" + } + //check if ingress controller deployment is ready + const deploymentStatus = await checkResourceStatus({ + api, + namespace: "ingress-nginx", + manifest: nginxKindMainResource, + log, + }) + log.debug(chalk.yellow(`Status of minikube ingress controller addon: ${deploymentStatus.state}`)) + return deploymentStatus.state + } + + override async uninstall(ctx: KubernetesPluginContext, log: Log): Promise { + const status = await this.getStatus(ctx, log) + if (status === "missing") { + return + } + log.info("Disabling minikube ingress controller addon") + await exec("minikube", ["addons", "disable", "ingress"]) + } +} + +interface MinikubeAddons { + [key: string]: { + Profile: string + Status: string + } +} + +const nginxKindMainResource = { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: "ingress-nginx-controller", + }, +} diff --git a/core/src/plugins/kubernetes/system.ts b/core/src/plugins/kubernetes/system.ts deleted file mode 100644 index 3c413c2f91..0000000000 --- a/core/src/plugins/kubernetes/system.ts +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (C) 2018-2023 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import { join } from "path" - -import { GardenApiVersion, STATIC_DIR } from "../../constants.js" -import { Garden } from "../../garden.js" -import type { KubernetesPluginContext, KubernetesConfig } from "./config.js" -import type { Log } from "../../logger/log-entry.js" -import { getSystemNamespace } from "./namespace.js" -import { PluginError } from "../../exceptions.js" -import type { DeepPrimitiveMap } from "../../config/common.js" -import { combineStates } from "../../types/service.js" -import { defaultDotIgnoreFile } from "../../util/fs.js" -import { LogLevel } from "../../logger/logger.js" -import { defaultNamespace } from "../../config/project.js" - -const systemProjectPath = join(STATIC_DIR, "kubernetes", "system") - -export const defaultSystemNamespace = "garden-system" - -export function getSystemMetadataNamespaceName(config: KubernetesConfig) { - return `${config.gardenSystemNamespace}--metadata` -} - -/** - * Note that we initialise system Garden with a custom Garden dir path. This is because - * the system modules are stored in the static directory but we want their build products - * stored at the project level. This way we can run several Garden processes at the same time - * without them all modifying the same system build directory, which can cause unexpected issues. - */ -export async function getSystemGarden( - ctx: KubernetesPluginContext, - variables: DeepPrimitiveMap, - log: Log -): Promise { - const systemNamespace = await getSystemNamespace(ctx, ctx.provider, log) - - // TODO: Find a better way to apply this. As it was, it was basically a circular dependency between these - // two providers. - // const conftestConfig = { - // environments: ["default"], - // name: "conftest-kubernetes", - // policyPath: "policy", - // testFailureThreshold: "warn", - // } - - const sysProvider: KubernetesConfig = { - ...ctx.provider.config, - environments: ["default"], - name: ctx.provider.name, - namespace: { name: systemNamespace }, - _systemServices: [], - } - - return Garden.factory(systemProjectPath, { - gardenDirPath: join(ctx.gardenDirPath, "kubernetes.garden"), - environmentString: "default", - noEnterprise: true, // we don't want to e.g. verify a client auth token or fetch secrets here - config: { - path: systemProjectPath, - apiVersion: GardenApiVersion.v1, - kind: "Project", - internal: { - basePath: systemProjectPath, - }, - name: systemNamespace, - defaultEnvironment: "default", - dotIgnoreFile: defaultDotIgnoreFile, - environments: [{ name: "default", defaultNamespace, variables: {} }], - providers: [sysProvider], - variables, - }, - commandInfo: ctx.command, - log: log - .createLog({ - name: "garden system", - fixLevel: LogLevel.debug, - }) - .info("Initializing..."), - }) -} - -interface GetSystemServicesStatusParams { - ctx: KubernetesPluginContext - sysGarden: Garden - log: Log - namespace: string - names: string[] -} - -export async function getSystemServiceStatus({ sysGarden, log, names }: GetSystemServicesStatusParams) { - const actions = await sysGarden.getActionRouter() - const graph = await sysGarden.getConfigGraph({ log, emit: false }) - - const serviceStatuses = await actions.getDeployStatuses({ - log: log.createLog({ fixLevel: LogLevel.verbose }), - graph, - names, - }) - const state = combineStates(Object.values(serviceStatuses).map((s) => s.detail?.state || "unknown")) - - return { - state, - serviceStatuses, - } -} - -interface PrepareSystemServicesParams extends GetSystemServicesStatusParams { - force: boolean -} - -export async function prepareSystemServices({ - ctx, - sysGarden, - log, - names: serviceNames, - force, -}: PrepareSystemServicesParams) { - const k8sCtx = ctx - const provider = k8sCtx.provider - - // Deploy enabled system services - if (serviceNames.length > 0) { - const actions = await sysGarden.getActionRouter() - const graph = await sysGarden.getConfigGraph({ log, emit: false }) - const { error } = await actions.deployMany({ - graph, - log, - deployNames: serviceNames, - force, - forceBuild: force, - }) - - if (error) { - throw new PluginError({ - message: `${provider.name} — an error occurred when configuring environment:\n${error}`, - }) - } - } -} diff --git a/core/src/plugins/kubernetes/types.ts b/core/src/plugins/kubernetes/types.ts index 335f0bcb21..4fe6453e45 100644 --- a/core/src/plugins/kubernetes/types.ts +++ b/core/src/plugins/kubernetes/types.ts @@ -16,6 +16,7 @@ import type { V1Pod, V1ListMeta, V1Ingress, + V1Service, } from "@kubernetes/client-node" import type { Omit } from "../../util/util.js" @@ -90,6 +91,7 @@ export type KubernetesDeployment = KubernetesResource export type KubernetesReplicaSet = KubernetesResource export type KubernetesStatefulSet = KubernetesResource export type KubernetesPod = KubernetesResource +export type KubernetesService = KubernetesResource export type KubernetesWorkload = KubernetesResource export type KubernetesIngress = KubernetesResource diff --git a/core/src/plugins/kubernetes/util.ts b/core/src/plugins/kubernetes/util.ts index 7fcf5bd6be..a9b3f62bf9 100644 --- a/core/src/plugins/kubernetes/util.ts +++ b/core/src/plugins/kubernetes/util.ts @@ -20,7 +20,7 @@ import type { SupportedRuntimeAction, } from "./types.js" import { isPodResource } from "./types.js" -import { findByName, exec } from "../../util/util.js" +import { findByName } from "../../util/util.js" import { KubeApi, KubernetesError } from "./api.js" import { gardenAnnotationKey, @@ -154,16 +154,6 @@ export interface K8sClientServerVersions { serverVersion: K8sVersion } -/** - * get objectyfied result of "kubectl version" - */ -export async function getK8sClientServerVersions(ctx: string): Promise { - const versions: K8sClientServerVersions = JSON.parse( - (await exec("kubectl", ["version", "--context", ctx, "--output", "json"])).stdout - ) - return versions -} - /** * Retrieve a list of pods based on the resource selector, deduplicated so that only the most recent * pod is returned when multiple pods with the same label are found. diff --git a/core/test/integ/src/plugins/kubernetes/container/ingress.ts b/core/test/integ/src/plugins/kubernetes/container/ingress.ts index 1b866291c2..fdd53583ef 100644 --- a/core/test/integ/src/plugins/kubernetes/container/ingress.ts +++ b/core/test/integ/src/plugins/kubernetes/container/ingress.ts @@ -20,7 +20,6 @@ import { } from "../../../../../../src/plugins/kubernetes/container/ingress.js" import type { ContainerDeployAction } from "../../../../../../src/plugins/container/moduleConfig.js" import type { ServicePortProtocol, ContainerIngressSpec } from "../../../../../../src/plugins/container/moduleConfig.js" -import { defaultSystemNamespace } from "../../../../../../src/plugins/kubernetes/system.js" import { getContainerTestGarden } from "./container.js" import type { PartialBy } from "../../../../../../src/util/util.js" import type { Resolved } from "../../../../../../src/actions/types.js" @@ -28,6 +27,7 @@ import { actionFromConfig } from "../../../../../../src/graph/actions.js" import type { DeployAction } from "../../../../../../src/actions/deploy.js" import { DEFAULT_DEPLOY_TIMEOUT_SEC } from "../../../../../../src/constants.js" import { uuidv4 } from "../../../../../../src/util/random.js" +import { defaultSystemNamespace } from "../../../../../../src/plugins/kubernetes/constants.js" const namespace = "my-namespace" const ports = [ @@ -62,7 +62,6 @@ const basicConfig: PartialConfig = { setupIngressController: null, systemNodeSelector: {}, tlsCertificates: [], - _systemServices: [], } const singleTlsConfig: PartialConfig = { diff --git a/core/test/integ/src/plugins/kubernetes/ingress-controller.ts b/core/test/integ/src/plugins/kubernetes/ingress-controller.ts new file mode 100644 index 0000000000..1e825e16b1 --- /dev/null +++ b/core/test/integ/src/plugins/kubernetes/ingress-controller.ts @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { expect } from "chai" +import type { Log } from "../../../../../src/logger/log-entry.js" +import type { KubernetesPluginContext, KubernetesProvider } from "../../../../../src/plugins/kubernetes/config.js" +import { ingressControllerReady } from "../../../../../src/plugins/kubernetes/nginx/ingress-controller.js" +import { uninstallGardenServices } from "../../../../../src/plugins/kubernetes/commands/uninstall-garden-services.js" +import { prepareEnvironment } from "../../../../../src/plugins/kubernetes/init.js" +import type { PrepareEnvironmentParams } from "../../../../../src/plugin/handlers/Provider/prepareEnvironment.js" +import { defaultEnvironmentStatus } from "../../../../../src/plugin/handlers/Provider/getEnvironmentStatus.js" +import { getContainerTestGarden } from "./container/container.js" +import type { Garden } from "../../../../../src/garden.js" + +describe("It should manage ingress controller for respective cluster type", () => { + let garden: Garden + let ctx: KubernetesPluginContext + let provider: KubernetesProvider + let log: Log + + before(async () => { + await init() + }) + + after(async () => { + await cleanup() + }) + + afterEach(async () => { + await cleanup() + }) + + const cleanup = async () => { + ctx.provider.config.setupIngressController = "nginx" + await uninstallGardenServices.handler({ + garden, + ctx, + log: garden.log, + args: [], + graph: await garden.getConfigGraph({ log: garden.log, emit: false }), + }) + } + + const init = async () => { + ;({ garden } = await getContainerTestGarden()) + provider = await garden.resolveProvider(garden.log, "local-kubernetes") + ctx = ( + await garden.getPluginContext({ provider, templateContext: undefined, events: undefined }) + ) + log = garden.log + } + + it("should install an nginx ingress controller during environment preparation when setupIngressController is set to nginx", async () => { + const params: PrepareEnvironmentParams = { + ctx, + log: garden.log, + status: defaultEnvironmentStatus, + force: false, + } + ctx.provider.config.setupIngressController = "nginx" + await prepareEnvironment(params) + const ingressControllerIsReady = await ingressControllerReady(ctx, log) + expect(ingressControllerIsReady).to.eql(true) + }) + + it("should not install an nginx ingress controller during environment preparation when setupIngressController is set to null", async () => { + const params: PrepareEnvironmentParams = { + ctx, + log: garden.log, + status: defaultEnvironmentStatus, + force: false, + } + ctx.provider.config.setupIngressController = "null" + await prepareEnvironment(params) + + const ingressControllerIsReady = await ingressControllerReady(ctx, log) + expect(ingressControllerIsReady).to.eql(false) + }) + + it("should remove an nginx ingress controller installed by garden when using plugin command", async () => { + const params: PrepareEnvironmentParams = { + ctx, + log: garden.log, + status: defaultEnvironmentStatus, + force: false, + } + ctx.provider.config.setupIngressController = "nginx" + await prepareEnvironment(params) + + const ingressControllerIsReadyAfterInstall = await ingressControllerReady(ctx, log) + expect(ingressControllerIsReadyAfterInstall).to.eql(true) + await cleanup() + const ingressControllerIsReadyAfterUninstall = await ingressControllerReady(ctx, log) + expect(ingressControllerIsReadyAfterUninstall).to.eql(false) + }) +}) diff --git a/core/test/integ/src/plugins/kubernetes/system.ts b/core/test/integ/src/plugins/kubernetes/system.ts deleted file mode 100644 index f23edb366a..0000000000 --- a/core/test/integ/src/plugins/kubernetes/system.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2018-2023 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import type { Garden } from "../../../../../src/garden.js" -import type { Provider } from "../../../../../src/config/provider.js" -import type { KubernetesConfig, KubernetesPluginContext } from "../../../../../src/plugins/kubernetes/config.js" -import { getDataDir, makeTestGarden } from "../../../../helpers.js" -import { expect } from "chai" -import { TestTask } from "../../../../../src/tasks/test.js" -import { getSystemGarden } from "../../../../../src/plugins/kubernetes/system.js" -import { getKubernetesSystemVariables } from "../../../../../src/plugins/kubernetes/init.js" -import { convertModules } from "../../../../../src/resolve-module.js" -import type { TestAction } from "../../../../../src/actions/test.js" -import { actionFromConfig } from "../../../../../src/graph/actions.js" - -describe("System services", () => { - let garden: Garden - let provider: Provider - - before(async () => { - const root = getDataDir("test-projects", "container") - garden = await makeTestGarden(root) - provider = (await garden.resolveProvider(garden.log, "local-kubernetes")) as Provider - }) - - after(async () => { - garden.close() - }) - - // TODO: Revisit this. Doesn't make sense to have the kubernetes provider depend on a provider that depends on - // the kubernetes provider. - it.skip("should use conftest to check whether system services have a valid config", async () => { - const ctx = ( - await garden.getPluginContext({ provider, templateContext: undefined, events: undefined }) - ) - const variables = getKubernetesSystemVariables(provider.config) - const systemGarden = await getSystemGarden(ctx, variables, garden.log) - const graph = await systemGarden.getConfigGraph({ log: garden.log, emit: false }) - const conftestModuleNames = (await graph.getModules()) - .filter((module) => module.name.startsWith("conftest-")) - .map((m) => m.name) - expect(conftestModuleNames.sort()).to.eql(["conftest-ingress-controller", "conftest-nginx-kind", "conftest-util"]) - }) - - it.skip("should check whether system modules pass the conftest test", async () => { - const ctx = ( - await garden.getPluginContext({ provider, templateContext: undefined, events: undefined }) - ) - const variables = getKubernetesSystemVariables(provider.config) - const systemGarden = await getSystemGarden(ctx, variables, garden.log) - const graph = await systemGarden.getConfigGraph({ log: garden.log, emit: false }) - const modules = graph.getModules().filter((module) => module.name.startsWith("conftest-")) - const actions = await convertModules(systemGarden, systemGarden.log, modules, graph.moduleGraph) - const router = await systemGarden.getActionRouter() - const tests = actions.actions.filter((a) => a.kind === "Test") - - await Promise.all( - tests.map(async (testConfig) => { - const action = (await actionFromConfig({ - config: testConfig, - configsByKey: {}, - garden: systemGarden, - graph, - log: systemGarden.log, - router, - mode: "default", - linkedSources: {}, - })) as TestAction - const resolved = await systemGarden.resolveAction({ action, graph, log: systemGarden.log }) - const testTask = new TestTask({ - garden: systemGarden, - log: garden.log, - action: resolved, - - force: false, - - graph, - }) - const key = testTask.getBaseKey() - const result = await systemGarden.processTasks({ - tasks: [testTask], - throwOnError: false, - log: systemGarden.log, - }) - expect(result[key]).to.exist - expect(result[key]?.error).to.not.exist - }) - ) - }) -}) diff --git a/core/test/unit/src/plugins/kubernetes/init.ts b/core/test/unit/src/plugins/kubernetes/init.ts index c85f6a9af2..c7c212ca7c 100644 --- a/core/test/unit/src/plugins/kubernetes/init.ts +++ b/core/test/unit/src/plugins/kubernetes/init.ts @@ -11,12 +11,11 @@ import { join } from "path" import * as td from "testdouble" import type { Garden } from "../../../../../src/garden.js" import { prepareDockerAuth, getIngressMisconfigurationWarnings } from "../../../../../src/plugins/kubernetes/init.js" -import { dockerAuthSecretKey } from "../../../../../src/plugins/kubernetes/constants.js" +import { defaultSystemNamespace, dockerAuthSecretKey } from "../../../../../src/plugins/kubernetes/constants.js" import { ConfigurationError } from "../../../../../src/exceptions.js" import type { KubernetesProvider, KubernetesConfig } from "../../../../../src/plugins/kubernetes/config.js" import { defaultResources } from "../../../../../src/plugins/kubernetes/config.js" import { gardenPlugin } from "../../../../../src/plugins/container/container.js" -import { defaultSystemNamespace } from "../../../../../src/plugins/kubernetes/system.js" import { KubeApi } from "../../../../../src/plugins/kubernetes/api.js" import { makeTestGarden, expectError, getDataDir } from "../../../../helpers.js" import type { KubernetesList, KubernetesResource } from "../../../../../src/plugins/kubernetes/types.js" @@ -62,7 +61,6 @@ const basicConfig: KubernetesConfig = { setupIngressController: null, systemNodeSelector: {}, tlsCertificates: [], - _systemServices: [], } const basicProvider: KubernetesProvider = { diff --git a/core/test/unit/src/plugins/kubernetes/kubernetes.ts b/core/test/unit/src/plugins/kubernetes/kubernetes.ts index 596949d214..511ab30394 100644 --- a/core/test/unit/src/plugins/kubernetes/kubernetes.ts +++ b/core/test/unit/src/plugins/kubernetes/kubernetes.ts @@ -9,13 +9,13 @@ import { configureProvider, gardenPlugin } from "../../../../../src/plugins/kubernetes/kubernetes.js" import type { KubernetesConfig } from "../../../../../src/plugins/kubernetes/config.js" import { defaultResources } from "../../../../../src/plugins/kubernetes/config.js" -import { defaultSystemNamespace } from "../../../../../src/plugins/kubernetes/system.js" import { expect } from "chai" import type { TempDirectory } from "../../../../helpers.js" import { makeTempDir } from "../../../../helpers.js" import { providerFromConfig } from "../../../../../src/config/provider.js" import type { Garden } from "../../../../../src/garden.js" import { makeDummyGarden } from "../../../../../src/garden.js" +import { defaultSystemNamespace } from "../../../../../src/plugins/kubernetes/constants.js" describe("kubernetes configureProvider", () => { const basicConfig: KubernetesConfig = { @@ -39,7 +39,6 @@ describe("kubernetes configureProvider", () => { setupIngressController: null, systemNodeSelector: {}, tlsCertificates: [], - _systemServices: [], } let tmpDir: TempDirectory diff --git a/docs/k8s-plugins/advanced/rbac-config.md b/docs/k8s-plugins/advanced/rbac-config.md index 4893ea92f3..3ae8869349 100644 --- a/docs/k8s-plugins/advanced/rbac-config.md +++ b/docs/k8s-plugins/advanced/rbac-config.md @@ -87,14 +87,6 @@ metadata: namespace: garden-system name: user--common rules: - # Allow port forward to build-sync services -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] - # Note: An upcoming release will remove the requirement -- apiGroups: [""] - resources: ["pods/portforward"] - verbs: ["get", "list", "create"] # Allow storing and reading test results - apiGroups: [""] resources: ["configmaps"] @@ -114,9 +106,6 @@ rules: - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["get", "list"] -- apiGroups: ["extensions", "apps"] - resources: ["deployments", "daemonsets"] - verbs: ["get", "list"] # Note: We do not store anything sensitive in secrets, aside from registry auth, # which users anyway need to be able to read and push built images. - apiGroups: [""] @@ -138,38 +127,3 @@ subjects: - namespace: kind: ServiceAccount name: user- - ---- - -# Allow building with kaniko in-cluster -# Note: An upcoming release will remove this required role -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: garden-system - name: user--kaniko -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: - - "get" - - "list" - - "create" - - "delete" - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: user--kaniko - namespace: garden-system -roleRef: - kind: Role - name: user--kaniko - apiGroup: "" -subjects: -- namespace: - kind: ServiceAccount - name: user- -``` diff --git a/static/garden.yml b/static/garden.yml deleted file mode 100644 index 340d82fbbd..0000000000 --- a/static/garden.yml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: garden.io/v1 -kind: Project -name: garden-framework -environments: - - name: local - variables: - # these are dummy variables, only used when resolving action versions - gateway-hostname: foo - function-namespace: boo -providers: - - name: local-kubernetes - # note: this context is not actually used for anything - context: docker-for-desktop \ No newline at end of file diff --git a/static/kubernetes/system/default-backend/garden.yml b/static/kubernetes/system/default-backend/garden.yml deleted file mode 100644 index 41738c9d9c..0000000000 --- a/static/kubernetes/system/default-backend/garden.yml +++ /dev/null @@ -1,10 +0,0 @@ -kind: Deploy -name: default-backend -description: Default backend ingress controller -type: container -spec: -# IMPORTANT: Please make sure to include the sha256 digest here - image: gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf - ports: - - name: http - containerPort: 80 diff --git a/static/kubernetes/system/ingress-class/garden.yml b/static/kubernetes/system/ingress-class/garden.yml deleted file mode 100644 index 704112c412..0000000000 --- a/static/kubernetes/system/ingress-class/garden.yml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Deploy -name: nginx-ingress-class -description: Special manifests for installing nginx ingress class -type: kubernetes -spec: - manifests: - - apiVersion: networking.k8s.io/v1 - kind: IngressClass - metadata: - name: nginx - spec: - controller: k8s.io/ingress-nginx diff --git a/static/kubernetes/system/ingress-controller/garden.yml b/static/kubernetes/system/ingress-controller/garden.yml deleted file mode 100644 index 3a8bfa956c..0000000000 --- a/static/kubernetes/system/ingress-controller/garden.yml +++ /dev/null @@ -1,39 +0,0 @@ -kind: Deploy -description: Ingress controller for garden development -name: ingress-controller -type: helm -dependencies: - - deploy.default-backend -spec: - chart: - name: ingress-nginx - repo: https://kubernetes.github.io/ingress-nginx - version: 4.0.13 - releaseName: garden-nginx - atomic: false - values: - name: ingress-controller - controller: - extraArgs: - default-backend-service: ${var.namespace}/default-backend - kind: DaemonSet - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - hostPort: - enabled: true - ports: - http: ${var.ingress-http-port} - https: ${var.ingress-https-port} - minReadySeconds: 1 - tolerations: ${var.system-tolerations} - nodeSelector: ${var.system-node-selector} - admissionWebhooks: - enabled: false - ingressClassResource: - name: nginx - enabled: true - default: true - defaultBackend: - enabled: false diff --git a/static/kubernetes/system/nginx-ephemeral/garden.yml b/static/kubernetes/system/nginx-ephemeral/garden.yml deleted file mode 100644 index c272fdea8e..0000000000 --- a/static/kubernetes/system/nginx-ephemeral/garden.yml +++ /dev/null @@ -1,41 +0,0 @@ -kind: Deploy -description: Ingress controller for garden development -name: nginx-ephemeral -type: helm -dependencies: - - deploy.default-backend -spec: - chart: - name: ingress-nginx - repo: https://kubernetes.github.io/ingress-nginx - version: 4.0.13 - releaseName: garden-nginx - atomic: false - values: - name: ingress-controller - controller: - extraArgs: - default-backend-service: ${var.namespace}/default-backend - kind: Deployment - replicaCount: 1 - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - minReadySeconds: 1 - tolerations: ${var.system-tolerations} - nodeSelector: ${var.system-node-selector} - admissionWebhooks: - enabled: false - ingressClassResource: - name: nginx - enabled: true - default: true - service: - annotations: - "kubernetes.namespace.so/expose": "true" - "kubernetes.namespace.so/exposed-port-80": "wildcard" - "kubernetes.namespace.so/exposed-port-443": "wildcard" - type: LoadBalancer - defaultBackend: - enabled: false diff --git a/static/kubernetes/system/nginx-k3s/garden.yml b/static/kubernetes/system/nginx-k3s/garden.yml deleted file mode 100644 index abc9926316..0000000000 --- a/static/kubernetes/system/nginx-k3s/garden.yml +++ /dev/null @@ -1,35 +0,0 @@ -kind: Deploy -description: Ingress controller for garden development -name: nginx-k3s -type: helm -dependencies: - - deploy.default-backend -spec: - chart: - name: ingress-nginx - repo: https://kubernetes.github.io/ingress-nginx - version: 4.0.13 - releaseName: garden-nginx - atomic: false - values: - name: ingress-controller - controller: - extraArgs: - default-backend-service: ${var.namespace}/default-backend - kind: Deployment - replicaCount: 1 - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - minReadySeconds: 1 - tolerations: ${var.system-tolerations} - nodeSelector: ${var.system-node-selector} - admissionWebhooks: - enabled: false - ingressClassResource: - name: nginx - enabled: true - default: true - defaultBackend: - enabled: false diff --git a/static/kubernetes/system/nginx-kind/nginx-kind-new.garden.yml b/static/kubernetes/system/nginx-kind/nginx-kind-new.garden.yml deleted file mode 100644 index 589c11fd57..0000000000 --- a/static/kubernetes/system/nginx-kind/nginx-kind-new.garden.yml +++ /dev/null @@ -1,636 +0,0 @@ -kind: Deploy -name: nginx-kind-new -description: Special manifests for installing nginx on KinD clusters -type: kubernetes -variables: - namespace: ingress-nginx -spec: - manifests: - - apiVersion: v1 - kind: Service - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-controller - namespace: ${var.namespace} - spec: - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: NodePort - - - apiVersion: v1 - kind: Service - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-controller-admission - namespace: ${var.namespace} - spec: - ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: ClusterIP - - - apiVersion: v1 - kind: Namespace - metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - name: ${var.namespace} - - - apiVersion: v1 - data: - allow-snippet-annotations: "true" - kind: ConfigMap - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-controller - namespace: ${var.namespace} - - - apiVersion: v1 - automountServiceAccountToken: true - kind: ServiceAccount - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx - namespace: ${var.namespace} - - - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission - namespace: ${var.namespace} - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx - rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update - - apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission - rules: - - apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx - namespace: ${var.namespace} - rules: - - apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update - - apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resourceNames: - - ingress-controller-leader - resources: - - configmaps - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission - namespace: ${var.namespace} - rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx - namespace: ${var.namespace} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx - subjects: - - kind: ServiceAccount - name: ingress-nginx - namespace: ${var.namespace} - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission - namespace: ${var.namespace} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission - subjects: - - kind: ServiceAccount - name: ingress-nginx-admission - namespace: ${var.namespace} - - - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx - subjects: - - kind: ServiceAccount - name: ingress-nginx - namespace: ${var.namespace} - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission - subjects: - - kind: ServiceAccount - name: ingress-nginx-admission - namespace: ${var.namespace} - - - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-controller - namespace: ${var.namespace} - spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 - selector: - matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - spec: - containers: - - args: - - /nginx-ingress-controller - - --election-id=ingress-controller-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - - --watch-ingress-without-class=true - - --publish-status-address=localhost - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.1.3@sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2 - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - hostPort: 80 - name: http - protocol: TCP - - containerPort: 443 - hostPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - runAsUser: 101 - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true - dnsPolicy: ClusterFirst - nodeSelector: - ingress-ready: "true" - kubernetes.io/os: linux - serviceAccountName: ingress-nginx - terminationGracePeriodSeconds: 0 - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Equal - volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission - - apiVersion: batch/v1 - kind: Job - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission-create - namespace: ${var.namespace} - spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - securityContext: - fsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 - serviceAccountName: ingress-nginx-admission - - apiVersion: batch/v1 - kind: Job - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission-patch - namespace: ${var.namespace} - spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - securityContext: - fsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 - serviceAccountName: ingress-nginx-admission - - apiVersion: networking.k8s.io/v1 - kind: IngressClass - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - annotations: - ingressclass.kubernetes.io/is-default-class: "true" - name: nginx - spec: - controller: k8s.io/ingress-nginx - - apiVersion: admissionregistration.k8s.io/v1 - kind: ValidatingWebhookConfiguration - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.1.3 - name: ingress-nginx-admission - webhooks: - - admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ${var.namespace} - path: /networking/v1/ingresses - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: - - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - ingresses - sideEffects: None diff --git a/static/kubernetes/system/nginx-kind/nginx-kind-old.garden.yml b/static/kubernetes/system/nginx-kind/nginx-kind-old.garden.yml deleted file mode 100644 index 7bcb6bba07..0000000000 --- a/static/kubernetes/system/nginx-kind/nginx-kind-old.garden.yml +++ /dev/null @@ -1,320 +0,0 @@ -kind: Deploy -name: nginx-kind-old -description: Special manifests for installing nginx on KinD clusters -type: kubernetes -spec: - manifests: - - apiVersion: v1 - kind: Service - metadata: - name: ingress-nginx - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - spec: - type: NodePort - ports: - - name: http - port: 80 - targetPort: 80 - protocol: TCP - - name: https - port: 443 - targetPort: 443 - protocol: TCP - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - - - apiVersion: v1 - kind: Namespace - metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - - - kind: ConfigMap - apiVersion: v1 - metadata: - name: nginx-configuration - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - - - kind: ConfigMap - apiVersion: v1 - metadata: - name: tcp-services - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - - - kind: ConfigMap - apiVersion: v1 - metadata: - name: udp-services - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - - - apiVersion: v1 - kind: ServiceAccount - metadata: - name: nginx-ingress-serviceaccount - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - - "networking.k8s.io" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - - "networking.k8s.io" - resources: - - ingresses/status - verbs: - - update - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: nginx-ingress-role - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: nginx-ingress-role-nisa-binding - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role - subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ${var.namespace} - - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: nginx-ingress-clusterrole-nisa-binding - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole - subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ${var.namespace} - - - apiVersion: apps/v1 - kind: Deployment - metadata: - name: nginx-ingress-controller - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - spec: - replicas: 1 - strategy: - # We only want one instance at a time, because we otherwise get a port conflict - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - spec: - # wait up to five minutes for the drain of connections - terminationGracePeriodSeconds: 300 - serviceAccountName: nginx-ingress-serviceaccount - nodeSelector: - kubernetes.io/os: linux - ingress-ready: "true" # <- kind customization - tolerations: # <- kind customization - - key: node-role.kubernetes.io/master - operator: Equal - effect: NoSchedule - containers: - - name: nginx-ingress-controller - # IMPORTANT: Please make sure to include the sha256 digest here - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0@sha256:b312c91d0de688a21075078982b5e3a48b13b46eda4df743317d3059fc3ca0d9 - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx - - --annotations-prefix=nginx.ingress.kubernetes.io - securityContext: - allowPrivilegeEscalation: true - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - # www-data -> 101 - runAsUser: 101 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - hostPort: 80 # <- kind customization - protocol: TCP - - name: https - containerPort: 443 - hostPort: 443 # <- kind customization - protocol: TCP - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - - - apiVersion: v1 - kind: LimitRange - metadata: - name: ingress-nginx - namespace: ${var.namespace} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - spec: - limits: - - min: - memory: 90Mi - cpu: 100m - type: Container diff --git a/static/kubernetes/system/policy/base_test.rego b/static/kubernetes/system/policy/base_test.rego deleted file mode 100644 index efb4698953..0000000000 --- a/static/kubernetes/system/policy/base_test.rego +++ /dev/null @@ -1,31 +0,0 @@ -package main - -empty(value) { - count(value) == 0 -} - -no_violations { - empty(deny) -} - -no_warnings { - empty(warn) -} - -test_deployment_without_security_context { - deny["Containers must not run as root in Deployment sample"] with input as {"kind": "Deployment", "metadata": { "name": "sample" }} -} - -test_deployment_with_security_context { - no_violations with input as {"kind": "Deployment", "metadata": {"name": "sample"}, "spec": { - "selector": { "matchLabels": { "app": "something", "release": "something" }}, - "template": { "spec": { "securityContext": { "runAsNonRoot": true }}}}} -} - -test_services_not_denied { - no_violations with input as {"kind": "Service", "metadata": { "name": "sample" }} -} - -test_services_issue_warning { - warn["Found service sample but services are not allowed"] with input as {"kind": "Service", "metadata": { "name": "sample" }} -} diff --git a/static/kubernetes/system/policy/deny.rego b/static/kubernetes/system/policy/deny.rego deleted file mode 100644 index 886fc4230d..0000000000 --- a/static/kubernetes/system/policy/deny.rego +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import data.kubernetes - -name = input.metadata.name - -deny[msg] { - kubernetes.is_deployment - toleration := { - "key": "garden-system", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule", - } - input.spec.template.spec.tolerations[_] != toleration - - msg = sprintf("Deployment %s is missing toleration of kind %v", [name, toleration]) -} diff --git a/static/kubernetes/system/policy/kubernetes.rego b/static/kubernetes/system/policy/kubernetes.rego deleted file mode 100644 index bb1671e712..0000000000 --- a/static/kubernetes/system/policy/kubernetes.rego +++ /dev/null @@ -1,9 +0,0 @@ -package kubernetes - -is_service { - input.kind = "Service" -} - -is_deployment { - input.kind = "Deployment" -} diff --git a/static/kubernetes/system/policy/labels.rego b/static/kubernetes/system/policy/labels.rego deleted file mode 100644 index 39b377ef3c..0000000000 --- a/static/kubernetes/system/policy/labels.rego +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import data.kubernetes - -name = input.metadata.name - -# TODO: Re-enable this policy (or some version thereof) -# labels { -# input.metadata.labels["app.kubernetes.io/name"] -# input.metadata.labels["app.kubernetes.io/instance"] -# input.metadata.labels["app.kubernetes.io/version"] -# input.metadata.labels["app.kubernetes.io/component"] -# input.metadata.labels["app.kubernetes.io/part-of"] -# input.metadata.labels["app.kubernetes.io/managed-by"] -# } -# -# deny[msg] { -# kubernetes.is_deployment -# not labels -# msg = sprintf("%s must include Kubernetes recommended labels: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels ", [name]) -# } diff --git a/static/kubernetes/system/policy/warn.rego b/static/kubernetes/system/policy/warn.rego deleted file mode 100644 index 192c6c540b..0000000000 --- a/static/kubernetes/system/policy/warn.rego +++ /dev/null @@ -1,6 +0,0 @@ -package main - -import data.kubernetes - -name = input.metadata.name - diff --git a/static/kubernetes/system/util/Chart.yaml b/static/kubernetes/system/util/Chart.yaml deleted file mode 100644 index 12c130d261..0000000000 --- a/static/kubernetes/system/util/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: util -description: A chart to run a persistent utility containers. -type: application -version: 0.1.1 -appVersion: 0.1.1 diff --git a/static/kubernetes/system/util/garden.yml b/static/kubernetes/system/util/garden.yml deleted file mode 100644 index 7082ef0663..0000000000 --- a/static/kubernetes/system/util/garden.yml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Deploy -name: util -description: Long lived utility containers. -type: helm -spec: - releaseName: garden-util-daemon diff --git a/static/kubernetes/system/util/templates/NOTES.txt b/static/kubernetes/system/util/templates/NOTES.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/static/kubernetes/system/util/templates/_helpers.tpl b/static/kubernetes/system/util/templates/_helpers.tpl deleted file mode 100644 index fd8a974045..0000000000 --- a/static/kubernetes/system/util/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "garden-util.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "garden-util.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "garden-util.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "garden-util.labels" -}} -helm.sh/chart: {{ include "garden-util.chart" . }} -{{ include "garden-util.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - -{{/* -Selector labels -*/}} -{{- define "garden-util.selectorLabels" -}} -app.kubernetes.io/name: {{ include "garden-util.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "garden-util.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "garden-util.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} diff --git a/static/kubernetes/system/util/templates/deployment.yaml b/static/kubernetes/system/util/templates/deployment.yaml deleted file mode 100644 index 200a809419..0000000000 --- a/static/kubernetes/system/util/templates/deployment.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "garden-util.fullname" . }} - labels: - app.kubernetes.io/name: {{ include "garden-util.name" . }} - helm.sh/chart: {{ include "garden-util.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: {{ include "garden-util.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "garden-util.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - volumes: - - name: docker-config - secret: - secretName: builder-docker-config - items: - - key: .dockerconfigjson - path: config.json - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.name }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 5 ; done"] - volumeMounts: - - name: docker-config - mountPath: /root/.docker - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/static/kubernetes/system/util/values.yaml b/static/kubernetes/system/util/values.yaml deleted file mode 100644 index da6a39f868..0000000000 --- a/static/kubernetes/system/util/values.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Default values for skopeo. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - name: gardendev/skopeo:1.41.0-3@sha256:082889982e35d7fda3b0ccd763312ceca21c0fce74b3e5e08c01081556e66533 - pullPolicy: IfNotPresent - -nameOverride: "garden-util-daemon" -fullnameOverride: "garden-util-daemon" - -resources: - limits: - cpu: "2" - memory: "4Gi" - requests: - cpu: 200m - memory: 256Mi - -registry: - hostname: garden-docker-registry - -nodeSelector: {} - -tolerations: [] - -affinity: {}