improvement(k8s): remove old system garden and improve local-k8s startup time

core/src/plugins/kubernetes/commands/cluster-init.ts

@@ -7,12 +7,13 @@
  */
 
 import type { PluginCommand } from "../../../plugin/command.js"
-import { prepareSystem, getEnvironmentStatus } from "../init.js"
+import { prepareEnvironment, getEnvironmentStatus } from "../init.js"
 import chalk from "chalk"
 
+// TODO: remove in 0.14
 export const clusterInit: PluginCommand = {
   name: "cluster-init",
-  description: "Initialize or update cluster-wide Garden services.",
+  description: "[DEPRECATED] Initialize or update cluster-wide Garden services.",
 
   title: ({ environmentName }) => {
     return `Initializing/updating cluster-wide services for ${chalk.white(environmentName)} environment`
@@ -25,12 +26,11 @@ export const clusterInit: PluginCommand = {
     if (status.ready) {
       log.info("All services already initialized!")
     } else {
-      result = await prepareSystem({
+      result = await prepareEnvironment({
         ctx,
         log,
         force: true,
         status,
-        clusterInit: true,
       })
     }
 

core/src/plugins/kubernetes/commands/uninstall-garden-services.ts

@@ -8,9 +8,8 @@
 
 import chalk from "chalk"
 import type { PluginCommand } from "../../../plugin/command.js"
-import { getKubernetesSystemVariables } from "../init.js"
 import type { KubernetesPluginContext } from "../config.js"
-import { getSystemGarden } from "../system.js"
+import { ingressControllerUninstall } from "../nginx/ingress-controller.js"
 
 export const uninstallGardenServices: PluginCommand = {
   name: "uninstall-garden-services",
@@ -22,25 +21,13 @@ export const uninstallGardenServices: PluginCommand = {
 
   handler: async ({ ctx, log }) => {
     const k8sCtx = <KubernetesPluginContext>ctx
-    const variables = getKubernetesSystemVariables(k8sCtx.provider.config)
 
-    const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log)
-    const actions = await sysGarden.getActionRouter()
-
-    const graph = await sysGarden.getConfigGraph({ log, emit: false })
-    const deploys = graph.getDeploys()
-
-    log.info("")
-
-    const deployNames = deploys.map((s) => s.name)
-    const statuses = await actions.deleteDeploys({ graph, log, names: deployNames })
-
-    log.info("")
-
-    const environmentStatuses = await actions.provider.cleanupAll(log)
+    if (k8sCtx.provider.config.setupIngressController === "nginx") {
+      await ingressControllerUninstall(k8sCtx, log)
+    }
 
     log.info(chalk.green("\nDone!"))
 
-    return { result: { serviceStatuses: statuses, environmentStatuses } }
+    return { result: {} }
   },
 }

core/src/plugins/kubernetes/config.ts

@@ -9,7 +9,6 @@
 import type { StringMap } from "../../config/common.js"
 import {
   joi,
-  joiArray,
   joiIdentifier,
   joiIdentifierDescription,
   joiProviderName,
@@ -28,7 +27,6 @@ import {
 } from "../container/moduleConfig.js"
 import type { PluginContext } from "../../plugin-context.js"
 import { dedent, deline } from "../../util/string.js"
-import { defaultSystemNamespace } from "./system.js"
 import type { SyncableKind } from "./types.js"
 import { syncableKinds } from "./types.js"
 import type { BaseTaskSpec } from "../../config/task.js"
@@ -42,7 +40,9 @@ import type { SyncDefaults } from "./sync.js"
 import { syncDefaultsSchema } from "./sync.js"
 import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl.js"
 import { DOCS_BASE_URL } from "../../constants.js"
-import { defaultKanikoImageName } from "./constants.js"
+import { defaultKanikoImageName, defaultSystemNamespace } from "./constants.js"
+import type { LocalKubernetesClusterType } from "./local/config.js"
+import type { EphemeralKubernetesClusterType } from "./ephemeral/config.js"
 
 export interface ProviderSecretRef {
   name: string
@@ -122,6 +122,8 @@ export interface ClusterBuildkitCacheConfig {
   registry?: ContainerRegistryConfig
 }
 
+export type KubernetesClusterType = LocalKubernetesClusterType | EphemeralKubernetesClusterType
+
 export interface KubernetesConfig extends BaseProviderConfig {
   buildMode: ContainerBuildMode
   clusterBuildkit?: {
@@ -173,8 +175,7 @@ export interface KubernetesConfig extends BaseProviderConfig {
   gardenSystemNamespace: string
   tlsCertificates: IngressTlsCertificate[]
   certManager?: CertManagerConfig
-  clusterType?: "kind" | "minikube" | "microk8s" | "k3s"
-  _systemServices: string[]
+  clusterType?: KubernetesClusterType
 }
 
 export type KubernetesProvider = Provider<KubernetesConfig>
@@ -633,7 +634,6 @@ export const kubernetesConfigBase = () =>
       tlsCertificates: joiSparseArray(tlsCertificateSchema())
         .unique("name")
         .description("One or more certificates to use for ingress."),
-      _systemServices: joiArray(joiIdentifier()).meta({ internal: true }),
       systemNodeSelector: joiStringMap(joi.string())
         .description(
           dedent`

core/src/plugins/kubernetes/constants.ts

@@ -42,6 +42,13 @@ export const buildkitRootlessImageName: DockerImageWithDigest =
   "gardendev/buildkit:v0.12.2-rootless@sha256:e30b7830078d51e66f1a861024dcc91f2ae5cb1108789c74d0e43ffe0d065b20"
 export const defaultKanikoImageName: DockerImageWithDigest =
   "gcr.io/kaniko-project/executor:v1.11.0-debug@sha256:32ba2214921892c2fa7b5f9c4ae6f8f026538ce6b2105a93a36a8b5ee50fe517"
+export const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest =
+  "gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf"
+export const defaultGardenIngressControllerImage: DockerImageWithDigest =
+  "k8s.gcr.io/ingress-nginx/controller:v1.1.3@sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2"
+export const defaultGardenIngressControllerKubeWebhookCertGenImage: DockerImageWithDigest =
+  "k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660"
 
 export const buildkitDeploymentName = "garden-buildkit"
 export const buildkitContainerName = "buildkitd"
+export const defaultSystemNamespace = "garden-system"

core/src/plugins/kubernetes/container/build/local.ts

@@ -93,7 +93,7 @@ export const localBuild: BuildHandler = async (params) => {
 }
 
 /**
- * Loads a built local image to a local Kubernetes instance
+ * Loads a built local image to a local Kubernetes instance.
  */
 export async function loadToLocalK8s(params: BuildActionParams<"build", ContainerBuildAction>) {
   const { ctx, log, action } = params

core/src/plugins/kubernetes/ephemeral/config.ts

@@ -10,7 +10,6 @@ import chalk from "chalk"
 import fsExtra from "fs-extra"
 const { mkdirp, writeFile } = fsExtra
 import { load } from "js-yaml"
-import { remove } from "lodash-es"
 import moment from "moment"
 import { join } from "path"
 import { joi, joiProviderName } from "../../../config/common.js"
@@ -19,9 +18,13 @@ import { ConfigurationError } from "../../../exceptions.js"
 import type { ConfigureProviderParams } from "../../../plugin/handlers/Provider/configureProvider.js"
 import { dedent } from "../../../util/string.js"
 import type { KubernetesConfig } from "../config.js"
+import { defaultResources } from "../config.js"
 import { namespaceSchema } from "../config.js"
 import { EPHEMERAL_KUBERNETES_PROVIDER_NAME } from "./ephemeral.js"
 import { DEFAULT_GARDEN_CLOUD_DOMAIN } from "../../../constants.js"
+import { defaultSystemNamespace } from "../constants.js"
+
+export type EphemeralKubernetesClusterType = "ephemeral"
 
 export const configSchema = () =>
   providerConfigBaseSchema()
@@ -43,12 +46,7 @@ export const configSchema = () =>
 
 export async function configureProvider(params: ConfigureProviderParams<KubernetesConfig>) {
   const { base, log, projectName, ctx, config: baseConfig } = params
-  if (projectName === "garden-system") {
-    // avoid configuring ephemeral-kubernetes provider and creating ephemeral-cluster for garden-system project
-    return {
-      config: baseConfig,
-    }
-  }
+
   log.info(`Configuring ${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider for project ${projectName}`)
   if (!ctx.cloudApi) {
     throw new ConfigurationError({
@@ -60,13 +58,16 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
       message: `${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider is currently not supported for ${ctx.cloudApi.distroName}.`,
     })
   }
+
   // creating tmp dir .garden/ephemeral-kubernetes for storing kubeconfig
   const ephemeralClusterDirPath = join(ctx.gardenDirPath, "ephemeral-kubernetes")
   await mkdirp(ephemeralClusterDirPath)
+
   log.info("Retrieving ephemeral Kubernetes cluster")
   const createEphemeralClusterResponse = await ctx.cloudApi.createEphemeralCluster()
   const clusterId = createEphemeralClusterResponse.instanceMetadata.instanceId
   log.info(`Ephemeral Kubernetes cluster retrieved successfully`)
+
   const deadlineDateTime = moment(createEphemeralClusterResponse.instanceMetadata.deadline)
   const diffInNowAndDeadline = moment.duration(deadlineDateTime.diff(moment())).asMinutes().toFixed(1)
   log.info(
@@ -76,6 +77,7 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
       )}`
     )
   )
+
   log.info("Fetching kubeconfig for the ephemeral cluster")
   const kubeConfig = await ctx.cloudApi.getKubeConfigForCluster(clusterId)
   const kubeconfigFileName = `${clusterId}-kubeconfig.yaml`
@@ -84,8 +86,7 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
   log.info(`Kubeconfig for ephemeral cluster saved at path: ${chalk.underline(kubeConfigPath)}`)
 
   const parsedKubeConfig: any = load(kubeConfig)
-  const currentContext = parsedKubeConfig["current-context"]
-  baseConfig.context = currentContext
+  baseConfig.context = parsedKubeConfig["current-context"]
   baseConfig.kubeconfig = kubeConfigPath
 
   // set deployment registry
@@ -94,15 +95,21 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
     namespace: createEphemeralClusterResponse.registry.repository,
     insecure: false,
   }
+
   // set imagePullSecrets
   baseConfig.imagePullSecrets = [
     {
       name: createEphemeralClusterResponse.registry.imagePullSecret.name,
       namespace: createEphemeralClusterResponse.registry.imagePullSecret.namespace,
     },
   ]
+
   // set build mode to kaniko
   baseConfig.buildMode = "kaniko"
+
+  // set resource requests and limits defaults for builder, sync and util
+  baseConfig.resources = defaultResources
+
   // set additional kaniko flags
   baseConfig.kaniko = {
     extraFlags: [
@@ -112,28 +119,24 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
       "--force",
     ],
   }
-  // set setupIngressController to null while initializing kubernetes plugin
-  // as we use it later and configure it separately for ephemeral-kubernetes
+
+  // set default hostname
+  baseConfig.defaultHostname = createEphemeralClusterResponse.ingressesHostname
+
+  // use garden-system as system namespace for ephemeral-kubernetes
+  baseConfig.gardenSystemNamespace = defaultSystemNamespace
+
+  // set the proper cluster type explicitly
+  baseConfig.clusterType = "ephemeral"
+
   const kubernetesPluginConfig = {
     ...params,
     config: {
       ...baseConfig,
-      setupIngressController: null,
     },
   }
   const { config: updatedConfig } = await base!(kubernetesPluginConfig)
 
-  // setup ingress controller unless setupIngressController is set to false/null in provider config
-  if (baseConfig.setupIngressController) {
-    const _systemServices = updatedConfig._systemServices
-    const nginxServices = ["ingress-controller", "default-backend"]
-    remove(_systemServices, (s) => nginxServices.includes(s))
-    _systemServices.push("nginx-ephemeral")
-    updatedConfig.setupIngressController = "nginx"
-    // set default hostname
-    updatedConfig.defaultHostname = createEphemeralClusterResponse.ingressesHostname
-  }
-
   return {
     config: updatedConfig,
   }

core/src/plugins/kubernetes/helm/status.ts

@@ -28,7 +28,7 @@ import { gardenAnnotationKey } from "../../../util/string.js"
 
 export const gardenCloudAECPauseAnnotation = gardenAnnotationKey("aec-status")
 
-const helmStatusMap: { [status: string]: DeployState } = {
+export const helmStatusMap: { [status: string]: DeployState } = {
   unknown: "unknown",
   deployed: "ready",
   deleted: "missing",