fix(k8s): ensure system namespace exists before using it

garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts

@@ -27,6 +27,7 @@ import { execInWorkload } from "../container/exec"
 import { dedent, deline } from "../../../util/string"
 import { execInBuilder, getBuilderPodName, BuilderExecParams, buildSyncDeploymentName } from "../container/build"
 import { getPods } from "../util"
+import { getSystemNamespace } from "../namespace"
 
 const workspaceSyncDirTtl = 0.5 * 86400 // 2 days
 
@@ -204,8 +205,7 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K
   })
 
   const provider = ctx.provider
-  const systemNamespace = provider.config.gardenSystemNamespace
-
+  const systemNamespace = await getSystemNamespace(provider, log)
   // Restart the registry in read-only mode
   // -> Get the original deployment
   log.info("Fetching original Deployment")
@@ -410,7 +410,7 @@ async function cleanupBuildSyncVolume(provider: KubernetesProvider, log: LogEntr
 // (doesn't matter which one, they all use the same volume)
 async function getBuildSyncPodName(provider: KubernetesProvider, log: LogEntry) {
   const api = await KubeApi.factory(log, provider)
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   const builderStatusRes = await api.apps.readNamespacedDeployment(buildSyncDeploymentName, systemNamespace)
   const builderPods = await getPods(api, systemNamespace, builderStatusRes.spec.selector.matchLabels)
@@ -428,7 +428,7 @@ async function getBuildSyncPodName(provider: KubernetesProvider, log: LogEntry)
 
 async function execInBuildSync({ provider, log, args, timeout, podName }: BuilderExecParams) {
   const execCmd = ["exec", "-i", podName, "--", ...args]
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   log.verbose(`Running: kubectl ${execCmd.join(" ")}`)
 

garden-service/src/plugins/kubernetes/commands/cluster-init.ts

@@ -11,6 +11,7 @@ import { prepareSystem, getEnvironmentStatus } from "../init"
 import chalk from "chalk"
 import { helm } from "../helm/helm-cli"
 import { KubernetesPluginContext, KubernetesProvider } from "../config"
+import { getSystemNamespace } from "../namespace"
 
 export const clusterInit: PluginCommand = {
   name: "cluster-init",
@@ -41,11 +42,12 @@ export const clusterInit: PluginCommand = {
 
     log.info("Cleaning up old resources...")
 
+    const systemNamespace = await getSystemNamespace(provider, log)
     try {
       await helm({
         ctx: k8sCtx,
         log,
-        namespace: provider.config.gardenSystemNamespace,
+        namespace: systemNamespace,
         args: ["delete", "--purge", "garden-nfs-provisioner"],
       })
     } catch (_) {}

garden-service/src/plugins/kubernetes/container/build.ts

@@ -30,6 +30,7 @@ import { Writable } from "stream"
 import { LogLevel } from "../../../logger/log-node"
 import { exec, renderOutputStream } from "../../../util/util"
 import { loadLocalImage } from "../local/kind"
+import { getSystemNamespace } from "../namespace"
 
 const dockerDaemonDeploymentName = "garden-docker-daemon"
 const dockerDaemonContainerName = "docker-daemon"
@@ -121,7 +122,7 @@ const localBuild: BuildHandler = async (params) => {
 const remoteBuild: BuildHandler = async (params) => {
   const { ctx, module, log } = params
   const provider = <KubernetesProvider>ctx.provider
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   if (!(await containerHelpers.hasDockerfile(module))) {
     return {}
@@ -285,7 +286,7 @@ const buildHandlers: { [mode in ContainerBuildMode]: BuildHandler } = {
 // TODO: we should make a simple service around this instead of execing into containers
 export async function execInBuilder({ provider, log, args, timeout, podName, stdout, stderr }: BuilderExecParams) {
   const execCmd = ["exec", "-i", podName, "-c", dockerDaemonContainerName, "--", ...args]
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   log.verbose(`Running: kubectl ${execCmd.join(" ")}`)
 
@@ -302,7 +303,7 @@ export async function execInBuilder({ provider, log, args, timeout, podName, std
 
 export async function getBuilderPodName(provider: KubernetesProvider, log: LogEntry) {
   const pod = await getRunningPodInDeployment(dockerDaemonDeploymentName, provider, log)
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   if (!pod) {
     throw new PluginError(`Could not find running image builder`, {
@@ -324,7 +325,7 @@ interface RunKanikoParams {
 
 async function runKaniko({ provider, log, module, args, outputStream }: RunKanikoParams) {
   const api = await KubeApi.factory(log, provider)
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   const podName = makePodName("kaniko", module.name, Math.round(new Date().getTime()).toString())
   const registryHostname = getRegistryHostname(provider.config)

garden-service/src/plugins/kubernetes/container/util.ts

@@ -12,10 +12,10 @@ import { getPortForward } from "../port-forward"
 import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../constants"
 import { containerHelpers } from "../../container/helpers"
 import { PluginError } from "../../../exceptions"
-import { PluginContext } from "../../../plugin-context"
 import { LogEntry } from "../../../logger/log-entry"
 import { KubernetesPluginContext } from "../config"
 import axios, { AxiosRequestConfig } from "axios"
+import { getSystemNamespace } from "../namespace"
 
 export async function queryRegistry(
   ctx: KubernetesPluginContext,
@@ -30,8 +30,8 @@ export async function queryRegistry(
   return axios({ url, ...opts })
 }
 
-export async function getRegistryPortForward(ctx: PluginContext, log: LogEntry) {
-  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+export async function getRegistryPortForward(ctx: KubernetesPluginContext, log: LogEntry) {
+  const systemNamespace = await getSystemNamespace(ctx.provider, log)
 
   return getPortForward({
     ctx,

garden-service/src/plugins/kubernetes/init.ts

@@ -7,7 +7,13 @@
  */
 
 import { KubeApi, KubernetesError } from "./api"
-import { getAppNamespace, prepareNamespaces, deleteNamespaces, getMetadataNamespace } from "./namespace"
+import {
+  getAppNamespace,
+  prepareNamespaces,
+  deleteNamespaces,
+  getMetadataNamespace,
+  getSystemNamespace,
+} from "./namespace"
 import { KubernetesPluginContext, KubernetesConfig, KubernetesProvider } from "./config"
 import { checkTillerStatus, migrateToHelm3 } from "./helm/tiller"
 import { prepareSystemServices, getSystemServiceStatus, getSystemGarden, systemNamespaceUpToDate } from "./system"
@@ -33,6 +39,7 @@ import { V1Secret } from "@kubernetes/client-node"
 import { KubernetesResource } from "./types"
 import { compareDeployedResources } from "./status/status"
 import { PrimitiveMap } from "../../config/common"
+import { LogEntry } from "../../logger/log-entry"
 
 // Note: We need to increment a version number here if we ever make breaking changes to the NFS provisioner StatefulSet
 const nfsStorageClassVersion = 2
@@ -95,7 +102,7 @@ export async function getEnvironmentStatus({
   }
 
   const systemServiceNames = k8sCtx.provider.config._systemServices
-  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(k8sCtx.provider, log)
 
   const detail: KubernetesEnvironmentDetail = {
     projectHelmMigrated,
@@ -182,7 +189,7 @@ export async function getEnvironmentStatus({
   let secretsUpToDate = true
 
   if (provider.config.buildMode !== "local-docker") {
-    const authSecret = await prepareDockerAuth(api, provider)
+    const authSecret = await prepareDockerAuth(api, provider, log)
     const comparison = await compareDeployedResources(k8sCtx, api, systemNamespace, [authSecret], log)
     secretsUpToDate = comparison.state === "ready"
   }
@@ -212,10 +219,7 @@ export async function getEnvironmentStatus({
 }
 
 /**
- * Performs the following actions to prepare the environment
- *  1. Installs Tiller in project namespace
- *  2. Installs Tiller in system namespace (if provider has system services)
- *  3. Deploys system services (if provider has system services)
+ * Deploys system services (if any)
  */
 export async function prepareEnvironment(
   params: PrepareEnvironmentParams<KubernetesEnvironmentStatus>
@@ -224,7 +228,7 @@ export async function prepareEnvironment(
   const k8sCtx = <KubernetesPluginContext>ctx
 
   // Migrate from Helm 2.x and remove Tiller from project namespace, if necessary
-  const systemNamespace = k8sCtx.provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(k8sCtx.provider, log)
 
   if (
     k8sCtx.provider.config.namespace !== systemNamespace &&
@@ -319,7 +323,7 @@ export async function prepareSystem({
   const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log)
   const sysProvider = <KubernetesProvider>await sysGarden.resolveProvider(provider.name)
   const sysCtx = <KubernetesPluginContext>sysGarden.getPluginContext(sysProvider)
-  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(sysProvider, log)
   const sysApi = await KubeApi.factory(log, sysProvider)
 
   await sysGarden.clearBuilds()
@@ -331,7 +335,7 @@ export async function prepareSystem({
 
   // Set auth secret for in-cluster builder
   if (provider.config.buildMode !== "local-docker") {
-    const authSecret = await prepareDockerAuth(sysApi, sysProvider)
+    const authSecret = await prepareDockerAuth(sysApi, sysProvider, log)
     await sysApi.upsert({ kind: "Secret", namespace: systemNamespace, obj: authSecret, log })
   }
 
@@ -461,7 +465,11 @@ export function getRegistryHostname(config: KubernetesConfig) {
   const systemNamespace = config.gardenSystemNamespace
   return `garden-docker-registry.${systemNamespace}.svc.cluster.local`
 }
-async function prepareDockerAuth(api: KubeApi, provider: KubernetesProvider): Promise<KubernetesResource<V1Secret>> {
+async function prepareDockerAuth(
+  api: KubeApi,
+  provider: KubernetesProvider,
+  log: LogEntry
+): Promise<KubernetesResource<V1Secret>> {
   // Read all configured imagePullSecrets and combine into a docker config file to use in the in-cluster builders.
   const auths: { [name: string]: any } = {}
 
@@ -525,7 +533,7 @@ async function prepareDockerAuth(api: KubeApi, provider: KubernetesProvider): Pr
   const config = { auths }
 
   // Store the config as a Secret (overwriting if necessary)
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   return {
     apiVersion: "v1",

garden-service/src/plugins/kubernetes/kubernetes.ts

@@ -10,7 +10,7 @@ import Bluebird from "bluebird"
 
 import { createGardenPlugin } from "../../types/plugin/plugin"
 import { helmHandlers } from "./helm/handlers"
-import { getAppNamespace, getMetadataNamespace } from "./namespace"
+import { getAppNamespace, getMetadataNamespace, getSystemNamespace } from "./namespace"
 import { getSecret, setSecret, deleteSecret } from "./secrets"
 import { getEnvironmentStatus, prepareEnvironment, cleanupEnvironment } from "./init"
 import { containerHandlers } from "./container/handlers"
@@ -126,7 +126,7 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams
   const provider = k8sCtx.provider
   const entry = log.info({ section: ctx.provider.name, msg: "collecting provider configuration", status: "active" })
 
-  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
   const systemMetadataNamespace = getSystemMetadataNamespaceName(provider.config)
 
   const namespacesList = [systemNamespace, systemMetadataNamespace]

garden-service/src/plugins/kubernetes/namespace.ts

@@ -88,6 +88,15 @@ export async function getNamespace({
   return namespace
 }
 
+export async function getSystemNamespace(provider: KubernetesProvider, log: LogEntry): Promise<string> {
+  const namespace = provider.config.gardenSystemNamespace
+
+  const api = await KubeApi.factory(log, provider)
+  await ensureNamespace(api, namespace)
+
+  return namespace
+}
+
 export async function getAppNamespace(ctx: PluginContext, log: LogEntry, provider: KubernetesProvider) {
   return getNamespace({
     log,

garden-service/src/plugins/kubernetes/system.ts

@@ -16,7 +16,7 @@ import { Garden } from "../../garden"
 import { KubernetesPluginContext, KubernetesConfig } from "./config"
 import { LogEntry } from "../../logger/log-entry"
 import { KubeApi } from "./api"
-import { createNamespace } from "./namespace"
+import { createNamespace, getSystemNamespace } from "./namespace"
 import { getPackageVersion } from "../../util/util"
 import { deline, gardenAnnotationKey } from "../../util/string"
 import { deleteNamespaces } from "./namespace"
@@ -50,7 +50,7 @@ export async function getSystemGarden(
   variables: PrimitiveMap,
   log: LogEntry
 ): Promise<Garden> {
-  const systemNamespace = ctx.provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(ctx.provider, log)
 
   const sysProvider: KubernetesConfig = {
     ...ctx.provider.config,

garden-service/src/plugins/kubernetes/test-results.ts

@@ -21,6 +21,7 @@ import hasha from "hasha"
 import { gardenAnnotationKey } from "../../util/string"
 import { upsertConfigMap } from "./util"
 import { trimRunOutput } from "./helm/common"
+import { getSystemNamespace } from "./namespace"
 
 export async function getTestResult({
   ctx,
@@ -31,7 +32,7 @@ export async function getTestResult({
 }: GetTestResultParams<ContainerModule | HelmModule | KubernetesModule>): Promise<TestResult | null> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const api = await KubeApi.factory(log, k8sCtx.provider)
-  const testResultNamespace = k8sCtx.provider.config.gardenSystemNamespace
+  const testResultNamespace = await getSystemNamespace(k8sCtx.provider, log)
 
   const resultKey = getTestResultKey(k8sCtx, module, testName, testVersion)
 
@@ -92,7 +93,7 @@ export async function storeTestResult({
 }: StoreTestResultParams): Promise<TestResult> {
   const k8sCtx = <KubernetesPluginContext>ctx
   const api = await KubeApi.factory(log, k8sCtx.provider)
-  const testResultNamespace = k8sCtx.provider.config.gardenSystemNamespace
+  const testResultNamespace = await getSystemNamespace(k8sCtx.provider, log)
 
   const data: TestResult = trimRunOutput(result)
 

garden-service/src/plugins/kubernetes/util.ts

@@ -27,6 +27,7 @@ import { HelmModule } from "./helm/config"
 import { KubernetesModule } from "./kubernetes-module/config"
 import { getChartPath, renderHelmTemplateString } from "./helm/common"
 import { HotReloadableResource } from "./hot-reload"
+import { getSystemNamespace } from "./namespace"
 
 const STATIC_LABEL_REGEX = /[0-9]/g
 export const workloadTypes = ["Deployment", "DaemonSet", "ReplicaSet", "StatefulSet"]
@@ -383,7 +384,7 @@ export function convertDeprecatedManifestVersion(manifest: KubernetesResource):
 
 export async function getRunningPodInDeployment(deploymentName: string, provider: KubernetesProvider, log: LogEntry) {
   const api = await KubeApi.factory(log, provider)
-  const systemNamespace = provider.config.gardenSystemNamespace
+  const systemNamespace = await getSystemNamespace(provider, log)
 
   const status = await api.apps.readNamespacedDeployment(deploymentName, systemNamespace)
   const pods = await getPods(api, systemNamespace, status.spec.selector.matchLabels)