fix: deploy namespaces during upgrade

helmfile.tpl/helmfile-init.yaml.gotmpl

@@ -23,6 +23,7 @@ releases:
     installed: true
     labels:
       init: true
+      upgrade: true
     namespace: kube-system
     chart: ../charts/raw
     values:

src/cmd/apply-teams.ts

@@ -1,56 +1,22 @@
-import { existsSync, mkdirSync, writeFileSync } from 'fs'
-import { resolve } from 'path'
 import { terminal } from 'src/common/debug'
-import { hf } from 'src/common/hf'
+import { deployEssential, hf } from 'src/common/hf'
 import { getFilename, rootDir } from 'src/common/utils'
-import { ProcessOutputTrimmed } from 'src/common/zx-enhance'
 import { CommandModule } from 'yargs'
-import { $ } from 'zx/core'
 
 const cmdName = getFilename(__filename)
-const dir = '/tmp/otomi/'
-const templateFile = `${dir}deploy-template.yaml`
 const d = terminal(`cmd:${cmdName}:apply-teams`)
 
 export const applyTeams = async (): Promise<boolean> => {
-  d.info(`Current working directory: ${process.cwd()}`)
-  const errors: Array<any> = []
+  d.info('Deploying team namespaces')
+  const result = await deployEssential(['team=true'])
 
-  const aplCoreDir = rootDir || resolve(process.cwd(), '../apl-core')
-  const helmfileSource = resolve(aplCoreDir, 'helmfile.tpl/helmfile-init.yaml.gotmpl')
-
-  if (!existsSync(helmfileSource)) {
-    errors.push(`Helmfile teams template not found at: ${helmfileSource}`)
-  }
-
-  d.info(`Parsing team namespaces defined in ${helmfileSource}`)
-
-  const output: ProcessOutputTrimmed = await hf(
-    { fileOpts: helmfileSource, args: 'template', labelOpts: ['team=true'] },
-    { streams: { stderr: d.stream.error } },
-  )
-  if (output.exitCode > 0) {
-    errors.push(output.stderr)
-  } else if (output.stderr.length > 0) {
-    errors.push(output.stderr)
-  }
-  const templateOutput = output.stdout
-  if (templateOutput) {
-    if (!existsSync(dir)) {
-      mkdirSync(dir, { recursive: true })
-    }
-    writeFileSync(templateFile, templateOutput)
-
-    await $`kubectl apply -f ${templateFile}`
-  }
-
-  if (errors.length === 0) d.info(`Teams applied`)
-  else {
-    errors.map((e) => d.error(e))
-    d.error(`Not all teams have been deployed successfully`)
+  if (result) {
+    d.info('Teams applied')
+  } else {
+    d.error('Not all teams have been deployed successfully')
   }
 
-  return true
+  return result
 }
 
 export const module: CommandModule = {

src/cmd/install.test.ts

@@ -28,6 +28,7 @@ jest.mock('src/common/values', () => ({
 
 jest.mock('src/common/hf', () => ({
   hf: jest.fn(),
+  deployEssential: jest.fn(),
   HF_DEFAULT_SYNC_ARGS: ['apply', '--sync-args', '--include-needs'],
 }))
 
@@ -88,6 +89,7 @@ describe('Install command', () => {
       writeValuesToFile: require('src/common/values').writeValuesToFile,
       applyServerSide: require('src/common/k8s').applyServerSide,
       hf: require('src/common/hf').hf,
+      deployEssential: require('src/common/hf').deployEssential,
       writeFileSync: require('fs').writeFileSync,
       $: require('zx').$,
     }
@@ -102,6 +104,7 @@ describe('Install command', () => {
       stdout: 'template-content',
       stderr: '',
     })
+    mockDeps.deployEssential.mockResolvedValue(true)
     mockDeps.$.mockResolvedValue(undefined)
   })
 
@@ -175,19 +178,15 @@ describe('Install command', () => {
       await installAll()
 
       expect(mockDeps.hf).toHaveBeenCalled()
-      expect(mockDeps.writeFileSync).toHaveBeenCalled()
+      expect(mockDeps.deployEssential).toHaveBeenCalled()
 
       process.env.DISABLE_SYNC = originalEnv
     })
 
-    test('should handle template generation errors', async () => {
-      const errorOutput = 'template generation failed'
+    test('should handle essential deployment errors', async () => {
+      const errorOutput = 'Failed to deploy essential manifests'
 
-      mockDeps.hf.mockResolvedValueOnce({
-        exitCode: 1,
-        stdout: '',
-        stderr: errorOutput,
-      })
+      mockDeps.deployEssential.mockResolvedValueOnce(false)
 
       await expect(installAll()).rejects.toThrow(errorOutput)
     })

src/cmd/install.ts

@@ -1,9 +1,9 @@
 import retry, { Options } from 'async-retry'
-import { mkdirSync, rmSync, writeFileSync } from 'fs'
+import { mkdirSync, rmSync } from 'fs'
 import { cleanupHandler, prepareEnvironment } from 'src/common/cli'
 import { logLevelString, terminal } from 'src/common/debug'
 import { env } from 'src/common/envalid'
-import { hf, HF_DEFAULT_SYNC_ARGS } from 'src/common/hf'
+import { deployEssential, hf, HF_DEFAULT_SYNC_ARGS } from 'src/common/hf'
 import {
   applyServerSide,
   getDeploymentState,
@@ -15,7 +15,6 @@ import {
 import { getFilename, rootDir } from 'src/common/utils'
 import { getCurrentVersion, getImageTag, writeValuesToFile } from 'src/common/values'
 import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs'
-import { ProcessOutputTrimmed } from 'src/common/zx-enhance'
 import { Argv, CommandModule } from 'yargs'
 import { $, cd } from 'zx'
 import {
@@ -29,7 +28,6 @@ import {
 
 const cmdName = getFilename(__filename)
 const dir = '/tmp/otomi/'
-const templateFile = `${dir}deploy-template.yaml`
 
 const cleanup = (argv: HelmArguments): void => {
   if (argv.skipCleanup) return
@@ -75,23 +73,15 @@ export const installAll = async () => {
   const releases = await getHelmReleases()
   await writeValuesToFile(`${env.ENV_DIR}/env/status.yaml`, { status: { otomi: state, helm: releases } }, true)
 
-  const output: ProcessOutputTrimmed = await hf(
-    { fileOpts: 'helmfile.tpl/helmfile-init.yaml.gotmpl', args: 'template' },
-    { streams: { stderr: d.stream.error } },
-  )
-  if (output.exitCode > 0) {
-    throw new Error(output.stderr)
-  } else if (output.stderr.length > 0) {
-    d.error(output.stderr)
+  d.info('Deploying essential manifests')
+  const essentialDeployResult = await deployEssential()
+  if (!essentialDeployResult) {
+    throw new Error('Failed to deploy essential manifests')
   }
-  const templateOutput = output.stdout
-  writeFileSync(templateFile, templateOutput)
 
   d.info('Deploying CRDs')
   await applyServerSide('charts/kube-prometheus-stack/charts/crds/crds')
   await $`kubectl apply -f charts/tekton-triggers/crds --server-side`
-  d.info('Deploying essential manifests')
-  await $`kubectl apply -f ${templateFile}`
 
   d.info('Deploying charts containing label stage=prep')
   await hf(

src/common/hf.ts

@@ -9,6 +9,8 @@ import { getFileMaps, setValuesFile } from './repo'
 import { asArray, extract, flattenObject, getValuesSchema, isCore, rootDir } from './utils'
 import { getParsedArgs, HelmArguments } from './yargs'
 import { ProcessOutputTrimmed, Streams } from './zx-enhance'
+import { resolve } from 'path'
+import { existsSync, mkdirSync, writeFileSync } from 'fs'
 
 const replaceHFPaths = (output: string, envDir = env.ENV_DIR): string => output.replaceAll('../env', envDir)
 export const HF_DEFAULT_SYNC_ARGS = ['sync', '--concurrency=1', '--sync-args', '--disable-openapi-validation --qps=20']
@@ -188,3 +190,33 @@ export const hfTemplate = async (
   template += outAll.stdout
   return template
 }
+
+export const deployEssential = async (labelOpts: string[] | null = null) => {
+  const d = terminal('common:hf:applyEssential')
+  const dir = '/tmp/otomi/'
+
+  const aplCoreDir = rootDir || resolve(process.cwd(), '../apl-core')
+  const helmfileSource = resolve(aplCoreDir, 'helmfile.tpl/helmfile-init.yaml.gotmpl')
+  const output: ProcessOutputTrimmed = await hf(
+    { fileOpts: helmfileSource, args: 'template', labelOpts },
+    { streams: { stderr: d.stream.error } },
+  )
+  if (output.exitCode > 0) {
+    d.error(output.stderr)
+    return false
+  } else if (output.stderr.length > 0) {
+    d.warn(output.stderr)
+  }
+  const templateOutput = output.stdout
+  if (templateOutput) {
+    const templateFile = `${dir}deploy-template.yaml`
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true })
+    }
+    writeFileSync(templateFile, templateOutput)
+
+    await $`kubectl apply -f ${templateFile}`
+  }
+
+  return true
+}

src/common/runtime-upgrade.test.ts

@@ -4,8 +4,10 @@ import { getDeploymentState, k8s, waitForArgoCDAppHealthy, waitForArgoCDAppSync
 import { filterRuntimeUpgrades, runtimeUpgrade } from './runtime-upgrade'
 import { RuntimeUpgrades } from './runtime-upgrades/runtime-upgrades'
 import { getCurrentVersion } from './values'
+import { deployEssential } from './hf'
 
 jest.mock('./k8s')
+jest.mock('./hf')
 jest.mock('./values')
 jest.mock('./debug')
 jest.mock('src/cmd/apply-as-apps')
@@ -24,8 +26,10 @@ const mockWaitForArgoCDAppSync = waitForArgoCDAppSync as jest.MockedFunction<typ
 const mockWaitForArgoCDAppHealthy = waitForArgoCDAppHealthy as jest.MockedFunction<typeof waitForArgoCDAppHealthy>
 const mockGetApplications = getApplications as jest.MockedFunction<typeof getApplications>
 const mockTerminal = terminal as jest.MockedFunction<typeof terminal>
+const mockDeployEssential = deployEssential as jest.MockedFunction<typeof deployEssential>
 const mockK8s = k8s as jest.Mocked<typeof k8s>
 
+mockDeployEssential.mockResolvedValue(true)
 // Mock the custom API
 const mockCustomApi = { mockCustomApi: true }
 mockK8s.custom.mockReturnValue(mockCustomApi as any)

src/common/runtime-upgrade.ts

@@ -5,6 +5,7 @@ import { terminal } from './debug'
 import { getDeploymentState, k8s, waitForArgoCDAppHealthy, waitForArgoCDAppSync } from './k8s'
 import { RuntimeUpgradeContext, RuntimeUpgrades, runtimeUpgrades } from './runtime-upgrades/runtime-upgrades'
 import { getCurrentVersion } from './values'
+import { deployEssential } from './hf'
 
 interface RuntimeUpgradeArgs {
   when: string
@@ -19,6 +20,12 @@ export async function runtimeUpgrade({ when }: RuntimeUpgradeArgs): Promise<void
     return
   }
 
+  d.info('Deploying essential manifests')
+  const essentialDeployResult = await deployEssential(['upgrade=true'])
+  if (!essentialDeployResult) {
+    throw new Error('Failed to update namespaces')
+  }
+
   const declaredVersion = await getCurrentVersion()
   const deployedVersion: string = deploymentState.version ?? declaredVersion
   const apps = await getApplications()