feat: download otomi values (#1510)

src/cmd/values.ts

@@ -2,7 +2,7 @@ import { prepareEnvironment } from 'src/common/cli'
 import { terminal } from 'src/common/debug'
 import { hfValues } from 'src/common/hf'
 import { getFilename } from 'src/common/utils'
-import { BasicArguments, getParsedArgs, setParsedArgs } from 'src/common/yargs'
+import { BasicArguments, setParsedArgs } from 'src/common/yargs'
 import { stringify } from 'yaml'
 import { Argv } from 'yargs'
 
@@ -11,21 +11,25 @@ const cmdName = getFilename(__filename)
 interface Arguments extends BasicArguments {
   filesOnly?: boolean
   excludeSecrets?: boolean
+  withWorkloadValues?: boolean
 }
 
-const values = async (): Promise<void> => {
+const values = async (argv: Arguments): Promise<void> => {
   const d = terminal(`cmd:${cmdName}:values`)
   d.info('Get values')
-  const argv: Arguments = getParsedArgs()
-  const hfVal = await hfValues({ filesOnly: argv.filesOnly, excludeSecrets: argv.excludeSecrets })
+  const hfVal = await hfValues({
+    filesOnly: argv.filesOnly,
+    excludeSecrets: argv.excludeSecrets,
+    withWorkloadValues: argv.withWorkloadValues,
+  })
   d.info('Print values')
   console.log(stringify(hfVal))
 }
 
 export const module = {
   command: cmdName,
   describe:
-    'Show helmfile values for target cluster (--files-only: only values stored on disk, --exclude-secrets: omit secrets)',
+    'Show helmfile values for target cluster (--files-only: only values stored on disk, --exclude-secrets: omit secrets --with-workload-values: include workload values)',
   builder: (parser: Argv): Argv =>
     parser.options({
       filesOnly: {
@@ -36,11 +40,15 @@ export const module = {
         boolean: true,
         default: false,
       },
+      withWorkloadValues: {
+        boolean: true,
+        default: false,
+      },
     }),
 
   handler: async (argv: Arguments): Promise<void> => {
     setParsedArgs(argv)
     await prepareEnvironment({ skipKubeContextCheck: true })
-    await values()
+    await values(argv)
   },
 }

src/common/hf.ts

@@ -1,10 +1,11 @@
 import { pathExists } from 'fs-extra'
-import { set } from 'lodash'
+import { readFile } from 'fs/promises'
+import { has, set } from 'lodash'
 import { parse } from 'yaml'
 import { $, ProcessOutput, ProcessPromise } from 'zx'
 import { logLevels, terminal } from './debug'
 import { env } from './envalid'
-import { asArray, extract, flattenObject, getValuesSchema, isCore, rootDir } from './utils'
+import { asArray, extract, flattenObject, getValuesSchema, isCore, readdirRecurse, rootDir } from './utils'
 import { HelmArguments, getParsedArgs } from './yargs'
 import { ProcessOutputTrimmed, Streams } from './zx-enhance'
 
@@ -67,13 +68,14 @@ export const hf = async (args: HFParams, opts?: HFOptions, envDir?: string): Pro
 }
 
 export interface ValuesArgs {
+  // Only files from values
   filesOnly?: boolean
+  withWorkloadValues?: boolean
   excludeSecrets?: boolean
   envDir?: string
 }
-
 export const hfValues = async (
-  { filesOnly = false, excludeSecrets = false }: ValuesArgs = {},
+  { filesOnly = false, excludeSecrets = false, withWorkloadValues = false }: ValuesArgs = {},
   envDir: string = env.ENV_DIR,
 ): Promise<Record<string, any> | undefined> => {
   const d = terminal('common:hf:hfValues')
@@ -96,10 +98,25 @@ export const hfValues = async (
     // strip secrets
     const schema = await getValuesSchema()
     const allSecrets = extract(schema, 'x-secret')
-    Object.keys(flattenObject(allSecrets)).forEach((path) => {
-      set(res, path, '<redacted>')
+    const allSecretsPaths = Object.keys(flattenObject(allSecrets))
+    allSecretsPaths.forEach((path) => {
+      if (has(res, path)) set(res, path, '<redacted>')
     })
-    return res
+  }
+
+  if (withWorkloadValues) {
+    const tragetDir = `${envDir}/env/teams/workloads`
+    const files = {}
+    if (await pathExists(tragetDir)) {
+      const paths = await readdirRecurse(tragetDir)
+      await Promise.allSettled(
+        paths.map(async (path) => {
+          const relativePath = path.replace(`${envDir}/`, '')
+          files[relativePath] = (await readFile(path)).toString()
+        }),
+      )
+      res.files = files
+    }
   }
   return res
 }

src/playground.ts

@@ -1,17 +1,19 @@
 #!/usr/bin/env node --nolazy -r ts-node/register
 
-import { getDeploymentState, getHelmReleases } from './common/k8s'
-import { writeValuesToFile } from './common/values'
+import { hfValues } from './common/hf'
 
 async function play() {
   // const version = await getCurrentVersion()
   // const prevVersion: string = (await getDeploymentState()).version ?? version
   // console.log(version)
 
-  const state = await getDeploymentState()
-  const releases = await getHelmReleases()
-
-  await writeValuesToFile(`/tmp/status.yaml`, { status: { otomi: state, helm: releases } }, true)
+  // const state = await getDeploymentState()
+  // const releases = await getHelmReleases()
+  const data = await hfValues(
+    { withWorkloadValues: true },
+    '/Users/jehoszafatzimnowoda/workspace/redkubes/otomi-core/tests/fixtures',
+  )
+  // await writeValuesToFile(`/tmp/status.yaml`, { status: { otomi: state, helm: releases } }, true)
 }
 
 play()

src/server.ts

@@ -5,6 +5,8 @@ import { bootstrapSops } from 'src/cmd/bootstrap'
 import { validateValues } from 'src/cmd/validate-values'
 import { decrypt, encrypt } from 'src/common/crypt'
 import { terminal } from 'src/common/debug'
+import { hfValues } from './common/hf'
+import { objectToYaml } from './common/values'
 
 const d = terminal('server')
 const app = express()
@@ -56,6 +58,28 @@ app.get('/prepare', async (req: Request, res: Response) => {
   }
 })
 
+function parseBoolean(string, defaultValue = false) {
+  return string === 'true' ? true : string === 'false' ? false : defaultValue
+}
+app.get('/otomi/values', async (req: Request, res: Response) => {
+  const { envDir } = req.query as QueryParams
+
+  const filesOnly = parseBoolean(req.query.filesOnly, true)
+  const excludeSecrets = parseBoolean(req.query.excludeSecrets, true)
+  const withWorkloadValues = parseBoolean(req.query.withWorkloadValues, true)
+  d.log('Get otomi values', req.query)
+  try {
+    const data = await hfValues({ filesOnly, excludeSecrets, withWorkloadValues }, envDir)
+    res.setHeader('Content-type', 'text/plain')
+    const yamlData = objectToYaml(data!)
+    res.status(200).send(yamlData)
+  } catch (error) {
+    const status = 500
+    d.error(error)
+    res.status(status).send(error)
+  }
+})
+
 export const startServer = (): void => {
   server = app.listen(17771, '0.0.0.0')
   d.log(`Server listening on http://0.0.0.0:17771`)

tests/fixtures/env/teams/workloads/admin/w1.yaml

@@ -0,0 +1,2 @@
+values: |
+    type: w1

tests/fixtures/env/teams/workloads/demo/w1.yaml

@@ -0,0 +1,2 @@
+values: |
+    type: w1-1

tests/fixtures/env/teams/workloads/demo/w2.yaml

@@ -0,0 +1,2 @@
+values: |
+    type: w2

versions.yaml

@@ -1,4 +1,4 @@
-api: 2.4.0
-console: 2.3.0
+api: 2.5.0
+console: 2.4.0
 tasks: 2.4.0
 tools: 1.6.0