diff --git a/.values/.secrets.sample b/.values/.secrets.sample index 69eee56476..b28af01bfa 100644 --- a/.values/.secrets.sample +++ b/.values/.secrets.sample @@ -1,20 +1,18 @@ -export USER_ID=${UID-} -export GROUP_ID=${GID-} -export CLUSTER_NAME= # k8s context -export CLUSTER_APISERVER= # k8s api server # GitOps values repo: -export GIT_USER= -export GIT_EMAIL= -export GIT_PASSWORD= +GIT_USER='' +GIT_EMAIL='' +GIT_PASSWORD='' # KMS access from here on # Google (paste json key here without newlines) -export GCLOUD_SERVICE_KEY='' +GCLOUD_SERVICE_KEY='' # Azure: -export AZURE_TENANT_ID='' -export AZURE_CLIENT_ID='' -export AZURE_CLIENT_SECRET='' +AZURE_TENANT_ID='' +AZURE_CLIENT_ID='' +AZURE_CLIENT_SECRET='' # AWS: -export AWS_DEFAULT_REGION='' -export AWS_REGION='' -export AWS_ACCESS_KEY_ID='' -export AWS_SECRET_ACCESS_KEY='' +AWS_DEFAULT_REGION='' +AWS_REGION='' +AWS_ACCESS_KEY_ID='' +AWS_SECRET_ACCESS_KEY='' +# Vault: +VAULT_TOKEN='' diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000..110568145f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "pwa-node", + "request": "launch", + "name": "Launch Program", + "skipFiles": ["/**"], + "program": "${workspaceFolder}/src/otomi.ts", + "args": ["--experimental-specifier-resolution=node", "bootstrap", "-vvv"], + "preLaunchTask": "tsc: build - tsconfig.json", + "outFiles": ["${workspaceFolder}/dist/**/*.js"] + } + ] +} diff --git a/bin/ci-tests.sh b/bin/ci-tests.sh index d47f8cba60..269246b4aa 100755 --- a/bin/ci-tests.sh +++ b/bin/ci-tests.sh @@ -7,7 +7,7 @@ set -e . bin/common.sh testEnv=$PWD/tests/fixtures -source $testEnv/env/.env +source $testEnv/env/.secrets echo "Validating $testEnv values" ln -s $testEnv env diff --git a/binzx/otomi b/binzx/otomi index 74d2bed270..c3fc62196b 100755 --- a/binzx/otomi +++ b/binzx/otomi @@ -28,7 +28,7 @@ elif [ -z "$ENV_DIR" ]; then else mkdir -p $ENV_DIR fi -# set -x + silent() { if [[ $calling_args == *'-v'* ]] && [ -t 1 ]; then "$@" @@ -152,7 +152,6 @@ check_update() { silent echo "Checking for updates" [ -z $CI ] && check_update -silent echo "Preparing docker environment variables" tmp_env=$(mktemp) function dump_vars() { @@ -180,7 +179,6 @@ vars=( DEBUG ENV_DIR GCLOUD_SERVICE_KEY - K8S_CONTEXT KUBE_VERSION_OVERRIDE OTOMI_DRY_RUN OTOMI_IN_TERMINAL @@ -196,6 +194,7 @@ vars=( TESTING TRACE VERBOSITY + VALUES_INPUT VAULT_TOKEN ) dump_vars "${vars[@]}" diff --git a/chart/otomi/Chart.yaml b/chart/otomi/Chart.yaml index 6e1e8b343e..de7127a63a 100644 --- a/chart/otomi/Chart.yaml +++ b/chart/otomi/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for installing otomi in Kubernetes home: https://otomi.io/ icon: https://otomi.io/img/otomi-logo.svg type: application -version: '0.2.0' +version: '0.2.1' appVersion: 'APP_VERSION_PLACEHOLDER' keywords: - otomi diff --git a/chart/otomi/localtest.sh b/chart/otomi/localtest.sh index a700f9513f..856d6ffca1 100755 --- a/chart/otomi/localtest.sh +++ b/chart/otomi/localtest.sh @@ -1,16 +1,7 @@ -# Usage: -# ENV_OUT=$PWD/../ENV_OUT VALUES_DIR=$PWD/../ chart/otomi/localtest.sh -# With VALUES_DIR holding a file named values.yaml holding the initial chart values set -e -docker run --rm -it \ - --env-file=../.env \ - -e VERBOSITY=1 \ - -e OTOMI_VALUES_INPUT=/secret/values.yaml \ - -e OTOMI_NON_INTERACTIVE='true' \ - -w ${WORKDIR:-$PWD} \ - -e ENV_DIR=/home/app/stack/env \ - -v $ENV_OUT:/home/app/stack/env \ - -v $PWD:$PWD -v $VALUES_DIR:/secret \ - -v /tmp:/tmp $image \ - "binzx/otomi chart bootstrap && binzx/otomi chart merge && binzx/otomi chart push && binzx/otomi apply" +export OTOMI_VALUES_INPUT=/tmp/otomi/secret/values.yaml +export CI=1 + +binzx/otomi bootstrap +binzx/otomi apply diff --git a/chart/otomi/templates/job.yaml b/chart/otomi/templates/job.yaml index c19d691fde..931e7be9e8 100644 --- a/chart/otomi/templates/job.yaml +++ b/chart/otomi/templates/job.yaml @@ -29,9 +29,11 @@ spec: command: [bash, -c] args: - | - binzx/otomi bootstrap values + binzx/otomi bootstrap binzx/otomi apply env: + - name: CI + value: '1' - name: VERBOSITY value: '1' - name: OTOMI_NON_INTERACTIVE diff --git a/chart/otomi/values.yaml b/chart/otomi/values.yaml index c55987a4e3..3a739f3546 100644 --- a/chart/otomi/values.yaml +++ b/chart/otomi/values.yaml @@ -2,8 +2,10 @@ cluster: apiName: '' apiServer: '' domainSuffix: '' + k8sContext: '' k8sVersion: '1.20' name: 'dev' + owner: '' provider: '' region: '' # kms: @@ -30,11 +32,9 @@ oidc: clientID: '' clientSecret: '' adminGroupID: '' - authUrl: '' issuer: '' teamAdminGroupID: '' tenantID: '' - tokenUrl: '' otomi: adminPassword: '' isMultitenant: true diff --git a/src/ci-tests.ts b/src/ci-tests.ts index b4e1265015..ada630988b 100755 --- a/src/ci-tests.ts +++ b/src/ci-tests.ts @@ -6,9 +6,8 @@ import { hf } from './cmd/hf' import { validateTemplates } from './cmd/validate-templates' import { validateValues } from './cmd/validate-values' import { x } from './cmd/x' -import { OtomiDebugger, terminal } from './common/debug' import { cleanupHandler } from './common/setup' -import { BasicArguments, getFilename, setParsedArgs, startingDir } from './common/utils' +import { BasicArguments, getFilename, OtomiDebugger, setParsedArgs, startingDir, terminal } from './common/utils' import { basicOptions } from './common/yargs-opts' const cmdName = getFilename(import.meta.url) diff --git a/src/cmd/EXAMPLE.ts b/src/cmd/EXAMPLE.ts deleted file mode 100644 index f8df79c407..0000000000 --- a/src/cmd/EXAMPLE.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' - -/* Steps: - * 1. Follow all TODO in this file - * 2. Update src/cmd/index.ts and add: - * `import Module from './'` - * `export { default as } from './' - * add `Module` to commands constant - */ - -// TODO: extend this interface with the HelmArguments from '../helm.opts.ts' or add the options that you define in the `builder` at the bottom -interface Arguments extends BasicArguments { - // TODO: Define custom options, if necessary - TODO?: string -} - -const cmdName = getFilename(import.meta.url) -let debug: OtomiDebugger - -/* eslint-disable no-useless-return */ -const cleanup = (argv: Arguments): void => { - if (argv.skipCleanup) return -} -/* eslint-enable no-useless-return */ - -const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) - debug = terminal(cmdName) - - if (options) await otomi.prepareEnvironment(options) -} - -// TODO: Rename function name to filename -export const example = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - await setup(argv, options) - - // TODO: Write your code here - debug.log(cmdName) - console.log(argv) -} - -export const module = { - command: cmdName, - describe: '', - builder: (parser: Argv): Argv => parser, - - handler: async (argv: Arguments): Promise => { - setParsedArgs(argv) - await example(argv, {}) // TODO: Replace with function name - }, -} - -export default module diff --git a/src/cmd/apply.ts b/src/cmd/apply.ts index adf5ed73d0..c9888dbe68 100644 --- a/src/cmd/apply.ts +++ b/src/cmd/apply.ts @@ -1,10 +1,9 @@ import { mkdirSync, rmdirSync, writeFileSync } from 'fs' import { Argv, CommandModule } from 'yargs' import { $ } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { hf, hfStream } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, logLevelString, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, getParsedArgs, logLevelString, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments as HelmArgs, helmOptions } from '../common/yargs-opts' import { ProcessOutputTrimmed } from '../common/zx-enhance' import { Arguments as DroneArgs } from './gen-drone' @@ -27,11 +26,12 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) mkdirSync(dir, { recursive: true }) } -const deployAll = async (argv: Arguments) => { +const applyAll = async (argv: Arguments) => { + debug.info('Start apply all') const output: ProcessOutputTrimmed = await hf( { fileOpts: 'helmfile.tpl/helmfile-init.yaml', args: 'template' }, { streams: { stdout: debug.stream.log, stderr: debug.stream.error } }, @@ -57,24 +57,23 @@ const deployAll = async (argv: Arguments) => { ) } -export const apply = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - await setup(argv, options) - if (argv._[0] === 'deploy' || (!argv.label && !argv.file)) { - debug.info('Start deploy') - await deployAll(argv) - } else { - debug.info('Start apply') - const skipCleanup = argv.skipCleanup ? '--skip-cleanup' : '' - await hfStream( - { - fileOpts: argv.file, - labelOpts: argv.label, - logLevel: logLevelString(), - args: ['apply', '--skip-deps', skipCleanup], - }, - { trim: true, streams: { stdout: debug.stream.log, stderr: debug.stream.error } }, - ) +export const apply = async (): Promise => { + const argv: Arguments = getParsedArgs() + if (!argv.label && !argv.file) { + await applyAll(argv) + return } + debug.info('Start apply') + const skipCleanup = argv.skipCleanup ? '--skip-cleanup' : '' + await hfStream( + { + fileOpts: argv.file, + labelOpts: argv.label, + logLevel: logLevelString(), + args: ['apply', '--skip-deps', skipCleanup], + }, + { trim: true, streams: { stdout: debug.stream.log, stderr: debug.stream.error } }, + ) } export const module: CommandModule = { @@ -84,7 +83,8 @@ export const module: CommandModule = { handler: async (argv: Arguments): Promise => { setParsedArgs(argv) - await apply(argv, {}) + await setup(argv, {}) + await apply() }, } diff --git a/src/cmd/bash.ts b/src/cmd/bash.ts index 98ea6850d6..1b614ec9a0 100644 --- a/src/cmd/bash.ts +++ b/src/cmd/bash.ts @@ -1,8 +1,7 @@ import { Argv, CommandModule } from 'yargs' import { $, nothrow } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, parser, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { BasicArguments, getFilename, OtomiDebugger, parser, setParsedArgs, terminal } from '../common/utils' const cmdName = getFilename(import.meta.url) let debug: OtomiDebugger @@ -17,7 +16,7 @@ const setup = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const bash = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts new file mode 100644 index 0000000000..c32a7a3068 --- /dev/null +++ b/src/cmd/bootstrap.ts @@ -0,0 +1,177 @@ +import { copyFileSync, existsSync, mkdirSync, writeFileSync } from 'fs' +import { copy } from 'fs-extra' +import { copyFile } from 'fs/promises' +import { Argv } from 'yargs' +import { $, cd } from 'zx' +import { encrypt } from '../common/crypt' +import { env } from '../common/envalid' +import { hfValues } from '../common/hf' +import { getImageTag } from '../common/setup' +import { BasicArguments, currDir, getFilename, loadYaml, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' +import { genSops } from './gen-sops' +import { getChartValues, mergeChartValues } from './lib/chart' + +export type Arguments = BasicArguments + +const cmdName = getFilename(import.meta.url) +const debug: OtomiDebugger = terminal(cmdName) + +const generateLooseSchema = (cwd: string) => { + const schemaPath = '.vscode/values-schema.yaml' + const targetPath = `${env.ENV_DIR}/${schemaPath}` + const sourcePath = `${cwd}/values-schema.yaml` + + const valuesSchema = loadYaml(sourcePath) + const trimmedVS = JSON.stringify(valuesSchema, (k, v) => (k === 'required' ? undefined : v), 2) + writeFileSync(targetPath, trimmedVS) + if (cwd !== '/home/app/stack' && !existsSync(`${cwd}/${schemaPath}`)) + writeFileSync(`${cwd}/.values/values-schema.yaml`, trimmedVS) + debug.info(`Stored YAML schema at: ${targetPath}`) +} + +export const bootstrapGit = async (): Promise => { + if (existsSync(`${env.ENV_DIR}/.git`)) { + // scenario 3: pull > bootstrap values + debug.info('Values repo already git initialized.') + } else { + // scenario 1 or 2 (2 will only be called upon first otomi commit) + debug.info('Initializing values repo.') + const cwd = await currDir() + cd(env.ENV_DIR) + + const values = getChartValues() ?? (await hfValues()) + await $`git init ${env.ENV_DIR}` + copyFileSync(`bin/hooks/pre-commit`, `${env.ENV_DIR}/.git/hooks/pre-commit`) + + const stage = values?.charts?.['cert-manager']?.stage ?? 'production' + if (stage === 'staging') process.env.GIT_SSL_NO_VERIFY = 'true' + + const giteaEnabled = values?.charts?.gitea?.enabled ?? true + const clusterDomain = values?.cluster?.domainSuffix + const byor = !!values?.charts?.['otomi-api']?.git + + if (!giteaEnabled && !byor) { + debug.error('Gitea was disabled but no charts.otomi-api.git config was given.') + process.exit(1) + } + let username = 'Otomi Admin' + let email + let password + let remote + const branch = 'main' + if (!giteaEnabled) { + const otomiApiGit = values?.charts?.['otomi-api']?.git + username = otomiApiGit?.user + password = otomiApiGit?.password + remote = otomiApiGit?.repoUrl + email = otomiApiGit?.email + } else { + username = 'otomi-admin' + password = values?.charts?.gitea?.adminPassword ?? values?.otomi?.adminPassword + email = `otomi-admin@${clusterDomain}` + const giteaUrl = `gitea.${clusterDomain}` + const giteaOrg = 'otomi' + const giteaRepo = 'values' + remote = `https://${username}:${password}@${giteaUrl}/${giteaOrg}/${giteaRepo}.git` + } + await $`git config --local user.name ${username}` + await $`git config --local user.password ${password}` + await $`git config --local user.email ${email}` + await $`git checkout -b ${branch}` + await $`git remote add origin ${remote}` + cd(cwd) + } +} + +export const bootstrapValues = async (argv: Arguments): Promise => { + const cwd = await currDir() + + const hasOtomi = existsSync(`${env.ENV_DIR}/bin/otomi`) + + const binPath = `${env.ENV_DIR}/bin` + mkdirSync(binPath, { recursive: true }) + const otomiImage = `otomi/core:${getImageTag()}` + debug.info(`Intalling artifacts from ${otomiImage}`) + + await Promise.allSettled([ + copyFile(`${cwd}/bin/aliases`, `${binPath}/aliases`), + copyFile(`${cwd}/binzx/otomi`, `${binPath}/otomi`), + ]) + debug.info('Copied bin files') + debug.info(cwd) + try { + mkdirSync(`${env.ENV_DIR}/.vscode`, { recursive: true }) + await copy(`${cwd}/.values/.vscode`, `${env.ENV_DIR}/.vscode`, { overwrite: false, recursive: true }) + debug.info('Copied vscode folder') + generateLooseSchema(cwd) + debug.info('Generated loose schema') + } catch (error) { + debug.error(error) + debug.error(`Could not copy from ${cwd}/.values/.vscode`) + process.exit(1) + } + + await Promise.allSettled( + ['.gitattributes', '.secrets.sample'] + .filter((val) => !existsSync(`${env.ENV_DIR}/${val.replace(/\.sample$/g, '')}`)) + .map(async (val) => copyFile(`${cwd}/.values/${val}`, `${env.ENV_DIR}/${val}`)), + ) + + await Promise.allSettled( + ['.gitignore', '.prettierrc.yml', 'README.md'].map(async (val) => + copyFile(`${cwd}/.values/${val}`, `${env.ENV_DIR}/${val}`), + ), + ) + if (!existsSync(`${env.ENV_DIR}/env`)) { + debug.log(`Copying basic values`) + await copy(`${cwd}/.values/env`, `${env.ENV_DIR}/env`, { overwrite: false, recursive: true }) + } + + if (env.GCLOUD_SERVICE_KEY) { + writeFileSync(`${env.ENV_DIR}/gcp-key.json`, JSON.stringify(env.GCLOUD_SERVICE_KEY, null, 2)) + } + + debug.log('Copying Otomi Console Setup') + mkdirSync(`${env.ENV_DIR}/docker-compose`, { recursive: true }) + await copy(`${cwd}/docker-compose`, `${env.ENV_DIR}/docker-compose`, { overwrite: true, recursive: true }) + await Promise.allSettled( + ['core.yaml', 'docker-compose.yml'].map((val) => copyFile(`${cwd}/${val}`, `${env.ENV_DIR}/${val}`)), + ) + + // If we run from chart installer, VALUES_INPUT will be set + if (env.VALUES_INPUT) await mergeChartValues() + + try { + await genSops({ ...argv, dryRun: false }, { skipAllPreChecks: true }) + } catch (error) { + debug.error(error.message) + } + + if (existsSync(`${env.ENV_DIR}/.sops.yaml`)) await encrypt() + + if (!hasOtomi) { + debug.log('You can now use the otomi CLI') + debug.log('Start by sourcing aliases:') + debug.log('. bin/aliases') + } + debug.log(`Done Bootstrapping`) +} + +export const module = { + command: cmdName, + hidden: true, + describe: 'Bootstrap all necessary settings and values', + builder: (parser: Argv): Argv => parser, + handler: async (argv: Arguments): Promise => { + setParsedArgs(argv) + /* + We have the following scenarios: + 1. chart install: assume empty env dir, so git init > bootstrap values (=load skeleton files, then merge chart values) > and commit + 2. cli install: first time, so git init > bootstrap values + 3. cli install: n-th time (.git exists), so pull > bootstrap values + */ + if (env.VALUES_INPUT) await bootstrapGit() + await bootstrapValues(argv) + }, +} +export default module diff --git a/src/cmd/bootstrap/git.ts b/src/cmd/bootstrap/git.ts deleted file mode 100644 index 011bf2d39b..0000000000 --- a/src/cmd/bootstrap/git.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { copyFileSync, existsSync } from 'fs' -import { Argv } from 'yargs' -import { $, cd } from 'zx' -import { OtomiDebugger, terminal } from '../../common/debug' -import { env } from '../../common/envalid' -import { hfValues } from '../../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../../common/setup' -import { BasicArguments, currDir, getFilename, gitPush, setParsedArgs } from '../../common/utils' - -type Arguments = BasicArguments - -const cmdName = getFilename(import.meta.url) -let debug: OtomiDebugger - -/* eslint-disable no-useless-return */ -const cleanup = (argv: Arguments): void => { - if (argv.skipCleanup) return -} -/* eslint-enable no-useless-return */ - -const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) - debug = terminal(cmdName) - - if (options) await otomi.prepareEnvironment(options) -} - -export const bootstrapGit = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - await setup(argv, options) - if (existsSync(`${env.ENV_DIR}/.git`)) { - debug.info('Values repo already git initialized.') - return - } - debug.info('Initializing values repo.') - const currDirVal = await currDir() - - await $`git init ${env.ENV_DIR}` - copyFileSync(`${currDirVal}/bin/hooks/pre-commit`, `${env.ENV_DIR}/.git/hooks/pre-commit`) - - const chartValues = await hfValues() - // now we can change pwd to avoid a lot of git boilerplating - cd(env.ENV_DIR) - - const stage = chartValues?.charts?.['cert-manager']?.stage ?? 'production' - if (stage === 'staging') process.env.GIT_SSL_NO_VERIFY = 'true' - - const giteaEnabled = chartValues?.charts?.gitea?.enabled ?? true - const clusterDomain = chartValues?.cluster?.domainSuffix - const byor = !!chartValues?.charts?.['otomi-api']?.git - - if (!giteaEnabled && !byor) { - debug.error('Gitea was disabled but no charts.otomi-api.git config was given.') - process.exit(1) - } - let username = 'Otomi Admin' - let email - let password - let remote - const branch = 'main' - let healthUrl - if (!giteaEnabled) { - const otomiApiGit = chartValues?.charts?.['otomi-api']?.git - username = otomiApiGit?.user - password = otomiApiGit?.password - email = `otomi-admin@${clusterDomain}` - remote = otomiApiGit?.repoUrl - healthUrl = remote - } else { - username = 'otomi-admin' - password = chartValues?.charts?.gitea?.adminPassword ?? chartValues?.otomi?.adminPassword - const giteaUrl = `gitea.${clusterDomain}` - const giteaOrg = 'otomi' - const giteaRepo = 'values' - healthUrl = `https://gitea.${clusterDomain}` - remote = `https://${username}:${password}@${giteaUrl}/${giteaOrg}/${giteaRepo}.git` - } - await $`git config --local user.name ${username}` - await $`git config --local user.password ${password}` - await $`git config --local user.email ${email}` - await $`git checkout -b ${branch}` - await $`git remote add origin ${remote}` - cd(currDirVal) -} - -export const module = { - command: cmdName, - describe: 'Bootstrap git settings', - builder: (parser: Argv): Argv => parser, - - handler: async (argv: Arguments): Promise => { - setParsedArgs(argv) - await bootstrapGit(argv, {}) - }, -} - -export default module diff --git a/src/cmd/bootstrap/index.ts b/src/cmd/bootstrap/index.ts deleted file mode 100644 index d4ee668693..0000000000 --- a/src/cmd/bootstrap/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Argv } from 'yargs' -import { BasicArguments, setParsedArgs } from '../../common/utils' -import { bootstrapGit, module as gitModule } from './git' -import { bootstrapValues, module as valuesModule } from './values' - -export { module as git } from './git' -export { module as values } from './values' -export const commands = [gitModule, valuesModule] -export type Arguments = BasicArguments - -const fileName = 'bootstrap' - -export const module = { - command: fileName, - hidden: true, - describe: 'Bootstrap all necessary settings and values', - builder: (parser: Argv): Argv => { - commands.map((cmd) => parser.command(cmd)) - return parser - }, - handler: async (argv: Arguments): Promise => { - setParsedArgs(argv) - await bootstrapValues(argv) - await bootstrapGit(argv) - }, -} -export default module diff --git a/src/cmd/bootstrap/values.ts b/src/cmd/bootstrap/values.ts deleted file mode 100644 index 22157cba51..0000000000 --- a/src/cmd/bootstrap/values.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'fs' -import { copy } from 'fs-extra' -import { copyFile } from 'fs/promises' -import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../../common/debug' -import { env } from '../../common/envalid' -import { cleanupHandler, otomi } from '../../common/setup' -import { BasicArguments, currDir, getFilename, loadYaml, setParsedArgs } from '../../common/utils' -import { genSops } from '../gen-sops' -import { merge } from './lib/chart' - -const cmdName = getFilename(import.meta.url) -let debug: OtomiDebugger - -export type Arguments = BasicArguments - -/* eslint-disable no-useless-return */ -const cleanup = (argv: Arguments): void => { - if (argv.skipCleanup) return -} -/* eslint-enable no-useless-return */ - -const setup = (argv: Arguments): void => { - if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) - debug = terminal(cmdName) -} - -const rollBack = (): void => { - const dirContent = readdirSync(env.ENV_DIR) - dirContent.map((item) => rmSync(item, { recursive: true, force: true })) -} - -const generateLooseSchema = (currDirVal: string) => { - const schemaPath = '.vscode/values-schema.yaml' - const targetPath = `${env.ENV_DIR}/${schemaPath}` - const sourcePath = `${currDirVal}/values-schema.yaml` - - const valuesSchema = loadYaml(sourcePath) - const trimmedVS = JSON.stringify(valuesSchema, (k, v) => (k === 'required' ? undefined : v), 2) - writeFileSync(targetPath, trimmedVS) - if (currDirVal !== '/home/app/stack' && !existsSync(`${currDirVal}/${schemaPath}`)) - writeFileSync(`${currDirVal}/.values/values-schema.yaml`, trimmedVS) - debug.info(`Stored YAML schema at: ${targetPath}`) -} - -export const bootstrapValues = async (argv: Arguments): Promise => { - const args = { ...argv } - setup(args) - - const currDirVal = await currDir() - - const hasOtomi = existsSync(`${env.ENV_DIR}/bin/otomi`) - - const binPath = `${env.ENV_DIR}/bin` - mkdirSync(binPath, { recursive: true }) - const otomiImage = `otomi/core:${otomi.imageTag()}` - debug.info(`Intalling artifacts from ${otomiImage}`) - - await Promise.allSettled([ - copyFile(`${currDirVal}/bin/aliases`, `${binPath}/aliases`), - copyFile(`${currDirVal}/binzx/otomi`, `${binPath}/otomi`), - ]) - debug.info('Copied bin files') - debug.info(currDirVal) - try { - mkdirSync(`${env.ENV_DIR}/.vscode`, { recursive: true }) - await copy(`${currDirVal}/.values/.vscode`, `${env.ENV_DIR}/.vscode`, { overwrite: false, recursive: true }) - debug.info('Copied vscode folder') - generateLooseSchema(currDirVal) - debug.info('Generated loose schema') - } catch (error) { - debug.error(error) - debug.error(`Could not copy from ${currDirVal}/.values/.vscode`) - process.exit(1) - } - - await Promise.allSettled( - ['.gitattributes', '.secrets.sample'] - .filter((val) => !existsSync(`${env.ENV_DIR}/${val.replace(/\.sample$/g, '')}`)) - .map(async (val) => copyFile(`${currDirVal}/.values/${val}`, `${env.ENV_DIR}/${val}`)), - ) - - await Promise.allSettled( - ['.gitignore', '.prettierrc.yml', 'README.md'].map(async (val) => - copyFile(`${currDirVal}/.values/${val}`, `${env.ENV_DIR}/${val}`), - ), - ) - if (!existsSync(`${env.ENV_DIR}/env`)) { - debug.log(`Copying basic values`) - await copy(`${currDirVal}/.values/env`, `${env.ENV_DIR}/env`, { overwrite: false, recursive: true }) - } - - try { - await genSops({ ...argv, dryRun: false }, { skipAllPreChecks: true }) - } catch (error) { - debug.error(error.message) - } - if (env.GCLOUD_SERVICE_KEY) { - writeFileSync(`${env.ENV_DIR}/gcp-key.json`, JSON.stringify(env.GCLOUD_SERVICE_KEY, null, 2)) - } - - debug.log('Copying Otomi Console Setup') - mkdirSync(`${env.ENV_DIR}/docker-compose`, { recursive: true }) - await copy(`${currDirVal}/docker-compose`, `${env.ENV_DIR}/docker-compose`, { overwrite: true, recursive: true }) - await Promise.allSettled( - ['core.yaml', 'docker-compose.yml'].map((val) => copyFile(`${currDirVal}/${val}`, `${env.ENV_DIR}/${val}`)), - ) - - // If we run from chart installer, VALUES_INPUT will be set - if (process.env.VALUES_INPUT) await merge() - - if (!hasOtomi) { - debug.log('You can now use the otomi CLI') - debug.log('Start by sourcing aliases:') - debug.log('. bin/aliases') - } - debug.log(`Done Bootstrapping`) -} - -export const module = { - command: cmdName, - describe: "Bootstrap values repo with artifacts corresponding to the cluster's stack version", - builder: (parser: Argv): Argv => parser, - handler: async (argv: Arguments): Promise => { - setParsedArgs(argv) - const envDirHasVals = existsSync(env.ENV_DIR) && readdirSync(env.ENV_DIR).length > 0 - try { - await bootstrapValues(argv) - } catch (error) { - debug.error('Error occurred, rolling back') - if (!envDirHasVals) rollBack() - debug.error(error) - process.exit(1) - } - }, -} - -export default module diff --git a/src/cmd/check-policies.ts b/src/cmd/check-policies.ts index 5666cef8d2..be67695662 100644 --- a/src/cmd/check-policies.ts +++ b/src/cmd/check-policies.ts @@ -1,11 +1,10 @@ import { rmSync } from 'fs' import { Argv } from 'yargs' import { $, nothrow } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { env } from '../common/envalid' import { hfTemplate } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, loadYaml, logLevel, logLevels, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, loadYaml, logLevel, logLevels, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' const cmdName = getFilename(import.meta.url) @@ -20,7 +19,7 @@ const cleanup = (argv: Arguments): void => { const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const checkPolicies = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index b1856cc426..45cf16b71d 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -1,38 +1,32 @@ +import { existsSync } from 'fs' import { Argv } from 'yargs' import { $, cd, nothrow } from 'zx' import { encrypt } from '../common/crypt' -import { OtomiDebugger, terminal } from '../common/debug' import { env } from '../common/envalid' import { hfValues } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { currDir, getFilename, gitPush, setParsedArgs } from '../common/utils' +import { exitIfInCore, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { currDir, getFilename, OtomiDebugger, setParsedArgs, terminal, waitTillAvailable } from '../common/utils' import { Arguments as HelmArgs } from '../common/yargs-opts' +import { bootstrapGit } from './bootstrap' import { Arguments as DroneArgs, genDrone } from './gen-drone' +import { getChartValues } from './lib/chart' +import { pull } from './pull' import { validateValues } from './validate-values' const cmdName = getFilename(import.meta.url) -let debug: OtomiDebugger +const debug: OtomiDebugger = terminal(cmdName) interface Arguments extends HelmArgs, DroneArgs {} -/* eslint-disable no-useless-return */ -const cleanup = (argv: Arguments): void => { - if (argv.skipCleanup) return -} -/* eslint-enable no-useless-return */ - const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) - debug = terminal(cmdName) - - if (options) await otomi.prepareEnvironment(options) - otomi.exitIfInCore(cmdName) + if (options) await prepareEnvironment(options) + exitIfInCore(cmdName) } export const preCommit = async (argv: DroneArgs): Promise => { const pcDebug = terminal('Pre Commit') pcDebug.info('Check for cluster diffs') - await nothrow($`git config diff.sopsdiffer.textconv "sops -d"`) + await nothrow($`git config --local diff.sopsdiffer.textconv "sops -d"`) const settingsDiff = (await $`git diff env/settings.yaml`).stdout.trim() const secretDiff = (await $`git diff env/secrets.settings.yaml`).stdout.trim() @@ -44,6 +38,33 @@ export const preCommit = async (argv: DroneArgs): Promise => { await genDrone(argv) } +export const gitPush = async ( + branch: string, + sslVerify: boolean, + giteaUrl: string | undefined = undefined, +): Promise => { + const gitDebug = terminal('gitPush') + gitDebug.info('Starting git push.') + let skipSslVerify = '' + if (giteaUrl) { + if (!sslVerify) skipSslVerify = '-c http.sslVerify=false' + await waitTillAvailable(giteaUrl) + } + + const cwd = await currDir() + cd(env.ENV_DIR) + try { + await $`git ${skipSslVerify} push -u origin ${branch} -f` + gitDebug.log('Otomi values have been pushed to git.') + return true + } catch (error) { + gitDebug.error(error) + return false + } finally { + cd(cwd) + } +} + export const commit = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { await setup(argv, options) @@ -51,60 +72,58 @@ export const commit = async (argv: Arguments, options?: PrepareEnvironmentOption debug.info('Preparing values') - const currDirVal = await currDir() + const cwd = await currDir() cd(env.ENV_DIR) - const vals = await hfValues() - const clusterDomain = vals.cluster.domainSuffix ?? vals.cluster.apiName + const values = getChartValues() ?? (await hfValues()) + const clusterDomain = values?.cluster.domainSuffix ?? values?.cluster.apiName preCommit(argv) await encrypt() - debug.info('Do commit') + debug.info('Committing values') await $`git add -A` await $`git commit -m 'otomi commit' --no-verify` - debug.info('Pulling latest values') - try { - await $`git pull` - } catch (error) { - debug.error( - `When trying to pull from ${clusterDomain} merge conflicts occured\nPlease resolve these and run \`otomi commit\` again.`, - ) - process.exit(1) - } + if (!env.CI) await pull(argv, options) let healthUrl let branch - if (!vals.charts?.gitea?.enabled) { + if (!values.charts?.gitea?.enabled) { healthUrl = `gitea.${clusterDomain}` branch = 'main' } else { // @ts-ignore - branch = vals.charts!['otomi-api']!.git!.branch ?? 'main' + branch = values.charts!['otomi-api']!.git!.branch ?? 'main' } try { - const stage = vals.charts?.['cert-manager']?.stage === 'staging' + const sslVerify = values.charts?.['cert-manager']?.stage === 'staging' await $`git remote show origin` - await gitPush(branch, stage, healthUrl) + await gitPush(branch, sslVerify, healthUrl) debug.log('Successfully pushed the updated values') } catch (error) { debug.error(error.stderr) debug.error('Pushing the values failed, please read the above error message and manually try again') process.exit(1) } finally { - cd(currDirVal) + cd(cwd) } } export const module = { command: cmdName, - // As discussed: https://otomi.slack.com/archives/C011D78FP47/p1623843840012900 describe: 'Execute wrapper for generate pipelines -> git commit changed files', builder: (parser: Argv): Argv => parser, handler: async (argv: Arguments): Promise => { setParsedArgs(argv) - await commit(argv, { skipKubeContextCheck: true }) + const options = { skipKubeContextCheck: true } + if (!env.CI && existsSync(`${env.ENV_DIR}/.git`)) { + debug.info('Values repo already git initialized.') + await pull(argv, options) + } else { + await bootstrapGit() + } + await commit(argv, options) }, } diff --git a/src/cmd/decrypt.ts b/src/cmd/decrypt.ts index dedc06db81..b9acc88379 100644 --- a/src/cmd/decrypt.ts +++ b/src/cmd/decrypt.ts @@ -1,8 +1,7 @@ import { Argv } from 'yargs' import { decrypt as decryptFunc } from '../common/crypt' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { BasicArguments, getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' interface Arguments extends BasicArguments { files?: string[] @@ -20,7 +19,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment({ ...options, skipDecrypt: true }) + if (options) await prepareEnvironment({ ...options, skipDecrypt: true }) } export const decrypt = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/destroy.ts b/src/cmd/destroy.ts index af645ac0d4..4f2a775f7d 100644 --- a/src/cmd/destroy.ts +++ b/src/cmd/destroy.ts @@ -1,10 +1,9 @@ import { unlinkSync, writeFileSync } from 'fs' import { Argv } from 'yargs' import { $ } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { hf, hfStream } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, logLevelString, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, logLevelString, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' import { ProcessOutputTrimmed, stream } from '../common/zx-enhance' @@ -21,7 +20,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } const destroyAll = async () => { diff --git a/src/cmd/diff.ts b/src/cmd/diff.ts index cec72610af..a4a3ee8c9d 100644 --- a/src/cmd/diff.ts +++ b/src/cmd/diff.ts @@ -1,8 +1,7 @@ import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' import { hfStream } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, logLevelString, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, logLevelString, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' import { ProcessOutputTrimmed } from '../common/zx-enhance' import { decrypt } from './decrypt' @@ -20,7 +19,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const diff = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/encrypt.ts b/src/cmd/encrypt.ts index 9820bc1e8a..c679d4c067 100644 --- a/src/cmd/encrypt.ts +++ b/src/cmd/encrypt.ts @@ -1,8 +1,7 @@ import { Argv } from 'yargs' import { encrypt as encryptFunc } from '../common/crypt' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { BasicArguments, getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' interface Arguments extends BasicArguments { files?: string[] @@ -21,7 +20,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const encrypt = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/gen-drone.ts b/src/cmd/gen-drone.ts index 97738a86a6..20ee4a7833 100644 --- a/src/cmd/gen-drone.ts +++ b/src/cmd/gen-drone.ts @@ -1,10 +1,23 @@ import { writeFileSync } from 'fs' import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' import { env } from '../common/envalid' import { hfValues } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, gucci, setParsedArgs, startingDir } from '../common/utils' +import { + cleanupHandler, + getClusterOwner, + getImageTag, + prepareEnvironment, + PrepareEnvironmentOptions, +} from '../common/setup' +import { + BasicArguments, + getFilename, + gucci, + OtomiDebugger, + setParsedArgs, + startingDir, + terminal, +} from '../common/utils' export interface Arguments extends BasicArguments { dryRun?: boolean @@ -23,35 +36,36 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const genDrone = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { await setup(argv, options) - const hfVals = await hfValues() - if (!hfVals.charts?.drone?.enabled) { + const allValues = await hfValues() + if (!allValues.charts?.drone?.enabled) { return } - const receiver = hfVals.alerts?.drone ?? 'slack' - const branch = hfVals.charts?.['otomi-api']?.git?.branch ?? 'main' + const receiver = allValues.alerts?.drone ?? 'slack' + const branch = allValues.charts?.['otomi-api']?.git?.branch ?? 'main' const key = receiver === 'slack' ? 'url' : 'lowPrio' - const channel = receiver === 'slack' ? hfVals.alerts?.[receiver]?.channel ?? 'dev-mon' : undefined + const channel = receiver === 'slack' ? allValues.alerts?.[receiver]?.channel ?? 'dev-mon' : undefined - const webhook = hfVals.alerts?.[receiver]?.[key] + const webhook = allValues.alerts?.[receiver]?.[key] if (!webhook) throw new Error(`Could not find webhook url in 'alerts.${receiver}.${key}'`) - const cluster = hfVals.cluster?.name - const globalPullSecret = hfVals.otomi?.globalPullSecret - const provider = hfVals.alerts.drone - const pullPolicy = otomi.imageTag().startsWith('v') ? 'if-not-exists' : 'always' + const cluster = allValues.cluster?.name + const globalPullSecret = allValues.otomi?.globalPullSecret + const provider = allValues.alerts.drone + const imageTag = getImageTag() + const pullPolicy = imageTag.startsWith('v') ? 'if-not-exists' : 'always' const obj = { - imageTag: otomi.imageTag(), + imageTag, branch, cluster, channel, - customer: otomi.clusterOwner(), + customer: getClusterOwner(), globalPullSecret, provider, webhook, diff --git a/src/cmd/gen-sops.ts b/src/cmd/gen-sops.ts index 5c9c1919ca..fc0f796965 100644 --- a/src/cmd/gen-sops.ts +++ b/src/cmd/gen-sops.ts @@ -1,10 +1,18 @@ import { existsSync, writeFileSync } from 'fs' import { Argv } from 'yargs' import { chalk } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { env } from '../common/envalid' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, gucci, loadYaml, setParsedArgs, startingDir } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { + BasicArguments, + getFilename, + gucci, + loadYaml, + OtomiDebugger, + setParsedArgs, + startingDir, + terminal, +} from '../common/utils' export interface Arguments extends BasicArguments { dryRun: boolean @@ -30,7 +38,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const genSops = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { @@ -39,7 +47,10 @@ export const genSops = async (argv: Arguments, options?: PrepareEnvironmentOptio const settingsFile = `${env.ENV_DIR}/env/settings.yaml` const settingsVals = loadYaml(settingsFile) const provider: string | undefined = settingsVals?.kms?.sops?.provider - if (!provider) throw new Error('No sops information given. Assuming no sops enc/decryption needed.') + if (!provider) { + debug.warn('No sops information given. Assuming no sops enc/decryption needed. Be careful!') + return + } const targetPath = `${env.ENV_DIR}/.sops.yaml` const templatePath = `${startingDir}/tpl/.sops.yaml.gotmpl` @@ -62,8 +73,9 @@ export const genSops = async (argv: Arguments, options?: PrepareEnvironmentOptio } if (!env.CI) { - if (!existsSync(`${env.ENV_DIR}/.secrets`)) { - debug.error(`Expecting ${env.ENV_DIR}/.secrets to exist and hold credentials for SOPS`) + const secretPath = `${env.ENV_DIR}/.secrets` + if (!existsSync(secretPath)) { + debug.error(`Expecting ${secretPath} to exist and hold credentials for SOPS`) return } } diff --git a/src/cmd/hf.ts b/src/cmd/hf.ts index 5adab80b82..5342ef4960 100644 --- a/src/cmd/hf.ts +++ b/src/cmd/hf.ts @@ -1,9 +1,8 @@ import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' import { env } from '../common/envalid' import { hfStream } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, logLevelString, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, logLevelString, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments as HelmArgs, helmOptions } from '../common/yargs-opts' interface Arguments extends HelmArgs { @@ -23,7 +22,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const hf = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/index.ts b/src/cmd/index.ts index 1f8ee8e588..898cd95d76 100644 --- a/src/cmd/index.ts +++ b/src/cmd/index.ts @@ -1,7 +1,7 @@ import { CommandModule } from 'yargs' import { module as applyModule } from './apply' import { module as bashModule } from './bash' -import { module as bootstrapModule } from './bootstrap/index' +import { module as bootstrapModule } from './bootstrap' import { module as checkPoliciesModule } from './check-policies' import { module as commitModule } from './commit' import { module as decryptModule } from './decrypt' @@ -14,7 +14,6 @@ import { module as hfModule } from './hf' import { module as lintModule } from './lint' import { module as playgroundModule } from './playground' import { module as pullModule } from './pull' -import { module as regCredModule } from './regcred' import { module as rotateKeysModule } from './rotate-keys' import { module as scoreTemplatesModule } from './score-templates' import { module as serverModule } from './server' @@ -29,7 +28,7 @@ import { module as xModule } from './x' export { module as apply } from './apply' export { module as bash } from './bash' -export { module as bootstrap } from './bootstrap/index' +export { module as bootstrap } from './bootstrap' export { module as checkPolicies } from './check-policies' export { module as commit } from './commit' export { module as decrypt } from './decrypt' @@ -41,7 +40,6 @@ export { module as genSops } from './gen-sops' export { module as hf } from './hf' export { module as lint } from './lint' export { module as pull } from './pull' -export { module as regCred } from './regcred' export { module as rotateKeys } from './rotate-keys' export { module as scoreTemplates } from './score-templates' export { module as server } from './server' @@ -70,7 +68,6 @@ export const commands: CommandModule[] = [ lintModule, playgroundModule, pullModule, - regCredModule, rotateKeysModule, scoreTemplatesModule, serverModule, diff --git a/src/cmd/bootstrap/lib/chart.ts b/src/cmd/lib/chart.ts similarity index 52% rename from src/cmd/bootstrap/lib/chart.ts rename to src/cmd/lib/chart.ts index f478e8ab32..9551f643ee 100644 --- a/src/cmd/bootstrap/lib/chart.ts +++ b/src/cmd/lib/chart.ts @@ -1,14 +1,12 @@ import $RefParser from '@apidevtools/json-schema-ref-parser' -import { cleanEnv, str } from 'envalid' import { existsSync } from 'fs' import { writeFile } from 'fs/promises' import yaml from 'js-yaml' -import { merge as _merge, omit, pick } from 'lodash-es' -import { terminal } from '../../../common/debug' -import { env } from '../../../common/envalid' -import { loadYaml } from '../../../common/utils' +import { merge, omit, pick } from 'lodash-es' +import { env } from '../../common/envalid' +import { loadYaml, terminal } from '../../common/utils' -const debug = terminal('mergeValues') +const debug = terminal('chart') let hasSops = false const extractSecrets = (schema: any, parentAddress?: string): Array => { const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] @@ -23,10 +21,11 @@ const extractSecrets = (schema: any, parentAddress?: string): Array => { else if (schemaKeywords.includes(key) || !Number.isNaN(Number(key))) address = parentAddress return extractSecrets(childObj, address) }) - .filter(Boolean) as Array + .filter(Boolean) + .map((s: string) => s.replace(/^properties\./, '')) } -const mergeValues = async (targetPath: string, newValues: Record): Promise => { +export const mergeFileValues = async (targetPath: string, newValues: Record): Promise => { debug.debug(`targetPath: ${targetPath}, values: ${JSON.stringify(newValues)}`) if (!existsSync(targetPath)) { // If the targetPath doesn't exist, just create it and write the valueObject in it. @@ -36,61 +35,59 @@ const mergeValues = async (targetPath: string, newValues: Record => { - const cleanedEnv = cleanEnv(process.env, { - VALUES_INPUT: str({ desc: 'The chart values.yaml file' }), - SCHEMA_PATH: str({ desc: 'The path to the values-schema.yaml schema file' }), - }) +export const getChartValues = (): any | undefined => { + return env.VALUES_INPUT ? loadYaml(env.VALUES_INPUT) : undefined +} + +export const mergeChartValues = async (): Promise => { hasSops = existsSync(`${env.ENV_DIR}/.sops.yaml`) - const values = loadYaml(cleanedEnv.VALUES_INPUT) + const values = getChartValues() // creating secret files const schema = loadYaml('values-schema.yaml') const derefSchema = await $RefParser.dereference(schema) - const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later + const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) const secretsJsonPath = extractSecrets(cleanSchema) + debug.debug('secretsJsonPath: ', secretsJsonPath) const secrets = pick(values, secretsJsonPath) // removing secrets const plainValues = omit(values, secretsJsonPath) as any const fieldsToOmit = ['cluster', 'policies', 'teamConfig', 'charts'] const secretSettings = omit(secrets, fieldsToOmit) const settings = omit(plainValues, fieldsToOmit) - // mergeValues(`${env.ENV_DIR}/env/secrets.teams.yaml`, { teamConfig: secrets.teamConfig }) // FIXME: lets fix the team part later - const individualPromises: Promise[] = [] + const promises: Promise[] = [] - if (settings) individualPromises.push(mergeValues(`${env.ENV_DIR}/env/settings.yaml`, settings)) - if (secretSettings) individualPromises.push(mergeValues(`${env.ENV_DIR}/env/secrets.settings.yaml`, secretSettings)) + if (settings) promises.push(mergeFileValues(`${env.ENV_DIR}/env/settings.yaml`, settings)) + if (secretSettings) promises.push(mergeFileValues(`${env.ENV_DIR}/env/secrets.settings.yaml`, secretSettings)) // creating non secret files if (plainValues.cluster) - individualPromises.push(mergeValues(`${env.ENV_DIR}/env/cluster.yaml`, { cluster: plainValues.cluster })) + promises.push(mergeFileValues(`${env.ENV_DIR}/env/cluster.yaml`, { cluster: plainValues.cluster })) if (plainValues.policies) - individualPromises.push(mergeValues(`${env.ENV_DIR}/env/policies.yaml`, { policies: plainValues.policies })) - if (plainValues.teamConfig) - individualPromises.push(mergeValues(`${env.ENV_DIR}/env/teams.yaml`, { teamConfig: plainValues.teamConfig })) + promises.push(mergeFileValues(`${env.ENV_DIR}/env/policies.yaml`, { policies: plainValues.policies })) - const plainChartPromises = Object.keys(plainValues.charts).map((chart) => { + const plainChartPromises = Object.keys(plainValues.charts || {}).map((chart) => { const valueObject = { charts: { [chart]: plainValues.charts[chart], }, } - return mergeValues(`${env.ENV_DIR}/env/charts/${chart}.yaml`, valueObject) + return mergeFileValues(`${env.ENV_DIR}/env/charts/${chart}.yaml`, valueObject) }) - const secretChartPromises = Object.keys(secrets.charts).map((chart) => { + const secretChartPromises = Object.keys(secrets.charts || {}).map((chart) => { const valueObject = { charts: { [chart]: values.charts[chart], }, } - return mergeValues(`${env.ENV_DIR}/env/charts/secrets.${chart}.yaml`, valueObject) + return mergeFileValues(`${env.ENV_DIR}/env/charts/secrets.${chart}.yaml`, valueObject) }) - await Promise.all([...individualPromises, ...secretChartPromises, ...plainChartPromises]) + await Promise.all([...promises, ...secretChartPromises, ...plainChartPromises]) - debug.log('otomi chart values merged with the bootstrapped values.') + debug.log('Chart values merged with the bootstrapped values.') } diff --git a/src/cmd/lint.ts b/src/cmd/lint.ts index f300a33a60..a5a88c297c 100644 --- a/src/cmd/lint.ts +++ b/src/cmd/lint.ts @@ -1,8 +1,7 @@ import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' import { hf } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, logLevelString, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, logLevelString, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' const cmdName = getFilename(import.meta.url) @@ -18,7 +17,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const lint = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/playground.ts b/src/cmd/playground.ts index e728dfa758..f14387ea64 100644 --- a/src/cmd/playground.ts +++ b/src/cmd/playground.ts @@ -1,8 +1,8 @@ import { Argv } from 'yargs' import { $ } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { BasicArguments, getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' + /** * This file is a scripting playground to test basic code * it's basically the same as EXAMPLE.ts @@ -22,7 +22,7 @@ const setup = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } // usage: diff --git a/src/cmd/pull.ts b/src/cmd/pull.ts index 123d82576b..186333bbbd 100644 --- a/src/cmd/pull.ts +++ b/src/cmd/pull.ts @@ -1,13 +1,13 @@ import { Argv } from 'yargs' -import { $ } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' +import { $, cd } from 'zx' import { env } from '../common/envalid' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, setParsedArgs } from '../common/utils' +import { hfValues } from '../common/hf' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions, scriptName } from '../common/setup' +import { currDir, getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments as HelmArgs } from '../common/yargs-opts' -import { Arguments as BootsrapArgs, bootstrapValues } from './bootstrap/values' +import { bootstrapValues } from './bootstrap' -interface Arguments extends HelmArgs, BootsrapArgs {} +type Arguments = HelmArgs const cmdName = getFilename(import.meta.url) let debug: OtomiDebugger @@ -22,20 +22,32 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const pull = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { await setup(argv, options) - otomi.exitIfInCore(cmdName) - debug.info('Pull latest values') - await $`git -C ${env.ENV_DIR} pull` + const allValues = await hfValues() + const branch = allValues.charts?.['otomi-api']?.git?.branch ?? 'main' + debug.info('Pulling latest values') + const cwd = await currDir() + cd(env.ENV_DIR) + try { + await $`git fetch` + await $`git merge origin/${branch}` + } catch (error) { + debug.error(`Merge conflicts occured when trying to pull.\nPlease resolve these and run \`otomi commit\` again.`) + process.exit(env.CI ? 0 : 1) + } finally { + cd(cwd) + } + await bootstrapValues(argv) } export const module = { command: cmdName, - describe: `Wrapper for git pull && ${otomi.scriptName} bootstrap`, + describe: `Wrapper for git pull && ${scriptName} bootstrap`, builder: (parser: Argv): Argv => parser, handler: async (argv: Arguments): Promise => { diff --git a/src/cmd/regcred.ts b/src/cmd/regcred.ts deleted file mode 100644 index c6bf7b436e..0000000000 --- a/src/cmd/regcred.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Argv } from 'yargs' -import { $ } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' -import { ask } from '../common/zx-enhance' - -interface Arguments extends BasicArguments { - server: string - username: string - password: string - docker: { - server: string - username: string - password: string - } -} - -const cmdName = getFilename(import.meta.url) -let debug: OtomiDebugger - -/* eslint-disable no-useless-return */ -const cleanup = (argv: Arguments): void => { - if (argv.skipCleanup) return -} -/* eslint-enable no-useless-return */ - -const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) - debug = terminal(cmdName) - - if (options) await otomi.prepareEnvironment(options) -} - -export const regCred = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { - await setup(argv, options) - - const server = - argv?.server || - (await ask('Please provide the docker server as it was not passed as an argument or environment variable')) - const username = - argv?.username || - (await ask('Please provide the docker username as it was not passed as an argument or environment variable')) - const password = - argv?.password || - (await ask('Please provide the docker password as it was not passed as an argument or environment variable')) - - const outputEnc = ( - await $`kubectl create secret docker-registry --dry-run=client regcred --docker-server="${server}" --docker-username="${username}" --docker-password="${password}" --docker-email=not@us.ed -ojsonpath='{.data.\.dockerconfigjson}'` - ).stdout - const output = Buffer.from(outputEnc, 'base64').toString() - debug.log(output) -} - -export const module = { - command: cmdName, - describe: undefined, - builder: (parser: Argv): Argv => - parser.options({ - server: { - describe: 'Docker server', - group: 'otomi regcred options', - }, - username: { - alias: ['u'], - describe: 'Docker username', - group: 'otomi regcred options', - }, - password: { - alias: ['p'], - describe: 'Docker password', - group: 'otomi regcred options', - }, - }), - handler: async (argv: Arguments): Promise => { - setParsedArgs(argv) - await regCred(argv, { skipKubeContextCheck: true }) - }, -} - -export default module diff --git a/src/cmd/rotate-keys.ts b/src/cmd/rotate-keys.ts index fb7e7b92ff..46fd1f0200 100644 --- a/src/cmd/rotate-keys.ts +++ b/src/cmd/rotate-keys.ts @@ -1,8 +1,7 @@ import { Argv } from 'yargs' import { rotate } from '../common/crypt' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { BasicArguments, getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' const cmdName = getFilename(import.meta.url) let debug: OtomiDebugger @@ -17,7 +16,7 @@ const setup = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const rotateKeys = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/score-templates.ts b/src/cmd/score-templates.ts index af7bc81c8a..5c836d8be3 100644 --- a/src/cmd/score-templates.ts +++ b/src/cmd/score-templates.ts @@ -1,10 +1,9 @@ import { existsSync, unlinkSync } from 'fs' import { Argv } from 'yargs' import { $, nothrow } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { hfTemplate } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' const cmdName = getFilename(import.meta.url) @@ -23,7 +22,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const scoreTemplate = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/server.ts b/src/cmd/server.ts index 17eab6e92a..a3edf9335b 100644 --- a/src/cmd/server.ts +++ b/src/cmd/server.ts @@ -1,7 +1,6 @@ import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { BasicArguments, getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { startServer, stopServer } from '../server/index' type Arguments = BasicArguments @@ -21,7 +20,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const server = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/status.ts b/src/cmd/status.ts index faa7a4037a..689d9d5d5f 100644 --- a/src/cmd/status.ts +++ b/src/cmd/status.ts @@ -1,6 +1,6 @@ import { Argv } from 'yargs' import { $ } from 'zx' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' type Arguments = BasicArguments @@ -16,7 +16,7 @@ const cleanup = (argv: Arguments): void => { const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const status = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/sync.ts b/src/cmd/sync.ts index 8bec261edb..dc99771acc 100644 --- a/src/cmd/sync.ts +++ b/src/cmd/sync.ts @@ -1,8 +1,7 @@ import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' import { hfStream } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, logLevelString, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, logLevelString, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' const cmdName = getFilename(import.meta.url) @@ -18,7 +17,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const sync = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/template.ts b/src/cmd/template.ts index 1437728925..007546867f 100644 --- a/src/cmd/template.ts +++ b/src/cmd/template.ts @@ -1,8 +1,7 @@ import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' import { hfTemplate } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments as HelmArgs, helmOptions } from '../common/yargs-opts' interface Arguments extends HelmArgs { @@ -22,7 +21,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const template = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/test.ts b/src/cmd/test.ts index 61e0cf9b73..573f53c256 100644 --- a/src/cmd/test.ts +++ b/src/cmd/test.ts @@ -1,11 +1,10 @@ import { unlinkSync, writeFileSync } from 'fs' import { Argv } from 'yargs' import { $ } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { env } from '../common/envalid' import { hf } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' import { ProcessOutputTrimmed } from '../common/zx-enhance' import { diff } from './diff' @@ -25,7 +24,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const test = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/validate-templates.ts b/src/cmd/validate-templates.ts index 637d20fabe..e3db916633 100644 --- a/src/cmd/validate-templates.ts +++ b/src/cmd/validate-templates.ts @@ -4,10 +4,9 @@ import { loadAll } from 'js-yaml' import tar from 'tar' import { Argv } from 'yargs' import { $, chalk, nothrow } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { hfTemplate } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { getFilename, readdirRecurse, setParsedArgs } from '../common/utils' +import { cleanupHandler, getK8sVersion, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { getFilename, OtomiDebugger, readdirRecurse, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' const cmdName = getFilename(import.meta.url) @@ -32,8 +31,8 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) - k8sVersion = otomi.getK8sVersion() + if (options) await prepareEnvironment(options) + k8sVersion = getK8sVersion() vk8sVersion = `v${k8sVersion}` let prep: Promise[] = [] diff --git a/src/cmd/validate-values.ts b/src/cmd/validate-values.ts index 2e925cb125..907f71115a 100644 --- a/src/cmd/validate-values.ts +++ b/src/cmd/validate-values.ts @@ -1,10 +1,9 @@ import Ajv, { DefinedError, ValidateFunction } from 'ajv' import { Argv } from 'yargs' import { chalk } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' import { hfValues } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { deletePropertyPath, getFilename, loadYaml, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { deletePropertyPath, getFilename, loadYaml, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' import { Arguments, helmOptions } from '../common/yargs-opts' const cmdName = getFilename(import.meta.url) @@ -22,7 +21,7 @@ const setup = async (argv: Arguments, options?: PrepareEnvironmentOptions): Prom if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const validateValues = async (argv: Arguments, options?: PrepareEnvironmentOptions): Promise => { @@ -36,11 +35,11 @@ export const validateValues = async (argv: Arguments, options?: PrepareEnvironme } debug.info('Getting values') - const hfVal = await hfValues() + const chartValues = await hfValues() // eslint-disable-next-line no-restricted-syntax for (const internalPath of internalPaths) { - deletePropertyPath(hfVal, internalPath) + deletePropertyPath(chartValues, internalPath) } try { @@ -57,7 +56,7 @@ export const validateValues = async (argv: Arguments, options?: PrepareEnvironme process.exit(1) } debug.info(`Validating values`) - const val = validate(hfVal) + const val = validate(chartValues) if (val) { debug.log('Values validation SUCCESSFUL') } else { diff --git a/src/cmd/values.ts b/src/cmd/values.ts index 5a32f450f7..8485c5a4e0 100644 --- a/src/cmd/values.ts +++ b/src/cmd/values.ts @@ -1,9 +1,8 @@ import { dump } from 'js-yaml' import { Argv } from 'yargs' -import { OtomiDebugger, terminal } from '../common/debug' import { values as valuesFunc } from '../common/hf' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { BasicArguments, getFilename, OtomiDebugger, setParsedArgs, terminal } from '../common/utils' const cmdName = getFilename(import.meta.url) let debug: OtomiDebugger @@ -18,7 +17,7 @@ const setup = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const values = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/cmd/x.ts b/src/cmd/x.ts index 2267df8442..81f80c1a0e 100644 --- a/src/cmd/x.ts +++ b/src/cmd/x.ts @@ -1,8 +1,15 @@ import { Argv } from 'yargs' import { $, nothrow } from 'zx' -import { OtomiDebugger, terminal } from '../common/debug' -import { cleanupHandler, otomi, PrepareEnvironmentOptions } from '../common/setup' -import { BasicArguments, getFilename, logLevel, logLevels, setParsedArgs } from '../common/utils' +import { cleanupHandler, prepareEnvironment, PrepareEnvironmentOptions } from '../common/setup' +import { + BasicArguments, + getFilename, + logLevel, + logLevels, + OtomiDebugger, + setParsedArgs, + terminal, +} from '../common/utils' import { stream } from '../common/zx-enhance' const cmdName = getFilename(import.meta.url) @@ -18,7 +25,7 @@ const setup = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): if (argv._[0] === cmdName) cleanupHandler(() => cleanup(argv)) debug = terminal(cmdName) - if (options) await otomi.prepareEnvironment(options) + if (options) await prepareEnvironment(options) } export const x = async (argv: BasicArguments, options?: PrepareEnvironmentOptions): Promise => { diff --git a/src/common/crypt.ts b/src/common/crypt.ts index 7ecf9dd30d..4abc52ab11 100644 --- a/src/common/crypt.ts +++ b/src/common/crypt.ts @@ -1,10 +1,8 @@ import { EventEmitter } from 'events' import { existsSync, statSync, utimesSync, writeFileSync } from 'fs' import { $, cd, chalk, nothrow, ProcessOutput } from 'zx' -import { OtomiDebugger, terminal } from './debug' -import { env, getEnv } from './envalid' -import { evaluateSecrets } from './secrets' -import { currDir, readdirRecurse } from './utils' +import { env } from './envalid' +import { currDir, OtomiDebugger, readdirRecurse, terminal } from './utils' EventEmitter.defaultMaxListeners = 20 @@ -16,23 +14,22 @@ enum CryptType { ROTATE = 'sops --input-type=yaml --output-type=yaml -i -r', } -const preCrypt = async (): Promise => { +const preCrypt = (): void => { debug.info('Checking prerequisites for the (de,en)crypt action') - await evaluateSecrets() - const secretEnv = getEnv() - if (secretEnv.GCLOUD_SERVICE_KEY) { + if (env.GCLOUD_SERVICE_KEY) { debug.debug('Writing GOOGLE_APPLICATION_CREDENTIAL') process.env.GOOGLE_APPLICATION_CREDENTIALS = '/tmp/key.json' - writeFileSync(process.env.GOOGLE_APPLICATION_CREDENTIALS, JSON.stringify(secretEnv.GCLOUD_SERVICE_KEY, null, 2)) + writeFileSync(process.env.GOOGLE_APPLICATION_CREDENTIALS, JSON.stringify(env.GCLOUD_SERVICE_KEY, null, 2)) } } const getAllSecretFiles = async () => { const files = await readdirRecurse(env.ENV_DIR, { skipHidden: true }) + debug.debug('files: ', files) return files .filter((file) => file.endsWith('.yaml') && file.includes('/secrets.')) .map((file) => file.replace(env.ENV_DIR, '.')) - .filter((file) => existsSync(`${env.ENV_DIR}/${file}`)) + // .filter((file) => existsSync(`${env.ENV_DIR}/${file}`)) } type CR = { @@ -42,16 +39,17 @@ type CR = { } const runOnSecretFiles = async (crypt: CR, filesArgs: string[] = []): Promise => { - const currDirVal = await currDir() + const cwd = await currDir() let files: string[] = filesArgs cd(env.ENV_DIR) if (files.length === 0) { files = await getAllSecretFiles() } - await preCrypt() + preCrypt() const eventEmitterDefaultListeners = EventEmitter.defaultMaxListeners EventEmitter.defaultMaxListeners = files.length + 5 + debug.debug('runOnSecretFiles - files: ', files) try { const commands = files.map(async (file) => { if (!crypt.condition || crypt.condition(env.ENV_DIR, file)) { @@ -69,7 +67,7 @@ const runOnSecretFiles = async (crypt: CR, filesArgs: string[] = []): Promise => { let encFiles = files if (encFiles.length === 0) encFiles = await getAllSecretFiles() + debug.debug('encFiles: ', encFiles) await runOnSecretFiles( { condition: (path: string, file: string): boolean => { const absFilePath = `${path}/${file}` - const encExists = existsSync(absFilePath) const decExists = existsSync(`${absFilePath}.dec`) - if (encExists !== decExists) return true - if (!encExists || !decExists) return false + if (!decExists) { + debug.debug(`Did not find decrypted ${file}.dec`) + return true + } // if there is a .dec && .dec is > 1s newer debug.debug(`Found decrypted ${file}.dec`) diff --git a/src/common/debug.ts b/src/common/debug.ts deleted file mode 100644 index 91ab258706..0000000000 --- a/src/common/debug.ts +++ /dev/null @@ -1,108 +0,0 @@ -import Debug, { Debugger as DebugDebugger } from 'debug' -import { Writable, WritableOptions } from 'stream' -import { env } from './envalid' -import { logLevel, logLevels } from './utils' - -const commonDebug: DebugDebugger = Debug('otomi') -commonDebug.enabled = true -export type DebuggerType = DebugDebugger | ((message?: any, ...optionalParams: any[]) => void) -export class DebugStream extends Writable { - output: DebuggerType - - constructor(output: DebuggerType, opts?: WritableOptions) { - super(opts) - this.output = output - } - - // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/explicit-module-boundary-types - _write(chunk: any, encoding: any, callback: (error?: Error | null) => void): void { - const data = chunk.toString().trim() - if (data.length > 0) this.output(data) - callback() - } -} - -export type OtomiStreamDebugger = { - log: DebugStream - trace: DebugStream - debug: DebugStream - info: DebugStream - warn: DebugStream - error: DebugStream -} -export type OtomiDebugger = { - enabled: boolean - base: DebuggerType - log: DebuggerType - trace: DebuggerType - debug: DebuggerType - info: DebuggerType - warn: DebuggerType - error: DebuggerType - stream: OtomiStreamDebugger -} - -const xtermColors = { - red: [52, 124, 9, 202, 211], - orange: [58, 130, 202, 208, 214], - green: [2, 28, 34, 46, 78, 119], -} -const setColor = (term: DebuggerType, color: number[]) => { - // Console.{log,warn,error} don't have namespace, so we know if it is in there that we use the DebugDebugger - if (!('namespace' in term && env.STATIC_COLORS)) return - const terminal: DebugDebugger = term - const colons = (terminal.namespace.match(/:/g) || ['']).length - 1 - terminal.color = color[Math.max(0, Math.min(colons, color.length - 1))].toString() -} -/* - * Must be function to be able to export overrides. - */ -/* eslint-disable no-redeclare */ -export function terminal(namespace: string): OtomiDebugger -export function terminal(namespace: string, terminalEnabled?: boolean): OtomiDebugger { - const newDebug = (baseNamespace: string, enabled = true, cons = console.log): DebuggerType => { - if (env.OTOMI_IN_TERMINAL) { - const newDebugObj: DebugDebugger = commonDebug.extend(baseNamespace) - newDebugObj.enabled = enabled - return newDebugObj - } - if (enabled) { - return cons - } - return () => { - /* Do nothing */ - } - } - const base = newDebug(`${namespace}`, terminalEnabled) - const log = newDebug(`${namespace}:log`, true) - const error = newDebug(`${namespace}:error`, true, console.error) - const trace = newDebug(`${namespace}:trace`, logLevel() >= logLevels.TRACE && terminalEnabled) - const debug = newDebug(`${namespace}:debug`, logLevel() >= logLevels.DEBUG && terminalEnabled) - const info = newDebug(`${namespace}:info`, logLevel() >= logLevels.INFO && terminalEnabled) - const warn = newDebug(`${namespace}:warn`, logLevel() >= logLevels.WARN && terminalEnabled, console.warn) - - setColor(error, xtermColors.red) - setColor(warn, xtermColors.orange) - setColor(info, xtermColors.green) - - const newDebugger: OtomiDebugger = { - enabled: terminalEnabled ?? true, - base, - log, - trace, - debug, - info, - warn, - error, - stream: { - log: new DebugStream(log), - trace: new DebugStream(trace), - debug: new DebugStream(debug), - info: new DebugStream(info), - warn: new DebugStream(warn), - error: new DebugStream(error), - }, - } - return newDebugger -} -/* eslint-enable no-redeclare */ diff --git a/src/common/envalid.ts b/src/common/envalid.ts index a7591d909c..18267242f8 100644 --- a/src/common/envalid.ts +++ b/src/common/envalid.ts @@ -1,3 +1,4 @@ +import { config } from 'dotenv' import { bool, cleanEnv, json, makeValidator, str } from 'envalid' import { existsSync, lstatSync } from 'fs' @@ -17,6 +18,13 @@ const cleanSpec = { STATIC_COLORS: bool({ default: false }), TESTING: bool({ default: false }), TRACE: bool({ default: false }), + VALUES_INPUT: str({ desc: 'The chart values.yaml file', default: undefined }), } -export const env = cleanEnv(process.env, cleanSpec) -export const getEnv = (): typeof env => cleanEnv(process.env, cleanSpec) +let pEnv: any = process.env +const path = `${pEnv.ENV_DIR}/.secrets` +if (pEnv.ENV_DIR && existsSync(path)) { + const result = config({ path }) // this sets vars from .env onto process.env + if (result.error) console.error(result.error) + pEnv = { ...pEnv, ...result.parsed } +} +export const env = cleanEnv(pEnv, cleanSpec) diff --git a/src/common/hf.ts b/src/common/hf.ts index 53634b88ab..cf09358907 100644 --- a/src/common/hf.ts +++ b/src/common/hf.ts @@ -1,9 +1,8 @@ import { dump, load } from 'js-yaml' import { Transform } from 'stream' import { $, ProcessOutput, ProcessPromise } from 'zx' -import { terminal } from './debug' import { env } from './envalid' -import { asArray, getParsedArgs, logLevels } from './utils' +import { asArray, getParsedArgs, logLevels, terminal } from './utils' import { Arguments } from './yargs-opts' import { ProcessOutputTrimmed, Streams } from './zx-enhance' diff --git a/src/common/secrets.ts b/src/common/secrets.ts deleted file mode 100644 index c4e2c39643..0000000000 --- a/src/common/secrets.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { existsSync } from 'fs' -import { terminal } from './debug' -import { env } from './envalid' -import { source } from './zx-enhance' - -/** - * Evaluate secrets, first check if .sops.yaml exists, then source the secrets path - which should exist - */ -export const evaluateSecrets = async (): Promise => { - const debug = terminal('evaluateSecrets') - if (!existsSync(`${env.ENV_DIR}/.sops.yaml`)) { - debug.log( - `Info: The 'secrets.*.yaml files' are not decrypted, because ${env.ENV_DIR}/.sops.yaml file is not present`, - ) - } - if (!env.CI) { - const secretPath = `${env.ENV_DIR}/.secrets` - if (existsSync(secretPath)) { - await source(secretPath) - } else { - debug.warn('%s\n', `Unable to find the '${secretPath}' file (while not in CI).`) - } - } -} - -export default evaluateSecrets diff --git a/src/common/setup.ts b/src/common/setup.ts index 14ce80cefe..3c3d512b1e 100644 --- a/src/common/setup.ts +++ b/src/common/setup.ts @@ -1,13 +1,11 @@ -import cliSelect from 'cli-select' -import { existsSync, readdirSync, writeFileSync } from 'fs' +import { existsSync, readdirSync } from 'fs' import { fileURLToPath } from 'url' -import { $, chalk, nothrow } from 'zx' +import { $, chalk } from 'zx' import { decrypt } from './crypt' -import { terminal } from './debug' import { env } from './envalid' -import { evaluateSecrets } from './secrets' -import { BasicArguments, loadYaml, parser } from './utils' -import { askYesNo, source } from './zx-enhance' +import { hfValues } from './hf' +import { BasicArguments, loadYaml, parser, terminal } from './utils' +import { askYesNo } from './zx-enhance' chalk.level = 2 const dirname = fileURLToPath(import.meta.url) @@ -17,7 +15,7 @@ let otomiClusterOwner: string let otomiK8sVersion: string /** - * Check whether the environment matches the configuration for the Kubernetes Context + * Check whether the environment matches the configuration for the kubernetes context * @returns */ const checkKubeContext = async (): Promise => { @@ -25,55 +23,24 @@ const checkKubeContext = async (): Promise => { const debug = terminal('checkKubeContext') debug.info('Validating kube context') - const envPath = `${env.ENV_DIR}/env/.env` - if (!existsSync(envPath)) { - const currentContext = (await $`kubectl config current-context`).stdout.trim() - const output = await $`kubectl config get-contexts -o=name` - const cancel = 'Cancel' - const out = output.stdout - .split('\n') - .map((val) => val.trim()) - .filter(Boolean) - debug.log(`No k8s context was defined in ${envPath} file , please select one.`) - const res = await cliSelect({ - values: [...out, cancel], - defaultValue: out.indexOf(currentContext), - valueRenderer: (value, selected) => (selected ? chalk.underline(value) : value), - }) - const val = res.value - if (val === cancel) { - debug.error('Please set an appropriate K8S context') - process.exit(0) - } - writeFileSync(envPath, `export K8S_CONTEXT="${val}"\n`) - process.env.K8S_CONTEXT = val - } else { - try { - await source(envPath) - } catch (error) { - debug.error(error) - process.exit(1) - } - } + const values: any = await hfValues() + const currentContext = (await $`kubectl config current-context`).stdout.trim() + const k8sContext = values?.cluster?.k8sContext + debug.debug('currentContext: ', currentContext) + debug.debug('k8sContext: ', k8sContext) - if (!('K8S_CONTEXT' in process.env)) { - debug.error(`K8S_CONTEXT is not defined in '${envPath}'`) - process.exit(1) - } - debug.info(`Using kube context: ${process.env.K8S_CONTEXT}`) + debug.info(`Using kube context: ${currentContext}`) - // TODO: Consider using the kubernetes-client: https://github.com/kubernetes-client/javascript - const runningContext = (await nothrow($`kubectl config current-context`)).stdout.trim() - if (process.env.K8S_CONTEXT !== runningContext) { + if (k8sContext !== currentContext) { let fixContext = false if (!(parser.argv as BasicArguments).setContext) { fixContext = await askYesNo( - `Warning: Your current kubernetes context (${runningContext}) does not match cluster context: ${process.env.K8S_CONTEXT}. Would you like to switch kube context to cluster first?`, + `Warning: Your current kubernetes context (${currentContext}) does not match cluster context: ${k8sContext}. Would you like to switch kube context to cluster first?`, { defaultYes: true }, ) } if (fixContext || (parser.argv as BasicArguments).setContext) { - await $`kubectl config use ${process.env.K8S_CONTEXT}` + await $`kubectl config use ${k8sContext}` } } } @@ -100,68 +67,65 @@ export type PrepareEnvironmentOptions = { } let clusterFile: any -export const otomi = { - scriptName: process.env.OTOMI_CALLER_COMMAND ?? 'otomi', - /** - * Find the cluster kubernetes version in the values - * @returns String of the kubernetes version on the cluster - */ - getK8sVersion: (): string => { - if (otomiK8sVersion) return otomiK8sVersion - if (!clusterFile) { - clusterFile = loadYaml(`${env.ENV_DIR}/env/cluster.yaml`) - } - otomiK8sVersion = clusterFile.cluster?.k8sVersion - return otomiK8sVersion - }, - /** - * Find what image tag is defined in configuration for otomi - * @returns string - */ - imageTag: (): string => { - if (otomiImageTag) return otomiImageTag - const file = `${env.ENV_DIR}/env/settings.yaml` - if (!existsSync(file)) return process.env.OTOMI_TAG ?? 'master' - const settingsFile = loadYaml(file) - otomiImageTag = settingsFile.otomi?.version ?? 'master' - return otomiImageTag - }, - /** - * Find the customer name that is defined in configuration for otomi - * @returns string - */ - clusterOwner: (): string => { - if (otomiClusterOwner) return otomiClusterOwner - if (!clusterFile) { - clusterFile = loadYaml(`${env.ENV_DIR}/env/cluster.yaml`) - } - otomiClusterOwner = clusterFile.cluster?.owner - return otomiClusterOwner - }, - /** - * Prepare environment when running an otomi command - */ - prepareEnvironment: async (options?: PrepareEnvironmentOptions): Promise => { - if (options?.skipAllPreChecks) return - const debug = terminal('prepareEnvironment') - debug.info('Checking environment') - if (!options?.skipEnvDirCheck && checkEnvDir()) { - if (!env.CI && !options?.skipEvaluateSecrets) await evaluateSecrets() - if (!env.CI && !options?.skipKubeContextCheck) await checkKubeContext() - if (!env.CI && !options?.skipDecrypt) await decrypt() - } - }, - /** - * If ran within otomi-core, stop execution as it should not be ran within that folder. - * @param command that is executed - */ - exitIfInCore: (command: string): void => { - if (dirname.includes('otomi-core') || env.ENV_DIR.includes('otomi-core')) { - const debug = terminal('exitIfInCore') - debug.error(`'otomi ${command}' should not be ran from otomi-core`) - process.exit(1) - } - }, +export const scriptName = process.env.OTOMI_CALLER_COMMAND ?? 'otomi' +/** + * Find the cluster kubernetes version in the values + * @returns String of the kubernetes version on the cluster + */ +export const getK8sVersion = (): string => { + if (otomiK8sVersion) return otomiK8sVersion + if (!clusterFile) { + clusterFile = loadYaml(`${env.ENV_DIR}/env/cluster.yaml`) + } + otomiK8sVersion = clusterFile.cluster?.k8sVersion + return otomiK8sVersion +} +/** + * Find what image tag is defined in configuration for otomi + * @returns string + */ +export const getImageTag = (): string => { + if (otomiImageTag) return otomiImageTag + const file = `${env.ENV_DIR}/env/settings.yaml` + if (!existsSync(file)) return process.env.OTOMI_TAG ?? 'master' + const settingsFile = loadYaml(file) + otomiImageTag = settingsFile.otomi?.version ?? 'master' + return otomiImageTag +} +/** + * Find the customer name that is defined in configuration for otomi + * @returns string + */ +export const getClusterOwner = (): string => { + if (otomiClusterOwner) return otomiClusterOwner + if (!clusterFile) { + clusterFile = loadYaml(`${env.ENV_DIR}/env/cluster.yaml`) + } + otomiClusterOwner = clusterFile.cluster?.owner + return otomiClusterOwner +} +/** + * Prepare environment when running an otomi command + */ +export const prepareEnvironment = async (options?: PrepareEnvironmentOptions): Promise => { + if (options?.skipAllPreChecks) return + const debug = terminal('prepareEnvironment') + debug.info('Checking environment') + if (!options?.skipEnvDirCheck && checkEnvDir()) { + if (!env.CI && !options?.skipKubeContextCheck) await checkKubeContext() + if (!env.CI && !options?.skipDecrypt) await decrypt() + } +} +/** + * If ran within otomi-core, stop execution as it should not be ran within that folder. + * @param command that is executed + */ +export const exitIfInCore = (command: string): void => { + if (dirname.includes('otomi-core') || env.ENV_DIR.includes('otomi-core')) { + const debug = terminal('exitIfInCore') + debug.error(`'otomi ${command}' should not be ran from otomi-core`) + process.exit(1) + } } /** diff --git a/src/common/utils.ts b/src/common/utils.ts index 57536cf6f3..8b89426c50 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,11 +1,12 @@ +import Debug, { Debugger as DebugDebugger } from 'debug' import { existsSync, readdirSync, readFileSync } from 'fs' import { load } from 'js-yaml' import fetch from 'node-fetch' import { resolve } from 'path' +import { Writable, WritableOptions } from 'stream' import { fileURLToPath } from 'url' import yargs, { Arguments as YargsArguments } from 'yargs' -import { $, cd, nothrow } from 'zx' -import { terminal } from './debug' +import { $, nothrow } from 'zx' import { env } from './envalid' process.stdin.isTTY = false @@ -46,6 +47,110 @@ export const getParsedArgs = (): BasicArguments => { return parsedArgs } +const commonDebug: DebugDebugger = Debug('otomi') +commonDebug.enabled = true +export type DebuggerType = DebugDebugger | ((message?: any, ...optionalParams: any[]) => void) +export class DebugStream extends Writable { + output: DebuggerType + + constructor(output: DebuggerType, opts?: WritableOptions) { + super(opts) + this.output = output + } + + // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/explicit-module-boundary-types + _write(chunk: any, encoding: any, callback: (error?: Error | null) => void): void { + const data = chunk.toString().trim() + if (data.length > 0) this.output(data) + callback() + } +} + +export type OtomiStreamDebugger = { + log: DebugStream + trace: DebugStream + debug: DebugStream + info: DebugStream + warn: DebugStream + error: DebugStream +} +export type OtomiDebugger = { + enabled: boolean + base: DebuggerType + log: DebuggerType + trace: DebuggerType + debug: DebuggerType + info: DebuggerType + warn: DebuggerType + error: DebuggerType + stream: OtomiStreamDebugger +} + +const xtermColors = { + red: [52, 124, 9, 202, 211], + orange: [58, 130, 202, 208, 214], + green: [2, 28, 34, 46, 78, 119], +} +const setColor = (term: DebuggerType, color: number[]) => { + // Console.{log,warn,error} don't have namespace, so we know if it is in there that we use the DebugDebugger + if (!('namespace' in term && env.STATIC_COLORS)) return + const terminal: DebugDebugger = term + const colons = (terminal.namespace.match(/:/g) || ['']).length - 1 + terminal.color = color[Math.max(0, Math.min(colons, color.length - 1))].toString() +} +/* + * Must be function to be able to export overrides. + */ +/* eslint-disable no-redeclare */ +export function terminal(namespace: string): OtomiDebugger +export function terminal(namespace: string, terminalEnabled?: boolean): OtomiDebugger { + const newDebug = (baseNamespace: string, enabled = true, cons = console.log): DebuggerType => { + if (env.OTOMI_IN_TERMINAL) { + const newDebugObj: DebugDebugger = commonDebug.extend(baseNamespace) + newDebugObj.enabled = enabled + return newDebugObj + } + if (enabled) { + return cons + } + return () => { + /* Do nothing */ + } + } + const base = newDebug(`${namespace}`, terminalEnabled) + const log = newDebug(`${namespace}:log`, true) + const error = newDebug(`${namespace}:error`, true, console.error) + const trace = newDebug(`${namespace}:trace`, logLevel() >= logLevels.TRACE && terminalEnabled) + const debug = newDebug(`${namespace}:debug`, logLevel() >= logLevels.DEBUG && terminalEnabled) + const info = newDebug(`${namespace}:info`, logLevel() >= logLevels.INFO && terminalEnabled) + const warn = newDebug(`${namespace}:warn`, logLevel() >= logLevels.WARN && terminalEnabled, console.warn) + + setColor(error, xtermColors.red) + setColor(warn, xtermColors.orange) + setColor(info, xtermColors.green) + + const newDebugger: OtomiDebugger = { + enabled: terminalEnabled ?? true, + base, + log, + trace, + debug, + info, + warn, + error, + stream: { + log: new DebugStream(log), + trace: new DebugStream(trace), + debug: new DebugStream(debug), + info: new DebugStream(info), + warn: new DebugStream(warn), + error: new DebugStream(error), + }, + } + return newDebugger +} +/* eslint-enable no-redeclare */ + export const asArray = (args: string | string[]): string[] => { return Array.isArray(args) ? args : [args] } @@ -69,34 +174,6 @@ export const capitalize = (s: string): string => .join(' ')) || '' -export const gitPush = async ( - branch: string, - sslVerify, - giteaUrl: string | undefined = undefined, -): Promise => { - const debug = terminal('Gitea Push') - debug.info('Gitea push') - let skipSslVerify = '' - if (giteaUrl) { - if (!sslVerify) skipSslVerify = '-c http.sslVerify=false' - await waitTillAvailable(giteaUrl) - } - - const currDirVal = await currDir() - - cd(env.ENV_DIR) - try { - await $`git ${skipSslVerify} push -u origin ${branch} -f` - debug.log('Otomi-values has been pushed to gitea') - return true - } catch (error) { - debug.error(error) - return false - } finally { - cd(currDirVal) - } -} - export const loadYaml = (path: string, opts?: { noError: boolean }): any => { if (!existsSync(path)) { if (opts?.noError) return null diff --git a/src/common/yargs-opts.ts b/src/common/yargs-opts.ts index 0c06423a5b..c2b4a01aab 100644 --- a/src/common/yargs-opts.ts +++ b/src/common/yargs-opts.ts @@ -41,51 +41,6 @@ const helmOpts: { [key: string]: Options } = { return files }, }, - // TODO: These options are defined, but not yet implemented! - // 'helm-binary': { - // alias: 'b', - // string: true, - // default: 'helm', - // describe: 'Path to the helm binary', - // }, - // environment: { - // alias: 'e', - // string: true, - // default: 'default', - // describe: 'Specify the environment name', - // }, - // quiet: { - // alias: 'q', - // boolean: true, - // }, - // 'state-value-set': { - // string: true, - // describe: - // 'set state values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)', - // coerce: (set: string) => { - // if (!set || set.length === 0) return set - // set.split(',').filter((val) => { - // if (!/\w+!?=\w+/.exec(val)) - // throw new Error(`Expected set in form key1=val1 or key1=val1,key2=val2, got "${chalk.italic(set)}"`) - // return true - // }) - // return set - // }, - // }, - // 'state-values-file': { - // string: true, - // describe: 'specify state values in a YAML file', - // coerce: (file: string) => { - // if (!file || file.length === 0) return file - // if (existsSync(file)) return file - // throw new Error(`State values file expected, but ${file} does not exist`) - // }, - // }, - // 'kube-context': { - // string: true, - // describe: 'Set kubectl context. Uses current context by default', - // default: process.env.K8S_CONTEXT, // Possibly empty if assuming the sourcing of env files from $env.ENV_DIR - // }, } Object.keys(helmOpts).map((k) => { helmOpts[k].group = 'Helmfile Options' diff --git a/src/common/zx-enhance.ts b/src/common/zx-enhance.ts index 1c697c16f8..e44908b1ba 100644 --- a/src/common/zx-enhance.ts +++ b/src/common/zx-enhance.ts @@ -1,36 +1,9 @@ -import * as dotenv from 'dotenv' -import { existsSync } from 'fs' -import { $, chalk, ProcessOutput, ProcessPromise, question } from 'zx' -import { DebugStream } from './debug' +import { chalk, ProcessOutput, ProcessPromise, question } from 'zx' import { env } from './envalid' -import { getParsedArgs } from './utils' +import { DebugStream, getParsedArgs } from './utils' const MAX_RETRIES_QUESTION = 3 -/** - * Do a bi-directional source. - * Sourcing using `bash` within zx, only applies to those commands, but are not avaialbe using `process.env.ENV_VAR_HERE` - * This function also maps that to process.env, making it bi-directional - * @param path - * @param force force sourcing of a file - even if it has previously been sourced - */ -export const source = async (path: string, force = false): Promise => { - if (!force && path in process.env) { - return - } - if (!existsSync(path)) { - throw new Error(`'${path}' does not exist`) - } - - const envVars = (await $`source ${path} && env`).stdout - const envVarAsObj = dotenv.parse(envVars) - Object.entries(envVarAsObj).map(([key, value]) => { - process.env[key] = value - return value - }) - process.env[path] = 'true' -} - export type AskType = { choices?: string[] matching?: string[] diff --git a/src/otomi.ts b/src/otomi.ts index bfce3a12a7..ac7c71a330 100755 --- a/src/otomi.ts +++ b/src/otomi.ts @@ -6,13 +6,12 @@ * node --experimental-specifier-resolution=node ./dist/otomi.js -- */ -import { lstatSync, readdirSync } from 'fs' +import { readdirSync } from 'fs' import { CommandModule } from 'yargs' import { bootstrap, commands, defaultCommand } from './cmd' -import { terminal } from './common/debug' import { env } from './common/envalid' -import { otomi } from './common/setup' -import { parser } from './common/utils' +import { scriptName } from './common/setup' +import { parser, terminal } from './common/utils' import { basicOptions } from './common/yargs-opts' const debug = terminal('global') @@ -25,28 +24,9 @@ if (!env.IN_DOCKER && !isAutoCompletion) { } const envDirContent = readdirSync(env.ENV_DIR) -if (envDirContent.length > 0 && !isAutoCompletion) { - try { - let errorMessage = '' - if (!lstatSync(`${env.ENV_DIR}/env`).isDirectory()) errorMessage += `\n${env.ENV_DIR}/env is not a directory` - if (!lstatSync(`${env.ENV_DIR}/env/charts`).isDirectory()) - errorMessage += `\n${env.ENV_DIR}/env/charts is not a directory` - if (!lstatSync(`${env.ENV_DIR}/env/cluster.yaml`).isFile()) - errorMessage += `\n${env.ENV_DIR}/env/cluster.yaml is not a file` - if (!lstatSync(`${env.ENV_DIR}/env/settings.yaml`).isFile()) - errorMessage += `\n${env.ENV_DIR}/env/settings.yaml is not a file` - if (errorMessage.trim().length > 0) { - debug.error(`It seems like '${env.ENV_DIR}' is not a valid values repo.${errorMessage}`) - process.exit(1) - } - } catch (error) { - debug.error(`It seems like '${env.ENV_DIR}' is not a valid values repo.\n${error.message}`) - process.exit(1) - } -} try { - parser.scriptName(otomi.scriptName) + parser.scriptName(scriptName) if (envDirContent.length === 0 && !isAutoCompletion) { parser.command(bootstrap) } else { diff --git a/src/server/index.ts b/src/server/index.ts index 28857e74ac..e76fb8b034 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,8 +3,7 @@ import express, { Request, Response } from 'express' import { Server } from 'http' import { commit } from '../cmd/commit' import { decrypt, encrypt } from '../common/crypt' -import { terminal } from '../common/debug' -import { defaultBasicArguments } from '../common/utils' +import { defaultBasicArguments, terminal } from '../common/utils' const debug = terminal('server') const app = express() diff --git a/tests/fixtures/.secrets.sample b/tests/fixtures/.secrets.sample deleted file mode 100644 index 69eee56476..0000000000 --- a/tests/fixtures/.secrets.sample +++ /dev/null @@ -1,20 +0,0 @@ -export USER_ID=${UID-} -export GROUP_ID=${GID-} -export CLUSTER_NAME= # k8s context -export CLUSTER_APISERVER= # k8s api server -# GitOps values repo: -export GIT_USER= -export GIT_EMAIL= -export GIT_PASSWORD= -# KMS access from here on -# Google (paste json key here without newlines) -export GCLOUD_SERVICE_KEY='' -# Azure: -export AZURE_TENANT_ID='' -export AZURE_CLIENT_ID='' -export AZURE_CLIENT_SECRET='' -# AWS: -export AWS_DEFAULT_REGION='' -export AWS_REGION='' -export AWS_ACCESS_KEY_ID='' -export AWS_SECRET_ACCESS_KEY='' diff --git a/values-schema.yaml b/values-schema.yaml index c38fe4ab02..25a9e350da 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -225,6 +225,8 @@ definitions: entrypoint: description: A Kubernetes API public IP address (onprem only). type: string + k8sContext: + $ref: '#/definitions/k8sContext' k8sVersion: $ref: '#/definitions/k8sVersion' name: @@ -245,6 +247,7 @@ definitions: description: AWS only. If provided will override autodiscovery from metadata. type: string required: + - k8sContext - k8sVersion - name - owner @@ -427,6 +430,9 @@ definitions: - script - type type: object + k8sContext: + description: The cluster k8s context as found in $KUBECONFIG. + type: string k8sVersion: description: The cluster k8s version. Otomi supports 2 minor versions backwards compatibility from the suggested default. enum: diff --git a/values/jobs/gitea-push.gotmpl b/values/jobs/gitea-push.gotmpl index ea95af4476..354829a98e 100644 --- a/values/jobs/gitea-push.gotmpl +++ b/values/jobs/gitea-push.gotmpl @@ -4,11 +4,12 @@ {{- $skipVerify := eq ($cm | get "stage" "production") "staging" }} {{- $otomiVersion := $v.otomi | get "version" "latest" }} {{- $pullPolicy := ternary "IfNotPresent" "Always" (regexMatch "^v\\d" $otomiVersion) }} +{{- $giteaUrl := print "https://gitea." $v.cluster.domainSuffix }} type: Job name: gitea-push env: - GITEA_URL: https://gitea.{{ $v.cluster.domainSuffix }} + GITEA_URL: {{ $giteaUrl }} DRONE_URL: https://drone.{{ $v.cluster.domainSuffix }} nativeSecrets: GITEA_PASSWORD: {{ $c | get "gitea.adminPassword" $v.otomi.adminPassword }} @@ -18,9 +19,24 @@ image: repository: otomi/core tag: {{ $otomiVersion }} pullPolicy: {{ $pullPolicy }} +init: + env: + GITEA_URL: {{ $giteaUrl }} + image: + repository: alpine/git + script: | + {{ if $skipVerify }}NOSSL=' -c http.sslVerify=false'{{ end }} + GITEA_URL=https://otomi-admin:$GITEA_PASSWORD@gitea.{{ $v.cluster.domainSuffix }}/otomi/values + echo "Waiting until gitea is accessible at {{ $giteaUrl }}/otomi/values" + until $(git $NOSSL ls-remote $GITEA_URL); do + printf '.' + sleep 5 + done + echo READY! script: | binzx/otomi bootstrap - binzx/otomi commit + # only run initial commit (when repo is empty)np + ! binzx/otomi pull && binzx/otomi commit runPolicy: OnSpecChange files: {{- $files := split "\n" (exec "bash" (list "-c" (printf "find %s -type f | grep -v '.git' | grep -v '.history'" (env "ENV_DIR"))) | trim) }}