From 99a100640884e36e2674a43f50eca3cdc927d230 Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Tue, 9 Jan 2024 09:52:03 +0200 Subject: [PATCH 01/14] skel for sandbox command --- .gitignore | 3 + src/bdcli/commands/bdcli-sandbox.ts | 14 ++++ .../commands/sandbox/bdcli-sandbox-deploy.ts | 34 +++++++++ .../commands/sandbox/bdcli-sandbox-destroy.ts | 34 +++++++++ .../commands/sandbox/bdcli-sandbox-diff.ts | 34 +++++++++ .../sandbox/bdcli-sandbox-download.ts | 56 +++++++++++++++ .../commands/sandbox/bdcli-sandbox-list.ts | 43 +++++++++++ .../commands/sandbox/bdcli-sandbox-plan.ts | 34 +++++++++ .../commands/sandbox/bdcli-sandbox-upload.ts | 41 +++++++++++ .../sandbox/bdcli-sandbox-validate.ts | 41 +++++++++++ src/index.ts | 3 +- .../boilingdata/boilingdata_api.ts | 2 + src/integration/boilingdata/sandbox.ts | 72 +++++++++++++++++++ 13 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 src/bdcli/commands/bdcli-sandbox.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-download.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-list.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts create mode 100644 src/integration/boilingdata/sandbox.ts diff --git a/.gitignore b/.gitignore index 4c6a49e..5386c6d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ dist/ datasource_config.yaml bdcli !bdcli/ +bd_querylog_config.yaml +boilingQueryLog_v2.yaml +boilingdata_user_x.yaml diff --git a/src/bdcli/commands/bdcli-sandbox.ts b/src/bdcli/commands/bdcli-sandbox.ts new file mode 100644 index 0000000..b3e82c1 --- /dev/null +++ b/src/bdcli/commands/bdcli-sandbox.ts @@ -0,0 +1,14 @@ +import { Command } from "commander"; + +const program = new Command("bdcli sandbox") + .executableDir("sandbox") + .command("list", "List deployed sandboxes") + .command("validate", "Validate sandbox template (e.g. for CI)") + .command("upload", "Upload sandbox template") + .command("download", "Download the deployed sandbox template") + .command("diff", "Yaml diff the current template file with the deployed one") + .command("plan", "List planned udpates/additions/deletions based on local sandbox definition and deployed sandbox") + .command("deploy", "Deploy the sandbox") + .command("destroy", "Destroy the sandbox"); + +program.parse(process.argv); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts new file mode 100644 index 0000000..ba9c2aa --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts @@ -0,0 +1,34 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; + +const logger = getLogger("bdcli-sandbox-deploy"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + console.log(options.template); + updateSpinnerText("TODO: Deploying sandbox"); + + spinnerWarn(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox deploy") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts new file mode 100644 index 0000000..466389c --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts @@ -0,0 +1,34 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; + +const logger = getLogger("bdcli-sandbox-destroy"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + console.log(options.template, options.name); + updateSpinnerText("TODO: Destroying sandbox"); + + spinnerWarn(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox destroy") + .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts new file mode 100644 index 0000000..6cd9ce1 --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts @@ -0,0 +1,34 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; + +const logger = getLogger("bdcli-sandbox-diff"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + console.log(options.template, options.name); + updateSpinnerText("TODO: Comparing local and deployed sandbox IaC template"); + + spinnerWarn(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox diff") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts new file mode 100644 index 0000000..b9d00ff --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts @@ -0,0 +1,56 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import * as fs from "fs/promises"; + +const logger = getLogger("bdcli-sandbox-validate"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + const filename = options.name + ".yaml"; + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + let fileAlreadyExists = false; + try { + if (await fs.lstat(filename)) { + fileAlreadyExists = true; + } + } catch (err) { + logger.debug({ err }); + } + if (fileAlreadyExists) { + spinnerError(`Local file ${filename} already exists`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Downloading sandbox IaC template"); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const template = await bdSandbox.downloadTemplate(options.name); + await fs.writeFile(filename, template); + spinnerSuccess(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox download") + .addOption(new cmd.Option("--name ", "template name from listing").makeOptionMandatory()) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts new file mode 100644 index 0000000..0cfee5c --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts @@ -0,0 +1,43 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-sandbox-list"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Listing sandboxes"); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const list = await bdSandbox.listSandboxes(); + spinnerSuccess(); + await outputResults(list, options.disableSpinner); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox list") + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts new file mode 100644 index 0000000..5df1ade --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts @@ -0,0 +1,34 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; + +const logger = getLogger("bdcli-sandbox-plan"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + console.log(options.region); + updateSpinnerText("TODO: Planning updates for deployment"); + + spinnerWarn(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox plan") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts new file mode 100644 index 0000000..f48301c --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts @@ -0,0 +1,41 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; + +const logger = getLogger("bdcli-sandbox-validate"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Uploading sandbox IaC template"); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + await bdSandbox.uploadTemplate(options.template); + spinnerSuccess(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox upload") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts new file mode 100644 index 0000000..92a7205 --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts @@ -0,0 +1,41 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; + +const logger = getLogger("bdcli-sandbox-validate"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Validating sandbox IaC template"); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + await bdSandbox.validateTemplate(options.template); + spinnerSuccess(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox validate") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/index.ts b/src/index.ts index 977c61b..1c098bb 100755 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,7 @@ program .allowExcessArguments(false) .command("account", "Setup and configure your BoilingData account") .command("aws", "Setup and configure your AWS account integration with BoilingData") - .command("domain", "Admin setup and configuration for your domain (.e.g @boilingdata.com, @mycompany.com)"); + .command("domain", "Admin setup and configuration for your domain (.e.g @boilingdata.com, @mycompany.com)") + .command("sandbox", "Managa Boiling S3 Sandboxes with IaC templates"); program.parse(process.argv); diff --git a/src/integration/boilingdata/boilingdata_api.ts b/src/integration/boilingdata/boilingdata_api.ts index a8e15d8..c11811d 100644 --- a/src/integration/boilingdata/boilingdata_api.ts +++ b/src/integration/boilingdata/boilingdata_api.ts @@ -6,10 +6,12 @@ export const dataSetsPath = "/data-sets"; export const accountPath = "/account"; export const tokenPath = "/token"; export const sharePath = "/share"; +export const sandboxPath = "/sandbox"; export const dataSetsUrl = baseApiUrl + dataSetsPath; export const accountUrl = baseApiUrl + accountPath; export const tokenUrl = baseApiUrl + tokenPath; export const tokenShareUrl = baseApiUrl + sharePath; +export const sandboxUrl = baseApiUrl + sandboxPath; // FIXME: get from bdAccount API export const bdAWSAccount = "589434896614"; diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts new file mode 100644 index 0000000..8f8d2eb --- /dev/null +++ b/src/integration/boilingdata/sandbox.ts @@ -0,0 +1,72 @@ +import { ILogger } from "../../bdcli/utils/logger_util.js"; +import { getReqHeaders, sandboxUrl } from "./boilingdata_api.js"; +import * as fs from "fs/promises"; + +// import { channel } from "node:diagnostics_channel"; + +export interface IBDConfig { + authToken: string; + logger: ILogger; +} + +export class BDSandbox { + private cognitoIdToken: string; + private logger: ILogger; + + constructor(private params: IBDConfig) { + this.logger = this.params.logger; + // this.logger.debug(this.params); + this.cognitoIdToken = this.params.authToken; + } + + public async downloadTemplate(templateName: string): Promise { + const headers = await getReqHeaders(this.cognitoIdToken); + this.logger.debug({ sandboxUrl, headers }); + const res = await fetch(sandboxUrl + "/" + templateName, { method: "GET", headers }); + const respBody = await res.json(); + this.logger.debug({ DownloadSandbox: { respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + return respBody?.template; + } + + public async uploadTemplate(templateFilename: string, validateOnly = false): Promise { + const headers = await getReqHeaders(this.cognitoIdToken); + this.logger.debug({ sandboxUrl, headers }); + const template = Buffer.from(await fs.readFile(templateFilename)).toString("base64"); + const body = JSON.stringify({ validateOnly, template }); + this.logger.debug({ body }); + const res = await fetch(sandboxUrl, { method: "PUT", headers, body }); + const respBody = await res.json(); + this.logger.debug({ ValidateSandbox: { respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (validateOnly && (!respBody?.isValidateOnly || respBody.validationResults != "OK")) { + throw new Error(`Validation failed: ${respBody.validationResults}`); + } + return respBody.isValidateOnly; + } + + public async validateTemplate(templateFilename: string): Promise { + return this.uploadTemplate(templateFilename, true); + } + + public async listSandboxes(): Promise> { + // channel("undici:request:create").subscribe(console.log); + // channel("undici:request:headers").subscribe(console.log); + const headers = await getReqHeaders(this.cognitoIdToken); + this.logger.debug({ sandboxUrl, headers }); + const res = await fetch(sandboxUrl, { method: "GET", headers }); + const body = await res.json(); + this.logger.debug({ listSandboxes: { body } }); + if (!body.ResponseCode || !body.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (Array.isArray(body.sandboxList)) { + return body.sandboxList; + } + throw new Error("Failed to list sandboxes"); + } +} From 12193634542a8bbe5c9f5c74a2c2ae81f52eda64 Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Tue, 9 Jan 2024 10:58:30 +0200 Subject: [PATCH 02/14] wiring more --- .../commands/sandbox/bdcli-sandbox-deploy.ts | 20 +++-- .../commands/sandbox/bdcli-sandbox-destroy.ts | 16 +++- .../commands/sandbox/bdcli-sandbox-diff.ts | 28 +++++-- .../sandbox/bdcli-sandbox-download.ts | 2 +- .../commands/sandbox/bdcli-sandbox-plan.ts | 20 +++-- .../sandbox/bdcli-sandbox-validate.ts | 4 +- src/integration/boilingdata/sandbox.ts | 81 +++++++++++++++---- 7 files changed, 135 insertions(+), 36 deletions(-) diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts index ba9c2aa..a525f04 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts @@ -1,8 +1,11 @@ import * as cmd from "commander"; import { getLogger } from "../../utils/logger_util.js"; -import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; import { addGlobalOptions } from "../../utils/options_util.js"; import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { outputResults } from "../../utils/output_util.js"; const logger = getLogger("bdcli-sandbox-deploy"); @@ -14,17 +17,24 @@ async function show(options: any, _command: cmd.Command): Promise { return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); } - console.log(options.template); - updateSpinnerText("TODO: Deploying sandbox"); + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); - spinnerWarn(); + updateSpinnerText(`Deploying sandbox ${options.name}`); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const results = await bdSandbox.deploySandbox(options.name); + spinnerSuccess(); + await outputResults(results, options.disableSpinner); } catch (err: any) { spinnerError(err?.message); } } const program = new cmd.Command("bdcli sandbox deploy") - .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) .action(async (options, command) => await show(options, command)); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts index 466389c..df1775a 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts @@ -1,8 +1,10 @@ import * as cmd from "commander"; import { getLogger } from "../../utils/logger_util.js"; -import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; import { addGlobalOptions } from "../../utils/options_util.js"; import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; const logger = getLogger("bdcli-sandbox-destroy"); @@ -14,10 +16,16 @@ async function show(options: any, _command: cmd.Command): Promise { return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); } - console.log(options.template, options.name); - updateSpinnerText("TODO: Destroying sandbox"); + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); - spinnerWarn(); + updateSpinnerText(`Destroying sandbox ${options.name}`); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + await bdSandbox.destroySandbox(options.name); + spinnerSuccess(); } catch (err: any) { spinnerError(err?.message); } diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts index 6cd9ce1..2e40cf0 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts @@ -1,12 +1,17 @@ import * as cmd from "commander"; import { getLogger } from "../../utils/logger_util.js"; -import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; import { addGlobalOptions } from "../../utils/options_util.js"; import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import * as fs from "fs/promises"; +import { outputResults } from "../../utils/output_util.js"; const logger = getLogger("bdcli-sandbox-diff"); async function show(options: any, _command: cmd.Command): Promise { + const tmpFileName = `/tmp/.${Date.now()}__${Math.random() * 1000}.yaml`; try { options = await combineOptsWithSettings(options, logger); @@ -14,17 +19,30 @@ async function show(options: any, _command: cmd.Command): Promise { return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); } - console.log(options.template, options.name); - updateSpinnerText("TODO: Comparing local and deployed sandbox IaC template"); + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); - spinnerWarn(); + updateSpinnerText(`Checking diff between template and deployment for sandbox ${options.name}`); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const diff = await bdSandbox.diffSandbox(options.name); + spinnerSuccess(); + await outputResults(diff, options.disableSpinner); } catch (err: any) { + try { + // cleanup + await fs.unlink(tmpFileName); + } catch (err) { + // ok, if it does not exist etc. + } spinnerError(err?.message); } } const program = new cmd.Command("bdcli sandbox diff") - .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--name ", "sandbox name to compare with").makeOptionMandatory()) .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) .action(async (options, command) => await show(options, command)); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts index b9d00ff..93a440b 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts @@ -35,7 +35,7 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); - updateSpinnerText("Downloading sandbox IaC template"); + updateSpinnerText(`Downloading sandbox IaC template of ${options.name}`); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); const template = await bdSandbox.downloadTemplate(options.name); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts index 5df1ade..d40f9e4 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts @@ -1,8 +1,11 @@ import * as cmd from "commander"; import { getLogger } from "../../utils/logger_util.js"; -import { spinnerError, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; import { addGlobalOptions } from "../../utils/options_util.js"; import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { outputResults } from "../../utils/output_util.js"; const logger = getLogger("bdcli-sandbox-plan"); @@ -14,17 +17,24 @@ async function show(options: any, _command: cmd.Command): Promise { return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); } - console.log(options.region); - updateSpinnerText("TODO: Planning updates for deployment"); + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); - spinnerWarn(); + updateSpinnerText(`Planning deployment for sandbox ${options.name}`); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const results = await bdSandbox.planSandbox(options.name); + spinnerSuccess(); + await outputResults(results, options.disableSpinner); } catch (err: any) { spinnerError(err?.message); } } const program = new cmd.Command("bdcli sandbox plan") - .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) .action(async (options, command) => await show(options, command)); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts index 92a7205..2686423 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts @@ -5,6 +5,7 @@ import { addGlobalOptions } from "../../utils/options_util.js"; import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; import { getIdToken } from "../../utils/auth_util.js"; import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { outputResults } from "../../utils/output_util.js"; const logger = getLogger("bdcli-sandbox-validate"); @@ -24,8 +25,9 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText("Validating sandbox IaC template"); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - await bdSandbox.validateTemplate(options.template); + const results = await bdSandbox.validateTemplate(options.template); spinnerSuccess(); + await outputResults(results, options.disableSpinner); } catch (err: any) { spinnerError(err?.message); } diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index 8f8d2eb..69c3fbf 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -19,38 +19,36 @@ export class BDSandbox { this.cognitoIdToken = this.params.authToken; } - public async downloadTemplate(templateName: string): Promise { + public async destroySandbox(sandboxName: string): Promise { const headers = await getReqHeaders(this.cognitoIdToken); this.logger.debug({ sandboxUrl, headers }); - const res = await fetch(sandboxUrl + "/" + templateName, { method: "GET", headers }); + const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "DELETE", headers }); const respBody = await res.json(); - this.logger.debug({ DownloadSandbox: { respBody } }); + this.logger.debug({ DeleteSandbox: { respBody } }); if (!respBody.ResponseCode || !respBody.ResponseText) { throw new Error("Malformed response from BD API"); } - return respBody?.template; } - public async uploadTemplate(templateFilename: string, validateOnly = false): Promise { + public async downloadTemplate(sandboxName: string): Promise { const headers = await getReqHeaders(this.cognitoIdToken); this.logger.debug({ sandboxUrl, headers }); - const template = Buffer.from(await fs.readFile(templateFilename)).toString("base64"); - const body = JSON.stringify({ validateOnly, template }); - this.logger.debug({ body }); - const res = await fetch(sandboxUrl, { method: "PUT", headers, body }); + const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "GET", headers }); const respBody = await res.json(); - this.logger.debug({ ValidateSandbox: { respBody } }); + this.logger.debug({ DownloadSandbox: { respBody } }); if (!respBody.ResponseCode || !respBody.ResponseText) { throw new Error("Malformed response from BD API"); } - if (validateOnly && (!respBody?.isValidateOnly || respBody.validationResults != "OK")) { - throw new Error(`Validation failed: ${respBody.validationResults}`); - } - return respBody.isValidateOnly; + return respBody?.template; + } + + public async uploadTemplate(templateFilename: string): Promise { + return this._uploadTemplate(templateFilename); } public async validateTemplate(templateFilename: string): Promise { - return this.uploadTemplate(templateFilename, true); + const validateOnly = true; + return this._uploadTemplate(templateFilename, validateOnly); } public async listSandboxes(): Promise> { @@ -69,4 +67,57 @@ export class BDSandbox { } throw new Error("Failed to list sandboxes"); } + + public async planSandbox(sandboxName: string): Promise { + const planOnly = true; + const diffOnly = false; + return this._deploySandbox(sandboxName, planOnly, diffOnly); + } + + public async diffSandbox(sandboxName: string): Promise { + const planOnly = false; + const diffOnly = true; + return this._deploySandbox(sandboxName, planOnly, diffOnly); + } + + public async deploySandbox(sandboxName: string): Promise { + return this._deploySandbox(sandboxName); + } + + // ---- private ---- + + private async _uploadTemplate(templateFilename: string, validateOnly = false): Promise { + const headers = await getReqHeaders(this.cognitoIdToken); + this.logger.debug({ sandboxUrl, headers }); + const template = Buffer.from(await fs.readFile(templateFilename)).toString("base64"); + const body = JSON.stringify({ validateOnly, template }); + this.logger.debug({ body }); + const res = await fetch(sandboxUrl, { method: "PUT", headers, body }); + const respBody = await res.json(); + this.logger.debug({ ValidateSandbox: { respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (validateOnly && (!respBody?.isValidateOnly || respBody.validationResults != "OK")) { + throw new Error(`Validation failed: ${respBody.validationResults}`); + } + return respBody.isValidateOnly; + } + + private async _deploySandbox(sandboxName: string, planOnly = false, diffOnly = false): Promise { + const action = planOnly ? "plan" : diffOnly ? "diff" : "deploy"; + const headers = await getReqHeaders(this.cognitoIdToken); + this.logger.debug({ sandboxUrl, headers }); + const body = JSON.stringify({ planOnly, diffOnly }); + const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "PUT", headers, body }); + const respBody = await res.json(); + this.logger.debug({ listSandboxes: { body: respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (respBody.ResponseCode != "00") { + throw new Error(`Failed to ${action} ${sandboxName}`); + } + return respBody; + } } From 4e5359a539f2373d150ade77f4147785573cbaaf Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Wed, 10 Jan 2024 13:24:10 +0200 Subject: [PATCH 03/14] Add bdcli account tap-token method --- .../account/bdcli-account-sts-token.ts | 2 +- .../account/bdcli-account-tap-token.ts | 58 ++++++++++++ src/bdcli/commands/bdcli-account.ts | 1 + .../commands/sandbox/bdcli-sandbox-deploy.ts | 2 +- .../commands/sandbox/bdcli-sandbox-destroy.ts | 4 +- .../commands/sandbox/bdcli-sandbox-diff.ts | 4 +- .../commands/sandbox/bdcli-sandbox-plan.ts | 2 +- src/bdcli/utils/config_util.ts | 1 + src/bdcli/utils/output_util.ts | 1 + src/integration/boilingdata/account.ts | 88 +++++++++++++++++-- .../boilingdata/boilingdata_api.ts | 6 +- src/integration/boilingdata/sandbox.ts | 9 +- 12 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 src/bdcli/commands/account/bdcli-account-tap-token.ts diff --git a/src/bdcli/commands/account/bdcli-account-sts-token.ts b/src/bdcli/commands/account/bdcli-account-sts-token.ts index 9e317f2..d7d5b24 100644 --- a/src/bdcli/commands/account/bdcli-account-sts-token.ts +++ b/src/bdcli/commands/account/bdcli-account-sts-token.ts @@ -56,7 +56,7 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Getting BoilingData STS token`); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdAccount = new BDAccount({ logger, authToken: token }); - const { bdStsToken, cached: stsCached, ...rest } = await bdAccount.getToken(options.lifetime ?? "1h"); + const { bdStsToken, cached: stsCached, ...rest } = await bdAccount.getStsToken(options.lifetime ?? "1h"); updateSpinnerText(`Getting BoilingData STS token: ${stsCached ? "cached" : "success"}`); spinnerSuccess(); diff --git a/src/bdcli/commands/account/bdcli-account-tap-token.ts b/src/bdcli/commands/account/bdcli-account-tap-token.ts new file mode 100644 index 0000000..c97eb40 --- /dev/null +++ b/src/bdcli/commands/account/bdcli-account-tap-token.ts @@ -0,0 +1,58 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { getIdToken, validateTokenLifetime } from "../../utils/auth_util.js"; +import { BDAccount } from "../../../integration/boilingdata/account.js"; +import { combineOptsWithSettings } from "../../utils/config_util.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-account-token"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (options.lifetime) await validateTokenLifetime(options.lifetime); + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText(`Getting BoilingData TAP token`); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdAccount = new BDAccount({ logger, authToken: token }); + const { + bdTapToken, + cached: tapCached, + ...rest + } = await bdAccount.getTapToken(options.lifetime ?? "24h", options.sharingUser); + updateSpinnerText(`Getting BoilingData TAP token: ${tapCached ? "cached" : "success"}`); + spinnerSuccess(); + await outputResults({ bdTapToken, cached: tapCached, ...rest }, options.disableSpinner); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli account tap-token") + .addOption( + new cmd.Option( + "--lifetime ", + "Expiration lifetime for the token, in string format, like '1h' (see https://github.com/vercel/ms)", + ), + ) + .addOption( + new cmd.Option( + "--sharing-user ", + "A user has shared Tap for you so that you can write to it." + + "\n\tExpects 'module: boilingdata' entry and upserts its config.token value", + ), + ) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/bdcli-account.ts b/src/bdcli/commands/bdcli-account.ts index 6a8ccf9..b092201 100755 --- a/src/bdcli/commands/bdcli-account.ts +++ b/src/bdcli/commands/bdcli-account.ts @@ -8,6 +8,7 @@ const program = new Command("bdcli account") .command("mfa", "Enable MFA") .command("migrate", "Migrate your account to another AWS region (not-yet-implemented)") .command("sts-token", "Exchange Cognito ID Token into shared or your own BoilingData Short-Term-Session (STS) token") + .command("tap-token", "Exchange Cognito ID Token into shared or your own BoilingData Stream Tap auth token") .command("token-share", "Share data sets via access tokens to other Boiling users") .command("token-unshare", "Unshare access tokens") .command("token-list-shares", "List shared data sets (access tokens) from and to you") diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts index a525f04..6e677c8 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts @@ -27,7 +27,7 @@ async function show(options: any, _command: cmd.Command): Promise { const bdSandbox = new BDSandbox({ logger, authToken: token }); const results = await bdSandbox.deploySandbox(options.name); spinnerSuccess(); - await outputResults(results, options.disableSpinner); + await outputResults(results?.deployResults, options.disableSpinner); } catch (err: any) { spinnerError(err?.message); } diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts index df1775a..fc67771 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts @@ -5,6 +5,7 @@ import { addGlobalOptions } from "../../utils/options_util.js"; import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; import { getIdToken } from "../../utils/auth_util.js"; import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { outputResults } from "../../utils/output_util.js"; const logger = getLogger("bdcli-sandbox-destroy"); @@ -24,8 +25,9 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Destroying sandbox ${options.name}`); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - await bdSandbox.destroySandbox(options.name); + const results = await bdSandbox.destroySandbox(options.name); spinnerSuccess(); + await outputResults(results?.destroyResults, options.disableSpinner); } catch (err: any) { spinnerError(err?.message); } diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts index 2e40cf0..95a165a 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts @@ -27,9 +27,9 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Checking diff between template and deployment for sandbox ${options.name}`); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - const diff = await bdSandbox.diffSandbox(options.name); + const results = await bdSandbox.diffSandbox(options.name); spinnerSuccess(); - await outputResults(diff, options.disableSpinner); + await outputResults(results?.diffResults, options.disableSpinner); } catch (err: any) { try { // cleanup diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts index d40f9e4..fd2bf04 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts @@ -27,7 +27,7 @@ async function show(options: any, _command: cmd.Command): Promise { const bdSandbox = new BDSandbox({ logger, authToken: token }); const results = await bdSandbox.planSandbox(options.name); spinnerSuccess(); - await outputResults(results, options.disableSpinner); + await outputResults(results?.planResults, options.disableSpinner); } catch (err: any) { spinnerError(err?.message); } diff --git a/src/bdcli/utils/config_util.ts b/src/bdcli/utils/config_util.ts index a57f12a..e10ed15 100644 --- a/src/bdcli/utils/config_util.ts +++ b/src/bdcli/utils/config_util.ts @@ -19,6 +19,7 @@ export interface ICredentials { accessToken?: string; refreshToken?: string; bdStsToken?: string; + bdTapToken?: string; sharedTokens?: string[]; region?: string; mfa?: boolean; diff --git a/src/bdcli/utils/output_util.ts b/src/bdcli/utils/output_util.ts index 115adc6..0105b58 100644 --- a/src/bdcli/utils/output_util.ts +++ b/src/bdcli/utils/output_util.ts @@ -1,5 +1,6 @@ import * as util from "node:util"; export async function outputResults(results: any, flat: boolean): Promise { + if (!results) return; console.log(flat ? JSON.stringify(results) : util.inspect(results, false, 20, true)); } diff --git a/src/integration/boilingdata/account.ts b/src/integration/boilingdata/account.ts index a4265d9..b6ad789 100644 --- a/src/integration/boilingdata/account.ts +++ b/src/integration/boilingdata/account.ts @@ -7,7 +7,7 @@ import { } from "../../bdcli/utils/config_util.js"; import { ILogger } from "../../bdcli/utils/logger_util.js"; import { spinnerError } from "../../bdcli/utils/spinner_util.js"; -import { accountUrl, getReqHeaders, tokenShareUrl, tokenUrl } from "./boilingdata_api.js"; +import { accountUrl, getReqHeaders, tokenShareUrl, stsTokenUrl, tapTokenUrl } from "./boilingdata_api.js"; import * as jwt from "jsonwebtoken"; // import { channel } from "node:diagnostics_channel"; @@ -25,9 +25,11 @@ interface IAPIAccountDetails { export class BDAccount { private cognitoIdToken: string; private bdStsToken: string | undefined; + private bdTapToken: string | undefined; private sharedTokens: IDecodedSession[]; private selectedToken: string | undefined; private decodedToken!: jwt.JwtPayload | null; + private decodedTapToken!: jwt.JwtPayload | null; private logger: ILogger; private accountDetails!: IAPIAccountDetails; @@ -101,7 +103,7 @@ export class BDAccount { this.decodedToken = jwt.decode(this.bdStsToken, { complete: true }); this.selectedToken = this.bdStsToken; } - if (!this.decodedToken || !this.selectedToken) throw new Error(`Could not find token (share id ${shareId})`); + if (!this.decodedToken || !this.selectedToken) throw new Error(`Could not find STS token (share id ${shareId})`); } private dumpSelectedToken(): void { @@ -109,6 +111,17 @@ export class BDAccount { this.logger.debug({ bdStsToken: this.selectedToken, decodedToken: this.decodedToken }); } + private decodeTapToken(): void { + if (!this.bdTapToken) throw new Error("No BD TAP token"); + this.decodedTapToken = jwt.decode(this.bdTapToken, { complete: true }); + if (!this.decodedTapToken) throw new Error(`Could not find TAP token`); + } + + private dumpTapToken(): void { + if (!this.bdTapToken) throw new Error("No BD TAP token"); + this.logger.debug({ bdTapToken: this.bdTapToken, decodedToken: this.decodedTapToken }); + } + private getHumanReadable(exp: number): string { const humanReadable = new Date(); humanReadable.setTime(exp * 1000); @@ -184,17 +197,43 @@ export class BDAccount { throw new Error("Failed to unshare token"); } + private async getTapTokenResp(): Promise< + { bdTapToken: string; cached: boolean; expiresIn: string; tokenLifetimeMins: number } | undefined + > { + this.decodeTapToken(); + this.dumpTapToken(); + if (!this.decodedTapToken || !this.bdTapToken) throw new Error("Unable to decode TAP token"); + const exp = this.decodedTapToken["payload"].exp; + const iat = this.decodedTapToken["payload"].iat; + const tapTokenLifetimeMins = Math.floor((exp - iat) / 60); + if (exp && this.checkExp(exp)) { + this.logger.debug({ cachedBdTapToken: true }); + // we clean up expired tokens at the same time + // clean first as we use deepmerge that merges lists and would otherwise cause duplicates + const credentials = { bdTapToken: this.bdTapToken }; + await updateConfig({ credentials }); // local config file + // NOTE: Even if the tokenLifetime would be different from the request, we return non-expired token + return { + bdTapToken: this.bdTapToken, + cached: true, + expiresIn: this.getHumanReadable(exp), + tokenLifetimeMins: tapTokenLifetimeMins, + }; + } + return; // expired + } + private async getTokenResp( shareId?: string, ): Promise<{ bdStsToken: string; cached: boolean; expiresIn: string; tokenLifetimeMins: number } | undefined> { this.selectAndDecodeToken(shareId); this.dumpSelectedToken(); - if (!this.decodedToken || !this.selectedToken) throw new Error("Unable to select/decode token"); + if (!this.decodedToken || !this.selectedToken) throw new Error("Unable to select/decode STS token"); const exp = this.decodedToken["payload"].exp; const iat = this.decodedToken["payload"].iat; const tokenLifetimeMins = Math.floor((exp - iat) / 60); if (exp && this.checkExp(exp)) { - this.logger.debug({ cachedBdStstToken: true }); + this.logger.debug({ cachedBdStsToken: true }); // we clean up expired tokens at the same time // clean first as we use deepmerge that merges lists and would otherwise cause duplicates await updateConfig({ credentials: { sharedTokens: undefined } }); @@ -206,7 +245,40 @@ export class BDAccount { return; // expired } - public async getToken(tokenLifetime: string, shareId?: string): Promise<{ bdStsToken: string; cached: boolean }> { + public async getTapToken( + tokenLifetime: string, + sharingUser?: string, + ): Promise<{ bdTapToken: string; cached: boolean }> { + if (this.bdTapToken) return { bdTapToken: this.bdTapToken, cached: true }; + const creds = await getConfigCredentials(); + this.bdTapToken = creds.bdTapToken; + // channel("undici:request:create").subscribe(console.log); + // channel("undici:request:headers").subscribe(console.log); + const headers = await getReqHeaders(this.cognitoIdToken); // , { tokenLifetime, vendingSchedule, shareId }); + const method = "POST"; + const body = JSON.stringify({ tokenLifetime, sharingUser }); + this.logger.debug({ method, tapTokenUrl, headers, body }); + const res = await fetch(tapTokenUrl, { method, headers, body }); + const resBody = await res.json(); + this.logger.debug({ getTapToken: { body: resBody } }); + if (!resBody.ResponseCode || !resBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (resBody.ResponseCode != "00") { + spinnerError(resBody.ResponseText); + throw new Error(`Failed to fetch token: ${resBody.ResponseText}`); + } + if (!resBody.bdTapToken) { + throw new Error("Missing bdStsToken in BD API Response"); + } + this.bdTapToken = resBody.bdTapToken; + + const resp = await this.getTapTokenResp(); + if (resp) return resp; + throw new Error(`Failed to get fresh TAP token from BD API`); + } + + public async getStsToken(tokenLifetime: string, shareId?: string): Promise<{ bdStsToken: string; cached: boolean }> { if (this.bdStsToken && !shareId) { this.selectedToken = this.bdStsToken; return { bdStsToken: this.bdStsToken, cached: true }; @@ -233,8 +305,8 @@ export class BDAccount { method = "POST"; body = JSON.stringify({ tokenLifetime, shareId }); } - this.logger.debug({ method, tokenUrl, headers, body }); - const res = await fetch(tokenUrl, { method, headers, body }); + this.logger.debug({ method, tokenUrl: stsTokenUrl, headers, body }); + const res = await fetch(stsTokenUrl, { method, headers, body }); const resBody = await res.json(); this.logger.debug({ getStsToken: { body: resBody } }); if (!resBody.ResponseCode || !resBody.ResponseText) { @@ -261,6 +333,6 @@ export class BDAccount { const resp = await this.getTokenResp(shareId); if (resp) return resp; - throw new Error(`Failed to get fresh token from BD API (with share id ${shareId})`); + throw new Error(`Failed to get fresh STS token from BD API (with share id ${shareId})`); } } diff --git a/src/integration/boilingdata/boilingdata_api.ts b/src/integration/boilingdata/boilingdata_api.ts index c11811d..3545fbe 100644 --- a/src/integration/boilingdata/boilingdata_api.ts +++ b/src/integration/boilingdata/boilingdata_api.ts @@ -4,12 +4,14 @@ import * as id from "amazon-cognito-identity-js"; export const baseApiUrl = "https://rest.api.test.boilingdata.com"; export const dataSetsPath = "/data-sets"; export const accountPath = "/account"; -export const tokenPath = "/token"; +export const stsTokenPath = "/token"; +export const tapTokenPath = "/taptoken"; export const sharePath = "/share"; export const sandboxPath = "/sandbox"; export const dataSetsUrl = baseApiUrl + dataSetsPath; export const accountUrl = baseApiUrl + accountPath; -export const tokenUrl = baseApiUrl + tokenPath; +export const stsTokenUrl = baseApiUrl + stsTokenPath; +export const tapTokenUrl = baseApiUrl + tapTokenPath; export const tokenShareUrl = baseApiUrl + sharePath; export const sandboxUrl = baseApiUrl + sandboxPath; // FIXME: get from bdAccount API diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index 69c3fbf..b88a8dd 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -19,7 +19,7 @@ export class BDSandbox { this.cognitoIdToken = this.params.authToken; } - public async destroySandbox(sandboxName: string): Promise { + public async destroySandbox(sandboxName: string): Promise { const headers = await getReqHeaders(this.cognitoIdToken); this.logger.debug({ sandboxUrl, headers }); const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "DELETE", headers }); @@ -28,6 +28,7 @@ export class BDSandbox { if (!respBody.ResponseCode || !respBody.ResponseText) { throw new Error("Malformed response from BD API"); } + return respBody; } public async downloadTemplate(sandboxName: string): Promise { @@ -68,19 +69,19 @@ export class BDSandbox { throw new Error("Failed to list sandboxes"); } - public async planSandbox(sandboxName: string): Promise { + public async planSandbox(sandboxName: string): Promise { const planOnly = true; const diffOnly = false; return this._deploySandbox(sandboxName, planOnly, diffOnly); } - public async diffSandbox(sandboxName: string): Promise { + public async diffSandbox(sandboxName: string): Promise { const planOnly = false; const diffOnly = true; return this._deploySandbox(sandboxName, planOnly, diffOnly); } - public async deploySandbox(sandboxName: string): Promise { + public async deploySandbox(sandboxName: string): Promise { return this._deploySandbox(sandboxName); } From 7ef57c926fd297c7d83b95449bef5caa28c929b4 Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Wed, 10 Jan 2024 13:51:29 +0200 Subject: [PATCH 04/14] fix tap token caching and scope checking --- .../account/bdcli-account-tap-token.ts | 3 +- src/integration/boilingdata/account.ts | 44 +++++++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/bdcli/commands/account/bdcli-account-tap-token.ts b/src/bdcli/commands/account/bdcli-account-tap-token.ts index c97eb40..71c05e9 100644 --- a/src/bdcli/commands/account/bdcli-account-tap-token.ts +++ b/src/bdcli/commands/account/bdcli-account-tap-token.ts @@ -46,8 +46,7 @@ const program = new cmd.Command("bdcli account tap-token") .addOption( new cmd.Option( "--sharing-user ", - "A user has shared Tap for you so that you can write to it." + - "\n\tExpects 'module: boilingdata' entry and upserts its config.token value", + "A user has shared Tap for you so that you can write to it.", ), ) .action(async (options, command) => await show(options, command)); diff --git a/src/integration/boilingdata/account.ts b/src/integration/boilingdata/account.ts index b6ad789..ab895e6 100644 --- a/src/integration/boilingdata/account.ts +++ b/src/integration/boilingdata/account.ts @@ -17,6 +17,16 @@ export interface IBDConfig { logger: ILogger; } +export interface ITapTokenResp { + bdTapToken: string; + cached: boolean; + expiresIn: string; + tokenLifetimeMins: number; + username: string; + email: string; + sharingUser: string; +} + interface IAPIAccountDetails { AccountAwsAccount: string; AccountExtId: string; @@ -128,14 +138,14 @@ export class BDAccount { return humanReadable.toISOString(); } - private checkExp(exp: number): boolean { - const twoMinsAgo = new Date(); - twoMinsAgo.setTime(Date.now() - 2 * 60 * 1000); - const diff = exp * 1000 - twoMinsAgo.getTime(); + private checkExp(exp: number, minutesAgo = 2): boolean { + const minsAgo = new Date(); + minsAgo.setTime(Date.now() - minutesAgo * 60 * 1000); + const diff = exp * 1000 - minsAgo.getTime(); this.logger.debug({ exp, diff, - twoMinsAgo: twoMinsAgo.toISOString(), + twoMinsAgo: minsAgo.toISOString(), expiresIn: this.getHumanReadable(exp), }); if (diff < 0) return false; @@ -197,27 +207,30 @@ export class BDAccount { throw new Error("Failed to unshare token"); } - private async getTapTokenResp(): Promise< - { bdTapToken: string; cached: boolean; expiresIn: string; tokenLifetimeMins: number } | undefined - > { + private async getTapTokenResp(updateConfigFile = true): Promise { this.decodeTapToken(); this.dumpTapToken(); if (!this.decodedTapToken || !this.bdTapToken) throw new Error("Unable to decode TAP token"); const exp = this.decodedTapToken["payload"].exp; const iat = this.decodedTapToken["payload"].iat; + const aud = this.decodedTapToken["payload"].aud; const tapTokenLifetimeMins = Math.floor((exp - iat) / 60); - if (exp && this.checkExp(exp)) { + const oneHourMin = 60; + if (exp && this.checkExp(exp, oneHourMin)) { this.logger.debug({ cachedBdTapToken: true }); // we clean up expired tokens at the same time // clean first as we use deepmerge that merges lists and would otherwise cause duplicates const credentials = { bdTapToken: this.bdTapToken }; - await updateConfig({ credentials }); // local config file + if (updateConfigFile) await updateConfig({ credentials }); // local config file // NOTE: Even if the tokenLifetime would be different from the request, we return non-expired token return { bdTapToken: this.bdTapToken, - cached: true, + cached: updateConfigFile != true, expiresIn: this.getHumanReadable(exp), tokenLifetimeMins: tapTokenLifetimeMins, + username: aud?.[0], + email: aud?.[1], + sharingUser: aud?.[2], }; } return; // expired @@ -249,9 +262,14 @@ export class BDAccount { tokenLifetime: string, sharingUser?: string, ): Promise<{ bdTapToken: string; cached: boolean }> { - if (this.bdTapToken) return { bdTapToken: this.bdTapToken, cached: true }; const creds = await getConfigCredentials(); this.bdTapToken = creds.bdTapToken; + if (this.bdTapToken) { + const cachedToken = await this.getTapTokenResp(false); + if (cachedToken && ((sharingUser && cachedToken?.sharingUser === sharingUser) || !sharingUser)) { + return cachedToken; // disk cached token is not yet expired.. + } + } // channel("undici:request:create").subscribe(console.log); // channel("undici:request:headers").subscribe(console.log); const headers = await getReqHeaders(this.cognitoIdToken); // , { tokenLifetime, vendingSchedule, shareId }); @@ -273,7 +291,7 @@ export class BDAccount { } this.bdTapToken = resBody.bdTapToken; - const resp = await this.getTapTokenResp(); + const resp = await this.getTapTokenResp(true); if (resp) return resp; throw new Error(`Failed to get fresh TAP token from BD API`); } From 4cdd5359a2635648bde91ef7e87fdad2c89e769a Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Thu, 11 Jan 2024 15:40:39 +0200 Subject: [PATCH 05/14] Enhance sandbox template listing --- src/bdcli/commands/bdcli-sandbox.ts | 1 - .../commands/sandbox/bdcli-sandbox-destroy.ts | 9 +++- .../commands/sandbox/bdcli-sandbox-diff.ts | 52 ------------------- .../sandbox/bdcli-sandbox-download.ts | 3 +- .../commands/sandbox/bdcli-sandbox-list.ts | 4 +- .../commands/sandbox/bdcli-sandbox-plan.ts | 5 ++ src/integration/boilingdata/sandbox.ts | 23 ++++++-- 7 files changed, 36 insertions(+), 61 deletions(-) delete mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts diff --git a/src/bdcli/commands/bdcli-sandbox.ts b/src/bdcli/commands/bdcli-sandbox.ts index b3e82c1..44fc906 100644 --- a/src/bdcli/commands/bdcli-sandbox.ts +++ b/src/bdcli/commands/bdcli-sandbox.ts @@ -6,7 +6,6 @@ const program = new Command("bdcli sandbox") .command("validate", "Validate sandbox template (e.g. for CI)") .command("upload", "Upload sandbox template") .command("download", "Download the deployed sandbox template") - .command("diff", "Yaml diff the current template file with the deployed one") .command("plan", "List planned udpates/additions/deletions based on local sandbox definition and deployed sandbox") .command("deploy", "Deploy the sandbox") .command("destroy", "Destroy the sandbox"); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts index fc67771..4e58ba7 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts @@ -25,7 +25,7 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Destroying sandbox ${options.name}`); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - const results = await bdSandbox.destroySandbox(options.name); + const results = await bdSandbox.destroySandbox(options.name, options.destroyAlsoInterfaces, options.deleteTemplate); spinnerSuccess(); await outputResults(results?.destroyResults, options.disableSpinner); } catch (err: any) { @@ -35,6 +35,13 @@ async function show(options: any, _command: cmd.Command): Promise { const program = new cmd.Command("bdcli sandbox destroy") .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) + .addOption(new cmd.Option("--destroy-also-interfaces", "Also delete interfaces like Tap URLs")) + .addOption( + new cmd.Option( + "--delete-template", + "Finally, delete template if all resources were destroyed, including interfaces", + ), + ) .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) .action(async (options, command) => await show(options, command)); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts deleted file mode 100644 index 95a165a..0000000 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-diff.ts +++ /dev/null @@ -1,52 +0,0 @@ -import * as cmd from "commander"; -import { getLogger } from "../../utils/logger_util.js"; -import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; -import { addGlobalOptions } from "../../utils/options_util.js"; -import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; -import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; -import { getIdToken } from "../../utils/auth_util.js"; -import * as fs from "fs/promises"; -import { outputResults } from "../../utils/output_util.js"; - -const logger = getLogger("bdcli-sandbox-diff"); - -async function show(options: any, _command: cmd.Command): Promise { - const tmpFileName = `/tmp/.${Date.now()}__${Math.random() * 1000}.yaml`; - try { - options = await combineOptsWithSettings(options, logger); - - if (!(await hasValidConfig())) { - return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); - } - - updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); - updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); - spinnerSuccess(); - - updateSpinnerText(`Checking diff between template and deployment for sandbox ${options.name}`); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); - const bdSandbox = new BDSandbox({ logger, authToken: token }); - const results = await bdSandbox.diffSandbox(options.name); - spinnerSuccess(); - await outputResults(results?.diffResults, options.disableSpinner); - } catch (err: any) { - try { - // cleanup - await fs.unlink(tmpFileName); - } catch (err) { - // ok, if it does not exist etc. - } - spinnerError(err?.message); - } -} - -const program = new cmd.Command("bdcli sandbox diff") - .addOption(new cmd.Option("--name ", "sandbox name to compare with").makeOptionMandatory()) - .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) - .action(async (options, command) => await show(options, command)); - -(async () => { - await addGlobalOptions(program, logger); - await program.parseAsync(process.argv); -})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts index 93a440b..65d9b4b 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts @@ -38,7 +38,7 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Downloading sandbox IaC template of ${options.name}`); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - const template = await bdSandbox.downloadTemplate(options.name); + const template = await bdSandbox.downloadTemplate(options.name, options.version); await fs.writeFile(filename, template); spinnerSuccess(); } catch (err: any) { @@ -48,6 +48,7 @@ async function show(options: any, _command: cmd.Command): Promise { const program = new cmd.Command("bdcli sandbox download") .addOption(new cmd.Option("--name ", "template name from listing").makeOptionMandatory()) + .addOption(new cmd.Option("--version ", "Download specific version from listing")) .action(async (options, command) => await show(options, command)); (async () => { diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts index 0cfee5c..5d867f2 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts @@ -25,7 +25,7 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText("Listing sandboxes"); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - const list = await bdSandbox.listSandboxes(); + const list = await bdSandbox.listSandboxes(options.listDeleted, options.listVersions); spinnerSuccess(); await outputResults(list, options.disableSpinner); } catch (err: any) { @@ -35,6 +35,8 @@ async function show(options: any, _command: cmd.Command): Promise { const program = new cmd.Command("bdcli sandbox list") .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .addOption(new cmd.Option("--list-deleted", "List also deleted templates")) + .addOption(new cmd.Option("--list-versions", "List all template versions")) .action(async (options, command) => await show(options, command)); (async () => { diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts index fd2bf04..8d1e136 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts @@ -33,6 +33,11 @@ async function show(options: any, _command: cmd.Command): Promise { } } +// TODO: +// - If the template is updated, like changing the name of a resource, it needs to be replaced? +// Like if the Tap name is changed the Lambda needs to be deleted and created again and then +// also the ingest URL changes. + const program = new cmd.Command("bdcli sandbox plan") .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index b88a8dd..9f8b7b0 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -19,8 +19,14 @@ export class BDSandbox { this.cognitoIdToken = this.params.authToken; } - public async destroySandbox(sandboxName: string): Promise { + public async destroySandbox( + sandboxName: string, + destroyAlsoInterfaces: boolean, + finallyDeleteTemplate: boolean, + ): Promise { const headers = await getReqHeaders(this.cognitoIdToken); + headers["x-bd-destroy-also-interfaces"] = `${destroyAlsoInterfaces}`; + headers["x-bd-finally-delete-template"] = `${finallyDeleteTemplate}`; this.logger.debug({ sandboxUrl, headers }); const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "DELETE", headers }); const respBody = await res.json(); @@ -28,11 +34,13 @@ export class BDSandbox { if (!respBody.ResponseCode || !respBody.ResponseText) { throw new Error("Malformed response from BD API"); } + if (respBody.ResponseCode != "00") throw new Error(respBody.ResponseText); return respBody; } - public async downloadTemplate(sandboxName: string): Promise { + public async downloadTemplate(sandboxName: string, version: string): Promise { const headers = await getReqHeaders(this.cognitoIdToken); + headers["x-bd-template-version"] = version; this.logger.debug({ sandboxUrl, headers }); const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "GET", headers }); const respBody = await res.json(); @@ -52,10 +60,15 @@ export class BDSandbox { return this._uploadTemplate(templateFilename, validateOnly); } - public async listSandboxes(): Promise> { + public async listSandboxes( + listDeleted: boolean, + listVersions: boolean, + ): Promise> { // channel("undici:request:create").subscribe(console.log); // channel("undici:request:headers").subscribe(console.log); const headers = await getReqHeaders(this.cognitoIdToken); + headers["x-bd-list-deleted"] = `${listDeleted}`; + headers["x-bd-list-versions"] = `${listVersions}`; this.logger.debug({ sandboxUrl, headers }); const res = await fetch(sandboxUrl, { method: "GET", headers }); const body = await res.json(); @@ -99,10 +112,10 @@ export class BDSandbox { if (!respBody.ResponseCode || !respBody.ResponseText) { throw new Error("Malformed response from BD API"); } - if (validateOnly && (!respBody?.isValidateOnly || respBody.validationResults != "OK")) { + if (respBody.ResponseCode != "00") { throw new Error(`Validation failed: ${respBody.validationResults}`); } - return respBody.isValidateOnly; + return respBody; } private async _deploySandbox(sandboxName: string, planOnly = false, diffOnly = false): Promise { From 99a9e65027acee12c9b75addfa9ce1f9e69d797e Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Fri, 12 Jan 2024 22:46:18 +0200 Subject: [PATCH 06/14] show sandbox template validation errors more nicely --- .../sandbox/bdcli-sandbox-validate.ts | 20 +++++++++++++++---- src/bdcli/utils/output_util.ts | 2 +- src/bdcli/utils/spinner_util.ts | 4 ++-- src/integration/boilingdata/sandbox.ts | 14 ++++++++----- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts index 2686423..a4ce7b1 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts @@ -25,16 +25,28 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText("Validating sandbox IaC template"); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - const results = await bdSandbox.validateTemplate(options.template); + await bdSandbox.validateTemplate(options.template, options.warningsAsErrors); spinnerSuccess(); - await outputResults(results, options.disableSpinner); - } catch (err: any) { - spinnerError(err?.message); + } catch (origErr: any) { + // try to decode the message + try { + spinnerError(origErr?.message, false); + await outputResults( + JSON.parse(origErr?.message) + ?.message?.split(";") + ?.map((msg: string) => msg.trim()), + false, + ); + } catch (err: any) { + spinnerError(origErr?.message, false); + console.error(err); + } } } const program = new cmd.Command("bdcli sandbox validate") .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--warnings-as-errors", "Treat any warning as an error")) .action(async (options, command) => await show(options, command)); (async () => { diff --git a/src/bdcli/utils/output_util.ts b/src/bdcli/utils/output_util.ts index 0105b58..f08fc24 100644 --- a/src/bdcli/utils/output_util.ts +++ b/src/bdcli/utils/output_util.ts @@ -1,6 +1,6 @@ import * as util from "node:util"; -export async function outputResults(results: any, flat: boolean): Promise { +export async function outputResults(results: any, flat = false): Promise { if (!results) return; console.log(flat ? JSON.stringify(results) : util.inspect(results, false, 20, true)); } diff --git a/src/bdcli/utils/spinner_util.ts b/src/bdcli/utils/spinner_util.ts index db54f5b..9a45229 100644 --- a/src/bdcli/utils/spinner_util.ts +++ b/src/bdcli/utils/spinner_util.ts @@ -39,9 +39,9 @@ export function spinnerWarn(message?: string): void { spinner.warn(message ? warning(message) : warning(spinner.text)); } -export function spinnerError(message?: string): void { +export function spinnerError(message?: string, forceExit = true): void { if (isEnabled) spinner.fail(message ? error(message) : undefined); - process.exit(1); // error + if (forceExit) process.exit(1); // error } export function spinnerSuccess(message?: string): void { diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index 9f8b7b0..3644b9b 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -55,9 +55,9 @@ export class BDSandbox { return this._uploadTemplate(templateFilename); } - public async validateTemplate(templateFilename: string): Promise { + public async validateTemplate(templateFilename: string, warningsAsErrors = false): Promise { const validateOnly = true; - return this._uploadTemplate(templateFilename, validateOnly); + return this._uploadTemplate(templateFilename, validateOnly, warningsAsErrors); } public async listSandboxes( @@ -100,11 +100,15 @@ export class BDSandbox { // ---- private ---- - private async _uploadTemplate(templateFilename: string, validateOnly = false): Promise { + private async _uploadTemplate( + templateFilename: string, + validateOnly = false, + warningsAsErrors = false, + ): Promise { const headers = await getReqHeaders(this.cognitoIdToken); this.logger.debug({ sandboxUrl, headers }); const template = Buffer.from(await fs.readFile(templateFilename)).toString("base64"); - const body = JSON.stringify({ validateOnly, template }); + const body = JSON.stringify({ validateOnly, warningsAsErrors, template }); this.logger.debug({ body }); const res = await fetch(sandboxUrl, { method: "PUT", headers, body }); const respBody = await res.json(); @@ -113,7 +117,7 @@ export class BDSandbox { throw new Error("Malformed response from BD API"); } if (respBody.ResponseCode != "00") { - throw new Error(`Validation failed: ${respBody.validationResults}`); + throw new Error(respBody?.validationResults ?? respBody); } return respBody; } From f82c421862aee5e95dc08868d4d9e3340e0bdb5a Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Tue, 16 Jan 2024 08:51:06 +0200 Subject: [PATCH 07/14] Add local sandbox template validation with ts-interface-checker as well --- package.json | 2 +- src/bdcli/commands/bdcli-sandbox.ts | 8 +- .../commands/sandbox/bdcli-sandbox-deploy.ts | 4 +- .../boilingdata/sandbox-template.types-ti.ts | 75 +++++++++++++++++++ .../boilingdata/sandbox-template.types.ts | 57 ++++++++++++++ src/integration/boilingdata/sandbox.ts | 24 +++++- 6 files changed, 161 insertions(+), 9 deletions(-) create mode 100644 src/integration/boilingdata/sandbox-template.types-ti.ts create mode 100644 src/integration/boilingdata/sandbox-template.types.ts diff --git a/package.json b/package.json index 436a120..cde99c5 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ }, "scripts": { "release": "standard-version -a", - "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts ", + "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts && npx ts-interface-builder src/integration/boilingdata/template.types.ts", "build": "rm -rf dist/ && tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && chmod 755 dist/esm/index.js", "prettier": "prettier --check 'src/**/*.{js,ts}'", "prettier:fix": "prettier --write 'src/**/*.{js,ts}'", diff --git a/src/bdcli/commands/bdcli-sandbox.ts b/src/bdcli/commands/bdcli-sandbox.ts index 44fc906..73a0890 100644 --- a/src/bdcli/commands/bdcli-sandbox.ts +++ b/src/bdcli/commands/bdcli-sandbox.ts @@ -3,11 +3,11 @@ import { Command } from "commander"; const program = new Command("bdcli sandbox") .executableDir("sandbox") .command("list", "List deployed sandboxes") - .command("validate", "Validate sandbox template (e.g. for CI)") + .command("validate", "Validate *local* sandbox template") .command("upload", "Upload sandbox template") - .command("download", "Download the deployed sandbox template") - .command("plan", "List planned udpates/additions/deletions based on local sandbox definition and deployed sandbox") - .command("deploy", "Deploy the sandbox") + .command("download", "Download the *uploaded* sandbox template") + .command("plan", "List planned udpates based on *uploaded* sandbox definition and deployed sandbox") + .command("deploy", "Deploy the *uploaded* sandbox") .command("destroy", "Destroy the sandbox"); program.parse(process.argv); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts index 6e677c8..a31d71e 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts @@ -18,12 +18,11 @@ async function show(options: any, _command: cmd.Command): Promise { } updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + const { idToken: token, cached: idCached } = await getIdToken(logger); updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); updateSpinnerText(`Deploying sandbox ${options.name}`); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); const results = await bdSandbox.deploySandbox(options.name); spinnerSuccess(); @@ -35,7 +34,6 @@ async function show(options: any, _command: cmd.Command): Promise { const program = new cmd.Command("bdcli sandbox deploy") .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) - .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) .action(async (options, command) => await show(options, command)); (async () => { diff --git a/src/integration/boilingdata/sandbox-template.types-ti.ts b/src/integration/boilingdata/sandbox-template.types-ti.ts new file mode 100644 index 0000000..4523eaa --- /dev/null +++ b/src/integration/boilingdata/sandbox-template.types-ti.ts @@ -0,0 +1,75 @@ +/** + * This module was automatically generated by `ts-interface-builder` + */ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const EModelFormat = t.enumtype({ + "NDJSON": "ndjson", + "AVRO": "avro", + "CSV": "csv", +}); + +export const EPermission = t.enumtype({ + "READ": "read", + "WRITE": "write", +}); + +export const ITemplateSandbox = t.iface([], { + "name": "string", + "urlPrefix": "string", + "uniqNamePart": t.opt("string"), + "permissions": t.array("EPermission"), +}); + +export const ITemplateTap = t.iface([], { + "name": "string", + "models": t.opt(t.array(t.iface([], { + "name": "string", + "model": t.opt(t.array("string")), + "format": t.opt("EModelFormat"), + }))), +}); + +export const ITemplateFlow = t.iface([], { + "name": "string", + "input": t.union("string", t.array("string")), + "keys": t.opt(t.array("string")), + "transformJs": t.opt("string"), + "transformSql": t.opt("string"), + "prefixFunc": t.opt("string"), + "output": t.opt(t.union("string", t.array("string"))), + "errors": t.opt("string"), +}); + +export const ITemplateShare = t.iface([], { + "name": "string", + "users": t.array("string"), + "sql": t.opt("string"), + "source": t.opt("string"), + "target": t.opt("string"), +}); + +export const ITemplate = t.iface([], { + "version": t.union("string", "number"), + "name": "string", + "environment": "string", + "region": "string", + "resources": t.iface([], { + "sandboxes": t.opt(t.array("ITemplateSandbox")), + "taps": t.opt(t.array("ITemplateTap")), + "flows": t.opt(t.array("ITemplateFlow")), + "shares": t.opt(t.array("ITemplateShare")), + }), +}); + +const exportedTypeSuite: t.ITypeSuite = { + EModelFormat, + EPermission, + ITemplateSandbox, + ITemplateTap, + ITemplateFlow, + ITemplateShare, + ITemplate, +}; +export default exportedTypeSuite; diff --git a/src/integration/boilingdata/sandbox-template.types.ts b/src/integration/boilingdata/sandbox-template.types.ts new file mode 100644 index 0000000..fd7d0f9 --- /dev/null +++ b/src/integration/boilingdata/sandbox-template.types.ts @@ -0,0 +1,57 @@ +export enum EModelFormat { + NDJSON = "ndjson", + AVRO = "avro", + CSV = "csv", +} +export enum EPermission { + READ = "read", + WRITE = "write", +} + +export interface ITemplateSandbox { + name: string; + urlPrefix: string; + uniqNamePart?: string; + permissions: EPermission[]; +} + +export interface ITemplateTap { + name: string; + models?: Array<{ + name: string; + model?: string[]; + format?: EModelFormat; + }>; +} + +export interface ITemplateFlow { + name: string; + input: string | string[]; + keys?: string[]; + transformJs?: string; + transformSql?: string; + prefixFunc?: string; + output?: string | string[]; + errors?: string; +} + +export interface ITemplateShare { + name: string; + users: string[]; + sql?: string; + source?: string; + target?: string; +} + +export interface ITemplate { + version: string | number; + name: string; + environment: string; + region: string; + resources: { + sandboxes?: ITemplateSandbox[]; + taps?: ITemplateTap[]; + flows?: ITemplateFlow[]; + shares?: ITemplateShare[]; + }; +} diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index 3644b9b..d6301dc 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -1,6 +1,10 @@ +import * as fs from "fs/promises"; +import * as yaml from "js-yaml"; import { ILogger } from "../../bdcli/utils/logger_util.js"; import { getReqHeaders, sandboxUrl } from "./boilingdata_api.js"; -import * as fs from "fs/promises"; +import { createCheckers } from "ts-interface-checker"; +import sandboxTemplateTI from "./sandbox-template.types-ti.js"; +import { ITemplate } from "./sandbox-template.types.js"; // import { channel } from "node:diagnostics_channel"; @@ -19,6 +23,17 @@ export class BDSandbox { this.cognitoIdToken = this.params.authToken; } + public isSandboxConfig(sandboxTemplate: unknown): sandboxTemplate is ITemplate { + try { + const { ITemplate } = createCheckers(sandboxTemplateTI); + ITemplate?.check(sandboxTemplate); + return true; + } catch (err) { + this.logger.error({ err }); + return false; + } + } + public async destroySandbox( sandboxName: string, destroyAlsoInterfaces: boolean, @@ -52,11 +67,18 @@ export class BDSandbox { } public async uploadTemplate(templateFilename: string): Promise { + // local validation + const sandboxTemplateConfig = yaml.load(await fs.readFile(templateFilename, "utf8")); + if (!this.isSandboxConfig(sandboxTemplateConfig)) throw new Error("sandbox template config schema not validated"); return this._uploadTemplate(templateFilename); } public async validateTemplate(templateFilename: string, warningsAsErrors = false): Promise { + // local validation + const sandboxTemplateConfig = yaml.load(await fs.readFile(templateFilename, "utf8")); + if (!this.isSandboxConfig(sandboxTemplateConfig)) throw new Error("sandbox template config schema not validated"); const validateOnly = true; + // remote validation return this._uploadTemplate(templateFilename, validateOnly, warningsAsErrors); } From 2ba3ab7b7a107c2f2250f4a772ddb45b7903c1d2 Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Tue, 16 Jan 2024 10:48:03 +0200 Subject: [PATCH 08/14] updates, types, naming --- package.json | 2 +- src/integration/boilingdata/sandbox-template.types-ti.ts | 9 +++++---- src/integration/boilingdata/sandbox-template.types.ts | 9 +++++---- src/integration/boilingdata/sandbox.ts | 5 +++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index cde99c5..7880401 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ }, "scripts": { "release": "standard-version -a", - "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts && npx ts-interface-builder src/integration/boilingdata/template.types.ts", + "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts && npx ts-interface-builder src/integration/boilingdata/sandbox-template.types.ts", "build": "rm -rf dist/ && tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && chmod 755 dist/esm/index.js", "prettier": "prettier --check 'src/**/*.{js,ts}'", "prettier:fix": "prettier --write 'src/**/*.{js,ts}'", diff --git a/src/integration/boilingdata/sandbox-template.types-ti.ts b/src/integration/boilingdata/sandbox-template.types-ti.ts index 4523eaa..419a6e6 100644 --- a/src/integration/boilingdata/sandbox-template.types-ti.ts +++ b/src/integration/boilingdata/sandbox-template.types-ti.ts @@ -17,9 +17,10 @@ export const EPermission = t.enumtype({ export const ITemplateSandbox = t.iface([], { "name": "string", - "urlPrefix": "string", - "uniqNamePart": t.opt("string"), - "permissions": t.array("EPermission"), + "permissions": t.array(t.iface([], { + "urlPrefix": "string", + "accessRights": t.opt(t.array("EPermission")), + })), }); export const ITemplateTap = t.iface([], { @@ -56,7 +57,7 @@ export const ITemplate = t.iface([], { "environment": "string", "region": "string", "resources": t.iface([], { - "sandboxes": t.opt(t.array("ITemplateSandbox")), + "sandbox": "ITemplateSandbox", "taps": t.opt(t.array("ITemplateTap")), "flows": t.opt(t.array("ITemplateFlow")), "shares": t.opt(t.array("ITemplateShare")), diff --git a/src/integration/boilingdata/sandbox-template.types.ts b/src/integration/boilingdata/sandbox-template.types.ts index fd7d0f9..1f15eb1 100644 --- a/src/integration/boilingdata/sandbox-template.types.ts +++ b/src/integration/boilingdata/sandbox-template.types.ts @@ -10,9 +10,10 @@ export enum EPermission { export interface ITemplateSandbox { name: string; - urlPrefix: string; - uniqNamePart?: string; - permissions: EPermission[]; + permissions: Array<{ + urlPrefix: string; + accessRights?: EPermission[]; + }>; } export interface ITemplateTap { @@ -49,7 +50,7 @@ export interface ITemplate { environment: string; region: string; resources: { - sandboxes?: ITemplateSandbox[]; + sandbox: ITemplateSandbox; taps?: ITemplateTap[]; flows?: ITemplateFlow[]; shares?: ITemplateShare[]; diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index d6301dc..7ca118f 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -25,8 +25,9 @@ export class BDSandbox { public isSandboxConfig(sandboxTemplate: unknown): sandboxTemplate is ITemplate { try { - const { ITemplate } = createCheckers(sandboxTemplateTI); - ITemplate?.check(sandboxTemplate); + const { ITemplate: ITemplateChecker } = createCheckers(sandboxTemplateTI); + if (!ITemplateChecker) throw new Error("ts-interface-check checker MISSING"); + ITemplateChecker.check(sandboxTemplate); return true; } catch (err) { this.logger.error({ err }); From 1cd67e736f2e3bdc249e2bf2b07002b0a716fbba Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Tue, 16 Jan 2024 12:49:18 +0200 Subject: [PATCH 09/14] update naming for role/policy --- src/bdcli/commands/aws/bdcli-aws-iam.ts | 2 +- .../commands/domain/bdcli-domain-setup.ts | 1 - .../aws/{util.test.ts => aws-region.test.ts} | 62 +++++++++---------- src/integration/aws/aws-region.ts | 14 +++++ src/integration/aws/iam_roles.test.ts | 16 ++--- src/integration/aws/iam_roles.ts | 49 ++++++++------- src/integration/aws/util.ts | 8 --- src/integration/bdIntegration.test.ts | 8 +-- src/integration/boilingdata/account.ts | 16 ++++- src/integration/boilingdata/dataset.ts | 12 ---- 10 files changed, 97 insertions(+), 91 deletions(-) rename src/integration/aws/{util.test.ts => aws-region.test.ts} (52%) create mode 100644 src/integration/aws/aws-region.ts delete mode 100644 src/integration/aws/util.ts diff --git a/src/bdcli/commands/aws/bdcli-aws-iam.ts b/src/bdcli/commands/aws/bdcli-aws-iam.ts index 1630112..44eca51 100644 --- a/src/bdcli/commands/aws/bdcli-aws-iam.ts +++ b/src/bdcli/commands/aws/bdcli-aws-iam.ts @@ -39,7 +39,7 @@ async function iamrole(options: any, _command: cmd.Command): Promise { logger, iamClient: new iam.IAMClient({ region }), stsClient: new sts.STSClient({ region }), - uniqNamePart: await bdDataSources.getUniqueNamePart(), + username: await bdAccount.getUsername(), assumeAwsAccount: await bdAccount.getAssumeAwsAccount(), assumeCondExternalId: await bdAccount.getExtId(), }); diff --git a/src/bdcli/commands/domain/bdcli-domain-setup.ts b/src/bdcli/commands/domain/bdcli-domain-setup.ts index e8bba8e..712bb7e 100644 --- a/src/bdcli/commands/domain/bdcli-domain-setup.ts +++ b/src/bdcli/commands/domain/bdcli-domain-setup.ts @@ -39,7 +39,6 @@ async function iamrole(options: any, _command: cmd.Command): Promise { logger, iamClient: new iam.IAMClient({ region }), stsClient: new sts.STSClient({ region }), - uniqNamePart: await bdDataSources.getUniqueNamePart(), assumeAwsAccount: await bdAccount.getAssumeAwsAccount(), assumeCondExternalId: await bdAccount.getExtId(), }); diff --git a/src/integration/aws/util.test.ts b/src/integration/aws/aws-region.test.ts similarity index 52% rename from src/integration/aws/util.test.ts rename to src/integration/aws/aws-region.test.ts index 3df8241..b9bb58f 100644 --- a/src/integration/aws/util.test.ts +++ b/src/integration/aws/aws-region.test.ts @@ -1,4 +1,4 @@ -import { getRegionShortName } from "./util.js"; +import { getAwsRegionShortName } from "./aws-region.js"; const aws_regions = [ "us-east-2", @@ -34,36 +34,36 @@ const aws_regions = [ describe("utils", () => { it("util", () => { - expect(aws_regions.map(getRegionShortName)).toEqual([ - "ue2", - "ue1", - "uw1", - "uw2", - "as1", - "ae1", - "as2", - "ase3", - "ase4", - "as1", - "ane3", - "ane2", - "ase1", - "ase2", - "ane1", - "cc1", - "ec1", - "ew1", - "ew2", - "es1", - "ew3", - "es2", - "en1", - "ec2", - "ms1", - "mc1", - "se1", - "uge1", - "ugw1", + expect(aws_regions.map(getAwsRegionShortName)).toEqual([ + "use2", + "use1", + "usw1", + "usw2", + "afs1", + "ape1", + "aps2", + "apse3", + "apse4", + "aps1", + "apne3", + "apne2", + "apse1", + "apse2", + "apne1", + "cac1", + "euc1", + "euw1", + "euw2", + "eus1", + "euw3", + "eus2", + "eun1", + "euc2", + "mes1", + "mec1", + "sae1", + "usge1", + "usgw1", ]); }); }); diff --git a/src/integration/aws/aws-region.ts b/src/integration/aws/aws-region.ts new file mode 100644 index 0000000..b90251d --- /dev/null +++ b/src/integration/aws/aws-region.ts @@ -0,0 +1,14 @@ +export function getAwsRegionShortName(region: string): string { + // a bit hacky + const splits = region + .replace("east-", "-east-") + .replace("--", "-") + .replace("west-", "-west-") + .replace("--", "-") + .split("-"); + if (splits.length != 3 && splits.length != 4) throw new Error(`Invalid AWS Region: ${region}`); + if (splits.length == 4) { + return `${splits[0]}${splits[1]?.[0]}${splits[2]?.[0]}${splits[3]}`; + } + return `${splits[0]}${splits[1]?.[0]}${splits[2]}`; +} diff --git a/src/integration/aws/iam_roles.test.ts b/src/integration/aws/iam_roles.test.ts index a9b39f5..a291e76 100644 --- a/src/integration/aws/iam_roles.test.ts +++ b/src/integration/aws/iam_roles.test.ts @@ -25,7 +25,7 @@ const roleParams: IBDIamRole = { iamClient: new IAMClient({ region }), stsClient: new STSClient({ region }), region, - uniqNamePart: "boilingdata-demo-isecurefi-dev-and-all-the-rest-of-the-buckets", + username: "aac5c1d9-a0a9-4855-b896-0f3998b2f16b", assumeAwsAccount: "123123123123", assumeCondExternalId: "abcdef123123", }; @@ -48,17 +48,17 @@ describe("iamRole", () => { it("getIamRoleName", async () => { const role = new BDIamRole(roleParams); - expect(role.iamRoleName).toEqual("bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-11843e6abe8afd6"); + expect(role.iamRoleName).toEqual("bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-role"); }); it("getIamRoleName with own prefix", async () => { - const role = new BDIamRole({ ...roleParams, roleNamePrefix: "myPrefix" }); - expect(role.iamRoleName).toEqual("myPrefix-ue1-boilingdata-demo-isecurefi-dev-and--1232b7ccb3bac8a"); + const role = new BDIamRole({ ...roleParams, roleNamePrefix: "my" }); + expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-role"); }); it("getIamRoleName with own path and prefix", async () => { - const role = new BDIamRole({ ...roleParams, roleNamePrefix: "myPrefix", path: "/bd-service/demo/" }); - expect(role.iamRoleName).toEqual("myPrefix-ue1-boilingdata-demo-isecurefi-dev-and--2cca7b8bb91fa2f"); + const role = new BDIamRole({ ...roleParams, roleNamePrefix: "my", path: "/bd-service/demo/" }); + expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-role"); }); it("getRole", async () => { @@ -75,7 +75,7 @@ describe("iamRole", () => { PolicyName: "bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", Arn: "arn:aws:iam::123123123123:policy/" + - "boilingdata/bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", + "boilingdata/bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-policy", Path: "/boilingdata/", DefaultVersionId: "v123", AttachmentCount: 1, @@ -115,7 +115,7 @@ describe("iamRole", () => { PolicyName: "bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", Arn: "arn:aws:iam::123123123123:policy/" + - "boilingdata/bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", + "boilingdata/bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-policy", Path: "/boilingdata/", DefaultVersionId: "v100", AttachmentCount: 1, diff --git a/src/integration/aws/iam_roles.ts b/src/integration/aws/iam_roles.ts index 9718aaa..d1599c3 100644 --- a/src/integration/aws/iam_roles.ts +++ b/src/integration/aws/iam_roles.ts @@ -1,7 +1,6 @@ import * as iam from "@aws-sdk/client-iam"; -import * as crypto from "crypto"; import * as sts from "@aws-sdk/client-sts"; -import { getRegionShortName } from "./util.js"; +import { getAwsRegionShortName } from "./aws-region.js"; import { ILogger } from "../../bdcli/utils/logger_util.js"; export interface Tag { @@ -19,7 +18,9 @@ export interface IBDIamRole { iamClient: iam.IAMClient; stsClient: sts.STSClient; region: string; - uniqNamePart: string; + username: string; + environment?: string; + templateName?: string; assumeCondExternalId: string; assumeAwsAccount: string; path?: string; @@ -50,24 +51,17 @@ export class BDIamRole { private getName(type: string): string { const prefix = this.params.roleNamePrefix ?? "bd"; - const regionShort = getRegionShortName(this.params.region ?? process.env["AWS_REGION"] ?? "eu-west-2"); - const hash = crypto - .createHash("md5") - .update( - type + - this.path + - prefix + - regionShort + - this.params.uniqNamePart + - this.params.assumeAwsAccount + - this.params.assumeCondExternalId + - `${this.params.maxSessionDuration ?? "N/A"}`, - ); - const reducedHashLength = 15; - const nameLenSoFar = prefix.length + regionShort.length + reducedHashLength; - const roomLeft = 64 - nameLenSoFar - 3; // "-" chars - const dataSetShort = this.params.uniqNamePart.substring(0, roomLeft); - return [prefix, regionShort, dataSetShort, hash.digest("hex").substring(0, 15)].join("-"); + const regionShort = getAwsRegionShortName(this.params.region ?? process.env["AWS_REGION"] ?? "eu-west-1"); + const env = this.params.environment ?? "noenv"; + const tmplName = this.params.templateName ?? "notmplname"; + const username = this.params.username.replaceAll("-", ""); + const nameLenSoFar = + prefix.length + regionShort.length + env.length + tmplName.length + username.length + type.length; + const roomLeft = 64 - nameLenSoFar - 5; // 5 '-' chars + if (roomLeft < 0) { + throw new Error(`${type} name too long, reduce prefix/env/tmplName lengths (roomLeft ${roomLeft})`); + } + return [prefix, regionShort, env, tmplName, username, type].join("-"); } public async getAwsAccountId(): Promise { @@ -83,14 +77,18 @@ export class BDIamRole { public get iamRoleName(): string { if (this._iamRoleName) return this._iamRoleName; this._iamRoleName = this.getName(ENameType.ROLE); - if (this._iamRoleName.length > 64) throw new Error("Role name too long, bugger in code."); + if (this._iamRoleName.length > 64) { + throw new Error(`Role name too long, bugger in code (${this._iamRoleName})`); + } return this._iamRoleName; } public get iamManagedPolicyName(): string { if (this._iamManagedPolicyName) return this._iamManagedPolicyName; this._iamManagedPolicyName = this.getName(ENameType.POLICY); - if (this._iamManagedPolicyName.length > 64) throw new Error("Role name too long, bugger in code."); + if (this._iamManagedPolicyName.length > 64) { + throw new Error(`Policy name too long, bugger in code (${this._iamManagedPolicyName})`); + } return this._iamManagedPolicyName; } @@ -193,7 +191,10 @@ export class BDIamRole { Tags: [...(this.params.tags ?? []), ...this.boilingDataTags], }; const resp = await this.iamClient.send(new iam.CreatePolicyCommand(commandParams)); - if (!resp?.Policy?.Arn) throw new Error("Policy creation failed"); + if (!resp?.Policy?.Arn) { + this.logger.error({ resp }); + throw new Error("Policy creation failed"); + } } public async upsertBdAccessPolicy(PolicyDocument: string): Promise { diff --git a/src/integration/aws/util.ts b/src/integration/aws/util.ts deleted file mode 100644 index bb86ec3..0000000 --- a/src/integration/aws/util.ts +++ /dev/null @@ -1,8 +0,0 @@ -export function getRegionShortName(region: string): string { - return region - .replace("east", "-east") - .replace("--", "-") - .split("-") - .map(part => part[0]) - .join(""); -} diff --git a/src/integration/bdIntegration.test.ts b/src/integration/bdIntegration.test.ts index 77fc5c3..f52bf85 100644 --- a/src/integration/bdIntegration.test.ts +++ b/src/integration/bdIntegration.test.ts @@ -37,13 +37,13 @@ describe("BDIntegration", () => { bdDataSets.readConfig("./example_datasource_config.yaml"); const assumeCondExternalId = await bdAccount.getExtId(); // FIXME: This calls real API const assumeAwsAccount = await bdAccount.getAssumeAwsAccount(); - const uniqNamePart = await bdDataSets.getUniqueNamePart(); + const username = await bdAccount.getUsername(); const bdRole = new BDIamRole({ logger: roleLogger, region, iamClient, stsClient, - uniqNamePart, + username, assumeAwsAccount, assumeCondExternalId, }); @@ -83,13 +83,13 @@ describe("BDIntegration", () => { bdDataSets.readConfig("./example_datasource_config.yaml"); const assumeCondExternalId = await bdAccount.getExtId(); // FIXME: This calls real API const assumeAwsAccount = await bdAccount.getAssumeAwsAccount(); - const uniqNamePart = await bdDataSets.getUniqueNamePart(); + const username = await bdAccount.getUsername(); const bdRole = new BDIamRole({ logger: roleLogger, region, iamClient, stsClient, - uniqNamePart, + username, assumeAwsAccount, assumeCondExternalId, }); diff --git a/src/integration/boilingdata/account.ts b/src/integration/boilingdata/account.ts index ab895e6..8a3f65c 100644 --- a/src/integration/boilingdata/account.ts +++ b/src/integration/boilingdata/account.ts @@ -30,6 +30,8 @@ export interface ITapTokenResp { interface IAPIAccountDetails { AccountAwsAccount: string; AccountExtId: string; + AccountUsername: string; + AccountEmail: string; } export class BDAccount { @@ -78,13 +80,23 @@ export class BDAccount { if (!body.ResponseCode || !body.ResponseText) { throw new Error("Malformed response from BD API"); } - if (!body.AccountDetails?.AccountAwsAccount || !body.AccountDetails?.AccountExtId) { - throw new Error("Missing AccountDetails from BD API Response"); + if ( + !body.AccountDetails?.AccountAwsAccount || + !body.AccountDetails?.AccountExtId || + !body.AccountDetails?.AccountUsername || + !body.AccountDetails?.AccountEmail + ) { + throw new Error("Missing some or all of AccountDetails from BD API Response"); } this.accountDetails = body.AccountDetails; this.logger.debug({ accountDetails: this.accountDetails }); } + public async getUsername(): Promise { + await this._getAccountDetails(); + return this.accountDetails.AccountUsername; + } + public async getAssumeAwsAccount(): Promise { await this._getAccountDetails(); return this.accountDetails.AccountAwsAccount; diff --git a/src/integration/boilingdata/dataset.ts b/src/integration/boilingdata/dataset.ts index 4a580d6..d28b879 100644 --- a/src/integration/boilingdata/dataset.ts +++ b/src/integration/boilingdata/dataset.ts @@ -17,18 +17,6 @@ export class BDDataSourceConfig { this.logger = this.params.logger; } - public async getUniqueNamePart(): Promise { - if (!this._dataSourcesConfig || this._dataSourcesConfig.dataSources.length <= 0) { - throw new Error("Set datasources config first"); - } - const uniqName = - this._dataSourcesConfig.uniqNamePart ?? - this._dataSourcesConfig.dataSources[this._dataSourcesConfig.dataSources.length - 1]?.name ?? - "bdIamRole"; - this.logger.debug({ uniqName }); - return uniqName; - } - public isDataSetsConfig(dataSourcesConfig: unknown): dataSourcesConfig is IDataSources { try { const { IDataSources } = createCheckers(DataSetInterfaceTI); From 142ec00a450c76f4c1e9e48dcc7bb160b08cb567 Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Thu, 18 Jan 2024 09:40:29 +0200 Subject: [PATCH 10/14] update sandbox cmd handling (no region param) --- src/bdcli/commands/bdcli-sandbox.ts | 1 + .../sandbox/bdcli-sandbox-create-role.ts | 74 +++++++++++++++++++ .../commands/sandbox/bdcli-sandbox-destroy.ts | 3 +- .../sandbox/bdcli-sandbox-download.ts | 3 +- .../commands/sandbox/bdcli-sandbox-list.ts | 3 +- .../commands/sandbox/bdcli-sandbox-plan.ts | 3 +- .../commands/sandbox/bdcli-sandbox-upload.ts | 3 +- .../sandbox/bdcli-sandbox-validate.ts | 3 +- src/integration/aws/iam_roles.ts | 15 ++-- src/integration/bdIntegration.ts | 44 ++++++----- .../boilingdata/dataset.interface-ti.ts | 10 +-- .../boilingdata/dataset.interface.ts | 10 +-- src/integration/boilingdata/dataset.ts | 5 ++ src/integration/boilingdata/sandbox.ts | 30 ++++++-- 14 files changed, 145 insertions(+), 62 deletions(-) create mode 100644 src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts diff --git a/src/bdcli/commands/bdcli-sandbox.ts b/src/bdcli/commands/bdcli-sandbox.ts index 73a0890..981c506 100644 --- a/src/bdcli/commands/bdcli-sandbox.ts +++ b/src/bdcli/commands/bdcli-sandbox.ts @@ -3,6 +3,7 @@ import { Command } from "commander"; const program = new Command("bdcli sandbox") .executableDir("sandbox") .command("list", "List deployed sandboxes") + .command("create-role", "Create Sandbox IAM Role with your AWS credentials") .command("validate", "Validate *local* sandbox template") .command("upload", "Upload sandbox template") .command("download", "Download the *uploaded* sandbox template") diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts new file mode 100644 index 0000000..6e7c845 --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts @@ -0,0 +1,74 @@ +import * as iam from "@aws-sdk/client-iam"; +import * as sts from "@aws-sdk/client-sts"; +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { BDIamRole } from "../../../integration/aws/iam_roles.js"; +import { BDAccount } from "../../../integration/boilingdata/account.js"; +import { BDIntegration } from "../../../integration/bdIntegration.js"; +import { BDDataSourceConfig } from "../../../integration/boilingdata/dataset.js"; + +const logger = getLogger("bdcli-sandbox-create-role"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + if (options.delete) { + updateSpinnerText("Not implemented yet. Please delete the IAM Role from AWS Console"); + spinnerWarn("Not implemented yet. Please delete the IAM Role from AWS Console"); + return; + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Creating sandbox IAM Role into *your* AWS Account"); + const bdSandbox = new BDSandbox({ logger, authToken: token }).withTemplate(options.template); + const region = bdSandbox.region; + const bdAccount = new BDAccount({ logger, authToken: token }); + const bdDataSources = new BDDataSourceConfig({ logger }); + bdDataSources.withConfig({ dataSources: bdSandbox.tmpl.resources.sandbox }); + const bdRole = new BDIamRole({ + ...options, + logger, + iamClient: new iam.IAMClient({ region }), + stsClient: new sts.STSClient({ region }), + environment: bdSandbox.tmpl.environment, + templateName: bdSandbox.tmpl.name, + username: await bdAccount.getUsername(), + assumeAwsAccount: await bdAccount.getAssumeAwsAccount(), + assumeCondExternalId: await bdAccount.getExtId(), + }); + const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources }); + const policyDocument = await bdIntegration.getPolicyDocument(options.listBucketsPermission); + let iamRoleArn; + if (!options.dryRun) iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument)); + updateSpinnerText(`Creating IAM Role: ${iamRoleArn}` + (options.dryRun ? "(dry-run)" : "")); + spinnerSuccess(); + } catch (err: any) { + // try to decode the message + spinnerError(err?.message, false); + } +} + +const program = new cmd.Command("bdcli sandbox create-role") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--delete", "Delete the sandbox IAM role from *your* AWS Account")) + .addOption(new cmd.Option("--no-list-buckets-permission", "Do NOT add s3:ListAllMyBuckets policy entry")) + .addOption(new cmd.Option("--dry-run", "Dry run, do not actually create the IAM role")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts index 4e58ba7..fb881b0 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts @@ -18,12 +18,11 @@ async function show(options: any, _command: cmd.Command): Promise { } updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + const { idToken: token, cached: idCached } = await getIdToken(logger); updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); updateSpinnerText(`Destroying sandbox ${options.name}`); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); const results = await bdSandbox.destroySandbox(options.name, options.destroyAlsoInterfaces, options.deleteTemplate); spinnerSuccess(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts index 65d9b4b..edf3e08 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts @@ -31,12 +31,11 @@ async function show(options: any, _command: cmd.Command): Promise { } updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + const { idToken: token, cached: idCached } = await getIdToken(logger); updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); updateSpinnerText(`Downloading sandbox IaC template of ${options.name}`); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); const template = await bdSandbox.downloadTemplate(options.name, options.version); await fs.writeFile(filename, template); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts index 5d867f2..6a16e93 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts @@ -18,12 +18,11 @@ async function show(options: any, _command: cmd.Command): Promise { } updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + const { idToken: token, cached: idCached } = await getIdToken(logger); updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); updateSpinnerText("Listing sandboxes"); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); const list = await bdSandbox.listSandboxes(options.listDeleted, options.listVersions); spinnerSuccess(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts index 8d1e136..a5d9f84 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts @@ -18,12 +18,11 @@ async function show(options: any, _command: cmd.Command): Promise { } updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + const { idToken: token, cached: idCached } = await getIdToken(logger); updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); updateSpinnerText(`Planning deployment for sandbox ${options.name}`); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); const results = await bdSandbox.planSandbox(options.name); spinnerSuccess(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts index f48301c..a9c9e8c 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts @@ -17,12 +17,11 @@ async function show(options: any, _command: cmd.Command): Promise { } updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + const { idToken: token, cached: idCached } = await getIdToken(logger); updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); updateSpinnerText("Uploading sandbox IaC template"); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); await bdSandbox.uploadTemplate(options.template); spinnerSuccess(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts index a4ce7b1..90c28ce 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts @@ -18,12 +18,11 @@ async function show(options: any, _command: cmd.Command): Promise { } updateSpinnerText("Authenticating"); - const { idToken: token, cached: idCached, region: region } = await getIdToken(logger); + const { idToken: token, cached: idCached } = await getIdToken(logger); updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); spinnerSuccess(); updateSpinnerText("Validating sandbox IaC template"); - if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdSandbox = new BDSandbox({ logger, authToken: token }); await bdSandbox.validateTemplate(options.template, options.warningsAsErrors); spinnerSuccess(); diff --git a/src/integration/aws/iam_roles.ts b/src/integration/aws/iam_roles.ts index d1599c3..4e389ab 100644 --- a/src/integration/aws/iam_roles.ts +++ b/src/integration/aws/iam_roles.ts @@ -55,13 +55,15 @@ export class BDIamRole { const env = this.params.environment ?? "noenv"; const tmplName = this.params.templateName ?? "notmplname"; const username = this.params.username.replaceAll("-", ""); - const nameLenSoFar = - prefix.length + regionShort.length + env.length + tmplName.length + username.length + type.length; - const roomLeft = 64 - nameLenSoFar - 5; // 5 '-' chars - if (roomLeft < 0) { - throw new Error(`${type} name too long, reduce prefix/env/tmplName lengths (roomLeft ${roomLeft})`); + const name = [prefix, regionShort, env, tmplName, username].join("-"); + if (name.length > 64) { + throw new Error( + `${type} name (${name}) too long (${name.length}), reduce prefix/env/tmplname lengths (roomLeft ${ + 64 - name.length + })`, + ); } - return [prefix, regionShort, env, tmplName, username, type].join("-"); + return name; } public async getAwsAccountId(): Promise { @@ -216,6 +218,7 @@ export class BDIamRole { public async upsertRole(policyDocument: string): Promise { let arn: string; + this.logger.debug({ policyDocument }); try { const resp = await this.getRole(); if (!resp.Arn) throw new Error("Could not find role ARN"); diff --git a/src/integration/bdIntegration.ts b/src/integration/bdIntegration.ts index 8b4e441..8e2deba 100644 --- a/src/integration/bdIntegration.ts +++ b/src/integration/bdIntegration.ts @@ -51,23 +51,17 @@ export class BDIntegration { public getGroupedBuckets(): IGroupedDataSources { const dataSourcesConfig = this.bdDatasets.getDatasourcesConfig(); - const allPolicies = dataSourcesConfig.dataSources - .map(d => { - console.log(d); - d.accessPolicy = d.accessPolicy.map(pol => { - if (!pol.permissions) pol.permissions = [GRANT_PERMISSION.G_READ]; // default - return pol; - }); - return d.accessPolicy; - }) - .flat(); + const allPolicies = dataSourcesConfig.dataSources.permissions.map(perm => { + if (!perm.accessRights) perm.accessRights = [GRANT_PERMISSION.G_READ]; // default + return perm; + }); this.logger.debug({ allPolicies }); - if (allPolicies.some(policy => !policy.permissions)) throw new Error("Missing policy permissions"); + if (allPolicies.some(policy => !policy.accessRights)) throw new Error("Missing policy permissions"); const readOnly = allPolicies .filter( policy => - !policy.permissions?.includes(GRANT_PERMISSION.G_WRITE) && - policy.permissions?.includes(GRANT_PERMISSION.G_READ), + !policy.accessRights?.includes(GRANT_PERMISSION.G_WRITE) && + policy.accessRights?.includes(GRANT_PERMISSION.G_READ), ) .map(policy => ({ ...policy, @@ -77,8 +71,8 @@ export class BDIntegration { const readWrite = allPolicies .filter( policy => - policy.permissions?.includes(GRANT_PERMISSION.G_WRITE) && - policy.permissions?.includes(GRANT_PERMISSION.G_READ), + policy.accessRights?.includes(GRANT_PERMISSION.G_WRITE) && + policy.accessRights?.includes(GRANT_PERMISSION.G_READ), ) .map(policy => ({ ...policy, @@ -88,8 +82,8 @@ export class BDIntegration { const writeOnly = allPolicies .filter( policy => - policy.permissions?.includes(GRANT_PERMISSION.G_WRITE) && - !policy.permissions?.includes(GRANT_PERMISSION.G_READ), + policy.accessRights?.includes(GRANT_PERMISSION.G_WRITE) && + !policy.accessRights?.includes(GRANT_PERMISSION.G_READ), ) .map(policy => ({ ...policy, @@ -100,7 +94,8 @@ export class BDIntegration { return { readOnly, readWrite, writeOnly }; } - public async getPolicyDocument(): Promise { + public async getPolicyDocument(haveListBucketsPolicy = true): Promise { + this.logger.debug({ haveListBucketsPolicy }); const grouped = this.getGroupedBuckets(); const allDatasets = [...grouped.readOnly, ...grouped.readWrite, ...grouped.writeOnly]; const Statements = []; @@ -108,11 +103,14 @@ export class BDIntegration { Statements.push(this.getStatement(grouped.readWrite, RW_ACTIONS, this.mapAccessPolicyToS3Resource.bind(this))); Statements.push(this.getStatement(grouped.writeOnly, WO_ACTIONS, this.mapAccessPolicyToS3Resource.bind(this))); Statements.push(this.getStatement(allDatasets, BUCKET_ACTIONS, this.mapDatasetsToUniqBuckets.bind(this))); - Statements.push({ - Effect: "Allow", - Action: "s3:ListAllMyBuckets", - Resource: "*", - }); + if (haveListBucketsPolicy) { + // This is so that you can run: SELECT * FROM list('s3://') + Statements.push({ + Effect: "Allow", + Action: "s3:ListAllMyBuckets", + Resource: "*", + }); + } const finalPolicy = { Version: "2012-10-17", Statement: Statements.filter(s => s?.Resource?.length) }; this.logger.debug({ getPolicyDocument: JSON.stringify(finalPolicy) }); return finalPolicy; diff --git a/src/integration/boilingdata/dataset.interface-ti.ts b/src/integration/boilingdata/dataset.interface-ti.ts index 2228e70..d7d5104 100644 --- a/src/integration/boilingdata/dataset.interface-ti.ts +++ b/src/integration/boilingdata/dataset.interface-ti.ts @@ -39,9 +39,8 @@ export const ULayout = t.union(t.enumlit("LAYOUT", "HIVE"), t.enumlit("LAYOUT", export const UFileType = t.union(t.enumlit("FILE_TYPE", "PARQUET"), t.enumlit("FILE_TYPE", "JSON"), t.enumlit("FILE_TYPE", "CSV")); export const IStatement = t.iface([], { - "id": "string", "urlPrefix": "string", - "permissions": t.opt(t.array("UGrant")), + "accessRights": t.opt(t.array("UGrant")), }); export const IStatementExt = t.iface(["IStatement"], { @@ -58,16 +57,13 @@ export const IDataSet = t.iface([], { export const IDataSource = t.iface([], { "name": "string", - "accessPolicy": t.array("IStatement"), - "type": t.opt("EDataSetType"), - "dataSets": t.opt(t.array("IDataSet")), - "sessionType": t.opt("USessionType"), + "permissions": t.array("IStatement"), }); export const IDataSources = t.iface([], { "version": t.opt(t.union("string", "number")), "uniqNamePart": t.opt("string"), - "dataSources": t.array("IDataSource"), + "dataSources": "IDataSource", }); const exportedTypeSuite: t.ITypeSuite = { diff --git a/src/integration/boilingdata/dataset.interface.ts b/src/integration/boilingdata/dataset.interface.ts index 0e0ce6a..44378d4 100644 --- a/src/integration/boilingdata/dataset.interface.ts +++ b/src/integration/boilingdata/dataset.interface.ts @@ -30,9 +30,8 @@ export type ULayout = LAYOUT.HIVE | LAYOUT.FOLDER | LAYOUT.FILE; export type UFileType = FILE_TYPE.PARQUET | FILE_TYPE.JSON | FILE_TYPE.CSV; export interface IStatement { - id: string; urlPrefix: string; - permissions?: UGrant[]; + accessRights?: UGrant[]; } export interface IStatementExt extends IStatement { @@ -49,14 +48,11 @@ export interface IDataSet { export interface IDataSource { name: string; - accessPolicy: Array; - type?: EDataSetType; - dataSets?: Array; - sessionType?: USessionType; + permissions: Array; } export interface IDataSources { version?: string | number; uniqNamePart?: string; - dataSources: Array; + dataSources: IDataSource; } diff --git a/src/integration/boilingdata/dataset.ts b/src/integration/boilingdata/dataset.ts index d28b879..1aad685 100644 --- a/src/integration/boilingdata/dataset.ts +++ b/src/integration/boilingdata/dataset.ts @@ -38,6 +38,11 @@ export class BDDataSourceConfig { return { ...this._dataSourcesConfig }; // make copy } + public withConfig(config: object): void { + if (!this.isDataSetsConfig(config)) throw new Error("datasources config schema not validated"); + this._dataSourcesConfig = config; + } + public async readConfig(filename: string): Promise { if (this._dataSourcesConfig) return this._dataSourcesConfig; try { diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index 7ca118f..18c4bc9 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -1,4 +1,5 @@ import * as fs from "fs/promises"; +import * as fsSync from "fs"; import * as yaml from "js-yaml"; import { ILogger } from "../../bdcli/utils/logger_util.js"; import { getReqHeaders, sandboxUrl } from "./boilingdata_api.js"; @@ -16,6 +17,7 @@ export interface IBDConfig { export class BDSandbox { private cognitoIdToken: string; private logger: ILogger; + private _tmpl!: ITemplate; constructor(private params: IBDConfig) { this.logger = this.params.logger; @@ -23,6 +25,19 @@ export class BDSandbox { this.cognitoIdToken = this.params.authToken; } + public get tmpl(): ITemplate { + return this._tmpl; + } + + public get region(): string | undefined { + return this._tmpl.region; + } + + public withTemplate(templateFilename: string): this { + this.validateTemplateLocal(templateFilename); + return this; + } + public isSandboxConfig(sandboxTemplate: unknown): sandboxTemplate is ITemplate { try { const { ITemplate: ITemplateChecker } = createCheckers(sandboxTemplateTI); @@ -67,19 +82,20 @@ export class BDSandbox { return respBody?.template; } - public async uploadTemplate(templateFilename: string): Promise { - // local validation - const sandboxTemplateConfig = yaml.load(await fs.readFile(templateFilename, "utf8")); + public validateTemplateLocal(templateFilename: string): void { + const sandboxTemplateConfig = yaml.load(fsSync.readFileSync(templateFilename, "utf8")); if (!this.isSandboxConfig(sandboxTemplateConfig)) throw new Error("sandbox template config schema not validated"); + this._tmpl = sandboxTemplateConfig; + } + + public async uploadTemplate(templateFilename: string): Promise { + this.validateTemplateLocal(templateFilename); return this._uploadTemplate(templateFilename); } public async validateTemplate(templateFilename: string, warningsAsErrors = false): Promise { - // local validation - const sandboxTemplateConfig = yaml.load(await fs.readFile(templateFilename, "utf8")); - if (!this.isSandboxConfig(sandboxTemplateConfig)) throw new Error("sandbox template config schema not validated"); + this.validateTemplateLocal(templateFilename); const validateOnly = true; - // remote validation return this._uploadTemplate(templateFilename, validateOnly, warningsAsErrors); } From af3326e7476a09b45aaf653d9f82c3a4d21755ec Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Fri, 2 Feb 2024 14:37:48 +0200 Subject: [PATCH 11/14] Renaming --- README.md | 5 ++ .../sandbox/bdcli-sandbox-create-role.ts | 4 +- .../sandbox/bdcli-sandbox-download.ts | 3 +- .../commands/sandbox/bdcli-sandbox-upload.ts | 23 +++++++-- .../sandbox/bdcli-sandbox-validate.ts | 3 +- .../boilingdata/sandbox-template.types-ti.ts | 24 +++++----- .../boilingdata/sandbox-template.types.ts | 18 ++++--- src/integration/boilingdata/sandbox.ts | 47 +++++++++++++++---- 8 files changed, 88 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 58cb8c6..c1ce042 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Commands: account Setup and configure your BoilingData account aws Setup and configure your AWS account integration with BoilingData domain Admin setup and configuration for your domain (.e.g @boilingdata.com, @mycompany.com) + sandbox Managa Boiling S3 Sandboxes with IaC templates help [command] display help for command ``` @@ -158,3 +159,7 @@ npm run build npm install -g . bdcli -h ``` + +### TODO + +- [_] Add shell auto completion with [omelette](https://github.com/f/omelette). diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts index 6e7c845..585f2fa 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts @@ -37,14 +37,14 @@ async function show(options: any, _command: cmd.Command): Promise { const region = bdSandbox.region; const bdAccount = new BDAccount({ logger, authToken: token }); const bdDataSources = new BDDataSourceConfig({ logger }); - bdDataSources.withConfig({ dataSources: bdSandbox.tmpl.resources.sandbox }); + bdDataSources.withConfig({ dataSources: bdSandbox.tmpl.resources.storage }); const bdRole = new BDIamRole({ ...options, logger, iamClient: new iam.IAMClient({ region }), stsClient: new sts.STSClient({ region }), environment: bdSandbox.tmpl.environment, - templateName: bdSandbox.tmpl.name, + templateName: bdSandbox.tmpl.id, username: await bdAccount.getUsername(), assumeAwsAccount: await bdAccount.getAssumeAwsAccount(), assumeCondExternalId: await bdAccount.getExtId(), diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts index edf3e08..157d59e 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts @@ -37,7 +37,7 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Downloading sandbox IaC template of ${options.name}`); const bdSandbox = new BDSandbox({ logger, authToken: token }); - const template = await bdSandbox.downloadTemplate(options.name, options.version); + const template = await bdSandbox.downloadTemplate(options.name, options.version, options?.status ?? "uploaded"); await fs.writeFile(filename, template); spinnerSuccess(); } catch (err: any) { @@ -47,6 +47,7 @@ async function show(options: any, _command: cmd.Command): Promise { const program = new cmd.Command("bdcli sandbox download") .addOption(new cmd.Option("--name ", "template name from listing").makeOptionMandatory()) + .addOption(new cmd.Option("--status ", "Download 'uploaded' (default) or 'deployed' template")) .addOption(new cmd.Option("--version ", "Download specific version from listing")) .action(async (options, command) => await show(options, command)); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts index a9c9e8c..69a70e2 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts @@ -5,6 +5,7 @@ import { addGlobalOptions } from "../../utils/options_util.js"; import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; import { getIdToken } from "../../utils/auth_util.js"; import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { outputResults } from "../../utils/output_util.js"; const logger = getLogger("bdcli-sandbox-validate"); @@ -23,15 +24,31 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText("Uploading sandbox IaC template"); const bdSandbox = new BDSandbox({ logger, authToken: token }); - await bdSandbox.uploadTemplate(options.template); + await bdSandbox.uploadTemplate(options.template, options.allowChangedFilename); spinnerSuccess(); - } catch (err: any) { - spinnerError(err?.message); + } catch (origErr: any) { + try { + spinnerError(origErr?.message, false); + await outputResults( + JSON.parse(origErr?.message) + ?.message?.split(";") + ?.map((msg: string) => msg.trim()), + false, + ); + } catch (err: any) { + if (err?.message && !origErr?.message) spinnerError(err?.message, false); + } } } const program = new cmd.Command("bdcli sandbox upload") .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption( + new cmd.Option( + "--allow-changed-filename", + "Allow local template filename chnage with an existing template ID (possible copy/paste error)", + ), + ) .action(async (options, command) => await show(options, command)); (async () => { diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts index 90c28ce..370625c 100644 --- a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts @@ -37,8 +37,7 @@ async function show(options: any, _command: cmd.Command): Promise { false, ); } catch (err: any) { - spinnerError(origErr?.message, false); - console.error(err); + if (err?.message && !origErr?.message) spinnerError(err?.message, false); } } } diff --git a/src/integration/boilingdata/sandbox-template.types-ti.ts b/src/integration/boilingdata/sandbox-template.types-ti.ts index 419a6e6..0a9f52e 100644 --- a/src/integration/boilingdata/sandbox-template.types-ti.ts +++ b/src/integration/boilingdata/sandbox-template.types-ti.ts @@ -15,7 +15,7 @@ export const EPermission = t.enumtype({ "WRITE": "write", }); -export const ITemplateSandbox = t.iface([], { +export const ITemplateStorage = t.iface([], { "name": "string", "permissions": t.array(t.iface([], { "urlPrefix": "string", @@ -32,18 +32,16 @@ export const ITemplateTap = t.iface([], { }))), }); -export const ITemplateFlow = t.iface([], { +export const ITemplatePipe = t.iface([], { "name": "string", - "input": t.union("string", t.array("string")), + "input": "string", "keys": t.opt(t.array("string")), - "transformJs": t.opt("string"), "transformSql": t.opt("string"), - "prefixFunc": t.opt("string"), "output": t.opt(t.union("string", t.array("string"))), "errors": t.opt("string"), }); -export const ITemplateShare = t.iface([], { +export const ITemplateACL = t.iface([], { "name": "string", "users": t.array("string"), "sql": t.opt("string"), @@ -53,24 +51,24 @@ export const ITemplateShare = t.iface([], { export const ITemplate = t.iface([], { "version": t.union("string", "number"), - "name": "string", + "id": "string", "environment": "string", "region": "string", "resources": t.iface([], { - "sandbox": "ITemplateSandbox", + "storage": "ITemplateStorage", "taps": t.opt(t.array("ITemplateTap")), - "flows": t.opt(t.array("ITemplateFlow")), - "shares": t.opt(t.array("ITemplateShare")), + "pipes": t.opt(t.array("ITemplatePipe")), + "acls": t.opt(t.array("ITemplateACL")), }), }); const exportedTypeSuite: t.ITypeSuite = { EModelFormat, EPermission, - ITemplateSandbox, + ITemplateStorage, ITemplateTap, - ITemplateFlow, - ITemplateShare, + ITemplatePipe, + ITemplateACL, ITemplate, }; export default exportedTypeSuite; diff --git a/src/integration/boilingdata/sandbox-template.types.ts b/src/integration/boilingdata/sandbox-template.types.ts index 1f15eb1..bb7c1ec 100644 --- a/src/integration/boilingdata/sandbox-template.types.ts +++ b/src/integration/boilingdata/sandbox-template.types.ts @@ -8,7 +8,7 @@ export enum EPermission { WRITE = "write", } -export interface ITemplateSandbox { +export interface ITemplateStorage { name: string; permissions: Array<{ urlPrefix: string; @@ -25,18 +25,16 @@ export interface ITemplateTap { }>; } -export interface ITemplateFlow { +export interface ITemplatePipe { name: string; - input: string | string[]; + input: string; keys?: string[]; - transformJs?: string; transformSql?: string; - prefixFunc?: string; output?: string | string[]; errors?: string; } -export interface ITemplateShare { +export interface ITemplateACL { name: string; users: string[]; sql?: string; @@ -46,13 +44,13 @@ export interface ITemplateShare { export interface ITemplate { version: string | number; - name: string; + id: string; environment: string; region: string; resources: { - sandbox: ITemplateSandbox; + storage: ITemplateStorage; taps?: ITemplateTap[]; - flows?: ITemplateFlow[]; - shares?: ITemplateShare[]; + pipes?: ITemplatePipe[]; + acls?: ITemplateACL[]; }; } diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts index 18c4bc9..4e9042e 100644 --- a/src/integration/boilingdata/sandbox.ts +++ b/src/integration/boilingdata/sandbox.ts @@ -6,6 +6,7 @@ import { getReqHeaders, sandboxUrl } from "./boilingdata_api.js"; import { createCheckers } from "ts-interface-checker"; import sandboxTemplateTI from "./sandbox-template.types-ti.js"; import { ITemplate } from "./sandbox-template.types.js"; +import * as path from "path"; // import { channel } from "node:diagnostics_channel"; @@ -18,6 +19,7 @@ export class BDSandbox { private cognitoIdToken: string; private logger: ILogger; private _tmpl!: ITemplate; + private tmplErrors: string[] = []; constructor(private params: IBDConfig) { this.logger = this.params.logger; @@ -38,14 +40,19 @@ export class BDSandbox { return this; } + public getSandboxTemplateErrors(): string { + return this.tmplErrors.map(e => "\t" + e).join("\n"); + } + public isSandboxConfig(sandboxTemplate: unknown): sandboxTemplate is ITemplate { try { const { ITemplate: ITemplateChecker } = createCheckers(sandboxTemplateTI); if (!ITemplateChecker) throw new Error("ts-interface-check checker MISSING"); ITemplateChecker.check(sandboxTemplate); return true; - } catch (err) { - this.logger.error({ err }); + } catch (err: any) { + //this.logger.error(`\n${err?.message}`); + this.tmplErrors.push(err?.message.replace("value.", "")); return false; } } @@ -69,9 +76,10 @@ export class BDSandbox { return respBody; } - public async downloadTemplate(sandboxName: string, version: string): Promise { + public async downloadTemplate(sandboxName: string, version: string, status?: string): Promise { const headers = await getReqHeaders(this.cognitoIdToken); headers["x-bd-template-version"] = version; + headers["x-bd-template-status"] = status ?? "uploaded"; this.logger.debug({ sandboxUrl, headers }); const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "GET", headers }); const respBody = await res.json(); @@ -79,18 +87,34 @@ export class BDSandbox { if (!respBody.ResponseCode || !respBody.ResponseText) { throw new Error("Malformed response from BD API"); } + if (respBody.ResponseCode != "00") { + throw new Error(respBody.ResponseText); + } return respBody?.template; } public validateTemplateLocal(templateFilename: string): void { - const sandboxTemplateConfig = yaml.load(fsSync.readFileSync(templateFilename, "utf8")); - if (!this.isSandboxConfig(sandboxTemplateConfig)) throw new Error("sandbox template config schema not validated"); + let fileContents; + let sandboxTemplateConfig; + try { + fileContents = fsSync.readFileSync(templateFilename, "utf8"); + } catch (err: any) { + throw new Error(`${err?.message}`); + } + try { + sandboxTemplateConfig = yaml.load(fileContents); + } catch (err: any) { + throw new Error(`${err?.message}`); + } + this.logger.debug(sandboxTemplateConfig); + if (!this.isSandboxConfig(sandboxTemplateConfig)) + throw new Error(`sandbox template config schema not validated:\n${this.getSandboxTemplateErrors()}`); this._tmpl = sandboxTemplateConfig; } - public async uploadTemplate(templateFilename: string): Promise { + public async uploadTemplate(templateFilename: string, allowChangedFilename = false): Promise { this.validateTemplateLocal(templateFilename); - return this._uploadTemplate(templateFilename); + return this._uploadTemplate(templateFilename, false, false, allowChangedFilename); } public async validateTemplate(templateFilename: string, warningsAsErrors = false): Promise { @@ -143,11 +167,18 @@ export class BDSandbox { templateFilename: string, validateOnly = false, warningsAsErrors = false, + allowChangedFilename = false, ): Promise { const headers = await getReqHeaders(this.cognitoIdToken); this.logger.debug({ sandboxUrl, headers }); const template = Buffer.from(await fs.readFile(templateFilename)).toString("base64"); - const body = JSON.stringify({ validateOnly, warningsAsErrors, template }); + const body = JSON.stringify({ + validateOnly, + warningsAsErrors, + template, + templateFilename: path.basename(templateFilename), + allowChangedFilename, + }); this.logger.debug({ body }); const res = await fetch(sandboxUrl, { method: "PUT", headers, body }); const respBody = await res.json(); From cae8767ad46e0167e37c3574e257357faa433f77 Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Fri, 2 Feb 2024 14:39:43 +0200 Subject: [PATCH 12/14] ignore generated code --- .prettierignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.prettierignore b/.prettierignore index 337d042..a2a7563 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,4 +2,5 @@ dist/ node_modules/ package.json src/VERSION.ts -src/integration/boilingdata/dataset.interface-ti.ts \ No newline at end of file +src/integration/boilingdata/dataset.interface-ti.ts +src/integration/boilingdata/sandbox-template.types-ti.ts \ No newline at end of file From 11fe9c2d085e2d523850dd5258a9d7b13e7d29af Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Fri, 2 Feb 2024 15:03:13 +0200 Subject: [PATCH 13/14] update tests and doc --- src/integration/aws/iam_roles.test.ts | 6 ++-- src/integration/bdIntegration.ts | 13 +++++--- .../boilingdata/dataset.interface-ti.ts | 31 +------------------ .../boilingdata/dataset.interface.ts | 22 +------------ .../fixtures/datasources_extra.test.yaml | 11 +++---- .../fixtures/datasources_valid.test.yaml | 17 ++++------ 6 files changed, 24 insertions(+), 76 deletions(-) diff --git a/src/integration/aws/iam_roles.test.ts b/src/integration/aws/iam_roles.test.ts index a291e76..d748c89 100644 --- a/src/integration/aws/iam_roles.test.ts +++ b/src/integration/aws/iam_roles.test.ts @@ -48,17 +48,17 @@ describe("iamRole", () => { it("getIamRoleName", async () => { const role = new BDIamRole(roleParams); - expect(role.iamRoleName).toEqual("bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-role"); + expect(role.iamRoleName).toEqual("bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b"); }); it("getIamRoleName with own prefix", async () => { const role = new BDIamRole({ ...roleParams, roleNamePrefix: "my" }); - expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-role"); + expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b"); }); it("getIamRoleName with own path and prefix", async () => { const role = new BDIamRole({ ...roleParams, roleNamePrefix: "my", path: "/bd-service/demo/" }); - expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-role"); + expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b"); }); it("getRole", async () => { diff --git a/src/integration/bdIntegration.ts b/src/integration/bdIntegration.ts index 8e2deba..3ef1519 100644 --- a/src/integration/bdIntegration.ts +++ b/src/integration/bdIntegration.ts @@ -1,7 +1,7 @@ import { ILogger } from "../bdcli/utils/logger_util.js"; import { BDIamRole } from "./aws/iam_roles.js"; import { BDAccount } from "./boilingdata/account.js"; -import { GRANT_PERMISSION, IStatementExt } from "./boilingdata/dataset.interface.js"; +import { GRANT_PERMISSION, IStatement, IStatementExt } from "./boilingdata/dataset.interface.js"; import { BDDataSourceConfig } from "./boilingdata/dataset.js"; const RO_ACTIONS = ["s3:GetObject"]; @@ -51,10 +51,13 @@ export class BDIntegration { public getGroupedBuckets(): IGroupedDataSources { const dataSourcesConfig = this.bdDatasets.getDatasourcesConfig(); - const allPolicies = dataSourcesConfig.dataSources.permissions.map(perm => { - if (!perm.accessRights) perm.accessRights = [GRANT_PERMISSION.G_READ]; // default - return perm; - }); + const allPolicies: IStatement[] = []; + dataSourcesConfig.dataSources.forEach(ds => + ds.permissions.forEach(perm => { + if (!perm.accessRights) perm.accessRights = [GRANT_PERMISSION.G_READ]; // default + allPolicies.push(perm); + }), + ); this.logger.debug({ allPolicies }); if (allPolicies.some(policy => !policy.accessRights)) throw new Error("Missing policy permissions"); const readOnly = allPolicies diff --git a/src/integration/boilingdata/dataset.interface-ti.ts b/src/integration/boilingdata/dataset.interface-ti.ts index d7d5104..a34384c 100644 --- a/src/integration/boilingdata/dataset.interface-ti.ts +++ b/src/integration/boilingdata/dataset.interface-ti.ts @@ -4,40 +4,19 @@ import * as t from "ts-interface-checker"; // tslint:disable:object-literal-key-quotes -export const EDataSetType = t.enumtype({ - "S3": "s3", -}); - export const GRANT_PERMISSION = t.enumtype({ "G_WRITE": "write", "G_READ": "read", }); -export const SESSION_TYPE = t.enumtype({ - "STS": "sts", - "ASSUME_ROLE": "assumeRole", -}); - -export const LAYOUT = t.enumtype({ - "HIVE": "hive", - "FOLDER": "folder", - "FILE": "file", -}); - export const FILE_TYPE = t.enumtype({ "PARQUET": "parquet", "JSON": "json", "CSV": "csv", }); -export const USessionType = t.union(t.enumlit("SESSION_TYPE", "STS"), t.enumlit("SESSION_TYPE", "ASSUME_ROLE")); - export const UGrant = t.union(t.enumlit("GRANT_PERMISSION", "G_READ"), t.enumlit("GRANT_PERMISSION", "G_WRITE")); -export const ULayout = t.union(t.enumlit("LAYOUT", "HIVE"), t.enumlit("LAYOUT", "FOLDER"), t.enumlit("LAYOUT", "FILE")); - -export const UFileType = t.union(t.enumlit("FILE_TYPE", "PARQUET"), t.enumlit("FILE_TYPE", "JSON"), t.enumlit("FILE_TYPE", "CSV")); - export const IStatement = t.iface([], { "urlPrefix": "string", "accessRights": t.opt(t.array("UGrant")), @@ -51,8 +30,6 @@ export const IStatementExt = t.iface(["IStatement"], { export const IDataSet = t.iface([], { "name": "string", "urlPrefix": "string", - "layout": t.opt("ULayout"), - "filetype": t.opt("UFileType"), }); export const IDataSource = t.iface([], { @@ -63,19 +40,13 @@ export const IDataSource = t.iface([], { export const IDataSources = t.iface([], { "version": t.opt(t.union("string", "number")), "uniqNamePart": t.opt("string"), - "dataSources": "IDataSource", + "dataSources": t.array("IDataSource"), }); const exportedTypeSuite: t.ITypeSuite = { - EDataSetType, GRANT_PERMISSION, - SESSION_TYPE, - LAYOUT, FILE_TYPE, - USessionType, UGrant, - ULayout, - UFileType, IStatement, IStatementExt, IDataSet, diff --git a/src/integration/boilingdata/dataset.interface.ts b/src/integration/boilingdata/dataset.interface.ts index 44378d4..06f66c0 100644 --- a/src/integration/boilingdata/dataset.interface.ts +++ b/src/integration/boilingdata/dataset.interface.ts @@ -1,33 +1,15 @@ -export enum EDataSetType { - S3 = "s3", -} - export enum GRANT_PERMISSION { G_WRITE = "write", G_READ = "read", } -export enum SESSION_TYPE { - STS = "sts", - ASSUME_ROLE = "assumeRole", -} - -export enum LAYOUT { - HIVE = "hive", - FOLDER = "folder", - FILE = "file", -} - export enum FILE_TYPE { PARQUET = "parquet", JSON = "json", CSV = "csv", } -export type USessionType = SESSION_TYPE.STS | SESSION_TYPE.ASSUME_ROLE; export type UGrant = GRANT_PERMISSION.G_READ | GRANT_PERMISSION.G_WRITE; -export type ULayout = LAYOUT.HIVE | LAYOUT.FOLDER | LAYOUT.FILE; -export type UFileType = FILE_TYPE.PARQUET | FILE_TYPE.JSON | FILE_TYPE.CSV; export interface IStatement { urlPrefix: string; @@ -42,8 +24,6 @@ export interface IStatementExt extends IStatement { export interface IDataSet { name: string; urlPrefix: string; - layout?: ULayout; - filetype?: UFileType; } export interface IDataSource { @@ -54,5 +34,5 @@ export interface IDataSource { export interface IDataSources { version?: string | number; uniqNamePart?: string; - dataSources: IDataSource; + dataSources: IDataSource[]; } diff --git a/src/integration/boilingdata/fixtures/datasources_extra.test.yaml b/src/integration/boilingdata/fixtures/datasources_extra.test.yaml index 885c1e8..2bb6c50 100644 --- a/src/integration/boilingdata/fixtures/datasources_extra.test.yaml +++ b/src/integration/boilingdata/fixtures/datasources_extra.test.yaml @@ -4,16 +4,15 @@ dataSources: - name: demo type: s3 sessionType: assumeRole - accessPolicy: - - id: bd-demo-policy - badKey: yesTrue + permissions: + - badKey: yesTrue urlPrefix: s3://boilingdata-demo/demo - permissions: + accessRights: - read - id: nyc-policy dummy: hereAndThere urlPrefix: s3://isecurefi-dev-test/nyc-tlc/ - permissions: + accessRights: - read - write dataSets: @@ -27,7 +26,7 @@ dataSources: filetype: parquet - name: logs type: s3 - accessPolicy: + permissions: - id: logs-policy urlPrefix: s3://logs-bucket/ dataSets: diff --git a/src/integration/boilingdata/fixtures/datasources_valid.test.yaml b/src/integration/boilingdata/fixtures/datasources_valid.test.yaml index ee20532..2707115 100644 --- a/src/integration/boilingdata/fixtures/datasources_valid.test.yaml +++ b/src/integration/boilingdata/fixtures/datasources_valid.test.yaml @@ -2,16 +2,12 @@ version: 1.0 uniqNamePart: demo2 dataSources: - name: demo - type: s3 - sessionType: assumeRole - accessPolicy: - - id: bd-demo-policy - urlPrefix: s3://boilingdata-demo/demo - permissions: + permissions: + - urlPrefix: s3://boilingdata-demo/demo + accessRights: - read - - id: nyc-policy - urlPrefix: s3://isecurefi-dev-test/nyc-tlc/ - permissions: + - urlPrefix: s3://isecurefi-dev-test/nyc-tlc/ + accessRights: - read - write dataSets: @@ -24,8 +20,7 @@ dataSources: layout: hive filetype: parquet - name: logs - type: s3 - accessPolicy: + permissions: - id: logs-policy urlPrefix: s3://logs-bucket/ dataSets: From 36a2627468b6e32ffa24cee758e117aaffd94532 Mon Sep 17 00:00:00 2001 From: Dan Forsberg Date: Fri, 2 Feb 2024 15:03:28 +0200 Subject: [PATCH 14/14] Update README --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c1ce042..f2742d9 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,16 @@ You can create BoilingData assumable IAM role into your AWS account with clear s % echo "version: 1.0 dataSources: - name: demo - type: s3 - accessPolicy: - - id: bd-test-policy - urlPrefix: s3://my-bucket/and/prefix + permissions: + - urlPrefix: s3://my-bucket/and/prefix + accessRights: + - read + - write " > datasource_config.yaml % bdcli aws iam -c datasource_config.yaml --region eu-west-1 --create-role-only ✔ Authenticating: success -✔ Creating IAM Role: arn:aws:iam::123123123123:role/boilingdata/bd-ew1-demo-0ccb08a39c45a24 +✔ Creating IAM Role: arn:aws:iam::589434896614:role/boilingdata/bd-euw1-noenv-notmplname-21346bf26c314caf8e7e9832205ffdee % echo "Now you can verify the generated IAM role" Now you can verify the generated IAM role