diff --git a/.gitignore b/.gitignore index 4c6a49e..5386c6d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ dist/ datasource_config.yaml bdcli !bdcli/ +bd_querylog_config.yaml +boilingQueryLog_v2.yaml +boilingdata_user_x.yaml diff --git a/.prettierignore b/.prettierignore index 337d042..a2a7563 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,4 +2,5 @@ dist/ node_modules/ package.json src/VERSION.ts -src/integration/boilingdata/dataset.interface-ti.ts \ No newline at end of file +src/integration/boilingdata/dataset.interface-ti.ts +src/integration/boilingdata/sandbox-template.types-ti.ts \ No newline at end of file diff --git a/README.md b/README.md index 58cb8c6..f2742d9 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Commands: account Setup and configure your BoilingData account aws Setup and configure your AWS account integration with BoilingData domain Admin setup and configuration for your domain (.e.g @boilingdata.com, @mycompany.com) + sandbox Managa Boiling S3 Sandboxes with IaC templates help [command] display help for command ``` @@ -28,15 +29,16 @@ You can create BoilingData assumable IAM role into your AWS account with clear s % echo "version: 1.0 dataSources: - name: demo - type: s3 - accessPolicy: - - id: bd-test-policy - urlPrefix: s3://my-bucket/and/prefix + permissions: + - urlPrefix: s3://my-bucket/and/prefix + accessRights: + - read + - write " > datasource_config.yaml % bdcli aws iam -c datasource_config.yaml --region eu-west-1 --create-role-only ✔ Authenticating: success -✔ Creating IAM Role: arn:aws:iam::123123123123:role/boilingdata/bd-ew1-demo-0ccb08a39c45a24 +✔ Creating IAM Role: arn:aws:iam::589434896614:role/boilingdata/bd-euw1-noenv-notmplname-21346bf26c314caf8e7e9832205ffdee % echo "Now you can verify the generated IAM role" Now you can verify the generated IAM role @@ -158,3 +160,7 @@ npm run build npm install -g . bdcli -h ``` + +### TODO + +- [_] Add shell auto completion with [omelette](https://github.com/f/omelette). diff --git a/package.json b/package.json index 436a120..7880401 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ }, "scripts": { "release": "standard-version -a", - "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts ", + "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts && npx ts-interface-builder src/integration/boilingdata/sandbox-template.types.ts", "build": "rm -rf dist/ && tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && chmod 755 dist/esm/index.js", "prettier": "prettier --check 'src/**/*.{js,ts}'", "prettier:fix": "prettier --write 'src/**/*.{js,ts}'", diff --git a/src/bdcli/commands/account/bdcli-account-sts-token.ts b/src/bdcli/commands/account/bdcli-account-sts-token.ts index 9e317f2..d7d5b24 100644 --- a/src/bdcli/commands/account/bdcli-account-sts-token.ts +++ b/src/bdcli/commands/account/bdcli-account-sts-token.ts @@ -56,7 +56,7 @@ async function show(options: any, _command: cmd.Command): Promise { updateSpinnerText(`Getting BoilingData STS token`); if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); const bdAccount = new BDAccount({ logger, authToken: token }); - const { bdStsToken, cached: stsCached, ...rest } = await bdAccount.getToken(options.lifetime ?? "1h"); + const { bdStsToken, cached: stsCached, ...rest } = await bdAccount.getStsToken(options.lifetime ?? "1h"); updateSpinnerText(`Getting BoilingData STS token: ${stsCached ? "cached" : "success"}`); spinnerSuccess(); diff --git a/src/bdcli/commands/account/bdcli-account-tap-token.ts b/src/bdcli/commands/account/bdcli-account-tap-token.ts new file mode 100644 index 0000000..71c05e9 --- /dev/null +++ b/src/bdcli/commands/account/bdcli-account-tap-token.ts @@ -0,0 +1,57 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { getIdToken, validateTokenLifetime } from "../../utils/auth_util.js"; +import { BDAccount } from "../../../integration/boilingdata/account.js"; +import { combineOptsWithSettings } from "../../utils/config_util.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-account-token"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (options.lifetime) await validateTokenLifetime(options.lifetime); + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached, region } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText(`Getting BoilingData TAP token`); + if (!region) throw new Error("Pass --region parameter or set AWS_REGION env"); + const bdAccount = new BDAccount({ logger, authToken: token }); + const { + bdTapToken, + cached: tapCached, + ...rest + } = await bdAccount.getTapToken(options.lifetime ?? "24h", options.sharingUser); + updateSpinnerText(`Getting BoilingData TAP token: ${tapCached ? "cached" : "success"}`); + spinnerSuccess(); + await outputResults({ bdTapToken, cached: tapCached, ...rest }, options.disableSpinner); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli account tap-token") + .addOption( + new cmd.Option( + "--lifetime ", + "Expiration lifetime for the token, in string format, like '1h' (see https://github.com/vercel/ms)", + ), + ) + .addOption( + new cmd.Option( + "--sharing-user ", + "A user has shared Tap for you so that you can write to it.", + ), + ) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/aws/bdcli-aws-iam.ts b/src/bdcli/commands/aws/bdcli-aws-iam.ts index 1630112..44eca51 100644 --- a/src/bdcli/commands/aws/bdcli-aws-iam.ts +++ b/src/bdcli/commands/aws/bdcli-aws-iam.ts @@ -39,7 +39,7 @@ async function iamrole(options: any, _command: cmd.Command): Promise { logger, iamClient: new iam.IAMClient({ region }), stsClient: new sts.STSClient({ region }), - uniqNamePart: await bdDataSources.getUniqueNamePart(), + username: await bdAccount.getUsername(), assumeAwsAccount: await bdAccount.getAssumeAwsAccount(), assumeCondExternalId: await bdAccount.getExtId(), }); diff --git a/src/bdcli/commands/bdcli-account.ts b/src/bdcli/commands/bdcli-account.ts index 6a8ccf9..b092201 100755 --- a/src/bdcli/commands/bdcli-account.ts +++ b/src/bdcli/commands/bdcli-account.ts @@ -8,6 +8,7 @@ const program = new Command("bdcli account") .command("mfa", "Enable MFA") .command("migrate", "Migrate your account to another AWS region (not-yet-implemented)") .command("sts-token", "Exchange Cognito ID Token into shared or your own BoilingData Short-Term-Session (STS) token") + .command("tap-token", "Exchange Cognito ID Token into shared or your own BoilingData Stream Tap auth token") .command("token-share", "Share data sets via access tokens to other Boiling users") .command("token-unshare", "Unshare access tokens") .command("token-list-shares", "List shared data sets (access tokens) from and to you") diff --git a/src/bdcli/commands/bdcli-sandbox.ts b/src/bdcli/commands/bdcli-sandbox.ts new file mode 100644 index 0000000..981c506 --- /dev/null +++ b/src/bdcli/commands/bdcli-sandbox.ts @@ -0,0 +1,14 @@ +import { Command } from "commander"; + +const program = new Command("bdcli sandbox") + .executableDir("sandbox") + .command("list", "List deployed sandboxes") + .command("create-role", "Create Sandbox IAM Role with your AWS credentials") + .command("validate", "Validate *local* sandbox template") + .command("upload", "Upload sandbox template") + .command("download", "Download the *uploaded* sandbox template") + .command("plan", "List planned udpates based on *uploaded* sandbox definition and deployed sandbox") + .command("deploy", "Deploy the *uploaded* sandbox") + .command("destroy", "Destroy the sandbox"); + +program.parse(process.argv); diff --git a/src/bdcli/commands/domain/bdcli-domain-setup.ts b/src/bdcli/commands/domain/bdcli-domain-setup.ts index e8bba8e..712bb7e 100644 --- a/src/bdcli/commands/domain/bdcli-domain-setup.ts +++ b/src/bdcli/commands/domain/bdcli-domain-setup.ts @@ -39,7 +39,6 @@ async function iamrole(options: any, _command: cmd.Command): Promise { logger, iamClient: new iam.IAMClient({ region }), stsClient: new sts.STSClient({ region }), - uniqNamePart: await bdDataSources.getUniqueNamePart(), assumeAwsAccount: await bdAccount.getAssumeAwsAccount(), assumeCondExternalId: await bdAccount.getExtId(), }); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts new file mode 100644 index 0000000..585f2fa --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts @@ -0,0 +1,74 @@ +import * as iam from "@aws-sdk/client-iam"; +import * as sts from "@aws-sdk/client-sts"; +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { BDIamRole } from "../../../integration/aws/iam_roles.js"; +import { BDAccount } from "../../../integration/boilingdata/account.js"; +import { BDIntegration } from "../../../integration/bdIntegration.js"; +import { BDDataSourceConfig } from "../../../integration/boilingdata/dataset.js"; + +const logger = getLogger("bdcli-sandbox-create-role"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + if (options.delete) { + updateSpinnerText("Not implemented yet. Please delete the IAM Role from AWS Console"); + spinnerWarn("Not implemented yet. Please delete the IAM Role from AWS Console"); + return; + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Creating sandbox IAM Role into *your* AWS Account"); + const bdSandbox = new BDSandbox({ logger, authToken: token }).withTemplate(options.template); + const region = bdSandbox.region; + const bdAccount = new BDAccount({ logger, authToken: token }); + const bdDataSources = new BDDataSourceConfig({ logger }); + bdDataSources.withConfig({ dataSources: bdSandbox.tmpl.resources.storage }); + const bdRole = new BDIamRole({ + ...options, + logger, + iamClient: new iam.IAMClient({ region }), + stsClient: new sts.STSClient({ region }), + environment: bdSandbox.tmpl.environment, + templateName: bdSandbox.tmpl.id, + username: await bdAccount.getUsername(), + assumeAwsAccount: await bdAccount.getAssumeAwsAccount(), + assumeCondExternalId: await bdAccount.getExtId(), + }); + const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources }); + const policyDocument = await bdIntegration.getPolicyDocument(options.listBucketsPermission); + let iamRoleArn; + if (!options.dryRun) iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument)); + updateSpinnerText(`Creating IAM Role: ${iamRoleArn}` + (options.dryRun ? "(dry-run)" : "")); + spinnerSuccess(); + } catch (err: any) { + // try to decode the message + spinnerError(err?.message, false); + } +} + +const program = new cmd.Command("bdcli sandbox create-role") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--delete", "Delete the sandbox IAM role from *your* AWS Account")) + .addOption(new cmd.Option("--no-list-buckets-permission", "Do NOT add s3:ListAllMyBuckets policy entry")) + .addOption(new cmd.Option("--dry-run", "Dry run, do not actually create the IAM role")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts new file mode 100644 index 0000000..a31d71e --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts @@ -0,0 +1,42 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-sandbox-deploy"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText(`Deploying sandbox ${options.name}`); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const results = await bdSandbox.deploySandbox(options.name); + spinnerSuccess(); + await outputResults(results?.deployResults, options.disableSpinner); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox deploy") + .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts new file mode 100644 index 0000000..fb881b0 --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts @@ -0,0 +1,50 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-sandbox-destroy"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText(`Destroying sandbox ${options.name}`); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const results = await bdSandbox.destroySandbox(options.name, options.destroyAlsoInterfaces, options.deleteTemplate); + spinnerSuccess(); + await outputResults(results?.destroyResults, options.disableSpinner); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox destroy") + .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) + .addOption(new cmd.Option("--destroy-also-interfaces", "Also delete interfaces like Tap URLs")) + .addOption( + new cmd.Option( + "--delete-template", + "Finally, delete template if all resources were destroyed, including interfaces", + ), + ) + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts new file mode 100644 index 0000000..157d59e --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-download.ts @@ -0,0 +1,57 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import * as fs from "fs/promises"; + +const logger = getLogger("bdcli-sandbox-validate"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + const filename = options.name + ".yaml"; + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + let fileAlreadyExists = false; + try { + if (await fs.lstat(filename)) { + fileAlreadyExists = true; + } + } catch (err) { + logger.debug({ err }); + } + if (fileAlreadyExists) { + spinnerError(`Local file ${filename} already exists`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText(`Downloading sandbox IaC template of ${options.name}`); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const template = await bdSandbox.downloadTemplate(options.name, options.version, options?.status ?? "uploaded"); + await fs.writeFile(filename, template); + spinnerSuccess(); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox download") + .addOption(new cmd.Option("--name ", "template name from listing").makeOptionMandatory()) + .addOption(new cmd.Option("--status ", "Download 'uploaded' (default) or 'deployed' template")) + .addOption(new cmd.Option("--version ", "Download specific version from listing")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts new file mode 100644 index 0000000..6a16e93 --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-list.ts @@ -0,0 +1,44 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-sandbox-list"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Listing sandboxes"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const list = await bdSandbox.listSandboxes(options.listDeleted, options.listVersions); + spinnerSuccess(); + await outputResults(list, options.disableSpinner); + } catch (err: any) { + spinnerError(err?.message); + } +} + +const program = new cmd.Command("bdcli sandbox list") + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .addOption(new cmd.Option("--list-deleted", "List also deleted templates")) + .addOption(new cmd.Option("--list-versions", "List all template versions")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts new file mode 100644 index 0000000..a5d9f84 --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-plan.ts @@ -0,0 +1,48 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-sandbox-plan"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText(`Planning deployment for sandbox ${options.name}`); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + const results = await bdSandbox.planSandbox(options.name); + spinnerSuccess(); + await outputResults(results?.planResults, options.disableSpinner); + } catch (err: any) { + spinnerError(err?.message); + } +} + +// TODO: +// - If the template is updated, like changing the name of a resource, it needs to be replaced? +// Like if the Tap name is changed the Lambda needs to be deleted and created again and then +// also the ingest URL changes. + +const program = new cmd.Command("bdcli sandbox plan") + .addOption(new cmd.Option("--name ", "sandbox name").makeOptionMandatory()) + .addOption(new cmd.Option("--region ", "AWS region (by default eu-west-1").default("eu-west-1")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts new file mode 100644 index 0000000..69a70e2 --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-upload.ts @@ -0,0 +1,57 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-sandbox-validate"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Uploading sandbox IaC template"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + await bdSandbox.uploadTemplate(options.template, options.allowChangedFilename); + spinnerSuccess(); + } catch (origErr: any) { + try { + spinnerError(origErr?.message, false); + await outputResults( + JSON.parse(origErr?.message) + ?.message?.split(";") + ?.map((msg: string) => msg.trim()), + false, + ); + } catch (err: any) { + if (err?.message && !origErr?.message) spinnerError(err?.message, false); + } + } +} + +const program = new cmd.Command("bdcli sandbox upload") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption( + new cmd.Option( + "--allow-changed-filename", + "Allow local template filename chnage with an existing template ID (possible copy/paste error)", + ), + ) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts new file mode 100644 index 0000000..370625c --- /dev/null +++ b/src/bdcli/commands/sandbox/bdcli-sandbox-validate.ts @@ -0,0 +1,53 @@ +import * as cmd from "commander"; +import { getLogger } from "../../utils/logger_util.js"; +import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js"; +import { addGlobalOptions } from "../../utils/options_util.js"; +import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js"; +import { getIdToken } from "../../utils/auth_util.js"; +import { BDSandbox } from "../../../integration/boilingdata/sandbox.js"; +import { outputResults } from "../../utils/output_util.js"; + +const logger = getLogger("bdcli-sandbox-validate"); + +async function show(options: any, _command: cmd.Command): Promise { + try { + options = await combineOptsWithSettings(options, logger); + + if (!(await hasValidConfig())) { + return spinnerError(`No valid bdcli configuration found for "${profile}" profile`); + } + + updateSpinnerText("Authenticating"); + const { idToken: token, cached: idCached } = await getIdToken(logger); + updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`); + spinnerSuccess(); + + updateSpinnerText("Validating sandbox IaC template"); + const bdSandbox = new BDSandbox({ logger, authToken: token }); + await bdSandbox.validateTemplate(options.template, options.warningsAsErrors); + spinnerSuccess(); + } catch (origErr: any) { + // try to decode the message + try { + spinnerError(origErr?.message, false); + await outputResults( + JSON.parse(origErr?.message) + ?.message?.split(";") + ?.map((msg: string) => msg.trim()), + false, + ); + } catch (err: any) { + if (err?.message && !origErr?.message) spinnerError(err?.message, false); + } + } +} + +const program = new cmd.Command("bdcli sandbox validate") + .addOption(new cmd.Option("--template ", "sandbox IaC file").makeOptionMandatory()) + .addOption(new cmd.Option("--warnings-as-errors", "Treat any warning as an error")) + .action(async (options, command) => await show(options, command)); + +(async () => { + await addGlobalOptions(program, logger); + await program.parseAsync(process.argv); +})(); diff --git a/src/bdcli/utils/config_util.ts b/src/bdcli/utils/config_util.ts index a57f12a..e10ed15 100644 --- a/src/bdcli/utils/config_util.ts +++ b/src/bdcli/utils/config_util.ts @@ -19,6 +19,7 @@ export interface ICredentials { accessToken?: string; refreshToken?: string; bdStsToken?: string; + bdTapToken?: string; sharedTokens?: string[]; region?: string; mfa?: boolean; diff --git a/src/bdcli/utils/output_util.ts b/src/bdcli/utils/output_util.ts index 115adc6..f08fc24 100644 --- a/src/bdcli/utils/output_util.ts +++ b/src/bdcli/utils/output_util.ts @@ -1,5 +1,6 @@ import * as util from "node:util"; -export async function outputResults(results: any, flat: boolean): Promise { +export async function outputResults(results: any, flat = false): Promise { + if (!results) return; console.log(flat ? JSON.stringify(results) : util.inspect(results, false, 20, true)); } diff --git a/src/bdcli/utils/spinner_util.ts b/src/bdcli/utils/spinner_util.ts index db54f5b..9a45229 100644 --- a/src/bdcli/utils/spinner_util.ts +++ b/src/bdcli/utils/spinner_util.ts @@ -39,9 +39,9 @@ export function spinnerWarn(message?: string): void { spinner.warn(message ? warning(message) : warning(spinner.text)); } -export function spinnerError(message?: string): void { +export function spinnerError(message?: string, forceExit = true): void { if (isEnabled) spinner.fail(message ? error(message) : undefined); - process.exit(1); // error + if (forceExit) process.exit(1); // error } export function spinnerSuccess(message?: string): void { diff --git a/src/index.ts b/src/index.ts index 977c61b..1c098bb 100755 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,7 @@ program .allowExcessArguments(false) .command("account", "Setup and configure your BoilingData account") .command("aws", "Setup and configure your AWS account integration with BoilingData") - .command("domain", "Admin setup and configuration for your domain (.e.g @boilingdata.com, @mycompany.com)"); + .command("domain", "Admin setup and configuration for your domain (.e.g @boilingdata.com, @mycompany.com)") + .command("sandbox", "Managa Boiling S3 Sandboxes with IaC templates"); program.parse(process.argv); diff --git a/src/integration/aws/util.test.ts b/src/integration/aws/aws-region.test.ts similarity index 52% rename from src/integration/aws/util.test.ts rename to src/integration/aws/aws-region.test.ts index 3df8241..b9bb58f 100644 --- a/src/integration/aws/util.test.ts +++ b/src/integration/aws/aws-region.test.ts @@ -1,4 +1,4 @@ -import { getRegionShortName } from "./util.js"; +import { getAwsRegionShortName } from "./aws-region.js"; const aws_regions = [ "us-east-2", @@ -34,36 +34,36 @@ const aws_regions = [ describe("utils", () => { it("util", () => { - expect(aws_regions.map(getRegionShortName)).toEqual([ - "ue2", - "ue1", - "uw1", - "uw2", - "as1", - "ae1", - "as2", - "ase3", - "ase4", - "as1", - "ane3", - "ane2", - "ase1", - "ase2", - "ane1", - "cc1", - "ec1", - "ew1", - "ew2", - "es1", - "ew3", - "es2", - "en1", - "ec2", - "ms1", - "mc1", - "se1", - "uge1", - "ugw1", + expect(aws_regions.map(getAwsRegionShortName)).toEqual([ + "use2", + "use1", + "usw1", + "usw2", + "afs1", + "ape1", + "aps2", + "apse3", + "apse4", + "aps1", + "apne3", + "apne2", + "apse1", + "apse2", + "apne1", + "cac1", + "euc1", + "euw1", + "euw2", + "eus1", + "euw3", + "eus2", + "eun1", + "euc2", + "mes1", + "mec1", + "sae1", + "usge1", + "usgw1", ]); }); }); diff --git a/src/integration/aws/aws-region.ts b/src/integration/aws/aws-region.ts new file mode 100644 index 0000000..b90251d --- /dev/null +++ b/src/integration/aws/aws-region.ts @@ -0,0 +1,14 @@ +export function getAwsRegionShortName(region: string): string { + // a bit hacky + const splits = region + .replace("east-", "-east-") + .replace("--", "-") + .replace("west-", "-west-") + .replace("--", "-") + .split("-"); + if (splits.length != 3 && splits.length != 4) throw new Error(`Invalid AWS Region: ${region}`); + if (splits.length == 4) { + return `${splits[0]}${splits[1]?.[0]}${splits[2]?.[0]}${splits[3]}`; + } + return `${splits[0]}${splits[1]?.[0]}${splits[2]}`; +} diff --git a/src/integration/aws/iam_roles.test.ts b/src/integration/aws/iam_roles.test.ts index a9b39f5..d748c89 100644 --- a/src/integration/aws/iam_roles.test.ts +++ b/src/integration/aws/iam_roles.test.ts @@ -25,7 +25,7 @@ const roleParams: IBDIamRole = { iamClient: new IAMClient({ region }), stsClient: new STSClient({ region }), region, - uniqNamePart: "boilingdata-demo-isecurefi-dev-and-all-the-rest-of-the-buckets", + username: "aac5c1d9-a0a9-4855-b896-0f3998b2f16b", assumeAwsAccount: "123123123123", assumeCondExternalId: "abcdef123123", }; @@ -48,17 +48,17 @@ describe("iamRole", () => { it("getIamRoleName", async () => { const role = new BDIamRole(roleParams); - expect(role.iamRoleName).toEqual("bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-11843e6abe8afd6"); + expect(role.iamRoleName).toEqual("bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b"); }); it("getIamRoleName with own prefix", async () => { - const role = new BDIamRole({ ...roleParams, roleNamePrefix: "myPrefix" }); - expect(role.iamRoleName).toEqual("myPrefix-ue1-boilingdata-demo-isecurefi-dev-and--1232b7ccb3bac8a"); + const role = new BDIamRole({ ...roleParams, roleNamePrefix: "my" }); + expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b"); }); it("getIamRoleName with own path and prefix", async () => { - const role = new BDIamRole({ ...roleParams, roleNamePrefix: "myPrefix", path: "/bd-service/demo/" }); - expect(role.iamRoleName).toEqual("myPrefix-ue1-boilingdata-demo-isecurefi-dev-and--2cca7b8bb91fa2f"); + const role = new BDIamRole({ ...roleParams, roleNamePrefix: "my", path: "/bd-service/demo/" }); + expect(role.iamRoleName).toEqual("my-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b"); }); it("getRole", async () => { @@ -75,7 +75,7 @@ describe("iamRole", () => { PolicyName: "bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", Arn: "arn:aws:iam::123123123123:policy/" + - "boilingdata/bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", + "boilingdata/bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-policy", Path: "/boilingdata/", DefaultVersionId: "v123", AttachmentCount: 1, @@ -115,7 +115,7 @@ describe("iamRole", () => { PolicyName: "bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", Arn: "arn:aws:iam::123123123123:policy/" + - "boilingdata/bd-ue1-boilingdata-demo-isecurefi-dev-and-all-th-acff8dae429911f", + "boilingdata/bd-use1-noenv-notmplname-aac5c1d9a0a94855b8960f3998b2f16b-policy", Path: "/boilingdata/", DefaultVersionId: "v100", AttachmentCount: 1, diff --git a/src/integration/aws/iam_roles.ts b/src/integration/aws/iam_roles.ts index 9718aaa..4e389ab 100644 --- a/src/integration/aws/iam_roles.ts +++ b/src/integration/aws/iam_roles.ts @@ -1,7 +1,6 @@ import * as iam from "@aws-sdk/client-iam"; -import * as crypto from "crypto"; import * as sts from "@aws-sdk/client-sts"; -import { getRegionShortName } from "./util.js"; +import { getAwsRegionShortName } from "./aws-region.js"; import { ILogger } from "../../bdcli/utils/logger_util.js"; export interface Tag { @@ -19,7 +18,9 @@ export interface IBDIamRole { iamClient: iam.IAMClient; stsClient: sts.STSClient; region: string; - uniqNamePart: string; + username: string; + environment?: string; + templateName?: string; assumeCondExternalId: string; assumeAwsAccount: string; path?: string; @@ -50,24 +51,19 @@ export class BDIamRole { private getName(type: string): string { const prefix = this.params.roleNamePrefix ?? "bd"; - const regionShort = getRegionShortName(this.params.region ?? process.env["AWS_REGION"] ?? "eu-west-2"); - const hash = crypto - .createHash("md5") - .update( - type + - this.path + - prefix + - regionShort + - this.params.uniqNamePart + - this.params.assumeAwsAccount + - this.params.assumeCondExternalId + - `${this.params.maxSessionDuration ?? "N/A"}`, + const regionShort = getAwsRegionShortName(this.params.region ?? process.env["AWS_REGION"] ?? "eu-west-1"); + const env = this.params.environment ?? "noenv"; + const tmplName = this.params.templateName ?? "notmplname"; + const username = this.params.username.replaceAll("-", ""); + const name = [prefix, regionShort, env, tmplName, username].join("-"); + if (name.length > 64) { + throw new Error( + `${type} name (${name}) too long (${name.length}), reduce prefix/env/tmplname lengths (roomLeft ${ + 64 - name.length + })`, ); - const reducedHashLength = 15; - const nameLenSoFar = prefix.length + regionShort.length + reducedHashLength; - const roomLeft = 64 - nameLenSoFar - 3; // "-" chars - const dataSetShort = this.params.uniqNamePart.substring(0, roomLeft); - return [prefix, regionShort, dataSetShort, hash.digest("hex").substring(0, 15)].join("-"); + } + return name; } public async getAwsAccountId(): Promise { @@ -83,14 +79,18 @@ export class BDIamRole { public get iamRoleName(): string { if (this._iamRoleName) return this._iamRoleName; this._iamRoleName = this.getName(ENameType.ROLE); - if (this._iamRoleName.length > 64) throw new Error("Role name too long, bugger in code."); + if (this._iamRoleName.length > 64) { + throw new Error(`Role name too long, bugger in code (${this._iamRoleName})`); + } return this._iamRoleName; } public get iamManagedPolicyName(): string { if (this._iamManagedPolicyName) return this._iamManagedPolicyName; this._iamManagedPolicyName = this.getName(ENameType.POLICY); - if (this._iamManagedPolicyName.length > 64) throw new Error("Role name too long, bugger in code."); + if (this._iamManagedPolicyName.length > 64) { + throw new Error(`Policy name too long, bugger in code (${this._iamManagedPolicyName})`); + } return this._iamManagedPolicyName; } @@ -193,7 +193,10 @@ export class BDIamRole { Tags: [...(this.params.tags ?? []), ...this.boilingDataTags], }; const resp = await this.iamClient.send(new iam.CreatePolicyCommand(commandParams)); - if (!resp?.Policy?.Arn) throw new Error("Policy creation failed"); + if (!resp?.Policy?.Arn) { + this.logger.error({ resp }); + throw new Error("Policy creation failed"); + } } public async upsertBdAccessPolicy(PolicyDocument: string): Promise { @@ -215,6 +218,7 @@ export class BDIamRole { public async upsertRole(policyDocument: string): Promise { let arn: string; + this.logger.debug({ policyDocument }); try { const resp = await this.getRole(); if (!resp.Arn) throw new Error("Could not find role ARN"); diff --git a/src/integration/aws/util.ts b/src/integration/aws/util.ts deleted file mode 100644 index bb86ec3..0000000 --- a/src/integration/aws/util.ts +++ /dev/null @@ -1,8 +0,0 @@ -export function getRegionShortName(region: string): string { - return region - .replace("east", "-east") - .replace("--", "-") - .split("-") - .map(part => part[0]) - .join(""); -} diff --git a/src/integration/bdIntegration.test.ts b/src/integration/bdIntegration.test.ts index 77fc5c3..f52bf85 100644 --- a/src/integration/bdIntegration.test.ts +++ b/src/integration/bdIntegration.test.ts @@ -37,13 +37,13 @@ describe("BDIntegration", () => { bdDataSets.readConfig("./example_datasource_config.yaml"); const assumeCondExternalId = await bdAccount.getExtId(); // FIXME: This calls real API const assumeAwsAccount = await bdAccount.getAssumeAwsAccount(); - const uniqNamePart = await bdDataSets.getUniqueNamePart(); + const username = await bdAccount.getUsername(); const bdRole = new BDIamRole({ logger: roleLogger, region, iamClient, stsClient, - uniqNamePart, + username, assumeAwsAccount, assumeCondExternalId, }); @@ -83,13 +83,13 @@ describe("BDIntegration", () => { bdDataSets.readConfig("./example_datasource_config.yaml"); const assumeCondExternalId = await bdAccount.getExtId(); // FIXME: This calls real API const assumeAwsAccount = await bdAccount.getAssumeAwsAccount(); - const uniqNamePart = await bdDataSets.getUniqueNamePart(); + const username = await bdAccount.getUsername(); const bdRole = new BDIamRole({ logger: roleLogger, region, iamClient, stsClient, - uniqNamePart, + username, assumeAwsAccount, assumeCondExternalId, }); diff --git a/src/integration/bdIntegration.ts b/src/integration/bdIntegration.ts index 8b4e441..3ef1519 100644 --- a/src/integration/bdIntegration.ts +++ b/src/integration/bdIntegration.ts @@ -1,7 +1,7 @@ import { ILogger } from "../bdcli/utils/logger_util.js"; import { BDIamRole } from "./aws/iam_roles.js"; import { BDAccount } from "./boilingdata/account.js"; -import { GRANT_PERMISSION, IStatementExt } from "./boilingdata/dataset.interface.js"; +import { GRANT_PERMISSION, IStatement, IStatementExt } from "./boilingdata/dataset.interface.js"; import { BDDataSourceConfig } from "./boilingdata/dataset.js"; const RO_ACTIONS = ["s3:GetObject"]; @@ -51,23 +51,20 @@ export class BDIntegration { public getGroupedBuckets(): IGroupedDataSources { const dataSourcesConfig = this.bdDatasets.getDatasourcesConfig(); - const allPolicies = dataSourcesConfig.dataSources - .map(d => { - console.log(d); - d.accessPolicy = d.accessPolicy.map(pol => { - if (!pol.permissions) pol.permissions = [GRANT_PERMISSION.G_READ]; // default - return pol; - }); - return d.accessPolicy; - }) - .flat(); + const allPolicies: IStatement[] = []; + dataSourcesConfig.dataSources.forEach(ds => + ds.permissions.forEach(perm => { + if (!perm.accessRights) perm.accessRights = [GRANT_PERMISSION.G_READ]; // default + allPolicies.push(perm); + }), + ); this.logger.debug({ allPolicies }); - if (allPolicies.some(policy => !policy.permissions)) throw new Error("Missing policy permissions"); + if (allPolicies.some(policy => !policy.accessRights)) throw new Error("Missing policy permissions"); const readOnly = allPolicies .filter( policy => - !policy.permissions?.includes(GRANT_PERMISSION.G_WRITE) && - policy.permissions?.includes(GRANT_PERMISSION.G_READ), + !policy.accessRights?.includes(GRANT_PERMISSION.G_WRITE) && + policy.accessRights?.includes(GRANT_PERMISSION.G_READ), ) .map(policy => ({ ...policy, @@ -77,8 +74,8 @@ export class BDIntegration { const readWrite = allPolicies .filter( policy => - policy.permissions?.includes(GRANT_PERMISSION.G_WRITE) && - policy.permissions?.includes(GRANT_PERMISSION.G_READ), + policy.accessRights?.includes(GRANT_PERMISSION.G_WRITE) && + policy.accessRights?.includes(GRANT_PERMISSION.G_READ), ) .map(policy => ({ ...policy, @@ -88,8 +85,8 @@ export class BDIntegration { const writeOnly = allPolicies .filter( policy => - policy.permissions?.includes(GRANT_PERMISSION.G_WRITE) && - !policy.permissions?.includes(GRANT_PERMISSION.G_READ), + policy.accessRights?.includes(GRANT_PERMISSION.G_WRITE) && + !policy.accessRights?.includes(GRANT_PERMISSION.G_READ), ) .map(policy => ({ ...policy, @@ -100,7 +97,8 @@ export class BDIntegration { return { readOnly, readWrite, writeOnly }; } - public async getPolicyDocument(): Promise { + public async getPolicyDocument(haveListBucketsPolicy = true): Promise { + this.logger.debug({ haveListBucketsPolicy }); const grouped = this.getGroupedBuckets(); const allDatasets = [...grouped.readOnly, ...grouped.readWrite, ...grouped.writeOnly]; const Statements = []; @@ -108,11 +106,14 @@ export class BDIntegration { Statements.push(this.getStatement(grouped.readWrite, RW_ACTIONS, this.mapAccessPolicyToS3Resource.bind(this))); Statements.push(this.getStatement(grouped.writeOnly, WO_ACTIONS, this.mapAccessPolicyToS3Resource.bind(this))); Statements.push(this.getStatement(allDatasets, BUCKET_ACTIONS, this.mapDatasetsToUniqBuckets.bind(this))); - Statements.push({ - Effect: "Allow", - Action: "s3:ListAllMyBuckets", - Resource: "*", - }); + if (haveListBucketsPolicy) { + // This is so that you can run: SELECT * FROM list('s3://') + Statements.push({ + Effect: "Allow", + Action: "s3:ListAllMyBuckets", + Resource: "*", + }); + } const finalPolicy = { Version: "2012-10-17", Statement: Statements.filter(s => s?.Resource?.length) }; this.logger.debug({ getPolicyDocument: JSON.stringify(finalPolicy) }); return finalPolicy; diff --git a/src/integration/boilingdata/account.ts b/src/integration/boilingdata/account.ts index a4265d9..8a3f65c 100644 --- a/src/integration/boilingdata/account.ts +++ b/src/integration/boilingdata/account.ts @@ -7,7 +7,7 @@ import { } from "../../bdcli/utils/config_util.js"; import { ILogger } from "../../bdcli/utils/logger_util.js"; import { spinnerError } from "../../bdcli/utils/spinner_util.js"; -import { accountUrl, getReqHeaders, tokenShareUrl, tokenUrl } from "./boilingdata_api.js"; +import { accountUrl, getReqHeaders, tokenShareUrl, stsTokenUrl, tapTokenUrl } from "./boilingdata_api.js"; import * as jwt from "jsonwebtoken"; // import { channel } from "node:diagnostics_channel"; @@ -17,17 +17,31 @@ export interface IBDConfig { logger: ILogger; } +export interface ITapTokenResp { + bdTapToken: string; + cached: boolean; + expiresIn: string; + tokenLifetimeMins: number; + username: string; + email: string; + sharingUser: string; +} + interface IAPIAccountDetails { AccountAwsAccount: string; AccountExtId: string; + AccountUsername: string; + AccountEmail: string; } export class BDAccount { private cognitoIdToken: string; private bdStsToken: string | undefined; + private bdTapToken: string | undefined; private sharedTokens: IDecodedSession[]; private selectedToken: string | undefined; private decodedToken!: jwt.JwtPayload | null; + private decodedTapToken!: jwt.JwtPayload | null; private logger: ILogger; private accountDetails!: IAPIAccountDetails; @@ -66,13 +80,23 @@ export class BDAccount { if (!body.ResponseCode || !body.ResponseText) { throw new Error("Malformed response from BD API"); } - if (!body.AccountDetails?.AccountAwsAccount || !body.AccountDetails?.AccountExtId) { - throw new Error("Missing AccountDetails from BD API Response"); + if ( + !body.AccountDetails?.AccountAwsAccount || + !body.AccountDetails?.AccountExtId || + !body.AccountDetails?.AccountUsername || + !body.AccountDetails?.AccountEmail + ) { + throw new Error("Missing some or all of AccountDetails from BD API Response"); } this.accountDetails = body.AccountDetails; this.logger.debug({ accountDetails: this.accountDetails }); } + public async getUsername(): Promise { + await this._getAccountDetails(); + return this.accountDetails.AccountUsername; + } + public async getAssumeAwsAccount(): Promise { await this._getAccountDetails(); return this.accountDetails.AccountAwsAccount; @@ -101,7 +125,7 @@ export class BDAccount { this.decodedToken = jwt.decode(this.bdStsToken, { complete: true }); this.selectedToken = this.bdStsToken; } - if (!this.decodedToken || !this.selectedToken) throw new Error(`Could not find token (share id ${shareId})`); + if (!this.decodedToken || !this.selectedToken) throw new Error(`Could not find STS token (share id ${shareId})`); } private dumpSelectedToken(): void { @@ -109,20 +133,31 @@ export class BDAccount { this.logger.debug({ bdStsToken: this.selectedToken, decodedToken: this.decodedToken }); } + private decodeTapToken(): void { + if (!this.bdTapToken) throw new Error("No BD TAP token"); + this.decodedTapToken = jwt.decode(this.bdTapToken, { complete: true }); + if (!this.decodedTapToken) throw new Error(`Could not find TAP token`); + } + + private dumpTapToken(): void { + if (!this.bdTapToken) throw new Error("No BD TAP token"); + this.logger.debug({ bdTapToken: this.bdTapToken, decodedToken: this.decodedTapToken }); + } + private getHumanReadable(exp: number): string { const humanReadable = new Date(); humanReadable.setTime(exp * 1000); return humanReadable.toISOString(); } - private checkExp(exp: number): boolean { - const twoMinsAgo = new Date(); - twoMinsAgo.setTime(Date.now() - 2 * 60 * 1000); - const diff = exp * 1000 - twoMinsAgo.getTime(); + private checkExp(exp: number, minutesAgo = 2): boolean { + const minsAgo = new Date(); + minsAgo.setTime(Date.now() - minutesAgo * 60 * 1000); + const diff = exp * 1000 - minsAgo.getTime(); this.logger.debug({ exp, diff, - twoMinsAgo: twoMinsAgo.toISOString(), + twoMinsAgo: minsAgo.toISOString(), expiresIn: this.getHumanReadable(exp), }); if (diff < 0) return false; @@ -184,17 +219,46 @@ export class BDAccount { throw new Error("Failed to unshare token"); } + private async getTapTokenResp(updateConfigFile = true): Promise { + this.decodeTapToken(); + this.dumpTapToken(); + if (!this.decodedTapToken || !this.bdTapToken) throw new Error("Unable to decode TAP token"); + const exp = this.decodedTapToken["payload"].exp; + const iat = this.decodedTapToken["payload"].iat; + const aud = this.decodedTapToken["payload"].aud; + const tapTokenLifetimeMins = Math.floor((exp - iat) / 60); + const oneHourMin = 60; + if (exp && this.checkExp(exp, oneHourMin)) { + this.logger.debug({ cachedBdTapToken: true }); + // we clean up expired tokens at the same time + // clean first as we use deepmerge that merges lists and would otherwise cause duplicates + const credentials = { bdTapToken: this.bdTapToken }; + if (updateConfigFile) await updateConfig({ credentials }); // local config file + // NOTE: Even if the tokenLifetime would be different from the request, we return non-expired token + return { + bdTapToken: this.bdTapToken, + cached: updateConfigFile != true, + expiresIn: this.getHumanReadable(exp), + tokenLifetimeMins: tapTokenLifetimeMins, + username: aud?.[0], + email: aud?.[1], + sharingUser: aud?.[2], + }; + } + return; // expired + } + private async getTokenResp( shareId?: string, ): Promise<{ bdStsToken: string; cached: boolean; expiresIn: string; tokenLifetimeMins: number } | undefined> { this.selectAndDecodeToken(shareId); this.dumpSelectedToken(); - if (!this.decodedToken || !this.selectedToken) throw new Error("Unable to select/decode token"); + if (!this.decodedToken || !this.selectedToken) throw new Error("Unable to select/decode STS token"); const exp = this.decodedToken["payload"].exp; const iat = this.decodedToken["payload"].iat; const tokenLifetimeMins = Math.floor((exp - iat) / 60); if (exp && this.checkExp(exp)) { - this.logger.debug({ cachedBdStstToken: true }); + this.logger.debug({ cachedBdStsToken: true }); // we clean up expired tokens at the same time // clean first as we use deepmerge that merges lists and would otherwise cause duplicates await updateConfig({ credentials: { sharedTokens: undefined } }); @@ -206,7 +270,45 @@ export class BDAccount { return; // expired } - public async getToken(tokenLifetime: string, shareId?: string): Promise<{ bdStsToken: string; cached: boolean }> { + public async getTapToken( + tokenLifetime: string, + sharingUser?: string, + ): Promise<{ bdTapToken: string; cached: boolean }> { + const creds = await getConfigCredentials(); + this.bdTapToken = creds.bdTapToken; + if (this.bdTapToken) { + const cachedToken = await this.getTapTokenResp(false); + if (cachedToken && ((sharingUser && cachedToken?.sharingUser === sharingUser) || !sharingUser)) { + return cachedToken; // disk cached token is not yet expired.. + } + } + // channel("undici:request:create").subscribe(console.log); + // channel("undici:request:headers").subscribe(console.log); + const headers = await getReqHeaders(this.cognitoIdToken); // , { tokenLifetime, vendingSchedule, shareId }); + const method = "POST"; + const body = JSON.stringify({ tokenLifetime, sharingUser }); + this.logger.debug({ method, tapTokenUrl, headers, body }); + const res = await fetch(tapTokenUrl, { method, headers, body }); + const resBody = await res.json(); + this.logger.debug({ getTapToken: { body: resBody } }); + if (!resBody.ResponseCode || !resBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (resBody.ResponseCode != "00") { + spinnerError(resBody.ResponseText); + throw new Error(`Failed to fetch token: ${resBody.ResponseText}`); + } + if (!resBody.bdTapToken) { + throw new Error("Missing bdStsToken in BD API Response"); + } + this.bdTapToken = resBody.bdTapToken; + + const resp = await this.getTapTokenResp(true); + if (resp) return resp; + throw new Error(`Failed to get fresh TAP token from BD API`); + } + + public async getStsToken(tokenLifetime: string, shareId?: string): Promise<{ bdStsToken: string; cached: boolean }> { if (this.bdStsToken && !shareId) { this.selectedToken = this.bdStsToken; return { bdStsToken: this.bdStsToken, cached: true }; @@ -233,8 +335,8 @@ export class BDAccount { method = "POST"; body = JSON.stringify({ tokenLifetime, shareId }); } - this.logger.debug({ method, tokenUrl, headers, body }); - const res = await fetch(tokenUrl, { method, headers, body }); + this.logger.debug({ method, tokenUrl: stsTokenUrl, headers, body }); + const res = await fetch(stsTokenUrl, { method, headers, body }); const resBody = await res.json(); this.logger.debug({ getStsToken: { body: resBody } }); if (!resBody.ResponseCode || !resBody.ResponseText) { @@ -261,6 +363,6 @@ export class BDAccount { const resp = await this.getTokenResp(shareId); if (resp) return resp; - throw new Error(`Failed to get fresh token from BD API (with share id ${shareId})`); + throw new Error(`Failed to get fresh STS token from BD API (with share id ${shareId})`); } } diff --git a/src/integration/boilingdata/boilingdata_api.ts b/src/integration/boilingdata/boilingdata_api.ts index a8e15d8..3545fbe 100644 --- a/src/integration/boilingdata/boilingdata_api.ts +++ b/src/integration/boilingdata/boilingdata_api.ts @@ -4,12 +4,16 @@ import * as id from "amazon-cognito-identity-js"; export const baseApiUrl = "https://rest.api.test.boilingdata.com"; export const dataSetsPath = "/data-sets"; export const accountPath = "/account"; -export const tokenPath = "/token"; +export const stsTokenPath = "/token"; +export const tapTokenPath = "/taptoken"; export const sharePath = "/share"; +export const sandboxPath = "/sandbox"; export const dataSetsUrl = baseApiUrl + dataSetsPath; export const accountUrl = baseApiUrl + accountPath; -export const tokenUrl = baseApiUrl + tokenPath; +export const stsTokenUrl = baseApiUrl + stsTokenPath; +export const tapTokenUrl = baseApiUrl + tapTokenPath; export const tokenShareUrl = baseApiUrl + sharePath; +export const sandboxUrl = baseApiUrl + sandboxPath; // FIXME: get from bdAccount API export const bdAWSAccount = "589434896614"; diff --git a/src/integration/boilingdata/dataset.interface-ti.ts b/src/integration/boilingdata/dataset.interface-ti.ts index 2228e70..a34384c 100644 --- a/src/integration/boilingdata/dataset.interface-ti.ts +++ b/src/integration/boilingdata/dataset.interface-ti.ts @@ -4,44 +4,22 @@ import * as t from "ts-interface-checker"; // tslint:disable:object-literal-key-quotes -export const EDataSetType = t.enumtype({ - "S3": "s3", -}); - export const GRANT_PERMISSION = t.enumtype({ "G_WRITE": "write", "G_READ": "read", }); -export const SESSION_TYPE = t.enumtype({ - "STS": "sts", - "ASSUME_ROLE": "assumeRole", -}); - -export const LAYOUT = t.enumtype({ - "HIVE": "hive", - "FOLDER": "folder", - "FILE": "file", -}); - export const FILE_TYPE = t.enumtype({ "PARQUET": "parquet", "JSON": "json", "CSV": "csv", }); -export const USessionType = t.union(t.enumlit("SESSION_TYPE", "STS"), t.enumlit("SESSION_TYPE", "ASSUME_ROLE")); - export const UGrant = t.union(t.enumlit("GRANT_PERMISSION", "G_READ"), t.enumlit("GRANT_PERMISSION", "G_WRITE")); -export const ULayout = t.union(t.enumlit("LAYOUT", "HIVE"), t.enumlit("LAYOUT", "FOLDER"), t.enumlit("LAYOUT", "FILE")); - -export const UFileType = t.union(t.enumlit("FILE_TYPE", "PARQUET"), t.enumlit("FILE_TYPE", "JSON"), t.enumlit("FILE_TYPE", "CSV")); - export const IStatement = t.iface([], { - "id": "string", "urlPrefix": "string", - "permissions": t.opt(t.array("UGrant")), + "accessRights": t.opt(t.array("UGrant")), }); export const IStatementExt = t.iface(["IStatement"], { @@ -52,16 +30,11 @@ export const IStatementExt = t.iface(["IStatement"], { export const IDataSet = t.iface([], { "name": "string", "urlPrefix": "string", - "layout": t.opt("ULayout"), - "filetype": t.opt("UFileType"), }); export const IDataSource = t.iface([], { "name": "string", - "accessPolicy": t.array("IStatement"), - "type": t.opt("EDataSetType"), - "dataSets": t.opt(t.array("IDataSet")), - "sessionType": t.opt("USessionType"), + "permissions": t.array("IStatement"), }); export const IDataSources = t.iface([], { @@ -71,15 +44,9 @@ export const IDataSources = t.iface([], { }); const exportedTypeSuite: t.ITypeSuite = { - EDataSetType, GRANT_PERMISSION, - SESSION_TYPE, - LAYOUT, FILE_TYPE, - USessionType, UGrant, - ULayout, - UFileType, IStatement, IStatementExt, IDataSet, diff --git a/src/integration/boilingdata/dataset.interface.ts b/src/integration/boilingdata/dataset.interface.ts index 0e0ce6a..06f66c0 100644 --- a/src/integration/boilingdata/dataset.interface.ts +++ b/src/integration/boilingdata/dataset.interface.ts @@ -1,38 +1,19 @@ -export enum EDataSetType { - S3 = "s3", -} - export enum GRANT_PERMISSION { G_WRITE = "write", G_READ = "read", } -export enum SESSION_TYPE { - STS = "sts", - ASSUME_ROLE = "assumeRole", -} - -export enum LAYOUT { - HIVE = "hive", - FOLDER = "folder", - FILE = "file", -} - export enum FILE_TYPE { PARQUET = "parquet", JSON = "json", CSV = "csv", } -export type USessionType = SESSION_TYPE.STS | SESSION_TYPE.ASSUME_ROLE; export type UGrant = GRANT_PERMISSION.G_READ | GRANT_PERMISSION.G_WRITE; -export type ULayout = LAYOUT.HIVE | LAYOUT.FOLDER | LAYOUT.FILE; -export type UFileType = FILE_TYPE.PARQUET | FILE_TYPE.JSON | FILE_TYPE.CSV; export interface IStatement { - id: string; urlPrefix: string; - permissions?: UGrant[]; + accessRights?: UGrant[]; } export interface IStatementExt extends IStatement { @@ -43,20 +24,15 @@ export interface IStatementExt extends IStatement { export interface IDataSet { name: string; urlPrefix: string; - layout?: ULayout; - filetype?: UFileType; } export interface IDataSource { name: string; - accessPolicy: Array; - type?: EDataSetType; - dataSets?: Array; - sessionType?: USessionType; + permissions: Array; } export interface IDataSources { version?: string | number; uniqNamePart?: string; - dataSources: Array; + dataSources: IDataSource[]; } diff --git a/src/integration/boilingdata/dataset.ts b/src/integration/boilingdata/dataset.ts index 4a580d6..1aad685 100644 --- a/src/integration/boilingdata/dataset.ts +++ b/src/integration/boilingdata/dataset.ts @@ -17,18 +17,6 @@ export class BDDataSourceConfig { this.logger = this.params.logger; } - public async getUniqueNamePart(): Promise { - if (!this._dataSourcesConfig || this._dataSourcesConfig.dataSources.length <= 0) { - throw new Error("Set datasources config first"); - } - const uniqName = - this._dataSourcesConfig.uniqNamePart ?? - this._dataSourcesConfig.dataSources[this._dataSourcesConfig.dataSources.length - 1]?.name ?? - "bdIamRole"; - this.logger.debug({ uniqName }); - return uniqName; - } - public isDataSetsConfig(dataSourcesConfig: unknown): dataSourcesConfig is IDataSources { try { const { IDataSources } = createCheckers(DataSetInterfaceTI); @@ -50,6 +38,11 @@ export class BDDataSourceConfig { return { ...this._dataSourcesConfig }; // make copy } + public withConfig(config: object): void { + if (!this.isDataSetsConfig(config)) throw new Error("datasources config schema not validated"); + this._dataSourcesConfig = config; + } + public async readConfig(filename: string): Promise { if (this._dataSourcesConfig) return this._dataSourcesConfig; try { diff --git a/src/integration/boilingdata/fixtures/datasources_extra.test.yaml b/src/integration/boilingdata/fixtures/datasources_extra.test.yaml index 885c1e8..2bb6c50 100644 --- a/src/integration/boilingdata/fixtures/datasources_extra.test.yaml +++ b/src/integration/boilingdata/fixtures/datasources_extra.test.yaml @@ -4,16 +4,15 @@ dataSources: - name: demo type: s3 sessionType: assumeRole - accessPolicy: - - id: bd-demo-policy - badKey: yesTrue + permissions: + - badKey: yesTrue urlPrefix: s3://boilingdata-demo/demo - permissions: + accessRights: - read - id: nyc-policy dummy: hereAndThere urlPrefix: s3://isecurefi-dev-test/nyc-tlc/ - permissions: + accessRights: - read - write dataSets: @@ -27,7 +26,7 @@ dataSources: filetype: parquet - name: logs type: s3 - accessPolicy: + permissions: - id: logs-policy urlPrefix: s3://logs-bucket/ dataSets: diff --git a/src/integration/boilingdata/fixtures/datasources_valid.test.yaml b/src/integration/boilingdata/fixtures/datasources_valid.test.yaml index ee20532..2707115 100644 --- a/src/integration/boilingdata/fixtures/datasources_valid.test.yaml +++ b/src/integration/boilingdata/fixtures/datasources_valid.test.yaml @@ -2,16 +2,12 @@ version: 1.0 uniqNamePart: demo2 dataSources: - name: demo - type: s3 - sessionType: assumeRole - accessPolicy: - - id: bd-demo-policy - urlPrefix: s3://boilingdata-demo/demo - permissions: + permissions: + - urlPrefix: s3://boilingdata-demo/demo + accessRights: - read - - id: nyc-policy - urlPrefix: s3://isecurefi-dev-test/nyc-tlc/ - permissions: + - urlPrefix: s3://isecurefi-dev-test/nyc-tlc/ + accessRights: - read - write dataSets: @@ -24,8 +20,7 @@ dataSources: layout: hive filetype: parquet - name: logs - type: s3 - accessPolicy: + permissions: - id: logs-policy urlPrefix: s3://logs-bucket/ dataSets: diff --git a/src/integration/boilingdata/sandbox-template.types-ti.ts b/src/integration/boilingdata/sandbox-template.types-ti.ts new file mode 100644 index 0000000..0a9f52e --- /dev/null +++ b/src/integration/boilingdata/sandbox-template.types-ti.ts @@ -0,0 +1,74 @@ +/** + * This module was automatically generated by `ts-interface-builder` + */ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const EModelFormat = t.enumtype({ + "NDJSON": "ndjson", + "AVRO": "avro", + "CSV": "csv", +}); + +export const EPermission = t.enumtype({ + "READ": "read", + "WRITE": "write", +}); + +export const ITemplateStorage = t.iface([], { + "name": "string", + "permissions": t.array(t.iface([], { + "urlPrefix": "string", + "accessRights": t.opt(t.array("EPermission")), + })), +}); + +export const ITemplateTap = t.iface([], { + "name": "string", + "models": t.opt(t.array(t.iface([], { + "name": "string", + "model": t.opt(t.array("string")), + "format": t.opt("EModelFormat"), + }))), +}); + +export const ITemplatePipe = t.iface([], { + "name": "string", + "input": "string", + "keys": t.opt(t.array("string")), + "transformSql": t.opt("string"), + "output": t.opt(t.union("string", t.array("string"))), + "errors": t.opt("string"), +}); + +export const ITemplateACL = t.iface([], { + "name": "string", + "users": t.array("string"), + "sql": t.opt("string"), + "source": t.opt("string"), + "target": t.opt("string"), +}); + +export const ITemplate = t.iface([], { + "version": t.union("string", "number"), + "id": "string", + "environment": "string", + "region": "string", + "resources": t.iface([], { + "storage": "ITemplateStorage", + "taps": t.opt(t.array("ITemplateTap")), + "pipes": t.opt(t.array("ITemplatePipe")), + "acls": t.opt(t.array("ITemplateACL")), + }), +}); + +const exportedTypeSuite: t.ITypeSuite = { + EModelFormat, + EPermission, + ITemplateStorage, + ITemplateTap, + ITemplatePipe, + ITemplateACL, + ITemplate, +}; +export default exportedTypeSuite; diff --git a/src/integration/boilingdata/sandbox-template.types.ts b/src/integration/boilingdata/sandbox-template.types.ts new file mode 100644 index 0000000..bb7c1ec --- /dev/null +++ b/src/integration/boilingdata/sandbox-template.types.ts @@ -0,0 +1,56 @@ +export enum EModelFormat { + NDJSON = "ndjson", + AVRO = "avro", + CSV = "csv", +} +export enum EPermission { + READ = "read", + WRITE = "write", +} + +export interface ITemplateStorage { + name: string; + permissions: Array<{ + urlPrefix: string; + accessRights?: EPermission[]; + }>; +} + +export interface ITemplateTap { + name: string; + models?: Array<{ + name: string; + model?: string[]; + format?: EModelFormat; + }>; +} + +export interface ITemplatePipe { + name: string; + input: string; + keys?: string[]; + transformSql?: string; + output?: string | string[]; + errors?: string; +} + +export interface ITemplateACL { + name: string; + users: string[]; + sql?: string; + source?: string; + target?: string; +} + +export interface ITemplate { + version: string | number; + id: string; + environment: string; + region: string; + resources: { + storage: ITemplateStorage; + taps?: ITemplateTap[]; + pipes?: ITemplatePipe[]; + acls?: ITemplateACL[]; + }; +} diff --git a/src/integration/boilingdata/sandbox.ts b/src/integration/boilingdata/sandbox.ts new file mode 100644 index 0000000..4e9042e --- /dev/null +++ b/src/integration/boilingdata/sandbox.ts @@ -0,0 +1,211 @@ +import * as fs from "fs/promises"; +import * as fsSync from "fs"; +import * as yaml from "js-yaml"; +import { ILogger } from "../../bdcli/utils/logger_util.js"; +import { getReqHeaders, sandboxUrl } from "./boilingdata_api.js"; +import { createCheckers } from "ts-interface-checker"; +import sandboxTemplateTI from "./sandbox-template.types-ti.js"; +import { ITemplate } from "./sandbox-template.types.js"; +import * as path from "path"; + +// import { channel } from "node:diagnostics_channel"; + +export interface IBDConfig { + authToken: string; + logger: ILogger; +} + +export class BDSandbox { + private cognitoIdToken: string; + private logger: ILogger; + private _tmpl!: ITemplate; + private tmplErrors: string[] = []; + + constructor(private params: IBDConfig) { + this.logger = this.params.logger; + // this.logger.debug(this.params); + this.cognitoIdToken = this.params.authToken; + } + + public get tmpl(): ITemplate { + return this._tmpl; + } + + public get region(): string | undefined { + return this._tmpl.region; + } + + public withTemplate(templateFilename: string): this { + this.validateTemplateLocal(templateFilename); + return this; + } + + public getSandboxTemplateErrors(): string { + return this.tmplErrors.map(e => "\t" + e).join("\n"); + } + + public isSandboxConfig(sandboxTemplate: unknown): sandboxTemplate is ITemplate { + try { + const { ITemplate: ITemplateChecker } = createCheckers(sandboxTemplateTI); + if (!ITemplateChecker) throw new Error("ts-interface-check checker MISSING"); + ITemplateChecker.check(sandboxTemplate); + return true; + } catch (err: any) { + //this.logger.error(`\n${err?.message}`); + this.tmplErrors.push(err?.message.replace("value.", "")); + return false; + } + } + + public async destroySandbox( + sandboxName: string, + destroyAlsoInterfaces: boolean, + finallyDeleteTemplate: boolean, + ): Promise { + const headers = await getReqHeaders(this.cognitoIdToken); + headers["x-bd-destroy-also-interfaces"] = `${destroyAlsoInterfaces}`; + headers["x-bd-finally-delete-template"] = `${finallyDeleteTemplate}`; + this.logger.debug({ sandboxUrl, headers }); + const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "DELETE", headers }); + const respBody = await res.json(); + this.logger.debug({ DeleteSandbox: { respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (respBody.ResponseCode != "00") throw new Error(respBody.ResponseText); + return respBody; + } + + public async downloadTemplate(sandboxName: string, version: string, status?: string): Promise { + const headers = await getReqHeaders(this.cognitoIdToken); + headers["x-bd-template-version"] = version; + headers["x-bd-template-status"] = status ?? "uploaded"; + this.logger.debug({ sandboxUrl, headers }); + const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "GET", headers }); + const respBody = await res.json(); + this.logger.debug({ DownloadSandbox: { respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (respBody.ResponseCode != "00") { + throw new Error(respBody.ResponseText); + } + return respBody?.template; + } + + public validateTemplateLocal(templateFilename: string): void { + let fileContents; + let sandboxTemplateConfig; + try { + fileContents = fsSync.readFileSync(templateFilename, "utf8"); + } catch (err: any) { + throw new Error(`${err?.message}`); + } + try { + sandboxTemplateConfig = yaml.load(fileContents); + } catch (err: any) { + throw new Error(`${err?.message}`); + } + this.logger.debug(sandboxTemplateConfig); + if (!this.isSandboxConfig(sandboxTemplateConfig)) + throw new Error(`sandbox template config schema not validated:\n${this.getSandboxTemplateErrors()}`); + this._tmpl = sandboxTemplateConfig; + } + + public async uploadTemplate(templateFilename: string, allowChangedFilename = false): Promise { + this.validateTemplateLocal(templateFilename); + return this._uploadTemplate(templateFilename, false, false, allowChangedFilename); + } + + public async validateTemplate(templateFilename: string, warningsAsErrors = false): Promise { + this.validateTemplateLocal(templateFilename); + const validateOnly = true; + return this._uploadTemplate(templateFilename, validateOnly, warningsAsErrors); + } + + public async listSandboxes( + listDeleted: boolean, + listVersions: boolean, + ): Promise> { + // channel("undici:request:create").subscribe(console.log); + // channel("undici:request:headers").subscribe(console.log); + const headers = await getReqHeaders(this.cognitoIdToken); + headers["x-bd-list-deleted"] = `${listDeleted}`; + headers["x-bd-list-versions"] = `${listVersions}`; + this.logger.debug({ sandboxUrl, headers }); + const res = await fetch(sandboxUrl, { method: "GET", headers }); + const body = await res.json(); + this.logger.debug({ listSandboxes: { body } }); + if (!body.ResponseCode || !body.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (Array.isArray(body.sandboxList)) { + return body.sandboxList; + } + throw new Error("Failed to list sandboxes"); + } + + public async planSandbox(sandboxName: string): Promise { + const planOnly = true; + const diffOnly = false; + return this._deploySandbox(sandboxName, planOnly, diffOnly); + } + + public async diffSandbox(sandboxName: string): Promise { + const planOnly = false; + const diffOnly = true; + return this._deploySandbox(sandboxName, planOnly, diffOnly); + } + + public async deploySandbox(sandboxName: string): Promise { + return this._deploySandbox(sandboxName); + } + + // ---- private ---- + + private async _uploadTemplate( + templateFilename: string, + validateOnly = false, + warningsAsErrors = false, + allowChangedFilename = false, + ): Promise { + const headers = await getReqHeaders(this.cognitoIdToken); + this.logger.debug({ sandboxUrl, headers }); + const template = Buffer.from(await fs.readFile(templateFilename)).toString("base64"); + const body = JSON.stringify({ + validateOnly, + warningsAsErrors, + template, + templateFilename: path.basename(templateFilename), + allowChangedFilename, + }); + this.logger.debug({ body }); + const res = await fetch(sandboxUrl, { method: "PUT", headers, body }); + const respBody = await res.json(); + this.logger.debug({ ValidateSandbox: { respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (respBody.ResponseCode != "00") { + throw new Error(respBody?.validationResults ?? respBody); + } + return respBody; + } + + private async _deploySandbox(sandboxName: string, planOnly = false, diffOnly = false): Promise { + const action = planOnly ? "plan" : diffOnly ? "diff" : "deploy"; + const headers = await getReqHeaders(this.cognitoIdToken); + this.logger.debug({ sandboxUrl, headers }); + const body = JSON.stringify({ planOnly, diffOnly }); + const res = await fetch(sandboxUrl + "/" + sandboxName, { method: "PUT", headers, body }); + const respBody = await res.json(); + this.logger.debug({ listSandboxes: { body: respBody } }); + if (!respBody.ResponseCode || !respBody.ResponseText) { + throw new Error("Malformed response from BD API"); + } + if (respBody.ResponseCode != "00") { + throw new Error(`Failed to ${action} ${sandboxName}`); + } + return respBody; + } +}