Add Initial Sandbox (Data Taps) support

.gitignore

@@ -6,3 +6,6 @@ dist/
 datasource_config.yaml
 bdcli
 !bdcli/
+bd_querylog_config.yaml
+boilingQueryLog_v2.yaml
+boilingdata_user_x.yaml

.prettierignore

@@ -2,4 +2,5 @@ dist/
 node_modules/
 package.json
 src/VERSION.ts
-src/integration/boilingdata/dataset.interface-ti.ts
+src/integration/boilingdata/dataset.interface-ti.ts
+src/integration/boilingdata/sandbox-template.types-ti.ts

README.md

@@ -19,6 +19,7 @@ Commands:
   account         Setup and configure your BoilingData account
   aws             Setup and configure your AWS account integration with BoilingData
   domain          Admin setup and configuration for your domain (.e.g @boilingdata.com, @mycompany.com)
+  sandbox         Managa Boiling S3 Sandboxes with IaC templates
   help [command]  display help for command
 ```
 
@@ -28,15 +29,16 @@ You can create BoilingData assumable IAM role into your AWS account with clear s
 % echo "version: 1.0
 dataSources:
   - name: demo
-    type: s3
-    accessPolicy:
-      - id: bd-test-policy
-        urlPrefix: s3://my-bucket/and/prefix
+    permissions:
+      - urlPrefix: s3://my-bucket/and/prefix
+        accessRights:
+          - read
+          - write
 " > datasource_config.yaml
 
 % bdcli aws iam -c datasource_config.yaml --region eu-west-1 --create-role-only
 ✔ Authenticating: success
-✔ Creating IAM Role: arn:aws:iam::123123123123:role/boilingdata/bd-ew1-demo-0ccb08a39c45a24
+✔ Creating IAM Role: arn:aws:iam::589434896614:role/boilingdata/bd-euw1-noenv-notmplname-21346bf26c314caf8e7e9832205ffdee
 
 % echo "Now you can verify the generated IAM role"
 Now you can verify the generated IAM role
@@ -158,3 +160,7 @@ npm run build
 npm install -g .
 bdcli -h
 ```
+
+### TODO
+
+- [_] Add shell auto completion with [omelette](https://github.com/f/omelette).

package.json

@@ -39,7 +39,7 @@
   },
   "scripts": {
     "release": "standard-version -a",
-    "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts ",
+    "prebuild": "rm -rf dist/ && node generateVersionFile.js && npx ts-interface-builder src/integration/boilingdata/dataset.interface.ts && npx ts-interface-builder src/integration/boilingdata/sandbox-template.types.ts",
     "build": "rm -rf dist/ && tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && chmod 755 dist/esm/index.js",
     "prettier": "prettier --check 'src/**/*.{js,ts}'",
     "prettier:fix": "prettier --write 'src/**/*.{js,ts}'",

src/bdcli/commands/account/bdcli-account-sts-token.ts

@@ -56,7 +56,7 @@ async function show(options: any, _command: cmd.Command): Promise<void> {
     updateSpinnerText(`Getting BoilingData STS token`);
     if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
     const bdAccount = new BDAccount({ logger, authToken: token });
-    const { bdStsToken, cached: stsCached, ...rest } = await bdAccount.getToken(options.lifetime ?? "1h");
+    const { bdStsToken, cached: stsCached, ...rest } = await bdAccount.getStsToken(options.lifetime ?? "1h");
     updateSpinnerText(`Getting BoilingData STS token: ${stsCached ? "cached" : "success"}`);
     spinnerSuccess();
 

src/bdcli/commands/account/bdcli-account-tap-token.ts

@@ -0,0 +1,57 @@
+import * as cmd from "commander";
+import { getLogger } from "../../utils/logger_util.js";
+import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js";
+import { addGlobalOptions } from "../../utils/options_util.js";
+import { getIdToken, validateTokenLifetime } from "../../utils/auth_util.js";
+import { BDAccount } from "../../../integration/boilingdata/account.js";
+import { combineOptsWithSettings } from "../../utils/config_util.js";
+import { outputResults } from "../../utils/output_util.js";
+
+const logger = getLogger("bdcli-account-token");
+
+async function show(options: any, _command: cmd.Command): Promise<void> {
+  try {
+    options = await combineOptsWithSettings(options, logger);
+
+    if (options.lifetime) await validateTokenLifetime(options.lifetime);
+
+    updateSpinnerText("Authenticating");
+    const { idToken: token, cached: idCached, region } = await getIdToken(logger);
+    updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`);
+    spinnerSuccess();
+
+    updateSpinnerText(`Getting BoilingData TAP token`);
+    if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
+    const bdAccount = new BDAccount({ logger, authToken: token });
+    const {
+      bdTapToken,
+      cached: tapCached,
+      ...rest
+    } = await bdAccount.getTapToken(options.lifetime ?? "24h", options.sharingUser);
+    updateSpinnerText(`Getting BoilingData TAP token: ${tapCached ? "cached" : "success"}`);
+    spinnerSuccess();
+    await outputResults({ bdTapToken, cached: tapCached, ...rest }, options.disableSpinner);
+  } catch (err: any) {
+    spinnerError(err?.message);
+  }
+}
+
+const program = new cmd.Command("bdcli account tap-token")
+  .addOption(
+    new cmd.Option(
+      "--lifetime <lifetime>",
+      "Expiration lifetime for the token, in string format, like '1h' (see https://github.com/vercel/ms)",
+    ),
+  )
+  .addOption(
+    new cmd.Option(
+      "--sharing-user <emailOfTapSharingUser>",
+      "A user has shared Tap for you so that you can write to it.",
+    ),
+  )
+  .action(async (options, command) => await show(options, command));
+
+(async () => {
+  await addGlobalOptions(program, logger);
+  await program.parseAsync(process.argv);
+})();

src/bdcli/commands/aws/bdcli-aws-iam.ts

@@ -39,7 +39,7 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
       logger,
       iamClient: new iam.IAMClient({ region }),
       stsClient: new sts.STSClient({ region }),
-      uniqNamePart: await bdDataSources.getUniqueNamePart(),
+      username: await bdAccount.getUsername(),
       assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
       assumeCondExternalId: await bdAccount.getExtId(),
     });

src/bdcli/commands/bdcli-account.ts

@@ -8,6 +8,7 @@ const program = new Command("bdcli account")
   .command("mfa", "Enable MFA")
   .command("migrate", "Migrate your account to another AWS region (not-yet-implemented)")
   .command("sts-token", "Exchange Cognito ID Token into shared or your own BoilingData Short-Term-Session (STS) token")
+  .command("tap-token", "Exchange Cognito ID Token into shared or your own BoilingData Stream Tap auth token")
   .command("token-share", "Share data sets via access tokens to other Boiling users")
   .command("token-unshare", "Unshare access tokens")
   .command("token-list-shares", "List shared data sets (access tokens) from and to you")

src/bdcli/commands/bdcli-sandbox.ts

@@ -0,0 +1,14 @@
+import { Command } from "commander";
+
+const program = new Command("bdcli sandbox")
+  .executableDir("sandbox")
+  .command("list", "List deployed sandboxes")
+  .command("create-role", "Create Sandbox IAM Role with your AWS credentials")
+  .command("validate", "Validate *local* sandbox template")
+  .command("upload", "Upload sandbox template")
+  .command("download", "Download the *uploaded* sandbox template")
+  .command("plan", "List planned udpates based on *uploaded* sandbox definition and deployed sandbox")
+  .command("deploy", "Deploy the *uploaded* sandbox")
+  .command("destroy", "Destroy the sandbox");
+
+program.parse(process.argv);

src/bdcli/commands/domain/bdcli-domain-setup.ts

@@ -39,7 +39,6 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
       logger,
       iamClient: new iam.IAMClient({ region }),
       stsClient: new sts.STSClient({ region }),
-      uniqNamePart: await bdDataSources.getUniqueNamePart(),
       assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
       assumeCondExternalId: await bdAccount.getExtId(),
     });

src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts

@@ -0,0 +1,74 @@
+import * as iam from "@aws-sdk/client-iam";
+import * as sts from "@aws-sdk/client-sts";
+import * as cmd from "commander";
+import { getLogger } from "../../utils/logger_util.js";
+import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js";
+import { addGlobalOptions } from "../../utils/options_util.js";
+import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js";
+import { getIdToken } from "../../utils/auth_util.js";
+import { BDSandbox } from "../../../integration/boilingdata/sandbox.js";
+import { BDIamRole } from "../../../integration/aws/iam_roles.js";
+import { BDAccount } from "../../../integration/boilingdata/account.js";
+import { BDIntegration } from "../../../integration/bdIntegration.js";
+import { BDDataSourceConfig } from "../../../integration/boilingdata/dataset.js";
+
+const logger = getLogger("bdcli-sandbox-create-role");
+
+async function show(options: any, _command: cmd.Command): Promise<void> {
+  try {
+    options = await combineOptsWithSettings(options, logger);
+
+    if (!(await hasValidConfig())) {
+      return spinnerError(`No valid bdcli configuration found for "${profile}" profile`);
+    }
+    if (options.delete) {
+      updateSpinnerText("Not implemented yet. Please delete the IAM Role from AWS Console");
+      spinnerWarn("Not implemented yet. Please delete the IAM Role from AWS Console");
+      return;
+    }
+
+    updateSpinnerText("Authenticating");
+    const { idToken: token, cached: idCached } = await getIdToken(logger);
+    updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`);
+    spinnerSuccess();
+
+    updateSpinnerText("Creating sandbox IAM Role into *your* AWS Account");
+    const bdSandbox = new BDSandbox({ logger, authToken: token }).withTemplate(options.template);
+    const region = bdSandbox.region;
+    const bdAccount = new BDAccount({ logger, authToken: token });
+    const bdDataSources = new BDDataSourceConfig({ logger });
+    bdDataSources.withConfig({ dataSources: bdSandbox.tmpl.resources.storage });
+    const bdRole = new BDIamRole({
+      ...options,
+      logger,
+      iamClient: new iam.IAMClient({ region }),
+      stsClient: new sts.STSClient({ region }),
+      environment: bdSandbox.tmpl.environment,
+      templateName: bdSandbox.tmpl.id,
+      username: await bdAccount.getUsername(),
+      assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
+      assumeCondExternalId: await bdAccount.getExtId(),
+    });
+    const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources });
+    const policyDocument = await bdIntegration.getPolicyDocument(options.listBucketsPermission);
+    let iamRoleArn;
+    if (!options.dryRun) iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
+    updateSpinnerText(`Creating IAM Role: ${iamRoleArn}` + (options.dryRun ? "(dry-run)" : ""));
+    spinnerSuccess();
+  } catch (err: any) {
+    // try to decode the message
+    spinnerError(err?.message, false);
+  }
+}
+
+const program = new cmd.Command("bdcli sandbox create-role")
+  .addOption(new cmd.Option("--template <templateFile>", "sandbox IaC file").makeOptionMandatory())
+  .addOption(new cmd.Option("--delete", "Delete the sandbox IAM role from *your* AWS Account"))
+  .addOption(new cmd.Option("--no-list-buckets-permission", "Do NOT add s3:ListAllMyBuckets policy entry"))
+  .addOption(new cmd.Option("--dry-run", "Dry run, do not actually create the IAM role"))
+  .action(async (options, command) => await show(options, command));
+
+(async () => {
+  await addGlobalOptions(program, logger);
+  await program.parseAsync(process.argv);
+})();

src/bdcli/commands/sandbox/bdcli-sandbox-deploy.ts

@@ -0,0 +1,42 @@
+import * as cmd from "commander";
+import { getLogger } from "../../utils/logger_util.js";
+import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js";
+import { addGlobalOptions } from "../../utils/options_util.js";
+import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js";
+import { BDSandbox } from "../../../integration/boilingdata/sandbox.js";
+import { getIdToken } from "../../utils/auth_util.js";
+import { outputResults } from "../../utils/output_util.js";
+
+const logger = getLogger("bdcli-sandbox-deploy");
+
+async function show(options: any, _command: cmd.Command): Promise<void> {
+  try {
+    options = await combineOptsWithSettings(options, logger);
+
+    if (!(await hasValidConfig())) {
+      return spinnerError(`No valid bdcli configuration found for "${profile}" profile`);
+    }
+
+    updateSpinnerText("Authenticating");
+    const { idToken: token, cached: idCached } = await getIdToken(logger);
+    updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`);
+    spinnerSuccess();
+
+    updateSpinnerText(`Deploying sandbox ${options.name}`);
+    const bdSandbox = new BDSandbox({ logger, authToken: token });
+    const results = await bdSandbox.deploySandbox(options.name);
+    spinnerSuccess();
+    await outputResults(results?.deployResults, options.disableSpinner);
+  } catch (err: any) {
+    spinnerError(err?.message);
+  }
+}
+
+const program = new cmd.Command("bdcli sandbox deploy")
+  .addOption(new cmd.Option("--name <sandboxName>", "sandbox name").makeOptionMandatory())
+  .action(async (options, command) => await show(options, command));
+
+(async () => {
+  await addGlobalOptions(program, logger);
+  await program.parseAsync(process.argv);
+})();

src/bdcli/commands/sandbox/bdcli-sandbox-destroy.ts

@@ -0,0 +1,50 @@
+import * as cmd from "commander";
+import { getLogger } from "../../utils/logger_util.js";
+import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js";
+import { addGlobalOptions } from "../../utils/options_util.js";
+import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js";
+import { getIdToken } from "../../utils/auth_util.js";
+import { BDSandbox } from "../../../integration/boilingdata/sandbox.js";
+import { outputResults } from "../../utils/output_util.js";
+
+const logger = getLogger("bdcli-sandbox-destroy");
+
+async function show(options: any, _command: cmd.Command): Promise<void> {
+  try {
+    options = await combineOptsWithSettings(options, logger);
+
+    if (!(await hasValidConfig())) {
+      return spinnerError(`No valid bdcli configuration found for "${profile}" profile`);
+    }
+
+    updateSpinnerText("Authenticating");
+    const { idToken: token, cached: idCached } = await getIdToken(logger);
+    updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`);
+    spinnerSuccess();
+
+    updateSpinnerText(`Destroying sandbox ${options.name}`);
+    const bdSandbox = new BDSandbox({ logger, authToken: token });
+    const results = await bdSandbox.destroySandbox(options.name, options.destroyAlsoInterfaces, options.deleteTemplate);
+    spinnerSuccess();
+    await outputResults(results?.destroyResults, options.disableSpinner);
+  } catch (err: any) {
+    spinnerError(err?.message);
+  }
+}
+
+const program = new cmd.Command("bdcli sandbox destroy")
+  .addOption(new cmd.Option("--name <sandboxName>", "sandbox name").makeOptionMandatory())
+  .addOption(new cmd.Option("--destroy-also-interfaces", "Also delete interfaces like Tap URLs"))
+  .addOption(
+    new cmd.Option(
+      "--delete-template",
+      "Finally, delete template if all resources were destroyed, including interfaces",
+    ),
+  )
+  .addOption(new cmd.Option("--region <region>", "AWS region (by default eu-west-1").default("eu-west-1"))
+  .action(async (options, command) => await show(options, command));
+
+(async () => {
+  await addGlobalOptions(program, logger);
+  await program.parseAsync(process.argv);
+})();

src/bdcli/commands/sandbox/bdcli-sandbox-download.ts

@@ -0,0 +1,57 @@
+import * as cmd from "commander";
+import { getLogger } from "../../utils/logger_util.js";
+import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js";
+import { addGlobalOptions } from "../../utils/options_util.js";
+import { combineOptsWithSettings, hasValidConfig, profile } from "../../utils/config_util.js";
+import { getIdToken } from "../../utils/auth_util.js";
+import { BDSandbox } from "../../../integration/boilingdata/sandbox.js";
+import * as fs from "fs/promises";
+
+const logger = getLogger("bdcli-sandbox-validate");
+
+async function show(options: any, _command: cmd.Command): Promise<void> {
+  try {
+    options = await combineOptsWithSettings(options, logger);
+    const filename = options.name + ".yaml";
+
+    if (!(await hasValidConfig())) {
+      return spinnerError(`No valid bdcli configuration found for "${profile}" profile`);
+    }
+
+    let fileAlreadyExists = false;
+    try {
+      if (await fs.lstat(filename)) {
+        fileAlreadyExists = true;
+      }
+    } catch (err) {
+      logger.debug({ err });
+    }
+    if (fileAlreadyExists) {
+      spinnerError(`Local file ${filename} already exists`);
+    }
+
+    updateSpinnerText("Authenticating");
+    const { idToken: token, cached: idCached } = await getIdToken(logger);
+    updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`);
+    spinnerSuccess();
+
+    updateSpinnerText(`Downloading sandbox IaC template of ${options.name}`);
+    const bdSandbox = new BDSandbox({ logger, authToken: token });
+    const template = await bdSandbox.downloadTemplate(options.name, options.version, options?.status ?? "uploaded");
+    await fs.writeFile(filename, template);
+    spinnerSuccess();
+  } catch (err: any) {
+    spinnerError(err?.message);
+  }
+}
+
+const program = new cmd.Command("bdcli sandbox download")
+  .addOption(new cmd.Option("--name <templateName>", "template name from listing").makeOptionMandatory())
+  .addOption(new cmd.Option("--status <status>", "Download 'uploaded' (default) or 'deployed' template"))
+  .addOption(new cmd.Option("--version <version>", "Download specific version from listing"))
+  .action(async (options, command) => await show(options, command));
+
+(async () => {
+  await addGlobalOptions(program, logger);
+  await program.parseAsync(process.argv);
+})();