feat: dab integration (#662)

* feat: Initial changes to download dab binary

* feat: add "data-api-location" flag support for 'swa start'

* feat: proxy requests made to "/data-api" to dab-cli

* fix: unzipping of DAB-package and routing issue

* nit: nit comments

* feat: add support for "staticwebapp.db.config.json" and nit changes

* chore: nit changes and handling staticwebapp.database.config.json

* chore: nit comments

* fix: temp url for data-api binary download
fix: authentication headers in dab routing
fix: swa start --help

* feat: add separate executable based on the OS and nit changes

* fix: ERR_STREAM_WRITE_AFTER_END error while writing stream

* nit: resolve comments

* fix: rename dab binary to new binary Name

* fix: Linux permission issue for dab-executable

* fix: fix naming while running dab command

* nit: resolve comments

* Added swa db init command (#6)

* feat: add 'swa db init' command

* added support for custom folder names, adjusted template for schema file, adjusted usage of cosmosdb params to fix bug

* Using execFileCommand instead of execSync to prevent shell injection attacks in case of untrusted user input

* merge: get latest changes

* fix: added cosmosdb_postgresql and refactored

* chore: nit comments

* fix: more checks to handle corner cases

* fix: add validDBcheck

* fix: nit comments

* Changed rest path in db init to /rest and fixed bug causing error to throw when valid database type is passed

---------

Co-authored-by: sgollapudi77 <85578033+sgollapudi77@users.noreply.github.com>

* updating register command changes for dab

* build error

* reverting few package-lock changes

* move registerdb out to keep it consistent with others

* refactor: moved dab init files to subfolder to accommodate dab add, dab update commands later

* nit: addressing comments from older PR

* fix: add support for downloading latest tag/specific version

* fix: comment pinned version code

* nit: triggering new build

---------

Co-authored-by: sgollapudi77 <85578033+sgollapudi77@users.noreply.github.com>
Co-authored-by: Sai Vamsi Krishna Gollapudi <sgollapudi@microsoft.com>
Co-authored-by: Thomas Gauvin <35609369+thomasgauvin@users.noreply.github.com>

docs/www/docs/contribute/99-troubleshooting.md

@@ -48,6 +48,7 @@ If you are having trouble accessing SWA CLI, the following domains need to be al
 - blob.core.windows.net
 - azurestaticapps.net
 - swalocaldeploy.azureedge.net
+- dataapibuilder.azureedge.net
 - functionscdn.azureedge.net
 
 ## `Unable to download StaticSitesClient binary (File Not Found 404 - 403)`

src/cli/commands/db/index.ts

@@ -0,0 +1 @@
+export * from "./init";

src/cli/commands/db/init/index.ts

@@ -0,0 +1,2 @@
+export * from "./init";
+export { default as registerDb } from "./register";

src/cli/commands/db/init/init.ts

@@ -0,0 +1,126 @@
+import fs from "fs";
+import path from "path";
+import {
+  DATA_API_BUILDER_BINARY_NAME,
+  DATA_API_BUILDER_DATABASE_TYPES,
+  DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME,
+  DATA_API_BUILDER_DEFAULT_FOLDER,
+  DATA_API_BUILDER_DEFAULT_REST_PATH,
+  DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME,
+  DEFAULT_DATA_API_BUILDER_SCHEMA_CONTENT,
+} from "../../../../core/constants";
+import { execFileCommand, logger } from "../../../../core";
+import { getDataApiBuilderBinaryPath } from "../../../../core/dataApiBuilder";
+
+export async function init(options: SWACLIConfig) {
+  let { databaseType, connectionString, cosmosdb_nosqlContainer, cosmosdb_nosqlDatabase } = options;
+
+  if (databaseType === undefined || !isValidDatabaseType(databaseType)) {
+    logger.error(
+      `--database-type is a required field. Please provide the type of the database you want to connect (mssql, postgresql, cosmosdb_nosql, mysql, cosmosdb_postgresql).`,
+      true
+    );
+    return;
+  }
+
+  // 1. create folder swa-db-connections if it doesn't exist
+  const folderName = options?.folderName ? options.folderName : DATA_API_BUILDER_DEFAULT_FOLDER;
+  const directory = path.join(process.cwd(), folderName);
+
+  if (!fs.existsSync(directory)) {
+    logger.log(`Creating database connections configuration folder ${folderName}`, "swa");
+    fs.mkdirSync(directory);
+  } else {
+    logger.log(`Folder ${folderName} already exists, using that folder for creating data-api files`, "swa");
+  }
+
+  // 2. create file staticwebapp.database.config.json by calling dab init and passing through options
+  const configFile = path.join(directory, DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME);
+
+  if (fs.existsSync(configFile)) {
+    logger.error(`Config file ${configFile} already exists. Please provide a different name or remove the existing config file.`, true);
+  }
+
+  logger.log(`Creating ${DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME} configuration file`, "swa");
+
+  const dataApiBinary = await getDataApiBuilderBinaryPath();
+  if (!dataApiBinary) {
+    logger.error(
+      `Could not find or install ${DATA_API_BUILDER_BINARY_NAME} binary.
+    If you already have data-api-builder installed, try running "dab init" directly to generate the config file. Exiting!!`,
+      true
+    );
+  }
+
+  let args: string[] = [
+    "init",
+    "--database-type",
+    databaseType,
+    "--config",
+    DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME,
+    "--rest.path",
+    DATA_API_BUILDER_DEFAULT_REST_PATH,
+  ];
+  if (connectionString) {
+    args = [...args, "--connection-string", connectionString];
+  }
+
+  if (cosmosdb_nosqlContainer) {
+    args = [...args, "--cosmosdb_nosql-container", cosmosdb_nosqlContainer];
+
+    if (databaseType != DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql) {
+      logger.warn(`Database type is not ${DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql}, --cosmosdb_nosql-container will be ignored.`);
+    }
+  }
+
+  if (cosmosdb_nosqlDatabase) {
+    args = [...args, "--cosmosdb_nosql-database", cosmosdb_nosqlDatabase];
+
+    if (databaseType != DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql) {
+      logger.warn(`Database type is not ${DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql}, --cosmosdb_nosql-database will be ignored.`);
+    }
+  }
+
+  if (databaseType === DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql) {
+    if (!cosmosdb_nosqlDatabase) {
+      logger.error(
+        `--cosmosdb_nosql-database is required when database-type is cosmosdb_nosql, ${DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME} will not be created`,
+        true
+      );
+    }
+    // create file staticwebapp.database.schema.json directly if database type cosmosdb_nosql since needed argument
+    const schemaFile = path.join(directory, DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME);
+
+    if (fs.existsSync(schemaFile)) {
+      logger.warn(`Schema file exists ${schemaFile}. This content will be replaced.`);
+    }
+
+    logger.info(`Creating ${DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME} schema file`, "swa");
+    try {
+      fs.writeFileSync(schemaFile, DEFAULT_DATA_API_BUILDER_SCHEMA_CONTENT);
+    } catch (ex) {
+      logger.warn(`Unable to write/modify schema file. Exception : ${ex}`);
+    }
+    args = [...args, "--graphql-schema", DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME];
+  }
+
+  // todo:DAB CLI doesn't return an error code when it fails, so we need to allow stdio to be inherited (this will be fixed in the March release)
+  // It would be better to have our own logs since DAB CLI refers to itself in its success messages
+  // which may lead to confusion for swa cli users ex: `SUGGESTION: Use 'dab add [entity-name] [options]' to add new entities in your config.`
+  execFileCommand(dataApiBinary, directory, args);
+
+  // not logging anything here since DAB CLI logs success messages or error messages and we can't catch an error
+}
+
+function isValidDatabaseType(databaseType: string): boolean {
+  if (
+    databaseType == DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql ||
+    databaseType == DATA_API_BUILDER_DATABASE_TYPES.CosmosDbPostGreSql ||
+    databaseType == DATA_API_BUILDER_DATABASE_TYPES.MsSql ||
+    databaseType == DATA_API_BUILDER_DATABASE_TYPES.MySql ||
+    databaseType == DATA_API_BUILDER_DATABASE_TYPES.PostGreSql
+  ) {
+    return true;
+  }
+  return false;
+}

src/cli/commands/db/init/register.ts

@@ -0,0 +1,43 @@
+import { Command } from "commander";
+import { configureOptions } from "../../../../core/utils";
+import { init } from "./init";
+import { DEFAULT_CONFIG } from "../../../../config";
+
+export default function registerCommand(program: Command) {
+  const dbCommand = program.command("db [command] [options]").description("Manage your database");
+  const dbInitCommand = new Command()
+    .command("init")
+    .usage("[options]")
+    .description("initialize database connection configurations for your static web app")
+    .requiredOption(
+      "-t, --database-type <database type>",
+      "(Required) The type of the database you want to connect (mssql, postgresql, cosmosdb_nosql, mysql, cosmosdb_postgresql)."
+    )
+    .option(
+      "-f, --folder-name <folder name>",
+      "A folder name to override the convention database connection configuration folder name (ensure that you update your CI/CD workflow files accordingly).",
+      DEFAULT_CONFIG.folderName
+    )
+    .option("-cs, --connection-string <connection string>", "The connection string of the database you want to connect.")
+    .option(
+      "-nd, --cosmosdb_nosql-database <cosmosdb nosql database>",
+      "The database of your cosmosdb account you want to connect (only needed if using cosmosdb_nosql database type)."
+    )
+    .option("-nc, --cosmosdb_nosql-container <cosmosdb nosql container>", "The container of your cosmosdb account you want to connect.")
+    .action(async (_options: SWACLIConfig, command: Command) => {
+      const options = await configureOptions(undefined, command.optsWithGlobals(), command, "db init", false);
+
+      await init(options);
+    })
+    .addHelpText(
+      "after",
+      `
+Examples:
+swa db init --database-type mssql --connection-string $YOUR_CONNECTION_STRING_ENV
+
+swa db init --database-type cosmosdb_nosql --cosmosdb_nosql-database myCosmosDB --connection-string $YOUR_CONNECTION_STRING_ENV
+  `
+    );
+
+  dbCommand.addCommand(dbInitCommand).copyInheritedSettings; // For supporting dab init
+}

src/cli/commands/deploy/deploy.ts

@@ -155,7 +155,7 @@ export async function deploy(options: SWACLIConfig) {
 
   // TODO: do that in options
   // mix CLI args with the project's build workflow configuration (if any)
-  // use any specific workflow config that the user might provide undef ".github/workflows/"
+  // use any specific workflow config that the user might provide under ".github/workflows/"
   // Note: CLI args will take precedence over workflow config
   let userWorkflowConfig: Partial<GithubActionWorkflow> | undefined = {
     appLocation,

src/cli/commands/start/register.ts

@@ -11,13 +11,15 @@ export default function registerCommand(program: Command) {
     .description("start the emulator from a directory or bind to a dev server")
     .option("-a, --app-location <path>", "the folder containing the source code of the front-end application", DEFAULT_CONFIG.appLocation)
     .option("-i, --api-location <path>", "the folder containing the source code of the API application", DEFAULT_CONFIG.apiLocation)
+    .option("-db, --data-api-location <path>", "the path to the data-api config file", DEFAULT_CONFIG.dataApiLocation)
     .option("-O, --output-location <path>", "the folder containing the built source of the front-end application", DEFAULT_CONFIG.outputLocation)
     .option(
       "-D, --app-devserver-url <url>",
       "connect to the app dev server at this URL instead of using output location",
       DEFAULT_CONFIG.appDevserverUrl
     )
     .option("-is, --api-devserver-url <url>", "connect to the api server at this URL instead of using api location", DEFAULT_CONFIG.apiDevserverUrl)
+    .option("-ds, --data-api-devserver-url <url>", "connect to the data-api server at this URL", DEFAULT_CONFIG.dataApiDevserverUrl)
     .option<number>("-j, --api-port <apiPort>", "the API server port passed to `func start`", parsePort, DEFAULT_CONFIG.apiPort)
     .option("-q, --host <host>", "the host address to use for the CLI dev server", DEFAULT_CONFIG.host)
     .option<number>("-p, --port <port>", "the port value to use for the CLI dev server", parsePort, DEFAULT_CONFIG.port)
@@ -88,6 +90,12 @@ swa start ./output-folder --api-location ./api
 Use a custom command to run framework development server at startup
 swa start http://localhost:3000 --run-build "npm start"
 
+Serve static content from a folder and start data-api-server from another folder
+swa start ./output-folder --data-api-location ./swa-db-connections
+
+Connect front-end to the data-api-dev-server running
+swa start ./output-folder --data-api-devserver-url http://localhost:5000
+
 Connect both front-end and the API to running development server
 swa start http://localhost:3000 --api-devserver-url http://localhost:7071
   `

src/cli/commands/start/start.ts

@@ -2,6 +2,7 @@ import concurrently, { CloseEvent } from "concurrently";
 import { CommandInfo } from "concurrently/dist/src/command";
 import fs from "fs";
 import path from "path";
+import { DEFAULT_CONFIG } from "../../../config";
 import {
   askNewPort,
   createStartupScriptCommand,
@@ -14,6 +15,8 @@ import {
   parseUrl,
   readWorkflowFile,
 } from "../../../core";
+import { DATA_API_BUILDER_BINARY_NAME, DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME } from "../../../core/constants";
+import { getDataApiBuilderBinaryPath } from "../../../core/dataApiBuilder";
 import { swaCLIEnv } from "../../../core/env";
 import { getCertificate } from "../../../core/ssl";
 const packageInfo = require("../../../../package.json");
@@ -28,10 +31,13 @@ export async function start(options: SWACLIConfig) {
   let {
     appLocation,
     apiLocation,
+    dataApiLocation,
     outputLocation,
     appDevserverUrl,
     apiDevserverUrl,
+    dataApiDevserverUrl,
     apiPort,
+    dataApiPort,
     devserverTimeout,
     ssl,
     sslCert,
@@ -46,6 +52,7 @@ export async function start(options: SWACLIConfig) {
   } = options;
 
   let useApiDevServer: string | undefined | null = undefined;
+  let useDataApiDevServer: string | undefined | null = undefined;
   let startupCommand: string | undefined | null = undefined;
 
   let resolvedPortNumber = await isAcceptingTcpConnections({ host, port });
@@ -94,15 +101,32 @@ export async function start(options: SWACLIConfig) {
     // TODO: properly refactor this after GA to send apiDevserverUrl to the server
     useApiDevServer = apiDevserverUrl;
     apiLocation = apiDevserverUrl;
+    logger.silly(`Api Dev Server found: ${apiDevserverUrl}`);
   } else if (apiLocation) {
     // resolves to the absolute path of the apiLocation
-    let resolvedApiLocation = path.resolve(apiLocation);
+    const resolvedApiLocation = path.resolve(apiLocation);
 
     // make sure api folder exists
     if (fs.existsSync(resolvedApiLocation)) {
       apiLocation = resolvedApiLocation;
+      logger.silly(`Api Folder found: ${apiLocation}`);
     } else {
-      logger.info(`Skipping API because folder "${resolvedApiLocation}" is missing`, "swa");
+      logger.info(`Skipping Api because folder "${resolvedApiLocation}" is missing`, "swa");
+    }
+  }
+
+  if (dataApiDevserverUrl) {
+    useDataApiDevServer = dataApiDevserverUrl;
+    dataApiLocation = dataApiDevserverUrl;
+    logger.silly(`Data Api Dev Server found: ${dataApiDevserverUrl}`);
+  } else if (dataApiLocation) {
+    const resolvedDataApiLocation = path.resolve(dataApiLocation);
+
+    if (fs.existsSync(resolvedDataApiLocation)) {
+      dataApiLocation = resolvedDataApiLocation;
+      logger.silly(`Data Api Folder found: ${dataApiLocation}`);
+    } else {
+      logger.info(`Skipping Data Api because folder "${resolvedDataApiLocation}" is missing`, "swa");
     }
   }
 
@@ -176,6 +200,32 @@ export async function start(options: SWACLIConfig) {
     }
   }
 
+  let serveDataApiCommand = "echo 'No Data API found'. Skipping";
+  let startDataApiBuilderNeeded = false;
+  if (useDataApiDevServer) {
+    serveDataApiCommand = `echo using Data API server at ${useDataApiDevServer}`;
+
+    dataApiPort = parseUrl(useDataApiDevServer)?.port;
+  } else {
+    if (dataApiLocation) {
+      const dataApiBinary = await getDataApiBuilderBinaryPath();
+      if (!dataApiBinary) {
+        logger.error(
+          `Could not find or install ${DATA_API_BUILDER_BINARY_NAME} binary.
+        If you already have data-api-builder installed, try connecting using --data-api-devserver-url by
+        starting data-api-builder engine separately. Exiting!!`,
+          true
+        );
+      } else {
+        serveDataApiCommand = `cd "${dataApiLocation}" && "${dataApiBinary}" start -c ${DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME} --no-https-redirect`;
+        dataApiPort = DEFAULT_CONFIG.dataApiPort;
+        startDataApiBuilderNeeded = true;
+      }
+    }
+
+    logger.silly(`Running ${serveDataApiCommand}`);
+  }
+
   if (ssl) {
     if (sslCert === undefined && sslKey === undefined) {
       logger.warn(`WARNING: Using built-in UNSIGNED certificate. DO NOT USE IN PRODUCTION!`);
@@ -215,6 +265,8 @@ export async function start(options: SWACLIConfig) {
     SWA_CLI_APP_LOCATION: userWorkflowConfig?.appLocation as string,
     SWA_CLI_OUTPUT_LOCATION: userWorkflowConfig?.outputLocation as string,
     SWA_CLI_API_LOCATION: userWorkflowConfig?.apiLocation as string,
+    SWA_CLI_DATA_API_LOCATION: dataApiLocation,
+    SWA_CLI_DATA_API_PORT: `${dataApiPort}`,
     SWA_CLI_HOST: `${host}`,
     SWA_CLI_PORT: `${port}`,
     SWA_CLI_APP_SSL: ssl ? "true" : "false",
@@ -248,6 +300,10 @@ export async function start(options: SWACLIConfig) {
     );
   }
 
+  if (startDataApiBuilderNeeded) {
+    concurrentlyCommands.push({ command: serveDataApiCommand, name: "dataApi", env });
+  }
+
   // run an external script, if it's available
   if (startupCommand) {
     let startupPath = userWorkflowConfig?.appLocation;
@@ -262,6 +318,7 @@ export async function start(options: SWACLIConfig) {
     commands: {
       swa: concurrentlyCommands.find((c) => c.name === "swa")?.command,
       api: concurrentlyCommands.find((c) => c.name === "api")?.command,
+      dataApi: concurrentlyCommands.find((c) => c.name == "dataApi")?.command,
       run: concurrentlyCommands.find((c) => c.name === "run")?.command,
     },
   });
@@ -288,6 +345,9 @@ export async function start(options: SWACLIConfig) {
           case "api":
             commandMessage = `API server exited with code ${exitCode}`;
             break;
+          case "dataApi":
+            commandMessage = `Data API server exited with code ${exitCode}`;
+            break;
           case "run":
             commandMessage = `the --run command exited with code ${exitCode}`;
             break;

src/cli/index.ts

@@ -14,6 +14,7 @@ import { registerLogin } from "./commands/login";
 import { registerStart } from "./commands/start";
 import { registerBuild } from "./commands/build";
 import { registerDocs } from "./commands/docs";
+import { registerDb } from "./commands/db/init";
 import { promptOrUseDefault } from "../core/prompts";
 
 export * from "./commands";
@@ -97,6 +98,7 @@ export async function run(argv?: string[]) {
   registerInit(program);
   registerBuild(program);
   registerDocs(program);
+  registerDb(program);
 
   program.showHelpAfterError();
   program.addOption(new Option("--ping").hideHelp());