From a77bd50aeb870069e52a6383c6f5614c301a46d0 Mon Sep 17 00:00:00 2001 From: Vivek Jilla Date: Tue, 14 Mar 2023 13:09:17 +0530 Subject: [PATCH] feat: dab integration (#662) * feat: Initial changes to download dab binary * feat: add "data-api-location" flag support for 'swa start' * feat: proxy requests made to "/data-api" to dab-cli * fix: unzipping of DAB-package and routing issue * nit: nit comments * feat: add support for "staticwebapp.db.config.json" and nit changes * chore: nit changes and handling staticwebapp.database.config.json * chore: nit comments * fix: temp url for data-api binary download fix: authentication headers in dab routing fix: swa start --help * feat: add separate executable based on the OS and nit changes * fix: ERR_STREAM_WRITE_AFTER_END error while writing stream * nit: resolve comments * fix: rename dab binary to new binary Name * fix: Linux permission issue for dab-executable * fix: fix naming while running dab command * nit: resolve comments * Added swa db init command (#6) * feat: add 'swa db init' command * added support for custom folder names, adjusted template for schema file, adjusted usage of cosmosdb params to fix bug * Using execFileCommand instead of execSync to prevent shell injection attacks in case of untrusted user input * merge: get latest changes * fix: added cosmosdb_postgresql and refactored * chore: nit comments * fix: more checks to handle corner cases * fix: add validDBcheck * fix: nit comments * Changed rest path in db init to /rest and fixed bug causing error to throw when valid database type is passed --------- Co-authored-by: sgollapudi77 <85578033+sgollapudi77@users.noreply.github.com> * updating register command changes for dab * build error * reverting few package-lock changes * move registerdb out to keep it consistent with others * refactor: moved dab init files to subfolder to accommodate dab add, dab update commands later * nit: addressing comments from older PR * fix: add support for downloading latest tag/specific version * fix: comment pinned version code * nit: triggering new build --------- Co-authored-by: sgollapudi77 <85578033+sgollapudi77@users.noreply.github.com> Co-authored-by: Sai Vamsi Krishna Gollapudi Co-authored-by: Thomas Gauvin <35609369+thomasgauvin@users.noreply.github.com> --- .../www/docs/contribute/99-troubleshooting.md | 1 + package-lock.json | 4 +- src/cli/commands/db/index.ts | 1 + src/cli/commands/db/init/index.ts | 2 + src/cli/commands/db/init/init.ts | 126 ++++++++++++ src/cli/commands/db/init/register.ts | 43 ++++ src/cli/commands/deploy/deploy.ts | 2 +- src/cli/commands/start/register.ts | 8 + src/cli/commands/start/start.ts | 64 +++++- src/cli/index.ts | 2 + src/config.ts | 10 + src/core/constants.ts | 53 ++++- src/core/dataApiBuilder/dab.ts | 186 ++++++++++++++++++ src/core/dataApiBuilder/index.ts | 14 ++ src/core/deploy-client.spec.ts | 4 +- src/core/deploy-client.ts | 140 +------------ src/core/download-binary-helper.ts | 149 ++++++++++++++ src/core/utils/command.ts | 16 +- src/core/utils/platform.ts | 23 +++ src/msha/handlers/dab.handler.ts | 102 ++++++++++ src/msha/middlewares/request.middleware.ts | 11 +- src/msha/middlewares/response.middleware.ts | 12 +- src/msha/routes-engine/rules/routes.spec.ts | 22 ++- src/msha/routes-engine/rules/routes.ts | 6 +- src/swa.d.ts | 68 ++++++- 25 files changed, 910 insertions(+), 159 deletions(-) create mode 100644 src/cli/commands/db/index.ts create mode 100644 src/cli/commands/db/init/index.ts create mode 100644 src/cli/commands/db/init/init.ts create mode 100644 src/cli/commands/db/init/register.ts create mode 100644 src/core/dataApiBuilder/dab.ts create mode 100644 src/core/dataApiBuilder/index.ts create mode 100644 src/core/download-binary-helper.ts create mode 100644 src/msha/handlers/dab.handler.ts diff --git a/docs/www/docs/contribute/99-troubleshooting.md b/docs/www/docs/contribute/99-troubleshooting.md index 3471069d..3f88562f 100644 --- a/docs/www/docs/contribute/99-troubleshooting.md +++ b/docs/www/docs/contribute/99-troubleshooting.md @@ -48,6 +48,7 @@ If you are having trouble accessing SWA CLI, the following domains need to be al - blob.core.windows.net - azurestaticapps.net - swalocaldeploy.azureedge.net +- dataapibuilder.azureedge.net - functionscdn.azureedge.net ## `Unable to download StaticSitesClient binary (File Not Found 404 - 403)` diff --git a/package-lock.json b/package-lock.json index 2ac8167a..e07248ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@azure/static-web-apps-cli", - "version": "1.0.6", + "version": "1.0.7-alpha", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@azure/static-web-apps-cli", - "version": "1.0.6", + "version": "1.0.7-alpha", "license": "MIT", "dependencies": { "@azure/arm-appservice": "^12.0.0", diff --git a/src/cli/commands/db/index.ts b/src/cli/commands/db/index.ts new file mode 100644 index 00000000..fccf9768 --- /dev/null +++ b/src/cli/commands/db/index.ts @@ -0,0 +1 @@ +export * from "./init"; diff --git a/src/cli/commands/db/init/index.ts b/src/cli/commands/db/init/index.ts new file mode 100644 index 00000000..f592539a --- /dev/null +++ b/src/cli/commands/db/init/index.ts @@ -0,0 +1,2 @@ +export * from "./init"; +export { default as registerDb } from "./register"; diff --git a/src/cli/commands/db/init/init.ts b/src/cli/commands/db/init/init.ts new file mode 100644 index 00000000..90759666 --- /dev/null +++ b/src/cli/commands/db/init/init.ts @@ -0,0 +1,126 @@ +import fs from "fs"; +import path from "path"; +import { + DATA_API_BUILDER_BINARY_NAME, + DATA_API_BUILDER_DATABASE_TYPES, + DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME, + DATA_API_BUILDER_DEFAULT_FOLDER, + DATA_API_BUILDER_DEFAULT_REST_PATH, + DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME, + DEFAULT_DATA_API_BUILDER_SCHEMA_CONTENT, +} from "../../../../core/constants"; +import { execFileCommand, logger } from "../../../../core"; +import { getDataApiBuilderBinaryPath } from "../../../../core/dataApiBuilder"; + +export async function init(options: SWACLIConfig) { + let { databaseType, connectionString, cosmosdb_nosqlContainer, cosmosdb_nosqlDatabase } = options; + + if (databaseType === undefined || !isValidDatabaseType(databaseType)) { + logger.error( + `--database-type is a required field. Please provide the type of the database you want to connect (mssql, postgresql, cosmosdb_nosql, mysql, cosmosdb_postgresql).`, + true + ); + return; + } + + // 1. create folder swa-db-connections if it doesn't exist + const folderName = options?.folderName ? options.folderName : DATA_API_BUILDER_DEFAULT_FOLDER; + const directory = path.join(process.cwd(), folderName); + + if (!fs.existsSync(directory)) { + logger.log(`Creating database connections configuration folder ${folderName}`, "swa"); + fs.mkdirSync(directory); + } else { + logger.log(`Folder ${folderName} already exists, using that folder for creating data-api files`, "swa"); + } + + // 2. create file staticwebapp.database.config.json by calling dab init and passing through options + const configFile = path.join(directory, DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME); + + if (fs.existsSync(configFile)) { + logger.error(`Config file ${configFile} already exists. Please provide a different name or remove the existing config file.`, true); + } + + logger.log(`Creating ${DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME} configuration file`, "swa"); + + const dataApiBinary = await getDataApiBuilderBinaryPath(); + if (!dataApiBinary) { + logger.error( + `Could not find or install ${DATA_API_BUILDER_BINARY_NAME} binary. + If you already have data-api-builder installed, try running "dab init" directly to generate the config file. Exiting!!`, + true + ); + } + + let args: string[] = [ + "init", + "--database-type", + databaseType, + "--config", + DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME, + "--rest.path", + DATA_API_BUILDER_DEFAULT_REST_PATH, + ]; + if (connectionString) { + args = [...args, "--connection-string", connectionString]; + } + + if (cosmosdb_nosqlContainer) { + args = [...args, "--cosmosdb_nosql-container", cosmosdb_nosqlContainer]; + + if (databaseType != DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql) { + logger.warn(`Database type is not ${DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql}, --cosmosdb_nosql-container will be ignored.`); + } + } + + if (cosmosdb_nosqlDatabase) { + args = [...args, "--cosmosdb_nosql-database", cosmosdb_nosqlDatabase]; + + if (databaseType != DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql) { + logger.warn(`Database type is not ${DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql}, --cosmosdb_nosql-database will be ignored.`); + } + } + + if (databaseType === DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql) { + if (!cosmosdb_nosqlDatabase) { + logger.error( + `--cosmosdb_nosql-database is required when database-type is cosmosdb_nosql, ${DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME} will not be created`, + true + ); + } + // create file staticwebapp.database.schema.json directly if database type cosmosdb_nosql since needed argument + const schemaFile = path.join(directory, DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME); + + if (fs.existsSync(schemaFile)) { + logger.warn(`Schema file exists ${schemaFile}. This content will be replaced.`); + } + + logger.info(`Creating ${DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME} schema file`, "swa"); + try { + fs.writeFileSync(schemaFile, DEFAULT_DATA_API_BUILDER_SCHEMA_CONTENT); + } catch (ex) { + logger.warn(`Unable to write/modify schema file. Exception : ${ex}`); + } + args = [...args, "--graphql-schema", DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME]; + } + + // todo:DAB CLI doesn't return an error code when it fails, so we need to allow stdio to be inherited (this will be fixed in the March release) + // It would be better to have our own logs since DAB CLI refers to itself in its success messages + // which may lead to confusion for swa cli users ex: `SUGGESTION: Use 'dab add [entity-name] [options]' to add new entities in your config.` + execFileCommand(dataApiBinary, directory, args); + + // not logging anything here since DAB CLI logs success messages or error messages and we can't catch an error +} + +function isValidDatabaseType(databaseType: string): boolean { + if ( + databaseType == DATA_API_BUILDER_DATABASE_TYPES.CosmosDbNoSql || + databaseType == DATA_API_BUILDER_DATABASE_TYPES.CosmosDbPostGreSql || + databaseType == DATA_API_BUILDER_DATABASE_TYPES.MsSql || + databaseType == DATA_API_BUILDER_DATABASE_TYPES.MySql || + databaseType == DATA_API_BUILDER_DATABASE_TYPES.PostGreSql + ) { + return true; + } + return false; +} diff --git a/src/cli/commands/db/init/register.ts b/src/cli/commands/db/init/register.ts new file mode 100644 index 00000000..13731590 --- /dev/null +++ b/src/cli/commands/db/init/register.ts @@ -0,0 +1,43 @@ +import { Command } from "commander"; +import { configureOptions } from "../../../../core/utils"; +import { init } from "./init"; +import { DEFAULT_CONFIG } from "../../../../config"; + +export default function registerCommand(program: Command) { + const dbCommand = program.command("db [command] [options]").description("Manage your database"); + const dbInitCommand = new Command() + .command("init") + .usage("[options]") + .description("initialize database connection configurations for your static web app") + .requiredOption( + "-t, --database-type ", + "(Required) The type of the database you want to connect (mssql, postgresql, cosmosdb_nosql, mysql, cosmosdb_postgresql)." + ) + .option( + "-f, --folder-name ", + "A folder name to override the convention database connection configuration folder name (ensure that you update your CI/CD workflow files accordingly).", + DEFAULT_CONFIG.folderName + ) + .option("-cs, --connection-string ", "The connection string of the database you want to connect.") + .option( + "-nd, --cosmosdb_nosql-database ", + "The database of your cosmosdb account you want to connect (only needed if using cosmosdb_nosql database type)." + ) + .option("-nc, --cosmosdb_nosql-container ", "The container of your cosmosdb account you want to connect.") + .action(async (_options: SWACLIConfig, command: Command) => { + const options = await configureOptions(undefined, command.optsWithGlobals(), command, "db init", false); + + await init(options); + }) + .addHelpText( + "after", + ` +Examples: +swa db init --database-type mssql --connection-string $YOUR_CONNECTION_STRING_ENV + +swa db init --database-type cosmosdb_nosql --cosmosdb_nosql-database myCosmosDB --connection-string $YOUR_CONNECTION_STRING_ENV + ` + ); + + dbCommand.addCommand(dbInitCommand).copyInheritedSettings; // For supporting dab init +} diff --git a/src/cli/commands/deploy/deploy.ts b/src/cli/commands/deploy/deploy.ts index eae7307d..962bd1d4 100644 --- a/src/cli/commands/deploy/deploy.ts +++ b/src/cli/commands/deploy/deploy.ts @@ -155,7 +155,7 @@ export async function deploy(options: SWACLIConfig) { // TODO: do that in options // mix CLI args with the project's build workflow configuration (if any) - // use any specific workflow config that the user might provide undef ".github/workflows/" + // use any specific workflow config that the user might provide under ".github/workflows/" // Note: CLI args will take precedence over workflow config let userWorkflowConfig: Partial | undefined = { appLocation, diff --git a/src/cli/commands/start/register.ts b/src/cli/commands/start/register.ts index fb92dc6d..8d2f9944 100644 --- a/src/cli/commands/start/register.ts +++ b/src/cli/commands/start/register.ts @@ -11,6 +11,7 @@ export default function registerCommand(program: Command) { .description("start the emulator from a directory or bind to a dev server") .option("-a, --app-location ", "the folder containing the source code of the front-end application", DEFAULT_CONFIG.appLocation) .option("-i, --api-location ", "the folder containing the source code of the API application", DEFAULT_CONFIG.apiLocation) + .option("-db, --data-api-location ", "the path to the data-api config file", DEFAULT_CONFIG.dataApiLocation) .option("-O, --output-location ", "the folder containing the built source of the front-end application", DEFAULT_CONFIG.outputLocation) .option( "-D, --app-devserver-url ", @@ -18,6 +19,7 @@ export default function registerCommand(program: Command) { DEFAULT_CONFIG.appDevserverUrl ) .option("-is, --api-devserver-url ", "connect to the api server at this URL instead of using api location", DEFAULT_CONFIG.apiDevserverUrl) + .option("-ds, --data-api-devserver-url ", "connect to the data-api server at this URL", DEFAULT_CONFIG.dataApiDevserverUrl) .option("-j, --api-port ", "the API server port passed to `func start`", parsePort, DEFAULT_CONFIG.apiPort) .option("-q, --host ", "the host address to use for the CLI dev server", DEFAULT_CONFIG.host) .option("-p, --port ", "the port value to use for the CLI dev server", parsePort, DEFAULT_CONFIG.port) @@ -88,6 +90,12 @@ swa start ./output-folder --api-location ./api Use a custom command to run framework development server at startup swa start http://localhost:3000 --run-build "npm start" +Serve static content from a folder and start data-api-server from another folder +swa start ./output-folder --data-api-location ./swa-db-connections + +Connect front-end to the data-api-dev-server running +swa start ./output-folder --data-api-devserver-url http://localhost:5000 + Connect both front-end and the API to running development server swa start http://localhost:3000 --api-devserver-url http://localhost:7071 ` diff --git a/src/cli/commands/start/start.ts b/src/cli/commands/start/start.ts index 8f6a1af2..b6b6834a 100644 --- a/src/cli/commands/start/start.ts +++ b/src/cli/commands/start/start.ts @@ -2,6 +2,7 @@ import concurrently, { CloseEvent } from "concurrently"; import { CommandInfo } from "concurrently/dist/src/command"; import fs from "fs"; import path from "path"; +import { DEFAULT_CONFIG } from "../../../config"; import { askNewPort, createStartupScriptCommand, @@ -14,6 +15,8 @@ import { parseUrl, readWorkflowFile, } from "../../../core"; +import { DATA_API_BUILDER_BINARY_NAME, DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME } from "../../../core/constants"; +import { getDataApiBuilderBinaryPath } from "../../../core/dataApiBuilder"; import { swaCLIEnv } from "../../../core/env"; import { getCertificate } from "../../../core/ssl"; const packageInfo = require("../../../../package.json"); @@ -28,10 +31,13 @@ export async function start(options: SWACLIConfig) { let { appLocation, apiLocation, + dataApiLocation, outputLocation, appDevserverUrl, apiDevserverUrl, + dataApiDevserverUrl, apiPort, + dataApiPort, devserverTimeout, ssl, sslCert, @@ -46,6 +52,7 @@ export async function start(options: SWACLIConfig) { } = options; let useApiDevServer: string | undefined | null = undefined; + let useDataApiDevServer: string | undefined | null = undefined; let startupCommand: string | undefined | null = undefined; let resolvedPortNumber = await isAcceptingTcpConnections({ host, port }); @@ -94,15 +101,32 @@ export async function start(options: SWACLIConfig) { // TODO: properly refactor this after GA to send apiDevserverUrl to the server useApiDevServer = apiDevserverUrl; apiLocation = apiDevserverUrl; + logger.silly(`Api Dev Server found: ${apiDevserverUrl}`); } else if (apiLocation) { // resolves to the absolute path of the apiLocation - let resolvedApiLocation = path.resolve(apiLocation); + const resolvedApiLocation = path.resolve(apiLocation); // make sure api folder exists if (fs.existsSync(resolvedApiLocation)) { apiLocation = resolvedApiLocation; + logger.silly(`Api Folder found: ${apiLocation}`); } else { - logger.info(`Skipping API because folder "${resolvedApiLocation}" is missing`, "swa"); + logger.info(`Skipping Api because folder "${resolvedApiLocation}" is missing`, "swa"); + } + } + + if (dataApiDevserverUrl) { + useDataApiDevServer = dataApiDevserverUrl; + dataApiLocation = dataApiDevserverUrl; + logger.silly(`Data Api Dev Server found: ${dataApiDevserverUrl}`); + } else if (dataApiLocation) { + const resolvedDataApiLocation = path.resolve(dataApiLocation); + + if (fs.existsSync(resolvedDataApiLocation)) { + dataApiLocation = resolvedDataApiLocation; + logger.silly(`Data Api Folder found: ${dataApiLocation}`); + } else { + logger.info(`Skipping Data Api because folder "${resolvedDataApiLocation}" is missing`, "swa"); } } @@ -176,6 +200,32 @@ export async function start(options: SWACLIConfig) { } } + let serveDataApiCommand = "echo 'No Data API found'. Skipping"; + let startDataApiBuilderNeeded = false; + if (useDataApiDevServer) { + serveDataApiCommand = `echo using Data API server at ${useDataApiDevServer}`; + + dataApiPort = parseUrl(useDataApiDevServer)?.port; + } else { + if (dataApiLocation) { + const dataApiBinary = await getDataApiBuilderBinaryPath(); + if (!dataApiBinary) { + logger.error( + `Could not find or install ${DATA_API_BUILDER_BINARY_NAME} binary. + If you already have data-api-builder installed, try connecting using --data-api-devserver-url by + starting data-api-builder engine separately. Exiting!!`, + true + ); + } else { + serveDataApiCommand = `cd "${dataApiLocation}" && "${dataApiBinary}" start -c ${DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME} --no-https-redirect`; + dataApiPort = DEFAULT_CONFIG.dataApiPort; + startDataApiBuilderNeeded = true; + } + } + + logger.silly(`Running ${serveDataApiCommand}`); + } + if (ssl) { if (sslCert === undefined && sslKey === undefined) { logger.warn(`WARNING: Using built-in UNSIGNED certificate. DO NOT USE IN PRODUCTION!`); @@ -215,6 +265,8 @@ export async function start(options: SWACLIConfig) { SWA_CLI_APP_LOCATION: userWorkflowConfig?.appLocation as string, SWA_CLI_OUTPUT_LOCATION: userWorkflowConfig?.outputLocation as string, SWA_CLI_API_LOCATION: userWorkflowConfig?.apiLocation as string, + SWA_CLI_DATA_API_LOCATION: dataApiLocation, + SWA_CLI_DATA_API_PORT: `${dataApiPort}`, SWA_CLI_HOST: `${host}`, SWA_CLI_PORT: `${port}`, SWA_CLI_APP_SSL: ssl ? "true" : "false", @@ -248,6 +300,10 @@ export async function start(options: SWACLIConfig) { ); } + if (startDataApiBuilderNeeded) { + concurrentlyCommands.push({ command: serveDataApiCommand, name: "dataApi", env }); + } + // run an external script, if it's available if (startupCommand) { let startupPath = userWorkflowConfig?.appLocation; @@ -262,6 +318,7 @@ export async function start(options: SWACLIConfig) { commands: { swa: concurrentlyCommands.find((c) => c.name === "swa")?.command, api: concurrentlyCommands.find((c) => c.name === "api")?.command, + dataApi: concurrentlyCommands.find((c) => c.name == "dataApi")?.command, run: concurrentlyCommands.find((c) => c.name === "run")?.command, }, }); @@ -288,6 +345,9 @@ export async function start(options: SWACLIConfig) { case "api": commandMessage = `API server exited with code ${exitCode}`; break; + case "dataApi": + commandMessage = `Data API server exited with code ${exitCode}`; + break; case "run": commandMessage = `the --run command exited with code ${exitCode}`; break; diff --git a/src/cli/index.ts b/src/cli/index.ts index 5709e6d4..f79bc7bf 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -14,6 +14,7 @@ import { registerLogin } from "./commands/login"; import { registerStart } from "./commands/start"; import { registerBuild } from "./commands/build"; import { registerDocs } from "./commands/docs"; +import { registerDb } from "./commands/db/init"; import { promptOrUseDefault } from "../core/prompts"; export * from "./commands"; @@ -97,6 +98,7 @@ export async function run(argv?: string[]) { registerInit(program); registerBuild(program); registerDocs(program); + registerDb(program); program.showHelpAfterError(); program.addOption(new Option("--ping").hideHelp()); diff --git a/src/config.ts b/src/config.ts index a405162b..cb1efdb8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,6 +4,7 @@ import { isRunningInDocker } from "./core/utils/docker"; const { SWA_CLI_APP_LOCATION, SWA_CLI_API_LOCATION, + SWA_CLI_DATA_API_LOCATION, SWA_CLI_SERVER_TIMEOUT, SWA_CLI_OUTPUT_LOCATION, SWA_CLI_OPEN_BROWSER, @@ -13,6 +14,7 @@ const { SWA_CLI_PORT, SWA_CLI_HOST, SWA_CLI_API_PORT, + SWA_CLI_DATA_API_PORT, SWA_CLI_DEBUG, SWA_RUNTIME_CONFIG_LOCATION, SWA_RUNTIME_WORKFLOW_LOCATION, @@ -31,6 +33,8 @@ const { SWA_CLI_LOGIN_CLEAR_CREDENTIALS, SWA_CLI_APP_DEVSERVER_URL, SWA_CLI_API_DEVSERVER_URL, + SWA_CLI_DATA_API_DEVSERVER_URL, + SWA_CLI_DATA_API_FOLDER, } = swaCLIEnv(); export const DEFAULT_CONFIG: SWACLIConfig = { @@ -38,8 +42,10 @@ export const DEFAULT_CONFIG: SWACLIConfig = { port: parseInt(SWA_CLI_PORT || "4280", 10), host: SWA_CLI_HOST || (isRunningInDocker() ? "0.0.0.0" : "localhost"), apiPort: parseInt(SWA_CLI_API_PORT || "7071", 10), + dataApiPort: parseInt(SWA_CLI_DATA_API_PORT || "5000", 10), appLocation: SWA_CLI_APP_LOCATION || `.`, apiLocation: SWA_CLI_API_LOCATION ? SWA_CLI_API_LOCATION : undefined, + dataApiLocation: SWA_CLI_DATA_API_LOCATION ? SWA_CLI_DATA_API_LOCATION : undefined, outputLocation: SWA_CLI_OUTPUT_LOCATION || `.`, swaConfigLocation: SWA_RUNTIME_CONFIG_LOCATION || undefined, ssl: useEnvVarOrUseDefault(SWA_CLI_APP_SSL, false), @@ -57,6 +63,7 @@ export const DEFAULT_CONFIG: SWACLIConfig = { dryRun: useEnvVarOrUseDefault(SWA_CLI_DEPLOY_DRY_RUN, false), appDevserverUrl: SWA_CLI_APP_DEVSERVER_URL || undefined, apiDevserverUrl: SWA_CLI_API_DEVSERVER_URL || undefined, + dataApiDevserverUrl: SWA_CLI_DATA_API_DEVSERVER_URL || undefined, // swa login options subscriptionId: AZURE_SUBSCRIPTION_ID || undefined, @@ -66,4 +73,7 @@ export const DEFAULT_CONFIG: SWACLIConfig = { clientSecret: AZURE_CLIENT_SECRET || undefined, useKeychain: useEnvVarOrUseDefault(SWA_CLI_LOGIN_USE_KEYCHAIN, true), clearCredentials: useEnvVarOrUseDefault(SWA_CLI_LOGIN_CLEAR_CREDENTIALS, false), + + // swa db options + folderName: SWA_CLI_DATA_API_FOLDER || "swa-db-connections", }; diff --git a/src/core/constants.ts b/src/core/constants.ts index 1b406dcc..201b90bd 100644 --- a/src/core/constants.ts +++ b/src/core/constants.ts @@ -1,9 +1,24 @@ import path from "path"; import { DEFAULT_CONFIG } from "../config"; import { address, isHttpUrl } from "./utils/net"; +import os from "os"; +export const DEPLOY_BINARY_NAME = "StaticSitesClient"; +export const DEPLOY_BINARY_STABLE_TAG = "stable"; +export const DATA_API_BUILDER_BINARY_NAME = "DataApiBuilder"; +export const DATA_API_BUILDER_COMMAND = "dab"; export const STATIC_SITE_CLIENT_RELEASE_METADATA_URL = "https://swalocaldeploy.azureedge.net/downloads/versions.json"; -export const SWA_COMMANDS = ["login", "init", "start", "deploy", "build"] as const; +export const DATA_API_BUILDER_RELEASE_METADATA_URL = "https://dataapibuilder.azureedge.net/releases/dab-manifest.json"; +export const DEPLOY_FOLDER = path.join(os.homedir(), ".swa", "deploy"); +export const DATA_API_BUILDER_FOLDER = path.join(os.homedir(), ".swa", "dataApiBuilder"); +export const DATA_API_BUILDER_RELEASE_TAG = "released"; +export const DATA_API_BUILDER_LATEST_TAG = "latest"; +export const DATA_API_BUILDER_DEFAULT_CONFIG_FILE_NAME = "staticwebapp.database.config.json"; +export const DATA_API_BUILDER_DEFAULT_SCHEMA_FILE_NAME = "staticwebapp.database.schema.gql"; +export const DATA_API_BUILDER_DEFAULT_FOLDER = "swa-db-connections"; +export const DATA_API_BUILDER_DEFAULT_REST_PATH = "/rest"; +export const DATA_API_BUILDER_VERSION_ID = "0.5.32"; +export const SWA_COMMANDS = ["login", "init", "start", "deploy", "build", "db init"] as const; // Type cannot be in swa.d.ts as it's inferred from SWA_COMMANDS export type SWACommand = typeof SWA_COMMANDS[number]; @@ -12,6 +27,20 @@ export const SWA_RUNTIME_CONFIG_MAX_SIZE_IN_KB = 20; // 20kb export const SWA_AUTH_COOKIE = `StaticWebAppsAuthCookie`; export const ALLOWED_HTTP_METHODS_FOR_STATIC_CONTENT = ["GET", "HEAD", "OPTIONS"]; +export const DEFAULT_DATA_API_BUILDER_BINARY = { + Windows: "Microsoft.DataApiBuilder.exe", + Linux: "Microsoft.DataApiBuilder", + MacOs: "Microsoft.DataApiBuilder", +}; + +export const DATA_API_BUILDER_DATABASE_TYPES = { + MsSql: "mssql", + CosmosDbNoSql: "cosmosdb_nosql", + CosmosDbPostGreSql: "cosmosdb_postgresql", + MySql: "mysql", + PostGreSql: "postgresql", +}; + export const AUTH_STATUS = { NoAuth: 0, HostNameAuthLogin: 1, @@ -164,6 +193,20 @@ export const MIME_TYPE_LIST: { [key: string]: string } = { ".7z": "application/x-7z-compressed", }; +export const DEFAULT_DATA_API_BUILDER_SCHEMA_CONTENT = ` +""" +Add your CosmosDB NoSQL database schema in this file + +For example: + +type Book @model { + id: ID + title: String +} + +""" +`; + export const DEFAULT_MIME_TYPE = "application/octet-stream"; export const HEADER_DELETE_KEYWORD = "@@HEADER_DELETE_KEYWORD@@"; @@ -194,3 +237,11 @@ export function IS_API_DEV_SERVER() { export function SWA_CLI_API_URI() { return IS_API_DEV_SERVER() ? DEFAULT_CONFIG.apiLocation : address(DEFAULT_CONFIG.host, DEFAULT_CONFIG.apiPort); } + +export function IS_DATA_API_DEV_SERVER() { + return isHttpUrl(DEFAULT_CONFIG.dataApiLocation); +} + +export function SWA_CLI_DATA_API_URI() { + return IS_DATA_API_DEV_SERVER() ? DEFAULT_CONFIG.dataApiLocation : address(DEFAULT_CONFIG.host, DEFAULT_CONFIG.dataApiPort, "http"); +} diff --git a/src/core/dataApiBuilder/dab.ts b/src/core/dataApiBuilder/dab.ts new file mode 100644 index 00000000..88f12028 --- /dev/null +++ b/src/core/dataApiBuilder/dab.ts @@ -0,0 +1,186 @@ +import { + DATA_API_BUILDER_BINARY_NAME, + DATA_API_BUILDER_COMMAND, + DATA_API_BUILDER_FOLDER, + DATA_API_BUILDER_LATEST_TAG, + DATA_API_BUILDER_RELEASE_METADATA_URL, + // DATA_API_BUILDER_VERSION_ID, + DEFAULT_DATA_API_BUILDER_BINARY, +} from "../constants"; +import fetch from "node-fetch"; +import { promisify } from "util"; +import { exec } from "child_process"; +import fs from "fs"; +import os from "os"; +import unzipper from "unzipper"; +import path from "path"; +import { PassThrough } from "stream"; +import { getPlatform, logger } from "../utils"; +import { downloadAndValidateBinary } from "../download-binary-helper"; + +/** + * Gets the filepath where the Microsoft.DataApiBuilder.exe is located + * - Gets the latest version from the manifest file + * - Checks if it is installed and is latest or not already + * - Gets the installed path if it is already present + * - Downloads, unzips and Installs if not already present + * @params null + * + * @returns binaryPath + */ +export async function installAndGetDataApiBuilder(): Promise<{ binaryPath: string }> { + const platform = getPlatform(); + if (!platform) { + throw new Error(`Unsupported platform: ${os.platform()}`); + } + + const releaseMetadata = (await getReleaseDataApiBuilderMetadata()).releaseMetadata; + if (releaseMetadata === undefined) { + throw new Error(`Could not load ${DATA_API_BUILDER_BINARY_NAME} metadata from remote. Please check your internet connection.`); // should we throw error and stop or can we allow users to use local version + } + const isLatestVersionInstalled = await isLocalVersionInstalledAndLatest(releaseMetadata.versionId); + + if (!isLatestVersionInstalled) { + const binaryPath = await downloadAndUnzipBinary(releaseMetadata, platform); + + if (binaryPath != undefined) { + return { + binaryPath: binaryPath, + }; + } + } + + return { + binaryPath: DATA_API_BUILDER_COMMAND, + }; +} + +/** + * + * @param releaseMetadata Release metadata obtained from DATA_API_BUILDER_RELEASE_METADATA_URL + * @param platform Current OS + * @returns Binary Path after downloading and extracting + */ +async function downloadAndUnzipBinary(releaseMetadata: DataApiBuilderReleaseMetadata, platform: "win-x64" | "osx-x64" | "linux-x64") { + try { + const destDirectory = path.join(DATA_API_BUILDER_FOLDER, releaseMetadata.versionId); + const binaryPath = path.join(destDirectory, getDefaultDataApiBuilderBinaryForOS(platform)); + + if (!fs.existsSync(binaryPath)) { + logger.silly(`Downloading the version ${releaseMetadata.versionId}`); + const zipFilePath = await downloadAndValidateBinary( + releaseMetadata, + DATA_API_BUILDER_BINARY_NAME, + DATA_API_BUILDER_FOLDER, + releaseMetadata?.versionId, + platform + ); + + await extractBinary(zipFilePath, destDirectory); + } + + if (platform == "linux-x64" || platform == "osx-x64") { + logger.silly(`Setting executable permissions for data-api-builder binary`); + fs.chmodSync(binaryPath, 0o755); + } + + return binaryPath; + } catch (ex) { + logger.error(`Unable to download/extract ${DATA_API_BUILDER_BINARY_NAME} binary. Exception ${ex}`); + return undefined; + } +} + +/** + * Fetches the latest version, metadata of Microsoft.DataApiBuilder.exe from DATA_API_BUILDER_RELEASE_METADATA_URL + * @returns DataApiBuilderReleaseMetadata + */ +async function getReleaseDataApiBuilderMetadata(): Promise<{ releaseMetadata: DataApiBuilderReleaseMetadata | undefined }> { + const response = await fetch(DATA_API_BUILDER_RELEASE_METADATA_URL); + const responseMetadata = (await response.json()) as DataApiBuilderReleaseMetadata; + + if (Array.isArray(responseMetadata)) { + const releaseMetadata = responseMetadata.find((c) => c.version === DATA_API_BUILDER_LATEST_TAG); // If we want to proceed with downloading the latest tag + // const releaseMetadata = responseMetadata.find((c) => c.versionId === DATA_API_BUILDER_VERSION_ID); // If we want to pin a specific version + + return { + releaseMetadata: releaseMetadata, + }; + } else { + return { + releaseMetadata: undefined, + }; + } +} + +/** + * Returns if the version installed locally is latest or not + * @param releaseVersion current released Version of the Microsoft.DataApiBuilder.exe + * @param platform current OS + * @returns true if latest Version of data-api-builder is installed else false + */ +async function isLocalVersionInstalledAndLatest(releaseVersion: string): Promise { + const versionInstalled = await getInstalledVersion(DATA_API_BUILDER_COMMAND); + + if (versionInstalled) { + logger.silly(`Installed version: ${versionInstalled}`); + return versionInstalled == releaseVersion; + } + + logger.silly(`${DATA_API_BUILDER_COMMAND} is not installed.`); + return undefined; +} + +/** + * Unzips the given file to destDirectory + * @param zipFilePath file to unzip + * @param destDirectory directory to extract + */ +async function extractBinary(zipFilePath: string, destDirectory: string) { + // todo: delete zip file after extraction + + const openAsStream = fs.createReadStream(zipFilePath).pipe(new PassThrough()); + const unzipPromise = new Promise((resolve, reject) => { + const unzipperInstance = unzipper.Extract({ path: destDirectory }); + unzipperInstance.promise().then(resolve, reject); + openAsStream.pipe(unzipperInstance); + }); + + await unzipPromise; +} + +/** + * the Data-api-builder binary for given OS + * @param platform current OS + * @returns the Data-api-builder binary for given OS + */ +function getDefaultDataApiBuilderBinaryForOS(platform: string): string { + switch (platform) { + case "win-x64": + return DEFAULT_DATA_API_BUILDER_BINARY.Windows; + case "osx-x64": + return DEFAULT_DATA_API_BUILDER_BINARY.MacOs; + case "linux-x64": + return DEFAULT_DATA_API_BUILDER_BINARY.Linux; + default: + return DEFAULT_DATA_API_BUILDER_BINARY.Windows; + } +} + +/** + * Returns installed version if installed else undefined + * @param command package to know the version + * @returns installed version + */ +async function getInstalledVersion(command: "dab"): Promise { + logger.silly(`Running ${DATA_API_BUILDER_COMMAND} --version`); + + try { + const { stdout } = await promisify(exec)(`${command} --version`); + const version = stdout.split(" ")[1].split("\r")[0]; // parsing output which looks like this "Microsoft.DataApiBuilder 0.5.0" (specific to dab) + + return version; + } catch { + return undefined; + } +} diff --git a/src/core/dataApiBuilder/index.ts b/src/core/dataApiBuilder/index.ts new file mode 100644 index 00000000..7749b7a6 --- /dev/null +++ b/src/core/dataApiBuilder/index.ts @@ -0,0 +1,14 @@ +import { DATA_API_BUILDER_BINARY_NAME } from "../constants"; +import { logger } from "../utils"; +import { installAndGetDataApiBuilder } from "./dab"; + +/** + * This function gets the Data-Api Builder binary path and returns it + * @returns DataApiBuilderBinary path + */ +export async function getDataApiBuilderBinaryPath(): Promise { + const binary = await installAndGetDataApiBuilder(); + logger.silly(`${DATA_API_BUILDER_BINARY_NAME} found: ${binary.binaryPath}. Using this to start data-api server`); + + return binary.binaryPath; +} diff --git a/src/core/deploy-client.spec.ts b/src/core/deploy-client.spec.ts index e2a20cfe..77250175 100644 --- a/src/core/deploy-client.spec.ts +++ b/src/core/deploy-client.spec.ts @@ -1,7 +1,9 @@ import mockFs from "mock-fs"; import os from "os"; import path from "path"; -import { DEPLOY_BINARY_NAME, DEPLOY_FOLDER, fetchClientVersionDefinition, getLocalClientMetadata, getPlatform } from "./deploy-client"; +import { DEPLOY_BINARY_NAME, DEPLOY_FOLDER } from "./constants"; +import { fetchClientVersionDefinition, getLocalClientMetadata } from "./deploy-client"; +import { getPlatform } from "./utils"; jest.mock("node-fetch", () => jest.fn()); jest.mock("os", () => ({ platform: () => "linux", homedir: () => "/home/user", tmpdir: () => "/tmp", release: () => "4.4.0-1-amd64" })); diff --git a/src/core/deploy-client.ts b/src/core/deploy-client.ts index d02183d1..20b704d7 100644 --- a/src/core/deploy-client.ts +++ b/src/core/deploy-client.ts @@ -1,34 +1,11 @@ -import chalk from "chalk"; -import crypto from "crypto"; import fs from "fs"; import fetch from "node-fetch"; -import ora from "ora"; import os from "os"; import path from "path"; -import { PassThrough } from "stream"; -import { STATIC_SITE_CLIENT_RELEASE_METADATA_URL } from "./constants"; +import { STATIC_SITE_CLIENT_RELEASE_METADATA_URL, DEPLOY_BINARY_NAME, DEPLOY_FOLDER, DEPLOY_BINARY_STABLE_TAG } from "./constants"; +import { downloadAndValidateBinary } from "./download-binary-helper"; import { swaCLIEnv } from "./env"; -import { logger } from "./utils"; - -type StaticSiteClientReleaseMetadata = { - version: "stable" | "latest"; - buildId: string; - publishDate: string; - files: { - ["linux-x64"]: { - url: string; - sha: string; - }; - ["win-x64"]: { - url: string; - sha: string; - }; - ["osx-x64"]: { - url: string; - sha: string; - }; - }; -}; +import { getPlatform, logger } from "./utils"; type StaticSiteClientLocalMetadata = { metadata: StaticSiteClientReleaseMetadata; @@ -36,9 +13,6 @@ type StaticSiteClientLocalMetadata = { checksum: string; }; -export const DEPLOY_BINARY_NAME = "StaticSitesClient"; -export const DEPLOY_FOLDER = path.join(os.homedir(), ".swa", "deploy"); - export async function getDeployClientPath(): Promise<{ binary: string; buildId: string }> { const platform = getPlatform(); if (!platform) { @@ -46,7 +20,7 @@ export async function getDeployClientPath(): Promise<{ binary: string; buildId: } const localClientMetadata = getLocalClientMetadata() as StaticSiteClientLocalMetadata; - const binaryVersion = swaCLIEnv().SWA_CLI_DEPLOY_BINARY_VERSION || "stable"; + const binaryVersion = swaCLIEnv().SWA_CLI_DEPLOY_BINARY_VERSION || DEPLOY_BINARY_STABLE_TAG; const remoteClientMetadata = await fetchClientVersionDefinition(binaryVersion); if (remoteClientMetadata === undefined) { throw new Error(`Could not load ${DEPLOY_BINARY_NAME} metadata from remote. Please check your internet connection.`); @@ -83,7 +57,7 @@ export async function getDeployClientPath(): Promise<{ binary: string; buildId: } return { - binary: await downloadAndValidateBinary(remoteClientMetadata, platform), + binary: await downloadAndValidateBinary(remoteClientMetadata, DEPLOY_BINARY_NAME, DEPLOY_FOLDER, remoteClientMetadata.buildId, platform), buildId: remoteClientMetadata.buildId, }; } @@ -117,34 +91,6 @@ export function getLocalClientMetadata(): StaticSiteClientLocalMetadata | null { return null; } -function computeChecksumfromFile(filePath: string | undefined): string { - if (!filePath || !fs.existsSync(filePath)) { - return ""; - } - - const buffer = fs.readFileSync(filePath); - const hash = crypto.createHash("sha256"); - hash.update(buffer); - return hash.digest("hex"); -} - -export function getPlatform(): "win-x64" | "osx-x64" | "linux-x64" | null { - switch (os.platform()) { - case "win32": - return "win-x64"; - case "darwin": - return "osx-x64"; - case "aix": - case "freebsd": - case "openbsd": - case "sunos": - case "linux": - return "linux-x64"; - default: - return null; - } -} - export async function fetchClientVersionDefinition(releaseVersion: string): Promise { logger.silly(`Fetching release metadata for version: ${releaseVersion}. Please wait...`); @@ -161,75 +107,6 @@ export async function fetchClientVersionDefinition(releaseVersion: string): Prom return undefined; } -async function downloadAndValidateBinary(release: StaticSiteClientReleaseMetadata, platform: "win-x64" | "osx-x64" | "linux-x64") { - const downloadUrl = release.files[platform!].url; - const downloadFilename = path.basename(downloadUrl); - - const url = release.files[platform].url; - const buildId = release.buildId; - - const spinner = ora({ prefixText: chalk.dim.gray(`[swa]`) }); - - spinner.start(`Downloading ${url}@${buildId}`); - - const response = await fetch(url); - - if (response.status !== 200) { - spinner.fail(); - throw new Error(`Failed to download ${DEPLOY_BINARY_NAME} binary. File not found (${response.status})`); - } - - const bodyStream = response?.body?.pipe(new PassThrough()); - - createDeployDirectoryIfNotExists(buildId); - - return await new Promise((resolve, reject) => { - const isPosix = platform === "linux-x64" || platform === "osx-x64"; - let outputFile = path.join(DEPLOY_FOLDER, buildId, downloadFilename); - - const writableStream = fs.createWriteStream(outputFile, { mode: isPosix ? 0o755 : undefined }); - bodyStream?.pipe(writableStream); - - writableStream.on("end", () => { - bodyStream?.end(); - }); - - writableStream.on("finish", () => { - const computedHash = computeChecksumfromFile(outputFile).toLowerCase(); - const releaseChecksum = release.files[platform].sha.toLocaleLowerCase(); - if (computedHash !== releaseChecksum) { - try { - // in case of a failure, we remove the file - fs.unlinkSync(outputFile); - } catch {} - - spinner.fail(); - reject(new Error(`Checksum mismatch! Expected ${computedHash}, got ${releaseChecksum}.`)); - } else { - spinner.succeed(); - - logger.silly(`Checksum match: ${computedHash}`); - logger.silly(`Saved binary to ${outputFile}`); - - saveMetadata(release, outputFile, computedHash); - - resolve(outputFile); - } - }); - }); -} - -function saveMetadata(release: StaticSiteClientReleaseMetadata, binaryFilename: string, sha: string) { - const metatdaFilename = path.join(DEPLOY_FOLDER, `${DEPLOY_BINARY_NAME}.json`); - const metdata: StaticSiteClientLocalMetadata = { - metadata: release, - binary: binaryFilename, - checksum: sha, - }; - fs.writeFileSync(metatdaFilename, JSON.stringify(metdata)); - logger.silly(`Saved metadata to ${metatdaFilename}`); -} - // TODO: get StaticSiteClient to remove zip files // TODO: can these ZIPs be created under /tmp? export function cleanUp() { @@ -245,10 +122,3 @@ export function cleanUp() { clean(".\\app.zip"); clean(".\\api.zip"); } - -function createDeployDirectoryIfNotExists(version: string) { - const deployPath = path.join(DEPLOY_FOLDER, version); - if (!fs.existsSync(deployPath)) { - fs.mkdirSync(deployPath, { recursive: true }); - } -} diff --git a/src/core/download-binary-helper.ts b/src/core/download-binary-helper.ts new file mode 100644 index 00000000..57cd5265 --- /dev/null +++ b/src/core/download-binary-helper.ts @@ -0,0 +1,149 @@ +import chalk from "chalk"; +import crypto from "crypto"; +import fs from "fs"; +import fetch from "node-fetch"; +import ora from "ora"; +import path from "path"; +import { PassThrough } from "stream"; +import { DATA_API_BUILDER_BINARY_NAME, DATA_API_BUILDER_FOLDER, DEPLOY_BINARY_NAME, DEPLOY_FOLDER } from "./constants"; +import { logger } from "./utils"; + +/** + * Downloads the binary to the given output folder + * @param releaseMetadata binary metadata + * @param binaryType StaticSiteClient or DataApiBuilder + * @param outputFolder path to download the binary + * @param id buildId or versionId + * @param platform os: win-x64 or linux-x64 or osx-x64 + * @returns + */ +export async function downloadAndValidateBinary( + releaseMetadata: BinaryMetadata, + binaryName: string, + outputFolder: string, + id: string, + platform: "win-x64" | "osx-x64" | "linux-x64" +) { + const downloadFilename = path.basename(releaseMetadata.files[platform].url); + const url = releaseMetadata.files[platform].url; + const spinner = ora({ prefixText: chalk.dim.gray(`[swa]`) }); + + spinner.start(`Downloading ${url}@${id}`); + + const response = await fetch(url); + + if (response.status !== 200) { + spinner.fail(); + throw new Error(`Failed to download ${binaryName} binary from url ${url}. File not found (${response.status})`); + } + + const bodyStream = response?.body?.pipe(new PassThrough()); + + createBinaryDirectoryIfNotExists(id, outputFolder); + + return await new Promise((resolve, reject) => { + const isPosix = platform === "linux-x64" || platform === "osx-x64"; + const outputFile = path.join(outputFolder, id, downloadFilename); + + const writableStream = fs.createWriteStream(outputFile, { mode: isPosix ? 0o755 : undefined }); + bodyStream?.pipe(writableStream); + + writableStream.on("end", () => { + bodyStream?.end(); + }); + + writableStream.on("finish", async () => { + const computedHash = computeChecksumfromFile(outputFile).toLowerCase(); + const releaseChecksum = releaseMetadata.files[platform].sha.toLowerCase(); + if (computedHash !== releaseChecksum) { + try { + // in case of a failure, we remove the file + fs.unlinkSync(outputFile); + } catch { + logger.silly(`Not able to delete ${downloadFilename}, please delete manually.`); + } + + spinner.fail(); + reject(new Error(`Checksum mismatch for ${binaryName}! Expected ${releaseChecksum}, got ${computedHash}.`)); + } else { + spinner.succeed(); + + logger.silly(`Checksum match: ${computedHash}`); + logger.silly(`Saved binary to ${outputFile}`); + + saveMetadata(releaseMetadata, outputFile, computedHash, binaryName); + + resolve(outputFile); + } + }); + + // writableStream.close(); + }); +} + +/** + * Creates the output folder for downloading the binary + * @param version version + * @param outputFolder path to download the binary + */ +function createBinaryDirectoryIfNotExists(version: string, outputFolder: string) { + const deployPath = path.join(outputFolder, version); + if (!fs.existsSync(deployPath)) { + fs.mkdirSync(deployPath, { recursive: true }); + } +} + +/** + * Computes and returns the sha256 hash value for the given file + * @param filePath filePath + * @returns sha256 checksum of the file + */ +function computeChecksumfromFile(filePath: string | undefined): string { + if (!filePath || !fs.existsSync(filePath)) { + return ""; + } + + const buffer = fs.readFileSync(filePath); + const hash = crypto.createHash("sha256"); + hash.update(buffer); + + return hash.digest("hex"); +} + +/** + * + * @param release binary Metadata + * @param binaryFileName binary file location + * @param sha hash value + * @param binaryType StaticSiteClient or DataApiBuilder + */ +function saveMetadata(release: BinaryMetadata, binaryFileName: string, sha: string, binaryName: string) { + const downloadFolder = getFolderForSavingMetadata(binaryName); + + if (downloadFolder != null) { + const metadataFileName = path.join(downloadFolder, `${binaryName}.json`); + const metdata: LocalBinaryMetadata = { + metadata: release, + binary: binaryFileName, + checksum: sha, + }; + fs.writeFileSync(metadataFileName, JSON.stringify(metdata)); + logger.silly(`Saved metadata to ${metadataFileName}`); + } +} + +/** + * Returns folder for saving binary metadata + * @param binaryName + * @returns folder + */ +function getFolderForSavingMetadata(binaryName: string): string | null { + switch (binaryName) { + case DEPLOY_BINARY_NAME: + return DEPLOY_FOLDER; + case DATA_API_BUILDER_BINARY_NAME: + return DATA_API_BUILDER_FOLDER; + default: + return null; + } +} diff --git a/src/core/utils/command.ts b/src/core/utils/command.ts index 6fcb6966..e07d7877 100644 --- a/src/core/utils/command.ts +++ b/src/core/utils/command.ts @@ -1,5 +1,5 @@ import process from "process"; -import { execSync } from "child_process"; +import { execFileSync, execSync } from "child_process"; export function runCommand(command: string, cwd?: string) { execSync(command, { @@ -13,3 +13,17 @@ export function runCommand(command: string, cwd?: string) { }, }); } + +export function execFileCommand(command: string, cwd?: string, args?: string[]){ + const child = execFileSync(command, args, { + stdio: "inherit", + cwd: cwd ?? process.cwd(), + // Set CI to avoid extra NPM logs and potentially unwanted interactive modes + env: { + ...process.env, + // Internal flag to avoid duplicating user messages + SWA_CLI_INTERNAL_COMMAND: "1", + }, + }); + return child; +} diff --git a/src/core/utils/platform.ts b/src/core/utils/platform.ts index 1dfce6e8..4fc4968d 100644 --- a/src/core/utils/platform.ts +++ b/src/core/utils/platform.ts @@ -1,3 +1,26 @@ +import os from "os"; + export function isWSL() { return process.env.WSL_DISTRO_NAME !== undefined; } + +/** + * Returns the os of the platform + * @returns current os + */ +export function getPlatform(): "win-x64" | "osx-x64" | "linux-x64" | null { + switch (os.platform()) { + case "win32": + return "win-x64"; + case "darwin": + return "osx-x64"; + case "aix": + case "freebsd": + case "openbsd": + case "sunos": + case "linux": + return "linux-x64"; + default: + return null; + } +} diff --git a/src/msha/handlers/dab.handler.ts b/src/msha/handlers/dab.handler.ts new file mode 100644 index 00000000..fd6babf8 --- /dev/null +++ b/src/msha/handlers/dab.handler.ts @@ -0,0 +1,102 @@ +import chalk from "chalk"; +import type http from "http"; +import httpProxy from "http-proxy"; +import { decodeCookie, logger, logRequest, registerProcessExit, validateCookie } from "../../core"; +import { SWA_CLI_DATA_API_URI } from "../../core/constants"; +import { onConnectionLost } from "../middlewares/request.middleware"; + +const proxyApi = httpProxy.createProxyServer({ autoRewrite: true }); +registerProcessExit(() => { + logger.silly(`killing SWA CLI`); + proxyApi.close(() => logger.log("Data-Api proxy stopped.")); + process.exit(0); +}); + +/** + * Gets response from the Data Api + * @param req http request url + * @param res http response after redirecting to Data api builder + */ +export function handleDataApiRequest(req: http.IncomingMessage, res: http.ServerResponse) { + const target = SWA_CLI_DATA_API_URI(); + + proxyApi.web( + req, + res, + { + target, + }, + onConnectionLost(req, res, target, "↳") + ); + + proxyApi.once("proxyReq", (proxyReq: http.ClientRequest) => { + injectHeaders(proxyReq, target); + injectClientPrincipalCookies(proxyReq); + }); + + proxyApi.once("proxyRes", (proxyRes: http.IncomingMessage) => { + logger.silly(`getting response from remote host`); + logRequest(req, "", proxyRes.statusCode); + }); + + logRequest(req, target); +} + +function injectHeaders(req: http.ClientRequest, host: string | undefined) { + const X_MS_ORIGINAL_URL_HEADER = "x-ms-original-url"; + const X_MS_REQUEST_ID_HEADER = "x-ms-request-id"; + + logger.silly(`injecting headers to Data-api request:`); + if (!req.getHeader(X_MS_ORIGINAL_URL_HEADER)) { + req.setHeader(X_MS_ORIGINAL_URL_HEADER, encodeURI(new URL(req.path!, host).toString())); + logger.silly(` - x-ms-original-url: ${chalk.yellow(req.getHeader(X_MS_ORIGINAL_URL_HEADER))}`); + } + // generate a fake correlation ID + req.setHeader(X_MS_REQUEST_ID_HEADER, `SWA-CLI-${Math.random().toString(36).substring(2).toUpperCase()}`); + logger.silly(` - x-ms-request-id: ${chalk.yellow(req.getHeader(X_MS_REQUEST_ID_HEADER))}`); +} + +/** + * Checks if the request is Data-api request or not + * @param req http request url + * @param rewritePath + * @returns true if the request is data-api Request else false + */ +export function isDataApiRequest(req: http.IncomingMessage, rewritePath?: string): boolean { + const path = rewritePath || req.url; + return Boolean(path?.toLowerCase().startsWith(`/data-api/`)); +} + +function injectClientPrincipalCookies(req: http.ClientRequest) { + const X_MS_CLIENT_PRINCIPAL_HEADER = "X-MS-CLIENT-PRINCIPAL"; + const AUTH_HEADER = "authorization"; + const COOKIE_HEADER = "cookie"; + const CLAIMS_HEADER = "claims"; + + logger.silly(`injecting client principal to Functions request:`); + + const cookie = req.getHeader(COOKIE_HEADER) as string; + if (cookie && validateCookie(cookie)) { + const user = decodeCookie(cookie); + + // Remove claims from client principal to match SWA behaviour. See https://github.com/MicrosoftDocs/azure-docs/issues/86803. + // The following property deletion can be removed depending on outcome of the above issue. + if (user) { + delete user[CLAIMS_HEADER]; + } + + const buff = Buffer.from(JSON.stringify(user), "utf-8"); + const token = buff.toString("base64"); + req.setHeader(X_MS_CLIENT_PRINCIPAL_HEADER, token); + logger.silly(` - X-MS-CLIENT-PRINCIPAL: ${chalk.yellow(req.getHeader(X_MS_CLIENT_PRINCIPAL_HEADER))}`); + + // locally, we set the JWT bearer token to be the same as the cookie value because we are not using the real auth flow. + // Note: on production, SWA uses a valid encrypted JWT token! + if (!req.getHeader(AUTH_HEADER)) { + req.setHeader(AUTH_HEADER, `Bearer ${token}`); + logger.silly(` - Authorization: ${chalk.yellow(req.getHeader(AUTH_HEADER))}`); + } + } else { + logger.silly(` - no valid cookie found`); + } +} diff --git a/src/msha/middlewares/request.middleware.ts b/src/msha/middlewares/request.middleware.ts index ed9eff4a..8fa960d4 100644 --- a/src/msha/middlewares/request.middleware.ts +++ b/src/msha/middlewares/request.middleware.ts @@ -10,6 +10,7 @@ import { DEFAULT_CONFIG } from "../../config"; import { findSWAConfigFile, logger, logRequest } from "../../core"; import { AUTH_STATUS, CUSTOM_URL_SCHEME, IS_APP_DEV_SERVER, SWA_PUBLIC_DIR } from "../../core/constants"; import { getAuthBlockResponse, handleAuthRequest, isAuthRequest, isLoginRequest, isLogoutRequest } from "../handlers/auth.handler"; +import { isDataApiRequest } from "../handlers/dab.handler"; import { handleErrorPage } from "../handlers/error-page.handler"; import { isFunctionRequest } from "../handlers/function.handler"; import { isRequestMethodValid, isRouteRequiringUserRolesCheck, tryGetMatchingRoute } from "../routes-engine"; @@ -244,7 +245,13 @@ export async function requestMiddleware( logger.silly(` - not a function request`); } - if (!isRequestMethodValid(req, isFunctionReq, isAuthReq)) { + logger.silly(`checking data-api request`); + const isDataApiReq = isDataApiRequest(req, matchingRouteRule?.rewrite); + if (!isDataApiReq) { + logger.silly(` - not a data Api request`); + } + + if (!isRequestMethodValid(req, isFunctionReq, isAuthReq, isDataApiReq)) { res.statusCode = 405; return res.end(); } @@ -284,7 +291,7 @@ export async function requestMiddleware( return await handleAuthRequest(req, res, matchingRouteRule, userConfig); } - if (!getResponse(req, res, matchingRouteRule, userConfig, isFunctionReq)) { + if (!getResponse(req, res, matchingRouteRule, userConfig, isFunctionReq, isDataApiReq)) { logger.silly(` - url: ${chalk.yellow(req.url)}`); logger.silly(` - target: ${chalk.yellow(target)}`); diff --git a/src/msha/middlewares/response.middleware.ts b/src/msha/middlewares/response.middleware.ts index 93ef685e..4af2a4cb 100644 --- a/src/msha/middlewares/response.middleware.ts +++ b/src/msha/middlewares/response.middleware.ts @@ -2,6 +2,7 @@ import chalk from "chalk"; import type http from "http"; import { isHttpUrl, isSWAConfigFileUrl, logger } from "../../core"; import { IS_APP_DEV_SERVER } from "../../core/constants"; +import { handleDataApiRequest } from "../handlers/dab.handler"; import { handleErrorPage } from "../handlers/error-page.handler"; import { handleFunctionRequest, isFunctionRequest } from "../handlers/function.handler"; import { @@ -19,7 +20,8 @@ export function getResponse( res: http.ServerResponse, matchedRoute: SWAConfigFileRoute | undefined, userConfig: SWAConfigFile | undefined, - isFunctionRequest: boolean + isFunctionRequest: boolean, + isDataApiRequest: boolean ): boolean { const statusCodeToServe = parseInt(`${matchedRoute?.statusCode}`, 10); const redirect = matchedRoute?.redirect; @@ -51,6 +53,14 @@ export function getResponse( return true; } + if (isDataApiRequest) { + if (req.url?.startsWith("/data-api/")) { + req.url = req.url?.replace("/data-api", ""); + } + handleDataApiRequest(req, res); + return true; + } + const storageResult = getStorageContent( req, res, diff --git a/src/msha/routes-engine/rules/routes.spec.ts b/src/msha/routes-engine/rules/routes.spec.ts index 8ab55703..dff1cd06 100644 --- a/src/msha/routes-engine/rules/routes.spec.ts +++ b/src/msha/routes-engine/rules/routes.spec.ts @@ -395,43 +395,49 @@ describe("route utilities", () => { describe("isRequestMethodValid()", () => { const testHttpMethods = ["GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"]; const req: Partial = {}; - function test(method: string, isFunctionRequest: boolean, isAuth: boolean, expectedValue: boolean) { + function test(method: string, isFunctionRequest: boolean, isAuth: boolean, isDataApiRequest: boolean, expectedValue: boolean) { return () => { req.method = method; - const isValid = isRequestMethodValid(req as http.IncomingMessage, isFunctionRequest, isAuth); + const isValid = isRequestMethodValid(req as http.IncomingMessage, isFunctionRequest, isAuth, isDataApiRequest); expect(isValid).toBe(expectedValue); }; } it("should return false when no method is provided", () => { - const isValid = isRequestMethodValid(req as http.IncomingMessage, false, false); + const isValid = isRequestMethodValid(req as http.IncomingMessage, false, false, false); expect(isValid).toBe(false); }); it("should return false when method is not valid", () => { req.method = "FOO"; - const isValid = isRequestMethodValid(req as http.IncomingMessage, false, false); + const isValid = isRequestMethodValid(req as http.IncomingMessage, false, false, false); expect(isValid).toBe(false); }); describe("when request is for static", () => { ["GET", "HEAD", "OPTIONS"].forEach((method) => { - it(`should return true when for valid method ${method}`, test(method, false, false, true)); + it(`should return true when for valid method ${method}`, test(method, false, false, false, true)); }); ["POST", "DELETE", "PUT", "PATCH"].forEach((method) => { - it(`should return true when for invalid method ${method}`, test(method, false, false, false)); + it(`should return true when for invalid method ${method}`, test(method, false, false, false, false)); }); }); describe("when request is for Functions", () => { testHttpMethods.forEach((method) => { - it(`should return true when method is ${method}`, test(method, true, false, true)); + it(`should return true when method is ${method}`, test(method, true, false, false, true)); + }); + }); + + describe("when request is for Data-api", () => { + testHttpMethods.forEach((method) => { + it(`should return true when method is ${method}`, test(method, false, false, true, true)); }); }); describe("when request is for auth", () => { testHttpMethods.forEach((method) => { - it(`should return true when method is ${method}`, test(method, false, true, true)); + it(`should return true when method is ${method}`, test(method, false, true, false, true)); }); }); }); diff --git a/src/msha/routes-engine/rules/routes.ts b/src/msha/routes-engine/rules/routes.ts index d3e2e1cb..ef93273a 100644 --- a/src/msha/routes-engine/rules/routes.ts +++ b/src/msha/routes-engine/rules/routes.ts @@ -164,11 +164,11 @@ export function tryGetMatchingRoute(req: http.IncomingMessage, userConfig: SWACo return; } -export function isRequestMethodValid(req: http.IncomingMessage, isFunctionRequest: boolean, isAuth: boolean) { +export function isRequestMethodValid(req: http.IncomingMessage, isFunctionRequest: boolean, isAuth: boolean, isDataApiReq: boolean) { logger.silly(`checking HTTP method: ${chalk.yellow(req.method)}`); - if (isFunctionRequest || isAuth) { - logger.silly(` - function or auth request detected, method is valid`); + if (isFunctionRequest || isAuth || isDataApiReq) { + logger.silly(` - function or auth or data-api request detected, method is valid`); return true; } diff --git a/src/swa.d.ts b/src/swa.d.ts index 48a2e4d8..648e4a38 100644 --- a/src/swa.d.ts +++ b/src/swa.d.ts @@ -31,10 +31,12 @@ declare interface SWACLIEnv extends StaticSiteClientEnv { SWA_RUNTIME_WORKFLOW_LOCATION?: string; // swa start - SWA_CLI_API_PORT?: string; SWA_CLI_APP_LOCATION?: string; SWA_CLI_OUTPUT_LOCATION?: string; SWA_CLI_API_LOCATION?: string; + SWA_CLI_DATA_API_LOCATION?: string; + SWA_CLI_API_PORT?: string; + SWA_CLI_DATA_API_PORT?: string; SWA_CLI_HOST?: string; SWA_CLI_PORT?: string; SWA_CLI_APP_SSL?: string; @@ -45,6 +47,7 @@ declare interface SWACLIEnv extends StaticSiteClientEnv { SWA_CLI_OPEN_BROWSER?: string; SWA_CLI_APP_DEVSERVER_URL?: string; SWA_CLI_API_DEVSERVER_URL?: string; + SWA_CLI_DATA_API_DEVSERVER_URL?: string; // swa deploy SWA_CLI_DEPLOY_DRY_RUN?: string; @@ -71,6 +74,9 @@ declare interface SWACLIEnv extends StaticSiteClientEnv { AZURE_TENANT_ID?: string; AZURE_CLIENT_ID?: string; AZURE_CLIENT_SECRET?: string; + + // swa db + SWA_CLI_DATA_API_FOLDER?: string; } declare interface Context { @@ -126,8 +132,11 @@ declare type SWACLIStartOptions = { appLocation?: string; outputLocation?: string; apiLocation?: string; + dataApiLocation?: string; appDevserverUrl?: string; apiDevserverUrl?: string; + dataApiDevserverUrl?: string; + dataApiPort?: number; apiPort?: number; host?: string; port?: number; @@ -153,6 +162,16 @@ declare type SWACLIBuildOptions = { auto?: boolean; }; +// -- CLI DB init options + +declare type SWACLIDBInitOptions = { + databaseType?: "mssql" | "postgresql" | "cosmosdb_nosql" | "mysql" | "cosmosdb_postgresql"; + connectionString?: string; + cosmosdb_nosqlDatabase?: string; + cosmosdb_nosqlContainer?: string; + folderName?: string; +}; + // -- CLI Deploy options ----------------------------------------------------- declare type SWACLIDeployOptions = SWACLISharedLoginOptions & { @@ -191,12 +210,14 @@ declare type SWACLIConfig = SWACLIGlobalOptions & SWACLIBuildOptions & SWACLIStartOptions & SWACLIDeployOptions & - SWACLIBuildOptions & { + SWACLIBuildOptions & + SWACLIDBInitOptions & { login?: SWACLIGlobalOptions & SWACLILoginOptions; init?: SWACLIGlobalOptions & SWACLIInitOptions; start?: SWACLIGlobalOptions & SWACLIStartOptions; deploy?: SWACLIGlobalOptions & SWACLIDeployOptions; build?: SWACLIGlobalOptions & SWACLIBuildOptions; + "db init"?: SWACLIGlobalOptions & SWACLIDBInitOptions; }; // Information about the loaded config @@ -294,3 +315,46 @@ declare interface CoreToolsZipInfo { } declare type NpmPackageManager = "npm" | "yarn" | "pnpm"; + +declare type BinaryMetadata = { + version: "stable" | "latest" | "old"; + files: { + ["linux-x64"]: { + url: string; + sha: string; + }; + ["win-x64"]: { + url: string; + sha: string; + }; + ["osx-x64"]: { + url: string; + sha: string; + }; + }; +}; + +declare type StaticSiteClientReleaseMetadata = BinaryMetadata & { + buildId: string; + publishDate: string; +}; + +declare type DataApiBuilderReleaseMetadata = BinaryMetadata & { + versionId: string; + releaseType: string; + releaseDate: string; +}; + +declare type LocalBinaryMetadata = { + metadata: BinaryMetadata; + binary: string; + checksum: string; +}; + +declare type StaticSiteClientLocalMetadata = LocalBinaryMetadata; +declare type DataApiBuilderLocalMetadata = LocalBinaryMetadata; + +const binaryType = { + StaticSiteClient: 1, + DataApiBuilder: 2, +};