diff --git a/packages/one-app-runner/src/asyncSpawn.js b/packages/one-app-runner/src/asyncSpawn.js new file mode 100644 index 00000000..6cd9824d --- /dev/null +++ b/packages/one-app-runner/src/asyncSpawn.js @@ -0,0 +1,25 @@ +const { spawn } = require('child_process'); + +module.exports = async function asyncSpawn(command, args) { + return new Promise((resolve, reject) => { + const spawnedProcess = spawn(command, args); + + let stdout = Buffer.from(''); + let stderr = Buffer.from(''); + + spawnedProcess.stdout.on('data', (chunk) => { + stdout = Buffer.concat([stdout, chunk]); + }); + + spawnedProcess.stderr.on('data', (chunk) => { + stderr = Buffer.concat([stderr, chunk]); + }); + + spawnedProcess.on('close', (code) => { + if (code !== 0) { + return reject(Object.assign(new Error('process exited with an error'), { code, stdout, stderr })); + } + return resolve({ code, stdout, stderr }); + }); + }); +}; diff --git a/packages/one-app-runner/src/generateContainerArgs.js b/packages/one-app-runner/src/generateContainerArgs.js new file mode 100644 index 00000000..2e22589c --- /dev/null +++ b/packages/one-app-runner/src/generateContainerArgs.js @@ -0,0 +1,89 @@ +const path = require('node:path'); + +const semver = require('semver'); + +function generateEnvironmentVariableArgs(envVars) { + return new Map([ + ['NODE_ENV', 'development'], + ...Object.entries(envVars), + process.env.HTTP_PROXY ? ['HTTP_PROXY', process.env.HTTP_PROXY] : null, + process.env.HTTPS_PROXY ? ['HTTPS_PROXY', process.env.HTTPS_PROXY] : null, + process.env.NO_PROXY ? ['NO_PROXY', process.env.NO_PROXY] : null, + process.env.HTTP_PORT ? ['HTTP_PORT', process.env.HTTP_PORT] : null, + process.env.HTTP_ONE_APP_DEV_CDN_PORT + ? ['HTTP_ONE_APP_DEV_CDN_PORT', process.env.HTTP_ONE_APP_DEV_CDN_PORT] + : null, + process.env.HTTP_ONE_APP_DEV_PROXY_SERVER_PORT + ? ['HTTP_ONE_APP_DEV_PROXY_SERVER_PORT', process.env.HTTP_ONE_APP_DEV_PROXY_SERVER_PORT] + : null, + process.env.HTTP_METRICS_PORT + ? ['HTTP_METRICS_PORT', process.env.HTTP_METRICS_PORT] + : null, + ].filter(Boolean)); +} + +function generateSetMiddlewareCommand(pathToMiddlewareFile) { + if (!pathToMiddlewareFile) { + return ''; + } + const pathArray = pathToMiddlewareFile.split(path.sep); + return `npm run set-middleware '/opt/module-workspace/${pathArray[pathArray.length - 2]}/${pathArray[pathArray.length - 1]}' &&`; +} + +function generateSetDevEndpointsCommand(pathToDevEndpointsFile) { + if (!pathToDevEndpointsFile) { + return ''; + } + const pathArray = pathToDevEndpointsFile.split(path.sep); + return `npm run set-dev-endpoints '/opt/module-workspace/${pathArray[pathArray.length - 2]}/${pathArray[pathArray.length - 1]}' &&`; +} + +function generateUseMocksFlag(shouldUseMocks) { return shouldUseMocks ? '-m' : ''; } + +function generateNpmConfigCommands() { return 'npm config set update-notifier false &&'; } + +function generateServeModuleCommands(modules) { + let command = ''; + if (modules && modules.length > 0) { + modules.forEach((modulePath) => { + const moduleRootDir = path.basename(modulePath); + command += `npm run serve-module '/opt/module-workspace/${moduleRootDir}' &&`; + }); + } + return command; +} + +function generateModuleMap(moduleMapUrl) { return moduleMapUrl ? `--module-map-url=${moduleMapUrl}` : ''; } + +function generateLogLevel(logLevel) { return logLevel ? `--log-level=${logLevel}` : ''; } + +function generateLogFormat(logFormat) { return logFormat ? `--log-format=${logFormat}` : ''; } + +function generateDebug(port, useDebug) { return useDebug ? `--inspect=0.0.0.0:${port}` : ''; } + +// NOTE: Node 12 does not support --dns-result-order or --no-experimental-fetch +// So we have to remove those flags if the one-app version is less than 5.13.0 +// 5.13.0 is when node 16 was introduced. +function generateNodeFlags(appVersion) { + if (semver.intersects(appVersion, '^5.13.0', { includePrerelease: true })) { + return '--dns-result-order=ipv4first --no-experimental-fetch'; + } + return ''; +} + +function generateUseHostFlag(useHost) { return useHost ? '--use-host' : ''; } + +module.exports = { + generateEnvironmentVariableArgs, + generateSetMiddlewareCommand, + generateSetDevEndpointsCommand, + generateUseMocksFlag, + generateNpmConfigCommands, + generateServeModuleCommands, + generateModuleMap, + generateLogLevel, + generateLogFormat, + generateDebug, + generateNodeFlags, + generateUseHostFlag, +}; diff --git a/packages/one-app-runner/src/spawnAndPipe.js b/packages/one-app-runner/src/spawnAndPipe.js new file mode 100644 index 00000000..9fcd1ce1 --- /dev/null +++ b/packages/one-app-runner/src/spawnAndPipe.js @@ -0,0 +1,22 @@ +const { spawn } = require('child_process'); + +module.exports = async function spawnAndPipe(command, args, logStream) { + return new Promise((resolve, reject) => { + const spawnedProcess = spawn(command, args); + + spawnedProcess.on('close', (code) => { + if (code !== 0) { + return reject(code); + } + return resolve(code); + }); + + if (logStream) { + spawnedProcess.stdout.pipe(logStream, { end: false }); + spawnedProcess.stderr.pipe(logStream, { end: false }); + } else { + spawnedProcess.stdout.pipe(process.stdout); + spawnedProcess.stderr.pipe(process.stderr); + } + }); +}; diff --git a/packages/one-app-runner/src/startApp.js b/packages/one-app-runner/src/startApp.js index bd1085c8..6ac65ed1 100644 --- a/packages/one-app-runner/src/startApp.js +++ b/packages/one-app-runner/src/startApp.js @@ -12,136 +12,33 @@ * the License. */ -const { spawn } = require('child_process'); const path = require('node:path'); const fs = require('node:fs'); const os = require('node:os'); const Docker = require('dockerode'); -const semver = require('semver'); -async function spawnAndPipe(command, args, logStream) { - return new Promise((resolve, reject) => { - const spawnedProcess = spawn(command, args); - - spawnedProcess.on('close', (code) => { - if (code !== 0) { - return reject(code); - } - return resolve(code); - }); - - if (logStream) { - spawnedProcess.stdout.pipe(logStream, { end: false }); - spawnedProcess.stderr.pipe(logStream, { end: false }); - } else { - spawnedProcess.stdout.pipe(process.stdout); - spawnedProcess.stderr.pipe(process.stderr); - } - }); -} +const asyncSpawn = require('./asyncSpawn'); +const spawnAndPipe = require('./spawnAndPipe'); +const startAppContainer = require('./startAppContainer'); +const { + generateEnvironmentVariableArgs, + generateSetMiddlewareCommand, + generateSetDevEndpointsCommand, + generateUseMocksFlag, + generateNpmConfigCommands, + generateServeModuleCommands, + generateModuleMap, + generateLogLevel, + generateLogFormat, + generateDebug, + generateNodeFlags, + generateUseHostFlag, +} = require('./generateContainerArgs'); async function dockerPull(imageReference, logStream) { return spawnAndPipe('docker', ['pull', imageReference], logStream); } -async function startAppContainer({ - imageReference, - containerShellCommand, - ports /* = [] */, - envVars /* = new Map() */, - mounts /* = new Map() */, - name, - network, - logStream, -}) { - return spawnAndPipe( - 'docker', - [ - 'run', - '-t', - ...ports.map((port) => `-p=${port}:${port}`), - ...[...envVars.entries()].map(([envVarName, envVarValue]) => `-e=${envVarName}=${envVarValue}`), - ...[...mounts.entries()].map(([hostPath, containerPath]) => `-v=${hostPath}:${containerPath}`), - name ? `--name=${name}` : null, - network ? `--network=${network}` : null, - imageReference, - '/bin/sh', - '-c', - containerShellCommand, - ].filter(Boolean), - logStream - ); -} - -function generateEnvironmentVariableArgs(envVars) { - return new Map([ - ['NODE_ENV', 'development'], - ...Object.entries(envVars), - process.env.HTTP_PROXY ? ['HTTP_PROXY', process.env.HTTP_PROXY] : null, - process.env.HTTPS_PROXY ? ['HTTPS_PROXY', process.env.HTTPS_PROXY] : null, - process.env.NO_PROXY ? ['NO_PROXY', process.env.NO_PROXY] : null, - process.env.HTTP_PORT ? ['HTTP_PORT', process.env.HTTP_PORT] : null, - process.env.HTTP_ONE_APP_DEV_CDN_PORT - ? ['HTTP_ONE_APP_DEV_CDN_PORT', process.env.HTTP_ONE_APP_DEV_CDN_PORT] - : null, - process.env.HTTP_ONE_APP_DEV_PROXY_SERVER_PORT - ? ['HTTP_ONE_APP_DEV_PROXY_SERVER_PORT', process.env.HTTP_ONE_APP_DEV_PROXY_SERVER_PORT] - : null, - process.env.HTTP_METRICS_PORT - ? ['HTTP_METRICS_PORT', process.env.HTTP_METRICS_PORT] - : null, - ].filter(Boolean)); -} - -const generateSetMiddlewareCommand = (pathToMiddlewareFile) => { - if (pathToMiddlewareFile) { - const pathArray = pathToMiddlewareFile.split(path.sep); - return `npm run set-middleware '/opt/module-workspace/${pathArray[pathArray.length - 2]}/${pathArray[pathArray.length - 1]}' &&`; - } - return ''; -}; - -const generateSetDevEndpointsCommand = (pathToDevEndpointsFile) => { - if (pathToDevEndpointsFile) { - const pathArray = pathToDevEndpointsFile.split(path.sep); - return `npm run set-dev-endpoints '/opt/module-workspace/${pathArray[pathArray.length - 2]}/${pathArray[pathArray.length - 1]}' &&`; - } - return ''; -}; - -const generateUseMocksFlag = (shouldUseMocks) => (shouldUseMocks ? '-m' : ''); - -const generateNpmConfigCommands = () => 'npm config set update-notifier false &&'; - -const generateServeModuleCommands = (modules) => { - let command = ''; - if (modules && modules.length > 0) { - modules.forEach((modulePath) => { - const moduleRootDir = path.basename(modulePath); - command += `npm run serve-module '/opt/module-workspace/${moduleRootDir}' &&`; - }); - } - return command; -}; - -const generateModuleMap = (moduleMapUrl) => (moduleMapUrl ? `--module-map-url=${moduleMapUrl}` : ''); - -const generateLogLevel = (logLevel) => (logLevel ? `--log-level=${logLevel}` : ''); - -const generateLogFormat = (logFormat) => (logFormat ? `--log-format=${logFormat}` : ''); - -const generateDebug = (port, useDebug) => (useDebug ? `--inspect=0.0.0.0:${port}` : ''); - -// NOTE: Node 12 does not support --dns-result-order or --no-experimental-fetch -// So we have to remove those flags if the one-app version is less than 5.13.0 -// 5.13.0 is when node 16 was introduced. -const generateNodeFlags = (appVersion) => { - if (semver.intersects(appVersion, '^5.13.0', { includePrerelease: true })) { - return '--dns-result-order=ipv4first --no-experimental-fetch'; - } - return ''; -}; - module.exports = async function startApp({ moduleMapUrl, rootModuleName, @@ -160,6 +57,15 @@ module.exports = async function startApp({ logLevel, logFormat, }) { + try { + // need a command that both invokes the CLI but also connects to the daemon + await asyncSpawn('docker', ['version']); + } catch (error) { + throw new Error( + `Error running docker. Are you sure you have it installed?\nFor installation and setup details see https://www.docker.com/products/docker-desktop\nExit code ${error.code}, error messages ${error.stderr.toString('utf8')}` + ); + } + if (createDockerNetwork) { if (!dockerNetworkToJoin) { throw new Error( @@ -176,8 +82,6 @@ module.exports = async function startApp({ } } - const generateUseHostFlag = () => (useHost ? '--use-host' : ''); - const appPort = Number.parseInt(process.env.HTTP_PORT, 10) || 3000; const devCDNPort = Number.parseInt(process.env.HTTP_ONE_APP_DEV_CDN_PORT, 10) || 3001; const devProxyServerPort = Number.parseInt( @@ -202,7 +106,7 @@ module.exports = async function startApp({ const hostNodeExtraCaCerts = envVars.NODE_EXTRA_CA_CERTS || process.env.NODE_EXTRA_CA_CERTS; if (hostNodeExtraCaCerts) { - console.log('mounting host NODE_EXTRA_CA_CERTS'); + console.log('adding NODE_EXTRA_CA_CERTS to the volume mount list'); const mountPath = '/opt/certs.pem'; mounts.set(hostNodeExtraCaCerts, mountPath); containerEnvVars.set('NODE_EXTRA_CA_CERTS', mountPath); @@ -221,7 +125,7 @@ module.exports = async function startApp({ `lib/server/index.js --root-module-name=${rootModuleName}`, generateModuleMap(moduleMapUrl), generateUseMocksFlag(parrotMiddlewareFile), - generateUseHostFlag(), + generateUseHostFlag(useHost), generateLogLevel(logLevel), generateLogFormat(logFormat), ].filter(Boolean).join(' '); @@ -254,22 +158,13 @@ module.exports = async function startApp({ logStream: logFileStream, }); } catch (error) { - throw new Error( - 'Error running docker. Are you sure you have it installed? For installation and setup details see https://www.docker.com/products/docker-desktop', - { cause: error } - ); + if (error.stderr) { + throw new Error(error.stderr.toString('utf8')); + } + throw error; } finally { if (logFileStream) { logFileStream.end(); } } - - [ - 'SIGINT', - 'SIGTERM', - ].forEach((signal) => { - // process is a global referring to current running process https://nodejs.org/api/globals.html#globals_process - /* istanbul ignore next */ - process.on(signal, () => 'noop - just need to pass signal to one app process so it can handle it'); - }); }; diff --git a/packages/one-app-runner/src/startAppContainer.js b/packages/one-app-runner/src/startAppContainer.js new file mode 100644 index 00000000..b6d6ca86 --- /dev/null +++ b/packages/one-app-runner/src/startAppContainer.js @@ -0,0 +1,210 @@ +const { createHash } = require('node:crypto'); + +const Docker = require('dockerode'); + +const spawn = require('./asyncSpawn'); +const spawnAndPipe = require('./spawnAndPipe'); + +const docker = new Docker(); + +function generateArgsHash({ + imageReference, // String + containerShellCommand, // String + ports, // array + envVars, // Map + mounts, // Map + network, // String +}) { + const hashing = createHash('md5'); + hashing.update(imageReference); + if (network) { + hashing.update(network); + } + hashing.update(ports.join(',')); + for (const [key, value] of envVars.entries()) { + hashing.update(`${key}=${value}`); + } + for (const [hostPath, containerPath] of mounts.entries()) { + hashing.update(`${hostPath}=${containerPath}`); + } + // TODO: remove this from the hash if we start to use docker exec to serve and unserve modules + hashing.update(containerShellCommand); + // though middleware is a startup option, so not all commands can be executed later + return hashing.digest('hex').slice(0, 7); +} + +async function pipeContainerLogs(containerName, logStream) { + try { + await spawnAndPipe( + 'docker', + [ + 'logs', + '--follow', + '--tail', '1', + containerName, + ], + logStream + ); + } catch (error) { + if (error === 255) { + // docker logs exits 255 when interrupted rather than 0 + return; + } + throw error; + } +} + +function setupPausingOnInterrupt(hashedName) { + [ + 'SIGINT', + 'SIGTERM', + ].forEach((signal) => { + process.once(signal, () => { + console.log(`pausing container ${hashedName}`); + docker.getContainer(hashedName).pause(); + }); + }); +} + +function setupStoppingOnInterrupt(name) { + [ + 'SIGINT', + 'SIGTERM', + ].forEach((signal) => { + process.once(signal, () => { + console.log(`stopping container ${name}`); + docker.getContainer(name).stop(); + }); + }); +} + +async function createAndStartContainer({ + imageReference, + containerShellCommand, + ports, + envVars, + mounts, + name, + network, +}) { + let createResult; + try { + createResult = await spawn( + 'docker', + [ + 'create', + ...ports.map((port) => `-p=${port}:${port}`), + ...[...envVars.entries()].map(([envVarName, envVarValue]) => `-e=${envVarName}=${envVarValue}`), + ...[...mounts.entries()].map(([hostPath, containerPath]) => `-v=${hostPath}:${containerPath}`), + name ? `--name=${name}` : null, + network ? `--network=${network}` : null, + imageReference, + '/bin/sh', + '-c', + containerShellCommand, + ].filter(Boolean) + ); + } catch (creationError) { + console.error(`unable to create image "${name}"`, creationError.code, creationError.stderr, creationError.stdout); + throw creationError; + } + + const containerId = createResult.stdout.toString('utf8').trim(); + await spawn('docker', ['start', containerId]); + return pipeContainerLogs(containerId); +} + +module.exports = async function startAppContainer({ + imageReference, + containerShellCommand, + ports /* = [] */, + envVars /* = new Map() */, + mounts /* = new Map() */, + name, + network, + logStream, +}) { + // don't mess with the name, can't store the hash of arguments + // so create as a new container each time + if (name) { + console.log(`Starting a new container with the name "${name}.`); + setupStoppingOnInterrupt(name); + return createAndStartContainer({ + imageReference, + containerShellCommand, + ports, + envVars, + mounts, + name, + network, + logStream, + }); + } + + // can we reuse an existing (paused) container? + // hash the args + const argsHash = generateArgsHash({ + imageReference, + containerShellCommand, + ports, + envVars, + mounts, + network, + }); + const hashedName = `${'one-app'}_argset-${argsHash}`; + // look for a container + const container = docker.getContainer(hashedName); + + let lowLevelInfo; + try { + lowLevelInfo = await container.inspect(); + } catch (inspectionError) { + if (inspectionError.reason === 'no such container' && inspectionError.statusCode === 404) { + console.log(`No container named ${hashedName} found to reuse. Starting a new container with this name.`); + setupPausingOnInterrupt(hashedName); + return createAndStartContainer({ + imageReference, + containerShellCommand, + ports, + envVars, + mounts, + name: hashedName, + network, + logStream, + }); + } + throw inspectionError; + } + + const containerStatus = lowLevelInfo.State.Status; + // check its status (is it paused?) + if (containerStatus === 'paused') { + console.log(`found paused container ${hashedName}, unpausing and streaming the logs`); + container.unpause(); + setupPausingOnInterrupt(hashedName); + return pipeContainerLogs(hashedName, logStream); + } + + if (containerStatus === 'created') { + console.log(`found created container ${hashedName}, starting and streaming the logs`); + container.start(); + setupPausingOnInterrupt(hashedName); + return pipeContainerLogs(hashedName, logStream); + } + + if (containerStatus === 'exited') { + // FIXME: inspect lowLevelInfo.State.Error and lowLevelInfo.State.ExitCode + console.log(`found stopped container ${hashedName}, starting and streaming the logs`); + container.start(); + setupPausingOnInterrupt(hashedName); + return pipeContainerLogs(hashedName, logStream); + } + + if (containerStatus === 'running') { + console.log(`found running container ${hashedName}, streaming the logs`); + setupPausingOnInterrupt(hashedName); + return pipeContainerLogs(hashedName, logStream); + } + + throw new Error(`Unknown state for container "${hashedName}", please save any work in the container, delete it, and try again.\nsaw ${JSON.stringify(lowLevelInfo.State)}`); +};