diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json index bd108ca4afb..8148a5f55d0 100644 --- a/creator-node/package-lock.json +++ b/creator-node/package-lock.json @@ -2951,7 +2951,7 @@ "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" }, "@protobufjs/base64": { "version": "1.1.2", @@ -2966,12 +2966,12 @@ "@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" }, "@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", "requires": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" @@ -2980,27 +2980,27 @@ "@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" }, "@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" }, "@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" }, "@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" }, "@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" }, "@sindresorhus/is": { "version": "0.14.0", @@ -4729,7 +4729,7 @@ "babel-plugin-syntax-jsx": { "version": "6.18.0", "resolved": "https://registry.npmjs.org/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz", - "integrity": "sha512-qrPaCSo9c8RHNRHIotaufGbuOBN8rtdC4QrrFFc43vyWCCz7Kl7GL1PGaXtMGQZUXrkCjNEgxDfmAuAabr/rlw==" + "integrity": "sha1-CvMqmm4Tyno/1QaeYtew9Y0NiUY=" }, "balanced-match": { "version": "1.0.2", @@ -5527,7 +5527,7 @@ "camelize": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/camelize/-/camelize-1.0.0.tgz", - "integrity": "sha512-W2lPwkBkMZwFlPCXhIlYgxu+7gC/NUlCtdK652DAJ1JdgV0sTrvuPFshNPrFa1TY2JOkLhgdeEBplB4ezEa+xg==" + "integrity": "sha1-FkpUg+Yw+kMh5a8HAg5TGDGyYJs=" }, "caniuse-lite": { "version": "1.0.30001241", @@ -6214,7 +6214,7 @@ "css-color-keywords": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/css-color-keywords/-/css-color-keywords-1.0.0.tgz", - "integrity": "sha512-FyyrDHZKEjXDpNJYvVsV960FiqQyXc/LlYmsxl2BcdMb2WPx0OGRVgTg55rPSyLSNMqP52R9r8geSp7apN3Ofg==" + "integrity": "sha1-/qJhbcZ2spYmhrOvjb2+GAskTgU=" }, "css-to-react-native": { "version": "3.0.0", @@ -8158,7 +8158,7 @@ "fast-stable-stringify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fast-stable-stringify/-/fast-stable-stringify-1.0.0.tgz", - "integrity": "sha512-wpYMUmFu5f00Sm0cj2pfivpmawLZ0NKdviQ4w9zJeR8JVtOpOxHmLaJuj0vxvGqMJQWyP/COUkF75/57OKyRag==" + "integrity": "sha1-XFVDRisiru79NtBbNOUceMuG0xM=" }, "fastq": { "version": "1.13.0", @@ -12712,7 +12712,7 @@ "qr.js": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/qr.js/-/qr.js-0.0.0.tgz", - "integrity": "sha512-c4iYnWb+k2E+vYpRimHqSu575b1/wKl4XFeJGpFmrJQz5I88v9aY2czh7s0w36srfCM1sXgC/xpoJz5dJfq+OQ==" + "integrity": "sha1-ys6GOG9ZoNuAUPqQ2baw6IoeNk8=" }, "qrcode.react": { "version": "1.0.1", @@ -13955,7 +13955,7 @@ "slide": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/slide/-/slide-1.1.6.tgz", - "integrity": "sha512-NwrtjCg+lZoqhFU8fOwl4ay2ei8PaqCBOUV3/ektPY9trO1yQ1oXEfmHAhKArUVUr/hOHvy5f6AdP17dCM0zMw==" + "integrity": "sha1-VusCfWW00tzmyy4tMsTUr8nh1wc=" }, "snake-case": { "version": "3.0.4", @@ -16332,7 +16332,7 @@ "wif": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/wif/-/wif-2.0.6.tgz", - "integrity": "sha512-HIanZn1zmduSF+BQhkE+YXIbEiH0xPr1012QbFEGB0xsKqJii0/SqJjyn8dFv6y36kOznMgMB+LGcbZTJ1xACQ==", + "integrity": "sha1-CNP1IFbGZnkplyb63g1DKudLRwQ=", "requires": { "bs58check": "<3.0.0" } @@ -16448,7 +16448,7 @@ "write-file-atomic": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-1.3.4.tgz", - "integrity": "sha512-SdrHoC/yVBPpV0Xq/mUZQIpW2sWXAShb/V4pomcJXh92RuaO+f3UTWItiR3Px+pLnV2PvC2/bfn5cwr5X6Vfxw==", + "integrity": "sha1-+Aek8LHZ6ROuekgRLmzDrxmRtF8=", "requires": { "graceful-fs": "^4.1.11", "imurmurhash": "^0.1.4", @@ -16529,7 +16529,7 @@ "xmlhttprequest": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz", - "integrity": "sha512-58Im/U0mlVBLM38NdZjHyhuMtCqa61469k2YP/AaPbvCoV9aQGUpbJBj1QRm2ytRiVQBD/fsw7L2bJGDVQswBA==" + "integrity": "sha1-Z/4HXFwk/vOfnWX197f+dRcZaPw=" }, "xtend": { "version": "4.0.2", diff --git a/creator-node/src/config.js b/creator-node/src/config.js index 8a1922f5a2f..efdd01ee953 100644 --- a/creator-node/src/config.js +++ b/creator-node/src/config.js @@ -265,11 +265,17 @@ const config = convict({ default: false }, expressAppConcurrency: { - doc: 'Number of processes to spawn, where each process runs its own Content Node. Default 0 to run one process per core (auto-detected)', + doc: 'Number of processes to spawn, where each process runs its own Content Node. Default 0 to run one process per core (auto-detected). Note that clusterModeEnabled must also be true for this to take effect', format: 'nat', env: 'expressAppConcurrency', default: 0 }, + clusterModeEnabled: { + doc: 'Whether or not cluster logic should be enabled (running multiple instances of the app to better utuilize multiple logical cores)', + format: Boolean, + env: 'clusterModeEnabled', + default: true + }, // Transcoding settings transcodingMaxConcurrency: { diff --git a/creator-node/src/index.ts b/creator-node/src/index.ts index 8c02ce4257d..49861f3149e 100644 --- a/creator-node/src/index.ts +++ b/creator-node/src/index.ts @@ -85,26 +85,26 @@ const verifyConfigAndDb = async () => { } } +/** + * Setting a different port is necessary for OpenResty to work. If OpenResty + * is enabled, have the app run on port 3000. Else, run on its configured port. + * @returns the port number to configure the Content Node app + */ +const getPort = () => { + const contentCacheLayerEnabled = config.get('contentCacheLayerEnabled') + + if (contentCacheLayerEnabled) { + return 3000 + } + + return config.get('port') +} + // The primary process performs one-time validation and spawns worker processes that each run the Express app const startAppForPrimary = async () => { logger.info(`Primary process with pid=${process.pid} is running`) - await verifyConfigAndDb() - await clearRunningQueries() - try { - logger.info('Executing database migrations...') - await runMigrations() - logger.info('Migrations completed successfully') - } catch (migrationError) { - exitWithError('Error in migrations:', migrationError) - } - - // Clear all redis locks - try { - await redisClient.WalletWriteLock.clearWriteLocks() - } catch (e: any) { - logger.warn(`Could not clear write locks. Skipping..: ${e.message}`) - } + await setupDbAndRedis() const numWorkers = clusterUtils.getNumWorkers() logger.info(`Spawning ${numWorkers} processes to run the Express app...`) @@ -161,27 +161,58 @@ const startAppForPrimary = async () => { // Workers don't share memory, so each one is its own Express instance with its own version of objects like serviceRegistry const startAppForWorker = async () => { - /** - * Setting a different port is necessary for OpenResty to work. If OpenResty - * is enabled, have the app run on port 3000. Else, run on its configured port. - * @returns the port number to configure the Content Node app - */ - const getPort = () => { - const contentCacheLayerEnabled = config.get('contentCacheLayerEnabled') - - if (contentCacheLayerEnabled) { - return 3000 + logger.info( + `Worker process with pid=${process.pid} and worker ID=${cluster.worker?.id} is running` + ) + await verifyConfigAndDb() + await startApp() + + cluster.worker!.on('message', (msg) => { + if (msg?.cmd === 'setSpecialWorkerId') { + clusterUtils.specialWorkerId = msg?.val + } else if (msg?.cmd === 'receiveAggregatePrometheusMetrics') { + try { + const { prometheusRegistry } = serviceRegistry + prometheusRegistry.resolvePromiseToGetAggregatedMetrics(msg?.val) + } catch (error: any) { + logger.error( + `Failed to send aggregated metrics data back to worker: ${error}` + ) + } } + }) - return config.get('port') + if (clusterUtils.isThisWorkerInit() && process.send) { + process.send({ cmd: 'initComplete' }) } +} - logger.info( - `Worker process with pid=${process.pid} and worker ID=${cluster.worker?.id} is running` - ) +const startAppWithoutCluster = async () => { + logger.info(`Starting app with cluster mode disabled`) + await setupDbAndRedis() + await startApp() +} +const setupDbAndRedis = async () => { await verifyConfigAndDb() + await clearRunningQueries() + try { + logger.info('Executing database migrations...') + await runMigrations() + logger.info('Migrations completed successfully') + } catch (migrationError) { + exitWithError('Error in migrations:', migrationError) + } + + // Clear all redis locks + try { + await redisClient.WalletWriteLock.clearWriteLocks() + } catch (e: any) { + logger.warn(`Could not clear write locks. Skipping..: ${e.message}`) + } +} +const startApp = async () => { // When app terminates, close down any open DB connections gracefully ON_DEATH((signal: any, error: any) => { // NOTE: log messages emitted here may be swallowed up if using the bunyan CLI (used by @@ -201,28 +232,12 @@ const startAppForWorker = async () => { const appInfo = initializeApp(getPort(), serviceRegistry) logger.info('Initialized app and server') await serviceRegistry.initServicesThatRequireServer(appInfo.app) - - cluster.worker!.on('message', (msg) => { - if (msg?.cmd === 'setSpecialWorkerId') { - clusterUtils.specialWorkerId = msg?.val - } else if (msg?.cmd === 'receiveAggregatePrometheusMetrics') { - try { - const { prometheusRegistry } = serviceRegistry - prometheusRegistry.resolvePromiseToGetAggregatedMetrics(msg?.val) - } catch (error: any) { - logger.error( - `Failed to send aggregated metrics data back to worker: ${error}` - ) - } - } - }) - - if (clusterUtils.isThisWorkerInit() && process.send) { - process.send({ cmd: 'initComplete' }) - } } -if (cluster.isMaster) { +if (!clusterUtils.isClusterEnabled()) { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + startAppWithoutCluster() +} else if (cluster.isMaster) { // eslint-disable-next-line @typescript-eslint/no-floating-promises startAppForPrimary() } else if (cluster.isWorker) { diff --git a/creator-node/src/routes/prometheusMetricsRoutes.js b/creator-node/src/routes/prometheusMetricsRoutes.js index 777681b369a..edf411b439b 100644 --- a/creator-node/src/routes/prometheusMetricsRoutes.js +++ b/creator-node/src/routes/prometheusMetricsRoutes.js @@ -1,24 +1,32 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ const express = require('express') +const { clusterUtils } = require('../utils') const router = express.Router() -/** - * Exposes Prometheus metrics for the worker (not aggregated) at `GET /prometheus_metrics_worker` - */ - -router.get('/prometheus_metrics_worker', async (req, res) => { +const returnMetricsForSingleProcess = async (req, res) => { const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry const metricData = await prometheusRegistry.getAllMetricData() res.setHeader('Content-Type', prometheusRegistry.registry.contentType) return res.end(metricData) +} + +/** + * Exposes Prometheus metrics for the worker (not aggregated) at `GET /prometheus_metrics_worker` + */ +router.get('/prometheus_metrics_worker', async (req, res) => { + return returnMetricsForSingleProcess(req, res) }) /** * Exposes Prometheus metrics aggregated across all workers at `GET /prometheus_metrics` */ router.get('/prometheus_metrics', async (req, res) => { + if (!clusterUtils.isClusterEnabled()) { + return returnMetricsForSingleProcess(req, res) + } + try { const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry const { metricsData, contentType } = diff --git a/creator-node/src/utils/clusterUtils.ts b/creator-node/src/utils/clusterUtils.ts index 6dd5c6ac858..4d292ae6d4d 100644 --- a/creator-node/src/utils/clusterUtils.ts +++ b/creator-node/src/utils/clusterUtils.ts @@ -20,16 +20,26 @@ class ClusterUtils { this._specialWorkerId = specialWorkerId } + /** + * Returns true if cluster mode is enabled. If it's disabled, then + * everything runs on one process with no primary or workers. + */ + isClusterEnabled() { + return config.get('clusterModeEnabled') + } + /** * Returns true if this current worker process is the first worker, which performs * some special initialization logic that other workers don't need to duplicate. */ isThisWorkerInit() { - return cluster.worker?.id === 1 + return !this.isClusterEnabled() || cluster.worker?.id === 1 } isThisWorkerSpecial() { - return cluster.worker?.id === this._specialWorkerId + return ( + !this.isClusterEnabled() || cluster.worker?.id === this._specialWorkerId + ) } getNumWorkers() {