diff --git a/config.json b/config.json index 1a6faf973a..7d0f7d9efe 100644 --- a/config.json +++ b/config.json @@ -14,29 +14,34 @@ "zenko-cloudserver-replicator": "us-east-1", "lb": "us-east-1" }, - "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com", - "s3-website.us-east-2.amazonaws.com", - "s3-website-us-west-1.amazonaws.com", - "s3-website-us-west-2.amazonaws.com", - "s3-website.ap-south-1.amazonaws.com", - "s3-website.ap-northeast-2.amazonaws.com", - "s3-website-ap-southeast-1.amazonaws.com", - "s3-website-ap-southeast-2.amazonaws.com", - "s3-website-ap-northeast-1.amazonaws.com", - "s3-website.eu-central-1.amazonaws.com", - "s3-website-eu-west-1.amazonaws.com", - "s3-website-sa-east-1.amazonaws.com", - "s3-website.localhost", - "s3-website.scality.test", - "zenkoazuretest.blob.core.windows.net"], - "replicationEndpoints": [{ - "site": "zenko", - "servers": ["127.0.0.1:8000"], - "default": true - }, { - "site": "us-east-2", - "type": "aws_s3" - }], + "websiteEndpoints": [ + "s3-website-us-east-1.amazonaws.com", + "s3-website.us-east-2.amazonaws.com", + "s3-website-us-west-1.amazonaws.com", + "s3-website-us-west-2.amazonaws.com", + "s3-website.ap-south-1.amazonaws.com", + "s3-website.ap-northeast-2.amazonaws.com", + "s3-website-ap-southeast-1.amazonaws.com", + "s3-website-ap-southeast-2.amazonaws.com", + "s3-website-ap-northeast-1.amazonaws.com", + "s3-website.eu-central-1.amazonaws.com", + "s3-website-eu-west-1.amazonaws.com", + "s3-website-sa-east-1.amazonaws.com", + "s3-website.localhost", + "s3-website.scality.test", + "zenkoazuretest.blob.core.windows.net" + ], + "replicationEndpoints": [ + { + "site": "zenko", + "servers": ["127.0.0.1:8000"], + "default": true + }, + { + "site": "us-east-2", + "type": "aws_s3" + } + ], "backbeat": { "host": "localhost", "port": 8900 @@ -135,7 +140,7 @@ "kmsHideScalityArn": false, "kmsAWS": { "providerName": "aws", - "region": "us-east-1", + "region": "us-east-1", "endpoint": "http://127.0.0.1:8080", "ak": "tbd", "sk": "tbd" @@ -147,9 +152,6 @@ "multiObjectDelete": 2097152, "bucketPutPolicy": 20480 }, - "integrityChecks": { - "objectPutRetention": true - }, "serverAccessLogs": { "mode": "DISABLED", "outputFile": "/logs/server-access.log", diff --git a/lib/Config.js b/lib/Config.js index ccfc5d11d3..3c9887fed1 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -35,10 +35,7 @@ const { const { parseRateLimitConfig } = require('./api/apiUtils/rateLimit/config'); // config paths -const configSearchPaths = [ - path.join(__dirname, '../conf'), - path.join(__dirname, '..'), -]; +const configSearchPaths = [path.join(__dirname, '../conf'), path.join(__dirname, '..')]; function findConfigFile(fileName) { if (fileName[0] === '/') { @@ -49,8 +46,10 @@ function findConfigFile(fileName) { return fs.existsSync(testFilePath); }); if (!containingPath) { - throw new Error(`Unable to find the configuration file "${fileName}" ` + - `under the paths: ${JSON.stringify(configSearchPaths)}`); + throw new Error( + `Unable to find the configuration file "${fileName}" ` + + `under the paths: ${JSON.stringify(configSearchPaths)}`, + ); } return path.join(containingPath, fileName); } @@ -85,26 +84,26 @@ function assertCertPaths(key, cert, ca, basePath) { certObj.certs = {}; if (key) { const keypath = key.startsWith('/') ? key : `${basePath}/${key}`; - assert.doesNotThrow(() => - fs.accessSync(keypath, fs.F_OK | fs.R_OK), - `File not found or unreachable: ${keypath}`); + assert.doesNotThrow( + () => fs.accessSync(keypath, fs.F_OK | fs.R_OK), + `File not found or unreachable: ${keypath}`, + ); certObj.paths.key = keypath; certObj.certs.key = fs.readFileSync(keypath, 'ascii'); } if (cert) { const certpath = cert.startsWith('/') ? cert : `${basePath}/${cert}`; - assert.doesNotThrow(() => - fs.accessSync(certpath, fs.F_OK | fs.R_OK), - `File not found or unreachable: ${certpath}`); + assert.doesNotThrow( + () => fs.accessSync(certpath, fs.F_OK | fs.R_OK), + `File not found or unreachable: ${certpath}`, + ); certObj.paths.cert = certpath; certObj.certs.cert = fs.readFileSync(certpath, 'ascii'); } if (ca) { const capath = ca.startsWith('/') ? ca : `${basePath}/${ca}`; - assert.doesNotThrow(() => - fs.accessSync(capath, fs.F_OK | fs.R_OK), - `File not found or unreachable: ${capath}`); + assert.doesNotThrow(() => fs.accessSync(capath, fs.F_OK | fs.R_OK), `File not found or unreachable: ${capath}`); certObj.paths.ca = capath; certObj.certs.ca = fs.readFileSync(capath, 'ascii'); } @@ -121,48 +120,56 @@ function parseSproxydConfig(configSproxyd) { } function parseRedisConfig(redisConfig) { - const joiSchema = joi.object({ - password: joi.string().allow(''), - host: joi.string(), - port: joi.number(), - retry: joi.object({ - connectBackoff: joi.object({ - min: joi.number().required(), - max: joi.number().required(), - jitter: joi.number().required(), - factor: joi.number().required(), - deadline: joi.number().required(), + const joiSchema = joi + .object({ + password: joi.string().allow(''), + host: joi.string(), + port: joi.number(), + retry: joi.object({ + connectBackoff: joi.object({ + min: joi.number().required(), + max: joi.number().required(), + jitter: joi.number().required(), + factor: joi.number().required(), + deadline: joi.number().required(), + }), }), - }), - // sentinel config - sentinels: joi.alternatives().try( - joi.string() - .pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/) - .custom(hosts => hosts.split(',').map(item => { - const [host, port] = item.split(':'); - return { host, port: Number.parseInt(port, 10) }; - })), - joi.array().items( - joi.object({ - host: joi.string().required(), - port: joi.number().required(), - }) - ).min(1), - ), - name: joi.string(), - sentinelPassword: joi.string().allow(''), - }) - .and('host', 'port') - .and('sentinels', 'name') - .xor('host', 'sentinels') - .without('sentinels', ['host', 'port']) - .without('host', ['sentinels', 'sentinelPassword']); + // sentinel config + sentinels: joi.alternatives().try( + joi + .string() + .pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/) + .custom(hosts => + hosts.split(',').map(item => { + const [host, port] = item.split(':'); + return { host, port: Number.parseInt(port, 10) }; + }), + ), + joi + .array() + .items( + joi.object({ + host: joi.string().required(), + port: joi.number().required(), + }), + ) + .min(1), + ), + name: joi.string(), + sentinelPassword: joi.string().allow(''), + }) + .and('host', 'port') + .and('sentinels', 'name') + .xor('host', 'sentinels') + .without('sentinels', ['host', 'port']) + .without('host', ['sentinels', 'sentinelPassword']); return joi.attempt(redisConfig, joiSchema, 'bad config'); } function parseSupportedLifecycleRules(supportedLifecycleRulesConfig) { - const supportedLifecycleRulesSchema = joi.array() + const supportedLifecycleRulesSchema = joi + .array() .items(joi.string().valid(...supportedLifecycleRules)) .default(supportedLifecycleRules) .min(1); @@ -174,52 +181,38 @@ function parseSupportedLifecycleRules(supportedLifecycleRulesConfig) { } function restEndpointsAssert(restEndpoints, locationConstraints) { - assert(typeof restEndpoints === 'object', - 'bad config: restEndpoints must be an object of endpoints'); - assert(Object.keys(restEndpoints).every( - r => typeof restEndpoints[r] === 'string'), - 'bad config: each endpoint must be a string'); - assert(Object.keys(restEndpoints).every( - r => typeof locationConstraints[restEndpoints[r]] === 'object'), - 'bad config: rest endpoint target not in locationConstraints'); + assert(typeof restEndpoints === 'object', 'bad config: restEndpoints must be an object of endpoints'); + assert( + Object.keys(restEndpoints).every(r => typeof restEndpoints[r] === 'string'), + 'bad config: each endpoint must be a string', + ); + assert( + Object.keys(restEndpoints).every(r => typeof locationConstraints[restEndpoints[r]] === 'object'), + 'bad config: rest endpoint target not in locationConstraints', + ); } function gcpLocationConstraintAssert(location, locationObj) { - const { - gcpEndpoint, - bucketName, - mpuBucketName, - } = locationObj.details; - const stringFields = [ - gcpEndpoint, - bucketName, - mpuBucketName, - ]; + const { gcpEndpoint, bucketName, mpuBucketName } = locationObj.details; + const stringFields = [gcpEndpoint, bucketName, mpuBucketName]; stringFields.forEach(field => { if (field !== undefined) { - assert(typeof field === 'string', - `bad config: ${field} must be a string`); + assert(typeof field === 'string', `bad config: ${field} must be a string`); } }); } function azureGetStorageAccountName(location, locationDetails) { const { azureStorageAccountName } = locationDetails; - const storageAccountNameFromEnv = - process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`]; + const storageAccountNameFromEnv = process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`]; return storageAccountNameFromEnv || azureStorageAccountName; } function azureGetLocationCredentials(location, locationDetails) { const storageAccessKey = - process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`] || - locationDetails.azureStorageAccessKey; - const sasToken = - process.env[`${location}_AZURE_SAS_TOKEN`] || - locationDetails.sasToken; - const clientKey = - process.env[`${location}_AZURE_CLIENT_KEY`] || - locationDetails.clientKey; + process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`] || locationDetails.azureStorageAccessKey; + const sasToken = process.env[`${location}_AZURE_SAS_TOKEN`] || locationDetails.sasToken; + const clientKey = process.env[`${location}_AZURE_CLIENT_KEY`] || locationDetails.clientKey; const authMethod = process.env[`${location}_AZURE_AUTH_METHOD`] || @@ -230,32 +223,27 @@ function azureGetLocationCredentials(location, locationDetails) { 'shared-key'; switch (authMethod) { - case 'shared-key': - default: - return { - authMethod, - storageAccountName: - azureGetStorageAccountName(location, locationDetails), - storageAccessKey, - }; + case 'shared-key': + default: + return { + authMethod, + storageAccountName: azureGetStorageAccountName(location, locationDetails), + storageAccessKey, + }; - case 'shared-access-signature': - return { - authMethod, - sasToken, - }; + case 'shared-access-signature': + return { + authMethod, + sasToken, + }; - case 'client-secret': - return { - authMethod, - tenantId: - process.env[`${location}_AZURE_TENANT_ID`] || - locationDetails.tenantId, - clientId: - process.env[`${location}_AZURE_CLIENT_ID`] || - locationDetails.clientId, - clientKey, - }; + case 'client-secret': + return { + authMethod, + tenantId: process.env[`${location}_AZURE_TENANT_ID`] || locationDetails.tenantId, + clientId: process.env[`${location}_AZURE_CLIENT_ID`] || locationDetails.clientId, + clientKey, + }; } } @@ -263,120 +251,128 @@ function azureLocationConstraintAssert(location, locationObj) { const locationParams = { ...azureGetLocationCredentials(location, locationObj.details), azureStorageEndpoint: - process.env[`${location}_AZURE_STORAGE_ENDPOINT`] || - locationObj.details.azureStorageEndpoint, + process.env[`${location}_AZURE_STORAGE_ENDPOINT`] || locationObj.details.azureStorageEndpoint, azureContainerName: locationObj.details.azureContainerName, }; Object.keys(locationParams).forEach(param => { const value = locationParams[param]; - assert.notEqual(value, undefined, + assert.notEqual( + value, + undefined, `bad location constraint: "${location}" ${param} ` + - 'must be set in locationConfig or environment variable'); - assert.strictEqual(typeof value, 'string', - `bad location constraint: "${location}" ${param} ` + - `"${value}" must be a string`); + 'must be set in locationConfig or environment variable', + ); + assert.strictEqual( + typeof value, + 'string', + `bad location constraint: "${location}" ${param} ` + `"${value}" must be a string`, + ); }); if (locationParams.authMethod === 'shared-key') { - assert(azureAccountNameRegex.test(locationParams.storageAccountName), + assert( + azureAccountNameRegex.test(locationParams.storageAccountName), `bad location constraint: "${location}" azureStorageAccountName ` + - `"${locationParams.storageAccountName}" is an invalid value`); - assert(base64Regex.test(locationParams.storageAccessKey), - `bad location constraint: "${location}" ` + - 'azureStorageAccessKey is not a valid base64 string'); + `"${locationParams.storageAccountName}" is an invalid value`, + ); + assert( + base64Regex.test(locationParams.storageAccessKey), + `bad location constraint: "${location}" ` + 'azureStorageAccessKey is not a valid base64 string', + ); } - assert(isValidBucketName(locationParams.azureContainerName, []), - `bad location constraint: "${location}" ` + - 'azureContainerName is an invalid container name'); + assert( + isValidBucketName(locationParams.azureContainerName, []), + `bad location constraint: "${location}" ` + 'azureContainerName is an invalid container name', + ); } function hdClientLocationConstraintAssert(configHd) { const hdclientFields = []; if (configHd.bootstrap !== undefined) { - assert(Array.isArray(configHd.bootstrap) - && configHd.bootstrap - .every(e => typeof e === 'string'), - 'bad config: hdclient.bootstrap must be an array of strings'); - assert(configHd.bootstrap.length > 0, - 'bad config: hdclient bootstrap list is empty'); + assert( + Array.isArray(configHd.bootstrap) && configHd.bootstrap.every(e => typeof e === 'string'), + 'bad config: hdclient.bootstrap must be an array of strings', + ); + assert(configHd.bootstrap.length > 0, 'bad config: hdclient bootstrap list is empty'); hdclientFields.push('bootstrap'); } return hdclientFields; } function locationConstraintAssert(locationConstraints) { - const supportedBackends = [ - 'mem', 'file', 'scality', 'mongodb', 'tlp', 'crr' - ].concat(Object.keys(validExternalBackends)); - assert(typeof locationConstraints === 'object', - 'bad config: locationConstraints must be an object'); + const supportedBackends = ['mem', 'file', 'scality', 'mongodb', 'tlp', 'crr'].concat( + Object.keys(validExternalBackends), + ); + assert(typeof locationConstraints === 'object', 'bad config: locationConstraints must be an object'); Object.keys(locationConstraints).forEach(l => { - assert(typeof locationConstraints[l] === 'object', - 'bad config: locationConstraints[region] must be an object'); - assert(typeof locationConstraints[l].type === 'string', - 'bad config: locationConstraints[region].type is ' + - 'mandatory and must be a string'); - assert(supportedBackends.indexOf(locationConstraints[l].type) > -1, - 'bad config: locationConstraints[region].type must ' + - `be one of ${supportedBackends}`); - assert(typeof locationConstraints[l].objectId === 'string', - 'bad config: locationConstraints[region].objectId is ' + - 'mandatory and must be a unique string across locations'); - assert(Object.keys(locationConstraints) - .filter(loc => (locationConstraints[loc].objectId === - locationConstraints[l].objectId)) - .length === 1, - 'bad config: location constraint objectId ' + - `"${locationConstraints[l].objectId}" is not unique across ` + - 'configured locations'); - assert(typeof locationConstraints[l].legacyAwsBehavior - === 'boolean', - 'bad config: locationConstraints[region]' + - '.legacyAwsBehavior is mandatory and must be a boolean'); - assert(['undefined', 'boolean'].includes( - typeof locationConstraints[l].isTransient), - 'bad config: locationConstraints[region]' + - '.isTransient must be a boolean'); + assert(typeof locationConstraints[l] === 'object', 'bad config: locationConstraints[region] must be an object'); + assert( + typeof locationConstraints[l].type === 'string', + 'bad config: locationConstraints[region].type is ' + 'mandatory and must be a string', + ); + assert( + supportedBackends.indexOf(locationConstraints[l].type) > -1, + 'bad config: locationConstraints[region].type must ' + `be one of ${supportedBackends}`, + ); + assert( + typeof locationConstraints[l].objectId === 'string', + 'bad config: locationConstraints[region].objectId is ' + + 'mandatory and must be a unique string across locations', + ); + assert( + Object.keys(locationConstraints).filter( + loc => locationConstraints[loc].objectId === locationConstraints[l].objectId, + ).length === 1, + 'bad config: location constraint objectId ' + + `"${locationConstraints[l].objectId}" is not unique across ` + + 'configured locations', + ); + assert( + typeof locationConstraints[l].legacyAwsBehavior === 'boolean', + 'bad config: locationConstraints[region]' + '.legacyAwsBehavior is mandatory and must be a boolean', + ); + assert( + ['undefined', 'boolean'].includes(typeof locationConstraints[l].isTransient), + 'bad config: locationConstraints[region]' + '.isTransient must be a boolean', + ); if (locationConstraints[l].sizeLimitGB !== undefined) { - assert(typeof locationConstraints[l].sizeLimitGB === 'number' || - locationConstraints[l].sizeLimitGB === null, - 'bad config: locationConstraints[region].sizeLimitGB ' + - 'must be a number (in gigabytes)'); + assert( + typeof locationConstraints[l].sizeLimitGB === 'number' || locationConstraints[l].sizeLimitGB === null, + 'bad config: locationConstraints[region].sizeLimitGB ' + 'must be a number (in gigabytes)', + ); } const details = locationConstraints[l].details; - assert(typeof details === 'object', - 'bad config: locationConstraints[region].details is ' + - 'mandatory and must be an object'); + assert( + typeof details === 'object', + 'bad config: locationConstraints[region].details is ' + 'mandatory and must be an object', + ); if (details.serverSideEncryption !== undefined) { - assert(typeof details.serverSideEncryption === 'boolean', - 'bad config: locationConstraints[region]' + - '.details.serverSideEncryption must be a boolean'); - } - const stringFields = [ - 'awsEndpoint', - 'bucketName', - 'credentialsProfile', - 'region', - ]; + assert( + typeof details.serverSideEncryption === 'boolean', + 'bad config: locationConstraints[region]' + '.details.serverSideEncryption must be a boolean', + ); + } + const stringFields = ['awsEndpoint', 'bucketName', 'credentialsProfile', 'region']; stringFields.forEach(field => { if (details[field] !== undefined) { - assert(typeof details[field] === 'string', - `bad config: ${field} must be a string`); + assert(typeof details[field] === 'string', `bad config: ${field} must be a string`); } }); if (details.bucketMatch !== undefined) { - assert(typeof details.bucketMatch === 'boolean', - 'bad config: details.bucketMatch must be a boolean'); + assert(typeof details.bucketMatch === 'boolean', 'bad config: details.bucketMatch must be a boolean'); } if (details.credentials !== undefined) { - assert(typeof details.credentials === 'object', - 'bad config: details.credentials must be an object'); - assert(typeof details.credentials.accessKey === 'string', - 'bad config: credentials must include accessKey as string'); - assert(typeof details.credentials.secretKey === 'string', - 'bad config: credentials must include secretKey as string'); + assert(typeof details.credentials === 'object', 'bad config: details.credentials must be an object'); + assert( + typeof details.credentials.accessKey === 'string', + 'bad config: credentials must include accessKey as string', + ); + assert( + typeof details.credentials.secretKey === 'string', + 'bad config: credentials must include secretKey as string', + ); } if (locationConstraints[l].type === 'tlp') { @@ -388,25 +384,30 @@ function locationConstraintAssert(locationConstraints) { } if (details.https !== undefined) { - assert(typeof details.https === 'boolean', 'bad config: ' + - 'locationConstraints[region].details https must be a boolean'); + assert( + typeof details.https === 'boolean', + 'bad config: ' + 'locationConstraints[region].details https must be a boolean', + ); } else { // eslint-disable-next-line no-param-reassign locationConstraints[l].details.https = true; } if (details.pathStyle !== undefined) { - assert(typeof details.pathStyle === 'boolean', 'bad config: ' + - 'locationConstraints[region].pathStyle must be a boolean'); + assert( + typeof details.pathStyle === 'boolean', + 'bad config: ' + 'locationConstraints[region].pathStyle must be a boolean', + ); } else { // eslint-disable-next-line no-param-reassign locationConstraints[l].details.pathStyle = false; } if (details.supportsVersioning !== undefined) { - assert(typeof details.supportsVersioning === 'boolean', - 'bad config: locationConstraints[region].supportsVersioning' + - 'must be a boolean'); + assert( + typeof details.supportsVersioning === 'boolean', + 'bad config: locationConstraints[region].supportsVersioning' + 'must be a boolean', + ); } else { // default to true // eslint-disable-next-line no-param-reassign @@ -420,52 +421,45 @@ function locationConstraintAssert(locationConstraints) { gcpLocationConstraintAssert(l, locationConstraints[l]); } if (locationConstraints[l].type === 'pfs') { - assert(typeof details.pfsDaemonEndpoint === 'object', - 'bad config: pfsDaemonEndpoint is mandatory and must be an object'); + assert( + typeof details.pfsDaemonEndpoint === 'object', + 'bad config: pfsDaemonEndpoint is mandatory and must be an object', + ); } - if (locationConstraints[l].type === 'scality' && + if ( + locationConstraints[l].type === 'scality' && locationConstraints[l].details.connector !== undefined && - locationConstraints[l].details.connector.hdclient !== undefined) { - hdClientLocationConstraintAssert( - locationConstraints[l].details.connector.hdclient); + locationConstraints[l].details.connector.hdclient !== undefined + ) { + hdClientLocationConstraintAssert(locationConstraints[l].details.connector.hdclient); } }); - assert(Object.keys(locationConstraints) - .includes('us-east-1'), 'bad locationConfig: must ' + - 'include us-east-1 as a locationConstraint'); + assert( + Object.keys(locationConstraints).includes('us-east-1'), + 'bad locationConfig: must ' + 'include us-east-1 as a locationConstraint', + ); } function parseUtapiReindex(config) { - const { - enabled, - schedule, - redis, - bucketd, - onlyCountLatestWhenObjectLocked, - } = config; - assert(typeof enabled === 'boolean', - 'bad config: utapi.reindex.enabled must be a boolean'); + const { enabled, schedule, redis, bucketd, onlyCountLatestWhenObjectLocked } = config; + assert(typeof enabled === 'boolean', 'bad config: utapi.reindex.enabled must be a boolean'); const parsedRedis = parseRedisConfig(redis); - assert(Array.isArray(parsedRedis.sentinels), - 'bad config: utapi reindex redis config requires a list of sentinels'); - - assert(typeof bucketd === 'object', - 'bad config: utapi.reindex.bucketd must be an object'); - assert(typeof bucketd.port === 'number', - 'bad config: utapi.reindex.bucketd.port must be a number'); - assert(typeof schedule === 'string', - 'bad config: utapi.reindex.schedule must be a string'); + assert(Array.isArray(parsedRedis.sentinels), 'bad config: utapi reindex redis config requires a list of sentinels'); + + assert(typeof bucketd === 'object', 'bad config: utapi.reindex.bucketd must be an object'); + assert(typeof bucketd.port === 'number', 'bad config: utapi.reindex.bucketd.port must be a number'); + assert(typeof schedule === 'string', 'bad config: utapi.reindex.schedule must be a string'); if (onlyCountLatestWhenObjectLocked !== undefined) { - assert(typeof onlyCountLatestWhenObjectLocked === 'boolean', - 'bad config: utapi.reindex.onlyCountLatestWhenObjectLocked must be a boolean'); + assert( + typeof onlyCountLatestWhenObjectLocked === 'boolean', + 'bad config: utapi.reindex.onlyCountLatestWhenObjectLocked must be a boolean', + ); } try { cronParser.parseExpression(schedule); } catch (e) { - assert(false, - 'bad config: utapi.reindex.schedule must be a valid ' + - `cron schedule. ${e.message}.`); + assert(false, 'bad config: utapi.reindex.schedule must be a valid ' + `cron schedule. ${e.message}.`); } return { enabled, @@ -478,81 +472,64 @@ function parseUtapiReindex(config) { function requestsConfigAssert(requestsConfig) { if (requestsConfig.viaProxy !== undefined) { - assert(typeof requestsConfig.viaProxy === 'boolean', - 'config: invalid requests configuration. viaProxy must be a ' + - 'boolean'); + assert( + typeof requestsConfig.viaProxy === 'boolean', + 'config: invalid requests configuration. viaProxy must be a ' + 'boolean', + ); if (requestsConfig.viaProxy) { - assert(Array.isArray(requestsConfig.trustedProxyCIDRs) && - requestsConfig.trustedProxyCIDRs.length > 0 && - requestsConfig.trustedProxyCIDRs - .every(ip => typeof ip === 'string'), - 'config: invalid requests configuration. ' + - 'trustedProxyCIDRs must be set if viaProxy is set to true ' + - 'and must be an array'); - - assert(typeof requestsConfig.extractClientIPFromHeader === 'string' - && requestsConfig.extractClientIPFromHeader.length > 0, - 'config: invalid requests configuration. ' + - 'extractClientIPFromHeader must be set if viaProxy is ' + - 'set to true and must be a string'); - - assert(typeof requestsConfig.extractProtocolFromHeader === 'string' - && requestsConfig.extractProtocolFromHeader.length > 0, - 'config: invalid requests configuration. ' + - 'extractProtocolFromHeader must be set if viaProxy is ' + - 'set to true and must be a string'); + assert( + Array.isArray(requestsConfig.trustedProxyCIDRs) && + requestsConfig.trustedProxyCIDRs.length > 0 && + requestsConfig.trustedProxyCIDRs.every(ip => typeof ip === 'string'), + 'config: invalid requests configuration. ' + + 'trustedProxyCIDRs must be set if viaProxy is set to true ' + + 'and must be an array', + ); + + assert( + typeof requestsConfig.extractClientIPFromHeader === 'string' && + requestsConfig.extractClientIPFromHeader.length > 0, + 'config: invalid requests configuration. ' + + 'extractClientIPFromHeader must be set if viaProxy is ' + + 'set to true and must be a string', + ); + + assert( + typeof requestsConfig.extractProtocolFromHeader === 'string' && + requestsConfig.extractProtocolFromHeader.length > 0, + 'config: invalid requests configuration. ' + + 'extractProtocolFromHeader must be set if viaProxy is ' + + 'set to true and must be a string', + ); } // All headers in NodeJS are lowercase: to be exploitable // we need to lowercase the value. // eslint-disable-next-line no-param-reassign - requestsConfig.extractClientIPFromHeader = - requestsConfig.extractClientIPFromHeader?.toLowerCase(); + requestsConfig.extractClientIPFromHeader = requestsConfig.extractClientIPFromHeader?.toLowerCase(); // eslint-disable-next-line no-param-reassign - requestsConfig.extractProtocolFromHeader = - requestsConfig.extractProtocolFromHeader?.toLowerCase(); + requestsConfig.extractProtocolFromHeader = requestsConfig.extractProtocolFromHeader?.toLowerCase(); } } function bucketNotifAssert(bucketNotifConfig) { - assert(Array.isArray(bucketNotifConfig), - 'bad config: bucket notification configuration must be an array'); + assert(Array.isArray(bucketNotifConfig), 'bad config: bucket notification configuration must be an array'); bucketNotifConfig.forEach(c => { const { resource, type, host, port, auth } = c; - assert(typeof resource === 'string', - 'bad config: bucket notification configuration resource must be a string'); - assert(typeof type === 'string', - 'bad config: bucket notification configuration type must be a string'); - assert(typeof host === 'string' && host !== '', - 'bad config: hostname must be a non-empty string'); + assert(typeof resource === 'string', 'bad config: bucket notification configuration resource must be a string'); + assert(typeof type === 'string', 'bad config: bucket notification configuration type must be a string'); + assert(typeof host === 'string' && host !== '', 'bad config: hostname must be a non-empty string'); if (port) { - assert(Number.isInteger(port, 10) && port > 0, - 'bad config: port must be a positive integer'); + assert(Number.isInteger(port, 10) && port > 0, 'bad config: port must be a positive integer'); } if (auth) { - assert(typeof auth === 'object', - 'bad config: bucket notification auth must be an object'); + assert(typeof auth === 'object', 'bad config: bucket notification auth must be an object'); } }); return bucketNotifConfig; } -function parseIntegrityChecks(config) { - const integrityChecks = {}; - - if (config && config.integrityChecks) { - for (const method in integrityChecks) { - if (method in config.integrityChecks) { - assert(typeof config.integrityChecks[method] == 'boolean', `bad config: ${method} not boolean`); - integrityChecks[method] = config.integrityChecks[method]; - } - } - } - - return integrityChecks; -} - const serverAccessLogsModes = { DISABLED: 'DISABLED', LOG_ONLY: 'LOG_ONLY', @@ -580,21 +557,27 @@ function parseServerAccessLogs(config) { settings.forEach(setting => { if (setting.key in config.serverAccessLogs) { - assert(typeof config.serverAccessLogs[setting.key] === setting.type, - `bad config: serverAccessLogs.${setting.key} is not a ${setting.type}`); + assert( + typeof config.serverAccessLogs[setting.key] === setting.type, + `bad config: serverAccessLogs.${setting.key} is not a ${setting.type}`, + ); res[setting.key] = config.serverAccessLogs[setting.key]; } }); if ('mode' in config.serverAccessLogs) { - assert(validModes.includes(config.serverAccessLogs.mode), - `bad config: serverAccessLogs.mode must be one of: ${validModes.join(', ')}`); + assert( + validModes.includes(config.serverAccessLogs.mode), + `bad config: serverAccessLogs.mode must be one of: ${validModes.join(', ')}`, + ); } } if (process.env.S3_SERVER_ACCESS_LOGS_MODE) { - assert(validModes.includes(process.env.S3_SERVER_ACCESS_LOGS_MODE), - `bad config: S3_SERVER_ACCESS_LOGS_MODE must be one of: ${validModes.join(', ')}`); + assert( + validModes.includes(process.env.S3_SERVER_ACCESS_LOGS_MODE), + `bad config: S3_SERVER_ACCESS_LOGS_MODE must be one of: ${validModes.join(', ')}`, + ); res.mode = process.env.S3_SERVER_ACCESS_LOGS_MODE; } @@ -621,16 +604,13 @@ class Config extends EventEmitter { * the S3_LOCATION_FILE environment var. */ this._basePath = path.join(__dirname, '..'); - this.configPath = findConfigFile(process.env.S3_CONFIG_FILE || - 'config.json'); + this.configPath = findConfigFile(process.env.S3_CONFIG_FILE || 'config.json'); let locationConfigFileName = 'locationConfig.json'; if (process.env.CI === 'true' && !process.env.S3_END_TO_END) { - locationConfigFileName = - 'tests/locationConfig/locationConfigTests.json'; + locationConfigFileName = 'tests/locationConfig/locationConfigTests.json'; } - this.locationConfigPath = findConfigFile(process.env.S3_LOCATION_FILE || - locationConfigFileName); + this.locationConfigPath = findConfigFile(process.env.S3_LOCATION_FILE || locationConfigFileName); if (process.env.S3_REPLICATION_FILE !== undefined) { this.replicationConfigPath = process.env.S3_REPLICATION_FILE; @@ -652,13 +632,17 @@ class Config extends EventEmitter { const { providerName, region, endpoint, ak, sk, tls, noAwsArn } = config.kmsAWS; assert(providerName, 'Configuration Error: providerName must be defined in kmsAWS'); - assert(isValidProvider(providerName), - 'Configuration Error: kmsAWS.providerNamer must be lowercase alphanumeric only'); + assert( + isValidProvider(providerName), + 'Configuration Error: kmsAWS.providerNamer must be lowercase alphanumeric only', + ); assert(endpoint, 'Configuration Error: endpoint must be defined in kmsAWS'); assert(ak, 'Configuration Error: ak must be defined in kmsAWS'); assert(sk, 'Configuration Error: sk must be defined in kmsAWS'); - assert(['undefined', 'boolean'].some(type => type === typeof noAwsArn), - 'Configuration Error:: kmsAWS.noAwsArn must be a boolean or not set'); + assert( + ['undefined', 'boolean'].some(type => type === typeof noAwsArn), + 'Configuration Error:: kmsAWS.noAwsArn must be a boolean or not set', + ); kmsAWS = { providerName, @@ -684,13 +668,11 @@ class Config extends EventEmitter { // min & max TLS: One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' // (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) if (tls.minVersion !== undefined) { - assert(typeof tls.minVersion === 'string', - 'bad config: KMS AWS TLS minVersion must be a string'); + assert(typeof tls.minVersion === 'string', 'bad config: KMS AWS TLS minVersion must be a string'); kmsAWS.tls.minVersion = tls.minVersion; } if (tls.maxVersion !== undefined) { - assert(typeof tls.maxVersion === 'string', - 'bad config: KMS AWS TLS maxVersion must be a string'); + assert(typeof tls.maxVersion === 'string', 'bad config: KMS AWS TLS maxVersion must be a string'); kmsAWS.tls.maxVersion = tls.maxVersion; } if (tls.ca !== undefined) { @@ -725,11 +707,8 @@ class Config extends EventEmitter { // for customization per host host: process.env.S3KMIP_HOSTS || process.env.S3KMIP_HOST, key: this._loadTlsFile(process.env.S3KMIP_KEY || undefined), - cert: this._loadTlsFile(process.env.S3KMIP_CERT || - undefined), - ca: (process.env.S3KMIP_CA - ? process.env.S3KMIP_CA.split(',') - : []).map(ca => this._loadTlsFile(ca)), + cert: this._loadTlsFile(process.env.S3KMIP_CERT || undefined), + ca: (process.env.S3KMIP_CA ? process.env.S3KMIP_CA.split(',') : []).map(ca => this._loadTlsFile(ca)), }, }; if (transportKmip.pipelineDepth) { @@ -739,17 +718,14 @@ class Config extends EventEmitter { if (transportKmip.tls) { const { host, port, key, cert, ca } = transportKmip.tls; if (!!key !== !!cert) { - throw new Error('bad config: KMIP TLS certificate ' + - 'and key must come along'); + throw new Error('bad config: KMIP TLS certificate ' + 'and key must come along'); } if (port) { - assert(typeof port === 'number', - 'bad config: KMIP TLS Port must be a number'); + assert(typeof port === 'number', 'bad config: KMIP TLS Port must be a number'); transport.tls.port = port; } if (host) { - assert(typeof host === 'string', - 'bad config: KMIP TLS Host must be a string'); + assert(typeof host === 'string', 'bad config: KMIP TLS Host must be a string'); transport.tls.host = host; } if (key) { @@ -777,52 +753,50 @@ class Config extends EventEmitter { * time for `now' instead of client specified activation date * which also targets the present instant. */ - compoundCreateActivate: - (process.env.S3KMIP_COMPOUND_CREATE === 'true') || false, + compoundCreateActivate: process.env.S3KMIP_COMPOUND_CREATE === 'true' || false, /** Set the bucket name attribute name here if the KMIP * server supports storing custom attributes along * with the keys. */ - bucketNameAttributeName: - process.env.S3KMIP_BUCKET_ATTRIBUTE_NAME || '', + bucketNameAttributeName: process.env.S3KMIP_BUCKET_ATTRIBUTE_NAME || '', }, transport: this._parseKmipTransport({}), retries: 0, }; if (config.kmip) { assert(config.kmip.providerName, 'config.kmip.providerName must be defined'); - assert(isValidProvider(config.kmip.providerName), - 'config.kmip.providerName must be lowercase alphanumeric only'); + assert( + isValidProvider(config.kmip.providerName), + 'config.kmip.providerName must be lowercase alphanumeric only', + ); this.kmip.providerName = config.kmip.providerName; if (config.kmip.client) { if (config.kmip.client.compoundCreateActivate) { - assert(typeof config.kmip.client.compoundCreateActivate === - 'boolean'); - this.kmip.client.compoundCreateActivate = - config.kmip.client.compoundCreateActivate; + assert(typeof config.kmip.client.compoundCreateActivate === 'boolean'); + this.kmip.client.compoundCreateActivate = config.kmip.client.compoundCreateActivate; } if (config.kmip.client.bucketNameAttributeName) { - assert(typeof config.kmip.client.bucketNameAttributeName === - 'string'); - this.kmip.client.bucketNameAttributeName = - config.kmip.client.bucketNameAttributeName; + assert(typeof config.kmip.client.bucketNameAttributeName === 'string'); + this.kmip.client.bucketNameAttributeName = config.kmip.client.bucketNameAttributeName; } } if (config.kmip.transport) { if (Array.isArray(config.kmip.transport)) { - this.kmip.transport = config.kmip.transport.map(t => - this._parseKmipTransport(t)); + this.kmip.transport = config.kmip.transport.map(t => this._parseKmipTransport(t)); if (config.kmip.retries) { - assert(typeof config.kmip.retries === 'number', - 'bad config: KMIP Cluster retries must be a number'); - assert(config.kmip.retries <= this.kmip.transport.length - 1, - 'bad config: KMIP Cluster retries must be lower or equal to the number of hosts - 1'); + assert( + typeof config.kmip.retries === 'number', + 'bad config: KMIP Cluster retries must be a number', + ); + assert( + config.kmip.retries <= this.kmip.transport.length - 1, + 'bad config: KMIP Cluster retries must be lower or equal to the number of hosts - 1', + ); } else { this.kmip.retries = this.kmip.transport.length - 1; } } else { - this.kmip.transport = - this._parseKmipTransport(config.kmip.transport); + this.kmip.transport = this._parseKmipTransport(config.kmip.transport); } } } @@ -831,8 +805,7 @@ class Config extends EventEmitter { _getLocationConfig() { let locationConfig; try { - const data = fs.readFileSync(this.locationConfigPath, - { encoding: 'utf-8' }); + const data = fs.readFileSync(this.locationConfigPath, { encoding: 'utf-8' }); locationConfig = JSON.parse(data); } catch (err) { throw new Error(`could not parse location config file: @@ -845,12 +818,12 @@ class Config extends EventEmitter { Object.keys(locationConfig).forEach(l => { const details = this.locationConstraints[l].details; if (locationConfig[l].details.connector !== undefined) { - assert(typeof locationConfig[l].details.connector === - 'object', 'bad config: connector must be an object'); - if (locationConfig[l].details.connector.sproxyd !== - undefined) { - details.connector.sproxyd = parseSproxydConfig( - locationConfig[l].details.connector.sproxyd); + assert( + typeof locationConfig[l].details.connector === 'object', + 'bad config: connector must be an object', + ); + if (locationConfig[l].details.connector.sproxyd !== undefined) { + details.connector.sproxyd = parseSproxydConfig(locationConfig[l].details.connector.sproxyd); } } }); @@ -861,18 +834,14 @@ class Config extends EventEmitter { return undefined; } if (typeof tlsFileName !== 'string') { - throw new Error( - 'bad config: TLS file specification must be a string'); + throw new Error('bad config: TLS file specification must be a string'); } - const tlsFilePath = (tlsFileName[0] === '/') - ? tlsFileName - : path.join(this._basePath, tlsFileName); + const tlsFilePath = tlsFileName[0] === '/' ? tlsFileName : path.join(this._basePath, tlsFileName); let tlsFileContent; try { tlsFileContent = fs.readFileSync(tlsFilePath); } catch (err) { - throw new Error(`Could not load tls file '${tlsFileName}':` + - ` ${err.message}`); + throw new Error(`Could not load tls file '${tlsFileName}':` + ` ${err.message}`); } return tlsFileContent; } @@ -899,20 +868,18 @@ class Config extends EventEmitter { _parseEndpoints(listenOn, fieldName) { let result = []; if (listenOn !== undefined) { - assert(Array.isArray(listenOn) - && listenOn.every(e => typeof e === 'string'), - `bad config: ${fieldName} must be a list of strings`); + assert( + Array.isArray(listenOn) && listenOn.every(e => typeof e === 'string'), + `bad config: ${fieldName} must be a list of strings`, + ); result = listenOn.map(item => { const lastColon = item.lastIndexOf(':'); // if address is IPv6 format, it includes brackets // that have to be removed from the final IP address - const ipAddress = item.indexOf(']') > 0 ? - item.substr(1, lastColon - 2) : - item.substr(0, lastColon); + const ipAddress = item.indexOf(']') > 0 ? item.substr(1, lastColon - 2) : item.substr(0, lastColon); // the port should not include the colon const port = item.substr(lastColon + 1); - assert(Number.parseInt(port, 10), - `bad config: ${fieldName} port must be a positive integer`); + assert(Number.parseInt(port, 10), `bad config: ${fieldName} port must be a positive integer`); return { ip: ipAddress, port }; }); } @@ -922,32 +889,30 @@ class Config extends EventEmitter { _getConfig() { let config; try { - const data = fs.readFileSync(this.configPath, - { encoding: 'utf-8' }); + const data = fs.readFileSync(this.configPath, { encoding: 'utf-8' }); config = JSON.parse(data); } catch (err) { throw new Error(`could not parse config file: ${err.message}`); } if (this.replicationConfigPath) { try { - const repData = fs.readFileSync(this.replicationConfigPath, - { encoding: 'utf-8' }); + const repData = fs.readFileSync(this.replicationConfigPath, { encoding: 'utf-8' }); const replicationEndpoints = JSON.parse(repData); config.replicationEndpoints.push(...replicationEndpoints); } catch (err) { - throw new Error( - `could not parse replication file: ${err.message}`); + throw new Error(`could not parse replication file: ${err.message}`); } } if (config.port !== undefined) { - assert(Number.isInteger(config.port) && config.port > 0, - 'bad config: port must be a positive integer'); + assert(Number.isInteger(config.port) && config.port > 0, 'bad config: port must be a positive integer'); } if (config.internalPort !== undefined) { - assert(Number.isInteger(config.internalPort) && config.internalPort > 0, - 'bad config: internalPort must be a positive integer'); + assert( + Number.isInteger(config.internalPort) && config.internalPort > 0, + 'bad config: internalPort must be a positive integer', + ); } this.serverHeader = config.serverHeader || 'S3 Server'; @@ -965,16 +930,17 @@ class Config extends EventEmitter { this.metricsPort = 8002; if (config.metricsPort !== undefined) { - assert(Number.isInteger(config.metricsPort) && config.metricsPort > 0, - 'bad config: metricsPort must be a positive integer'); + assert( + Number.isInteger(config.metricsPort) && config.metricsPort > 0, + 'bad config: metricsPort must be a positive integer', + ); this.metricsPort = config.metricsPort; } this.metricsListenOn = this._parseEndpoints(config.metricsListenOn, 'metricsListenOn'); if (config.replicationGroupId) { - assert(typeof config.replicationGroupId === 'string', - 'bad config: replicationGroupId must be a string'); + assert(typeof config.replicationGroupId === 'string', 'bad config: replicationGroupId must be a string'); this.replicationGroupId = config.replicationGroupId; } else { this.replicationGroupId = 'RG001'; @@ -982,12 +948,10 @@ class Config extends EventEmitter { const instanceId = process.env.CLOUDSERVER_INSTANCE_ID || config.instanceId; if (instanceId) { - assert(typeof instanceId === 'string', - 'bad config: instanceId must be a string'); + assert(typeof instanceId === 'string', 'bad config: instanceId must be a string'); // versionID generation code will truncate instanceId to 6 characters // so we enforce this limit here to make the behavior predictable - assert(instanceId.length <= 6, - 'bad config: instanceId must be at most 6 characters long'); + assert(instanceId.length <= 6, 'bad config: instanceId must be at most 6 characters long'); this.instanceId = instanceId; } else { this.instanceId = uuidv4().replace(/-/g, '').slice(0, 6); @@ -996,38 +960,59 @@ class Config extends EventEmitter { this.replicationEndpoints = []; if (config.replicationEndpoints) { const { replicationEndpoints } = config; - assert(replicationEndpoints instanceof Array, 'bad config: ' + - '`replicationEndpoints` property must be an array'); + assert( + replicationEndpoints instanceof Array, + 'bad config: ' + '`replicationEndpoints` property must be an array', + ); replicationEndpoints.forEach(replicationEndpoint => { - assert.strictEqual(typeof replicationEndpoint, 'object', - 'bad config: `replicationEndpoints` property must be an ' + - 'array of objects'); + assert.strictEqual( + typeof replicationEndpoint, + 'object', + 'bad config: `replicationEndpoints` property must be an ' + 'array of objects', + ); const { site, servers, type } = replicationEndpoint; - assert.notStrictEqual(site, undefined, 'bad config: each ' + - 'object of `replicationEndpoints` array must have a ' + - '`site` property'); - assert.strictEqual(typeof site, 'string', 'bad config: ' + - '`site` property of object in `replicationEndpoints` ' + - 'must be a string'); - assert.notStrictEqual(site, '', 'bad config: `site` property ' + - "of object in `replicationEndpoints` must not be ''"); + assert.notStrictEqual( + site, + undefined, + 'bad config: each ' + 'object of `replicationEndpoints` array must have a ' + '`site` property', + ); + assert.strictEqual( + typeof site, + 'string', + 'bad config: ' + '`site` property of object in `replicationEndpoints` ' + 'must be a string', + ); + assert.notStrictEqual( + site, + '', + 'bad config: `site` property ' + "of object in `replicationEndpoints` must not be ''", + ); if (type !== undefined) { - assert(validExternalBackends[type], 'bad config: `type` ' + - 'property of `replicationEndpoints` object must be ' + - 'a valid external backend (one of: "' + - `${Object.keys(validExternalBackends).join('", "')}")`); + assert( + validExternalBackends[type], + 'bad config: `type` ' + + 'property of `replicationEndpoints` object must be ' + + 'a valid external backend (one of: "' + + `${Object.keys(validExternalBackends).join('", "')}")`, + ); } else { - assert.notStrictEqual(servers, undefined, 'bad config: ' + - 'each object of `replicationEndpoints` array that is ' + - 'not an external backend must have `servers` property'); - assert(servers instanceof Array, 'bad config: ' + - '`servers` property of object in ' + - '`replicationEndpoints` must be an array'); + assert.notStrictEqual( + servers, + undefined, + 'bad config: ' + + 'each object of `replicationEndpoints` array that is ' + + 'not an external backend must have `servers` property', + ); + assert( + servers instanceof Array, + 'bad config: ' + '`servers` property of object in ' + '`replicationEndpoints` must be an array', + ); servers.forEach(item => { - assert(typeof item === 'string' && item !== '', + assert( + typeof item === 'string' && item !== '', 'bad config: each item of ' + - '`replicationEndpoints:servers` must be a ' + - 'non-empty string'); + '`replicationEndpoints:servers` must be a ' + + 'non-empty string', + ); }); } }); @@ -1036,27 +1021,31 @@ class Config extends EventEmitter { if (config.backbeat) { const { backbeat } = config; - assert.strictEqual(typeof backbeat.host, 'string', - 'bad config: backbeat host must be a string'); - assert(Number.isInteger(backbeat.port) && backbeat.port > 0, - 'bad config: backbeat port must be a positive integer'); + assert.strictEqual(typeof backbeat.host, 'string', 'bad config: backbeat host must be a string'); + assert( + Number.isInteger(backbeat.port) && backbeat.port > 0, + 'bad config: backbeat port must be a positive integer', + ); this.backbeat = backbeat; } if (config.workflowEngineOperator) { const { workflowEngineOperator } = config; - assert.strictEqual(typeof workflowEngineOperator.host, 'string', - 'bad config: workflowEngineOperator host must be a string'); - assert(Number.isInteger(workflowEngineOperator.port) && - workflowEngineOperator.port > 0, - 'bad config: workflowEngineOperator port not a positive integer'); + assert.strictEqual( + typeof workflowEngineOperator.host, + 'string', + 'bad config: workflowEngineOperator host must be a string', + ); + assert( + Number.isInteger(workflowEngineOperator.port) && workflowEngineOperator.port > 0, + 'bad config: workflowEngineOperator port not a positive integer', + ); this.workflowEngineOperator = workflowEngineOperator; } // legacy if (config.regions !== undefined) { - throw new Error('bad config: regions key is deprecated. ' + - 'Please use restEndpoints and locationConfig'); + throw new Error('bad config: regions key is deprecated. ' + 'Please use restEndpoints and locationConfig'); } if (config.restEndpoints !== undefined) { @@ -1071,16 +1060,19 @@ class Config extends EventEmitter { this.websiteEndpoints = []; if (config.websiteEndpoints !== undefined) { - assert(Array.isArray(config.websiteEndpoints) - && config.websiteEndpoints.every(e => typeof e === 'string'), - 'bad config: websiteEndpoints must be a list of strings'); + assert( + Array.isArray(config.websiteEndpoints) && config.websiteEndpoints.every(e => typeof e === 'string'), + 'bad config: websiteEndpoints must be a list of strings', + ); this.websiteEndpoints = config.websiteEndpoints; } this.clusters = false; if (config.clusters !== undefined) { - assert(Number.isInteger(config.clusters) && config.clusters > 0, - 'bad config: clusters must be a positive integer'); + assert( + Number.isInteger(config.clusters) && config.clusters > 0, + 'bad config: clusters must be a positive integer', + ); this.clusters = config.clusters; } if (process.env.S3BACKEND === 'mem') { @@ -1089,40 +1081,36 @@ class Config extends EventEmitter { this.isCluster = this.clusters > 1; if (config.usEastBehavior !== undefined) { - throw new Error('bad config: usEastBehavior key is deprecated. ' + - 'Please use restEndpoints and locationConfig'); + throw new Error( + 'bad config: usEastBehavior key is deprecated. ' + 'Please use restEndpoints and locationConfig', + ); } // legacy if (config.sproxyd !== undefined) { - throw new Error('bad config: sproxyd key is deprecated. ' + - 'Please use restEndpoints and locationConfig'); + throw new Error('bad config: sproxyd key is deprecated. ' + 'Please use restEndpoints and locationConfig'); } this.cdmi = {}; if (config.cdmi !== undefined) { if (config.cdmi.host !== undefined) { - assert.strictEqual(typeof config.cdmi.host, 'string', - 'bad config: cdmi host must be a string'); + assert.strictEqual(typeof config.cdmi.host, 'string', 'bad config: cdmi host must be a string'); this.cdmi.host = config.cdmi.host; } if (config.cdmi.port !== undefined) { - assert(Number.isInteger(config.cdmi.port) - && config.cdmi.port > 0, - 'bad config: cdmi port must be a positive integer'); + assert( + Number.isInteger(config.cdmi.port) && config.cdmi.port > 0, + 'bad config: cdmi port must be a positive integer', + ); this.cdmi.port = config.cdmi.port; } if (config.cdmi.path !== undefined) { - assert(typeof config.cdmi.path === 'string', - 'bad config: cdmi.path must be a string'); - assert(config.cdmi.path.length > 0, - 'bad config: cdmi.path is empty'); - assert(config.cdmi.path.charAt(0) === '/', - 'bad config: cdmi.path should start with a "/"'); + assert(typeof config.cdmi.path === 'string', 'bad config: cdmi.path must be a string'); + assert(config.cdmi.path.length > 0, 'bad config: cdmi.path is empty'); + assert(config.cdmi.path.charAt(0) === '/', 'bad config: cdmi.path should start with a "/"'); this.cdmi.path = config.cdmi.path; } if (config.cdmi.readonly !== undefined) { - assert(typeof config.cdmi.readonly === 'boolean', - 'bad config: cdmi.readonly must be a boolean'); + assert(typeof config.cdmi.readonly === 'boolean', 'bad config: cdmi.readonly must be a boolean'); this.cdmi.readonly = config.cdmi.readonly; } else { this.cdmi.readonly = true; @@ -1130,88 +1118,98 @@ class Config extends EventEmitter { } this.bucketd = { bootstrap: [] }; - if (config.bucketd !== undefined - && config.bucketd.bootstrap !== undefined) { - assert(config.bucketd.bootstrap instanceof Array - && config.bucketd.bootstrap.every( - e => typeof e === 'string'), - 'bad config: bucketd.bootstrap must be a list of strings'); + if (config.bucketd !== undefined && config.bucketd.bootstrap !== undefined) { + assert( + config.bucketd.bootstrap instanceof Array && config.bucketd.bootstrap.every(e => typeof e === 'string'), + 'bad config: bucketd.bootstrap must be a list of strings', + ); this.bucketd.bootstrap = config.bucketd.bootstrap; } this.vaultd = {}; if (config.vaultd) { if (config.vaultd.port !== undefined) { - assert(Number.isInteger(config.vaultd.port) - && config.vaultd.port > 0, - 'bad config: vaultd port must be a positive integer'); + assert( + Number.isInteger(config.vaultd.port) && config.vaultd.port > 0, + 'bad config: vaultd port must be a positive integer', + ); this.vaultd.port = config.vaultd.port; } if (config.vaultd.host !== undefined) { - assert.strictEqual(typeof config.vaultd.host, 'string', - 'bad config: vaultd host must be a string'); + assert.strictEqual(typeof config.vaultd.host, 'string', 'bad config: vaultd host must be a string'); this.vaultd.host = config.vaultd.host; } if (process.env.VAULTD_HOST !== undefined) { - assert.strictEqual(typeof process.env.VAULTD_HOST, 'string', - 'bad config: vaultd host must be a string'); + assert.strictEqual( + typeof process.env.VAULTD_HOST, + 'string', + 'bad config: vaultd host must be a string', + ); this.vaultd.host = process.env.VAULTD_HOST; } } if (config.dataClient) { this.dataClient = {}; - assert.strictEqual(typeof config.dataClient.host, 'string', - 'bad config: data client host must be ' + - 'a string'); + assert.strictEqual( + typeof config.dataClient.host, + 'string', + 'bad config: data client host must be ' + 'a string', + ); this.dataClient.host = config.dataClient.host; - assert(Number.isInteger(config.dataClient.port) - && config.dataClient.port > 0, - 'bad config: dataClient port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.dataClient.port) && config.dataClient.port > 0, + 'bad config: dataClient port must be a positive ' + 'integer', + ); this.dataClient.port = config.dataClient.port; } if (config.metadataClient) { this.metadataClient = {}; assert.strictEqual( - typeof config.metadataClient.host, 'string', - 'bad config: metadata client host must be a string'); + typeof config.metadataClient.host, + 'string', + 'bad config: metadata client host must be a string', + ); this.metadataClient.host = config.metadataClient.host; - assert(Number.isInteger(config.metadataClient.port) - && config.metadataClient.port > 0, - 'bad config: metadata client port must be a ' + - 'positive integer'); + assert( + Number.isInteger(config.metadataClient.port) && config.metadataClient.port > 0, + 'bad config: metadata client port must be a ' + 'positive integer', + ); this.metadataClient.port = config.metadataClient.port; } if (config.pfsClient) { this.pfsClient = {}; - assert.strictEqual(typeof config.pfsClient.host, 'string', - 'bad config: pfsClient host must be ' + - 'a string'); + assert.strictEqual( + typeof config.pfsClient.host, + 'string', + 'bad config: pfsClient host must be ' + 'a string', + ); this.pfsClient.host = config.pfsClient.host; - assert(Number.isInteger(config.pfsClient.port) && - config.pfsClient.port > 0, - 'bad config: pfsClient port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.pfsClient.port) && config.pfsClient.port > 0, + 'bad config: pfsClient port must be a positive ' + 'integer', + ); this.pfsClient.port = config.pfsClient.port; } if (config.dataDaemon) { this.dataDaemon = {}; assert.strictEqual( - typeof config.dataDaemon.bindAddress, 'string', - 'bad config: data daemon bind address must be a string'); + typeof config.dataDaemon.bindAddress, + 'string', + 'bad config: data daemon bind address must be a string', + ); this.dataDaemon.bindAddress = config.dataDaemon.bindAddress; - assert(Number.isInteger(config.dataDaemon.port) - && config.dataDaemon.port > 0, - 'bad config: data daemon port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.dataDaemon.port) && config.dataDaemon.port > 0, + 'bad config: data daemon port must be a positive ' + 'integer', + ); this.dataDaemon.port = config.dataDaemon.port; /** @@ -1219,9 +1217,7 @@ class Config extends EventEmitter { * backend. If no path provided, uses data at the root of * the S3 project directory. */ - this.dataDaemon.dataPath = - process.env.S3DATAPATH ? - process.env.S3DATAPATH : `${__dirname}/../localData`; + this.dataDaemon.dataPath = process.env.S3DATAPATH ? process.env.S3DATAPATH : `${__dirname}/../localData`; this.dataDaemon.noSync = process.env.S3DATA_NOSYNC === 'true'; this.dataDaemon.noCache = process.env.S3DATA_NOCACHE === 'true'; } @@ -1229,35 +1225,37 @@ class Config extends EventEmitter { if (config.pfsDaemon) { this.pfsDaemon = {}; assert.strictEqual( - typeof config.pfsDaemon.bindAddress, 'string', - 'bad config: data daemon bind address must be a string'); + typeof config.pfsDaemon.bindAddress, + 'string', + 'bad config: data daemon bind address must be a string', + ); this.pfsDaemon.bindAddress = config.pfsDaemon.bindAddress; - assert(Number.isInteger(config.pfsDaemon.port) - && config.pfsDaemon.port > 0, - 'bad config: data daemon port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.pfsDaemon.port) && config.pfsDaemon.port > 0, + 'bad config: data daemon port must be a positive ' + 'integer', + ); this.pfsDaemon.port = config.pfsDaemon.port; - this.pfsDaemon.dataPath = - process.env.PFSD_MOUNT_PATH ? - process.env.PFSD_MOUNT_PATH : `${__dirname}/../localPfs`; + this.pfsDaemon.dataPath = process.env.PFSD_MOUNT_PATH + ? process.env.PFSD_MOUNT_PATH + : `${__dirname}/../localPfs`; this.pfsDaemon.noSync = process.env.PFSD_NOSYNC === 'true'; this.pfsDaemon.noCache = process.env.PFSD_NOCACHE === 'true'; - this.pfsDaemon.isReadOnly = - process.env.PFSD_READONLY === 'true'; + this.pfsDaemon.isReadOnly = process.env.PFSD_READONLY === 'true'; } if (config.metadataDaemon) { this.metadataDaemon = {}; assert.strictEqual( - typeof config.metadataDaemon.bindAddress, 'string', - 'bad config: metadata daemon bind address must be a string'); - this.metadataDaemon.bindAddress = - config.metadataDaemon.bindAddress; - - assert(Number.isInteger(config.metadataDaemon.port) - && config.metadataDaemon.port > 0, - 'bad config: metadata daemon port must be a ' + - 'positive integer'); + typeof config.metadataDaemon.bindAddress, + 'string', + 'bad config: metadata daemon bind address must be a string', + ); + this.metadataDaemon.bindAddress = config.metadataDaemon.bindAddress; + + assert( + Number.isInteger(config.metadataDaemon.port) && config.metadataDaemon.port > 0, + 'bad config: metadata daemon port must be a ' + 'positive integer', + ); this.metadataDaemon.port = config.metadataDaemon.port; /** @@ -1265,12 +1263,11 @@ class Config extends EventEmitter { * backend. If no path provided, uses data and metadata at * the root of the S3 project directory. */ - this.metadataDaemon.metadataPath = - process.env.S3METADATAPATH ? - process.env.S3METADATAPATH : `${__dirname}/../localMetadata`; + this.metadataDaemon.metadataPath = process.env.S3METADATAPATH + ? process.env.S3METADATAPATH + : `${__dirname}/../localMetadata`; - this.metadataDaemon.restEnabled = - config.metadataDaemon.restEnabled; + this.metadataDaemon.restEnabled = config.metadataDaemon.restEnabled; this.metadataDaemon.restPort = config.metadataDaemon.restPort; } @@ -1284,48 +1281,51 @@ class Config extends EventEmitter { this.localCache = defaultLocalCache; } if (config.localCache) { - assert(typeof config.localCache === 'object', - 'config: invalid local cache configuration. localCache must ' + - 'be an object'); + assert( + typeof config.localCache === 'object', + 'config: invalid local cache configuration. localCache must ' + 'be an object', + ); if (config.localCache.sentinels) { this.localCache = { sentinels: [], name: null }; - assert(typeof config.localCache.name === 'string', - 'bad config: localCache sentinel name must be a string'); + assert( + typeof config.localCache.name === 'string', + 'bad config: localCache sentinel name must be a string', + ); this.localCache.name = config.localCache.name; - assert(Array.isArray(config.localCache.sentinels) || - typeof config.localCache.sentinels === 'string', - 'bad config: localCache sentinels' + - 'must be an array or string'); + assert( + Array.isArray(config.localCache.sentinels) || typeof config.localCache.sentinels === 'string', + 'bad config: localCache sentinels' + 'must be an array or string', + ); if (typeof config.localCache.sentinels === 'string') { config.localCache.sentinels.split(',').forEach(item => { const [host, port] = item.split(':'); - this.localCache.sentinels.push({ host, - port: Number.parseInt(port, 10) }); + this.localCache.sentinels.push({ host, port: Number.parseInt(port, 10) }); }); } else if (Array.isArray(config.localCache.sentinels)) { config.localCache.sentinels.forEach(item => { const { host, port } = item; - assert(typeof host === 'string', - 'bad config: localCache' + - 'sentinel host must be a string'); - assert(typeof port === 'number', - 'bad config: localCache' + - 'sentinel port must be a number'); + assert(typeof host === 'string', 'bad config: localCache' + 'sentinel host must be a string'); + assert(typeof port === 'number', 'bad config: localCache' + 'sentinel port must be a number'); this.localCache.sentinels.push({ host, port }); }); } } else { - assert(typeof config.localCache.host === 'string', - 'config: bad host for localCache. host must be a string'); - assert(typeof config.localCache.port === 'number', - 'config: bad port for localCache. port must be a number'); + assert( + typeof config.localCache.host === 'string', + 'config: bad host for localCache. host must be a string', + ); + assert( + typeof config.localCache.port === 'number', + 'config: bad port for localCache. port must be a number', + ); if (config.localCache.password !== undefined) { - assert(typeof config.localCache.password === 'string', - 'config: vad password for localCache. password must' + - ' be a string'); + assert( + typeof config.localCache.password === 'string', + 'config: vad password for localCache. password must' + ' be a string', + ); } this.localCache = { host: config.localCache.host, @@ -1337,11 +1337,10 @@ class Config extends EventEmitter { if (config.mongodb) { this.mongodb = config.mongodb; - if (process.env.MONGODB_AUTH_USERNAME && - process.env.MONGODB_AUTH_PASSWORD) { + if (process.env.MONGODB_AUTH_USERNAME && process.env.MONGODB_AUTH_PASSWORD) { this.mongodb.authCredentials = { - username: process.env.MONGODB_AUTH_USERNAME, - password: process.env.MONGODB_AUTH_PASSWORD, + username: process.env.MONGODB_AUTH_USERNAME, + password: process.env.MONGODB_AUTH_PASSWORD, }; } } else { @@ -1351,29 +1350,29 @@ class Config extends EventEmitter { if (config.redis) { // Fail fast to make sure we detect any bad config throw new Error( - 'config.redis is not supported anymore: it should be config.utapi.redis or config.localCache' + 'config.redis is not supported anymore: it should be config.utapi.redis or config.localCache', ); } if (config.scuba) { this.scuba = {}; if (config.scuba.host) { - assert(typeof config.scuba.host === 'string', - 'bad config: scuba host must be a string'); + assert(typeof config.scuba.host === 'string', 'bad config: scuba host must be a string'); this.scuba.host = config.scuba.host; } if (config.scuba.port) { - assert(Number.isInteger(config.scuba.port) - && config.scuba.port > 0, - 'bad config: scuba port must be a positive integer'); + assert( + Number.isInteger(config.scuba.port) && config.scuba.port > 0, + 'bad config: scuba port must be a positive integer', + ); this.scuba.port = config.scuba.port; } } if (process.env.SCUBA_HOST && process.env.SCUBA_PORT) { - assert(typeof process.env.SCUBA_HOST === 'string', - 'bad config: scuba host must be a string'); - assert(Number.isInteger(Number(process.env.SCUBA_PORT)) - && Number(process.env.SCUBA_PORT) > 0, - 'bad config: scuba port must be a positive integer'); + assert(typeof process.env.SCUBA_HOST === 'string', 'bad config: scuba host must be a string'); + assert( + Number.isInteger(Number(process.env.SCUBA_PORT)) && Number(process.env.SCUBA_PORT) > 0, + 'bad config: scuba port must be a positive integer', + ); this.scuba = { host: process.env.SCUBA_HOST, port: Number(process.env.SCUBA_PORT), @@ -1382,12 +1381,10 @@ class Config extends EventEmitter { if (this.scuba) { this.quotaEnabled = true; } - const maxStaleness = Number(process.env.QUOTA_MAX_STALENESS_MS) || - config.quota?.maxStatenessMS || - 24 * 60 * 60 * 1000; + const maxStaleness = + Number(process.env.QUOTA_MAX_STALENESS_MS) || config.quota?.maxStatenessMS || 24 * 60 * 60 * 1000; assert(Number.isInteger(maxStaleness), 'bad config: maxStalenessMS must be an integer'); - const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' || - config.quota?.enableInflights || false; + const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' || config.quota?.enableInflights || false; this.quota = { maxStaleness, enableInflights, @@ -1395,30 +1392,29 @@ class Config extends EventEmitter { if (config.utapi) { this.utapi = { component: 's3' }; if (config.utapi.host) { - assert(typeof config.utapi.host === 'string', - 'bad config: utapi host must be a string'); + assert(typeof config.utapi.host === 'string', 'bad config: utapi host must be a string'); this.utapi.host = config.utapi.host; } if (config.utapi.port) { - assert(Number.isInteger(config.utapi.port) - && config.utapi.port > 0, - 'bad config: utapi port must be a positive integer'); + assert( + Number.isInteger(config.utapi.port) && config.utapi.port > 0, + 'bad config: utapi port must be a positive integer', + ); this.utapi.port = config.utapi.port; } if (utapiVersion === 1) { if (config.utapi.workers !== undefined) { - assert(Number.isInteger(config.utapi.workers) - && config.utapi.workers > 0, - 'bad config: utapi workers must be a positive integer'); + assert( + Number.isInteger(config.utapi.workers) && config.utapi.workers > 0, + 'bad config: utapi workers must be a positive integer', + ); this.utapi.workers = config.utapi.workers; } // Utapi uses the same localCache config defined for S3 to avoid // config duplication. - assert(config.localCache, 'missing required property of utapi ' + - 'configuration: localCache'); + assert(config.localCache, 'missing required property of utapi ' + 'configuration: localCache'); this.utapi.localCache = this.localCache; - assert(config.utapi.redis, 'missing required property of utapi ' + - 'configuration: redis'); + assert(config.utapi.redis, 'missing required property of utapi ' + 'configuration: redis'); this.utapi.redis = parseRedisConfig(config.utapi.redis); if (this.utapi.redis.retry === undefined) { this.utapi.redis.retry = { @@ -1437,29 +1433,36 @@ class Config extends EventEmitter { this.utapi.enabledOperationCounters = []; if (config.utapi.enabledOperationCounters !== undefined) { const { enabledOperationCounters } = config.utapi; - assert(Array.isArray(enabledOperationCounters), - 'bad config: utapi.enabledOperationCounters must be an ' + - 'array'); - assert(enabledOperationCounters.length > 0, - 'bad config: utapi.enabledOperationCounters cannot be ' + - 'empty'); + assert( + Array.isArray(enabledOperationCounters), + 'bad config: utapi.enabledOperationCounters must be an ' + 'array', + ); + assert( + enabledOperationCounters.length > 0, + 'bad config: utapi.enabledOperationCounters cannot be ' + 'empty', + ); this.utapi.enabledOperationCounters = enabledOperationCounters; } this.utapi.disableOperationCounters = false; if (config.utapi.disableOperationCounters !== undefined) { const { disableOperationCounters } = config.utapi; - assert(typeof disableOperationCounters === 'boolean', - 'bad config: utapi.disableOperationCounters must be a ' + - 'boolean'); + assert( + typeof disableOperationCounters === 'boolean', + 'bad config: utapi.disableOperationCounters must be a ' + 'boolean', + ); this.utapi.disableOperationCounters = disableOperationCounters; } - if (config.utapi.disableOperationCounters !== undefined && - config.utapi.enabledOperationCounters !== undefined) { - assert(config.utapi.disableOperationCounters === false, + if ( + config.utapi.disableOperationCounters !== undefined && + config.utapi.enabledOperationCounters !== undefined + ) { + assert( + config.utapi.disableOperationCounters === false, 'bad config: conflicting rules: ' + - 'utapi.disableOperationCounters and ' + - 'utapi.enabledOperationCounters cannot both be ' + - 'specified'); + 'utapi.disableOperationCounters and ' + + 'utapi.enabledOperationCounters cannot both be ' + + 'specified', + ); } if (config.utapi.component) { this.utapi.component = config.utapi.component; @@ -1467,17 +1470,23 @@ class Config extends EventEmitter { // (optional) The value of the replay schedule should be cron-style // scheduling. For example, every five minutes: '*/5 * * * *'. if (config.utapi.replaySchedule) { - assert(typeof config.utapi.replaySchedule === 'string', 'bad' + - 'config: utapi.replaySchedule must be a string'); + assert( + typeof config.utapi.replaySchedule === 'string', + 'bad' + 'config: utapi.replaySchedule must be a string', + ); this.utapi.replaySchedule = config.utapi.replaySchedule; } // (optional) The number of elements processed by each call to the // Redis local cache during a replay. For example, 50. if (config.utapi.batchSize) { - assert(typeof config.utapi.batchSize === 'number', 'bad' + - 'config: utapi.batchSize must be a number'); - assert(config.utapi.batchSize > 0, 'bad config:' + - 'utapi.batchSize must be a number greater than 0'); + assert( + typeof config.utapi.batchSize === 'number', + 'bad' + 'config: utapi.batchSize must be a number', + ); + assert( + config.utapi.batchSize > 0, + 'bad config:' + 'utapi.batchSize must be a number greater than 0', + ); this.utapi.batchSize = config.utapi.batchSize; } @@ -1485,16 +1494,20 @@ class Config extends EventEmitter { // Disabled by default this.utapi.expireMetrics = false; if (config.utapi.expireMetrics !== undefined) { - assert(typeof config.utapi.expireMetrics === 'boolean', 'bad' + - 'config: utapi.expireMetrics must be a boolean'); + assert( + typeof config.utapi.expireMetrics === 'boolean', + 'bad' + 'config: utapi.expireMetrics must be a boolean', + ); this.utapi.expireMetrics = config.utapi.expireMetrics; } // (optional) TTL controlling the expiry for bucket level metrics // keys when expireMetrics is enabled this.utapi.expireMetricsTTL = 0; if (config.utapi.expireMetricsTTL !== undefined) { - assert(typeof config.utapi.expireMetricsTTL === 'number', - 'bad config: utapi.expireMetricsTTL must be a number'); + assert( + typeof config.utapi.expireMetricsTTL === 'number', + 'bad config: utapi.expireMetricsTTL must be a number', + ); this.utapi.expireMetricsTTL = config.utapi.expireMetricsTTL; } @@ -1506,40 +1519,42 @@ class Config extends EventEmitter { if (utapiVersion === 2 && config.utapi.filter) { const { filter: filterConfig } = config.utapi; const utapiResourceFilters = {}; - allowedUtapiEventFilterFields.forEach( - field => allowedUtapiEventFilterStates.forEach( - state => { - const resources = (filterConfig[state] && filterConfig[state][field]) || null; - if (resources) { - assert.strictEqual(utapiResourceFilters[field], undefined, - `bad config: utapi.filter.${state}.${field} can't define an allow and a deny list`); - assert(resources.every(r => typeof r === 'string'), - `bad config: utapi.filter.${state}.${field} must be an array of strings`); - utapiResourceFilters[field] = { [state]: new Set(resources) }; - } + allowedUtapiEventFilterFields.forEach(field => + allowedUtapiEventFilterStates.forEach(state => { + const resources = (filterConfig[state] && filterConfig[state][field]) || null; + if (resources) { + assert.strictEqual( + utapiResourceFilters[field], + undefined, + `bad config: utapi.filter.${state}.${field} can't define an allow and a deny list`, + ); + assert( + resources.every(r => typeof r === 'string'), + `bad config: utapi.filter.${state}.${field} must be an array of strings`, + ); + utapiResourceFilters[field] = { [state]: new Set(resources) }; } - )); + }), + ); this.utapi.filter = utapiResourceFilters; } } - if (Object.keys(this.locationConstraints).some( - loc => this.locationConstraints[loc].sizeLimitGB)) { - assert(this.utapi && this.utapi.metrics && - this.utapi.metrics.includes('location'), + if (Object.keys(this.locationConstraints).some(loc => this.locationConstraints[loc].sizeLimitGB)) { + assert( + this.utapi && this.utapi.metrics && this.utapi.metrics.includes('location'), 'bad config: if storage size limit set on a location ' + - 'constraint, Utapi must also be configured correctly'); + 'constraint, Utapi must also be configured correctly', + ); } this.log = { logLevel: 'debug', dumpLevel: 'error' }; if (config.log !== undefined) { if (config.log.logLevel !== undefined) { - assert(typeof config.log.logLevel === 'string', - 'bad config: log.logLevel must be a string'); + assert(typeof config.log.logLevel === 'string', 'bad config: log.logLevel must be a string'); this.log.logLevel = config.log.logLevel; } if (config.log.dumpLevel !== undefined) { - assert(typeof config.log.dumpLevel === 'string', - 'bad config: log.dumpLevel must be a string'); + assert(typeof config.log.dumpLevel === 'string', 'bad config: log.dumpLevel must be a string'); this.log.dumpLevel = config.log.dumpLevel; } } @@ -1547,8 +1562,10 @@ class Config extends EventEmitter { this.kms = {}; if (config.kms) { assert(config.kms.providerName, 'config.kms.providerName must be provided'); - assert(isValidProvider(config.kms.providerName), - 'config.kms.providerName must be lowercase alphanumeric only'); + assert( + isValidProvider(config.kms.providerName), + 'config.kms.providerName must be lowercase alphanumeric only', + ); assert(typeof config.kms.userName === 'string'); assert(typeof config.kms.password === 'string'); this.kms.providerName = config.kms.providerName; @@ -1573,13 +1590,14 @@ class Config extends EventEmitter { const globalEncryptionEnabled = config.globalEncryptionEnabled; this.globalEncryptionEnabled = globalEncryptionEnabled || false; - assert(typeof this.globalEncryptionEnabled === 'boolean', - 'config.globalEncryptionEnabled must be a boolean'); + assert(typeof this.globalEncryptionEnabled === 'boolean', 'config.globalEncryptionEnabled must be a boolean'); const defaultEncryptionKeyPerAccount = config.defaultEncryptionKeyPerAccount; this.defaultEncryptionKeyPerAccount = defaultEncryptionKeyPerAccount || false; - assert(typeof this.defaultEncryptionKeyPerAccount === 'boolean', - 'config.defaultEncryptionKeyPerAccount must be a boolean'); + assert( + typeof this.defaultEncryptionKeyPerAccount === 'boolean', + 'config.defaultEncryptionKeyPerAccount must be a boolean', + ); this.kmsHideScalityArn = Object.hasOwnProperty.call(config, 'kmsHideScalityArn') ? config.kmsHideScalityArn @@ -1588,16 +1606,17 @@ class Config extends EventEmitter { this.healthChecks = defaultHealthChecks; if (config.healthChecks && config.healthChecks.allowFrom) { - assert(config.healthChecks.allowFrom instanceof Array, - 'config: invalid healthcheck configuration. allowFrom must ' + - 'be an array'); + assert( + config.healthChecks.allowFrom instanceof Array, + 'config: invalid healthcheck configuration. allowFrom must ' + 'be an array', + ); config.healthChecks.allowFrom.forEach(item => { - assert(typeof item === 'string', - 'config: invalid healthcheck configuration. allowFrom IP ' + - 'address must be a string'); + assert( + typeof item === 'string', + 'config: invalid healthcheck configuration. allowFrom IP ' + 'address must be a string', + ); }); - this.healthChecks.allowFrom = defaultHealthChecks.allowFrom - .concat(config.healthChecks.allowFrom); + this.healthChecks.allowFrom = defaultHealthChecks.allowFrom.concat(config.healthChecks.allowFrom); } /** * CLDSRV-740: S3C with nginx s3-frontend needs the healthcheck on the @@ -1606,23 +1625,21 @@ class Config extends EventEmitter { this.healthChecks.enableInternalRoute = config.healthChecks?.enableInternalRoute || false; if (config.certFilePaths) { - assert(typeof config.certFilePaths === 'object' && - typeof config.certFilePaths.key === 'string' && - typeof config.certFilePaths.cert === 'string' && (( - config.certFilePaths.ca && - typeof config.certFilePaths.ca === 'string') || - !config.certFilePaths.ca) - ); + assert( + typeof config.certFilePaths === 'object' && + typeof config.certFilePaths.key === 'string' && + typeof config.certFilePaths.cert === 'string' && + ((config.certFilePaths.ca && typeof config.certFilePaths.ca === 'string') || + !config.certFilePaths.ca), + ); } - const { key, cert, ca } = config.certFilePaths ? - config.certFilePaths : {}; + const { key, cert, ca } = config.certFilePaths ? config.certFilePaths : {}; let certObj = undefined; if (key && cert) { certObj = assertCertPaths(key, cert, ca, this._basePath); } else if (key || cert) { - throw new Error('bad config: both certFilePaths.key and ' + - 'certFilePaths.cert must be defined'); + throw new Error('bad config: both certFilePaths.key and ' + 'certFilePaths.cert must be defined'); } if (certObj) { if (Object.keys(certObj.certs).length > 0) { @@ -1634,30 +1651,29 @@ class Config extends EventEmitter { } this.outboundProxy = {}; - const envProxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY - || process.env.http_proxy || process.env.https_proxy; + const envProxy = + process.env.HTTP_PROXY || process.env.HTTPS_PROXY || process.env.http_proxy || process.env.https_proxy; const p = config.outboundProxy; const proxyUrl = envProxy || (p ? p.url : ''); if (proxyUrl) { - assert(typeof proxyUrl === 'string', - 'bad proxy config: url must be a string'); + assert(typeof proxyUrl === 'string', 'bad proxy config: url must be a string'); const { protocol, hostname, port, auth } = url.parse(proxyUrl); - assert(protocol === 'http:' || protocol === 'https:', - 'bad proxy config: protocol must be http or https'); - assert(typeof hostname === 'string' && hostname !== '', - 'bad proxy config: hostname must be a non-empty string'); + assert(protocol === 'http:' || protocol === 'https:', 'bad proxy config: protocol must be http or https'); + assert( + typeof hostname === 'string' && hostname !== '', + 'bad proxy config: hostname must be a non-empty string', + ); if (port) { const portInt = Number.parseInt(port, 10); - assert(!Number.isNaN(portInt) && portInt > 0, - 'bad proxy config: port must be a number greater than 0'); + assert(!Number.isNaN(portInt) && portInt > 0, 'bad proxy config: port must be a number greater than 0'); } if (auth) { - assert(typeof auth === 'string', - 'bad proxy config: auth must be string'); + assert(typeof auth === 'string', 'bad proxy config: auth must be string'); const authArray = auth.split(':'); - assert(authArray.length === 2 && authArray[0].length > 0 - && authArray[1].length > 0, 'bad proxy config: ' + - 'auth must be of format username:password'); + assert( + authArray.length === 2 && authArray[0].length > 0 && authArray[1].length > 0, + 'bad proxy config: ' + 'auth must be of format username:password', + ); } this.outboundProxy.url = proxyUrl; this.outboundProxy.certs = {}; @@ -1666,23 +1682,18 @@ class Config extends EventEmitter { const cert = p ? p.cert : ''; const caBundle = envCert || (p ? p.caBundle : ''); if (p) { - assert(typeof p === 'object', - 'bad config: "proxy" should be an object'); + assert(typeof p === 'object', 'bad config: "proxy" should be an object'); } if (key) { - assert(typeof key === 'string', - 'bad config: proxy.key should be a string'); + assert(typeof key === 'string', 'bad config: proxy.key should be a string'); } if (cert) { - assert(typeof cert === 'string', - 'bad config: proxy.cert should be a string'); + assert(typeof cert === 'string', 'bad config: proxy.cert should be a string'); } if (caBundle) { - assert(typeof caBundle === 'string', - 'bad config: proxy.caBundle should be a string'); + assert(typeof caBundle === 'string', 'bad config: proxy.caBundle should be a string'); } - const certObj = - assertCertPaths(key, cert, caBundle, this._basePath); + const certObj = assertCertPaths(key, cert, caBundle, this._basePath); this.outboundProxy.certs = certObj.certs; } @@ -1691,16 +1702,18 @@ class Config extends EventEmitter { this.managementAgent.host = 'localhost'; if (config.managementAgent !== undefined) { if (config.managementAgent.port !== undefined) { - assert(Number.isInteger(config.managementAgent.port) - && config.managementAgent.port > 0, - 'bad config: managementAgent port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.managementAgent.port) && config.managementAgent.port > 0, + 'bad config: managementAgent port must be a positive ' + 'integer', + ); this.managementAgent.port = config.managementAgent.port; } if (config.managementAgent.host !== undefined) { - assert.strictEqual(typeof config.managementAgent.host, 'string', - 'bad config: management agent host must ' + - 'be a string'); + assert.strictEqual( + typeof config.managementAgent.host, + 'string', + 'bad config: management agent host must ' + 'be a string', + ); this.managementAgent.host = config.managementAgent.host; } } @@ -1708,10 +1721,7 @@ class Config extends EventEmitter { // Ephemeral token to protect the reporting endpoint: // try inherited from parent first, then hardcoded in conf file, // then create a fresh one as last resort. - this.reportToken = - process.env.REPORT_TOKEN || - config.reportToken || - uuidv4(); + this.reportToken = process.env.REPORT_TOKEN || config.reportToken || uuidv4(); // External backends // Currently supports configuring httpAgent(s) for keepAlive @@ -1720,29 +1730,28 @@ class Config extends EventEmitter { const extBackendsConfig = Object.keys(config.externalBackends); extBackendsConfig.forEach(b => { // assert that it's a valid backend - assert(validExternalBackends[b] !== undefined, + assert( + validExternalBackends[b] !== undefined, `bad config: ${b} is not one of valid external backends: ` + - `${Object.keys(validExternalBackends).join(', ')}`); + `${Object.keys(validExternalBackends).join(', ')}`, + ); const { httpAgent } = config.externalBackends[b]; - assert(typeof httpAgent === 'object', - `bad config: ${b} must have httpAgent object defined`); - const { keepAlive, keepAliveMsecs, maxFreeSockets, maxSockets } - = httpAgent; - assert(typeof keepAlive === 'boolean', - `bad config: ${b}.httpAgent.keepAlive must be a boolean`); - assert(typeof keepAliveMsecs === 'number' && - httpAgent.keepAliveMsecs > 0, - `bad config: ${b}.httpAgent.keepAliveMsecs must be` + - ' a number > 0'); - assert(typeof maxFreeSockets === 'number' && - httpAgent.maxFreeSockets >= 0, - `bad config: ${b}.httpAgent.maxFreeSockets must be ` + - 'a number >= 0'); - assert((typeof maxSockets === 'number' && maxSockets >= 0) || - maxSockets === null, - `bad config: ${b}.httpAgent.maxFreeSockets must be ` + - 'null or a number >= 0'); + assert(typeof httpAgent === 'object', `bad config: ${b} must have httpAgent object defined`); + const { keepAlive, keepAliveMsecs, maxFreeSockets, maxSockets } = httpAgent; + assert(typeof keepAlive === 'boolean', `bad config: ${b}.httpAgent.keepAlive must be a boolean`); + assert( + typeof keepAliveMsecs === 'number' && httpAgent.keepAliveMsecs > 0, + `bad config: ${b}.httpAgent.keepAliveMsecs must be` + ' a number > 0', + ); + assert( + typeof maxFreeSockets === 'number' && httpAgent.maxFreeSockets >= 0, + `bad config: ${b}.httpAgent.maxFreeSockets must be ` + 'a number >= 0', + ); + assert( + (typeof maxSockets === 'number' && maxSockets >= 0) || maxSockets === null, + `bad config: ${b}.httpAgent.maxFreeSockets must be ` + 'null or a number >= 0', + ); Object.assign(this.externalBackends[b].httpAgent, httpAgent); }); } @@ -1786,9 +1795,11 @@ class Config extends EventEmitter { // maxScannedLifecycleListingEntries > 2 is required as a minimum because we must // scan at least three entries to determine version eligibility. // Two entries representing the master key and the following one representing the non-current version. - assert(Number.isInteger(config.maxScannedLifecycleListingEntries) && - config.maxScannedLifecycleListingEntries > 2, - 'bad config: maxScannedLifecycleListingEntries must be greater than 2'); + assert( + Number.isInteger(config.maxScannedLifecycleListingEntries) && + config.maxScannedLifecycleListingEntries > 2, + 'bad config: maxScannedLifecycleListingEntries must be greater than 2', + ); this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries; } @@ -1798,29 +1809,33 @@ class Config extends EventEmitter { this.apiBodySizeLimits = { ...constants.defaultApiBodySizeLimits }; if (config.apiBodySizeLimits) { - assert(typeof config.apiBodySizeLimits === 'object' && - !Array.isArray(config.apiBodySizeLimits), - 'bad config: apiBodySizeLimits must be an object'); + assert( + typeof config.apiBodySizeLimits === 'object' && !Array.isArray(config.apiBodySizeLimits), + 'bad config: apiBodySizeLimits must be an object', + ); for (const [apiKey, limit] of Object.entries(config.apiBodySizeLimits)) { // Only allow modifications of predefined APIs from constants - assert(Object.hasOwn(constants.defaultApiBodySizeLimits, apiKey), + assert( + Object.hasOwn(constants.defaultApiBodySizeLimits, apiKey), `bad config: apiBodySizeLimits for "${apiKey}" cannot be configured. ` + - `Valid APIs are: ${Object.keys(constants.defaultApiBodySizeLimits).join(', ')}`); + `Valid APIs are: ${Object.keys(constants.defaultApiBodySizeLimits).join(', ')}`, + ); - assert(Number.isInteger(limit) && limit > 0, - `bad config: apiBodySizeLimits for "${apiKey}" must be a positive integer`); + assert( + Number.isInteger(limit) && limit > 0, + `bad config: apiBodySizeLimits for "${apiKey}" must be a positive integer`, + ); this.apiBodySizeLimits[apiKey] = limit; } } - this.integrityChecks = parseIntegrityChecks(config); this.serverAccessLogs = parseServerAccessLogs(config); /** * S3C-10336: PutObject max size of 5GB is new in 9.5.1 * Provides a way to bypass the new validation if it breaks customer workflows */ - this.bypassMaxPutObjectSize = process.env.BYPASS_MAX_PUT_OBJECT_SIZE === 'true' - || config.bypassMaxPutObjectSize || false; + this.bypassMaxPutObjectSize = + process.env.BYPASS_MAX_PUT_OBJECT_SIZE === 'true' || config.bypassMaxPutObjectSize || false; /** * S3C-10370: Before 9.5.1, there was no limit on the key length. @@ -1831,8 +1846,10 @@ class Config extends EventEmitter { process.env.OVERRIDE_OBJECT_KEY_BYTE_LIMIT || config.overrideObjectKeyByteLimit; if (overrideObjectKeyByteLimit !== null && overrideObjectKeyByteLimit !== undefined) { this.objectKeyByteLimit = parseInt(overrideObjectKeyByteLimit, 10); - assert(Number.isInteger(this.objectKeyByteLimit) && this.objectKeyByteLimit >= 0, - 'bad config: overrideObjectKeyByteLimit must be a positive integer'); + assert( + Number.isInteger(this.objectKeyByteLimit) && this.objectKeyByteLimit >= 0, + 'bad config: overrideObjectKeyByteLimit must be a positive integer', + ); } this.enableVeeamRoute = true; @@ -1871,9 +1888,12 @@ class Config extends EventEmitter { // decreases the weight attributed to a day in order to expedite the lifecycle of objects. const timeProgressionFactor = Number.parseInt(process.env.TIME_PROGRESSION_FACTOR, 10) || 1; - const isIncompatible = (expireOneDayEarlier || transitionOneDayEarlier) && (timeProgressionFactor > 1); - assert(!isIncompatible, 'The environment variables "EXPIRE_ONE_DAY_EARLIER" or ' + - '"TRANSITION_ONE_DAY_EARLIER" are not compatible with the "TIME_PROGRESSION_FACTOR" variable.'); + const isIncompatible = (expireOneDayEarlier || transitionOneDayEarlier) && timeProgressionFactor > 1; + assert( + !isIncompatible, + 'The environment variables "EXPIRE_ONE_DAY_EARLIER" or ' + + '"TRANSITION_ONE_DAY_EARLIER" are not compatible with the "TIME_PROGRESSION_FACTOR" variable.', + ); // The scaledMsPerDay value is initially set to the number of milliseconds per day // (24 * 60 * 60 * 1000) as the default value. @@ -1909,9 +1929,9 @@ class Config extends EventEmitter { let quota = 'none'; if (process.env.S3BACKEND) { const validBackends = ['mem', 'file', 'scality', 'cdmi']; - assert(validBackends.indexOf(process.env.S3BACKEND) > -1, - 'bad environment variable: S3BACKEND environment variable ' + - 'should be one of mem/file/scality/cdmi' + assert( + validBackends.indexOf(process.env.S3BACKEND) > -1, + 'bad environment variable: S3BACKEND environment variable ' + 'should be one of mem/file/scality/cdmi', ); auth = process.env.S3BACKEND; data = process.env.S3BACKEND; @@ -1925,11 +1945,11 @@ class Config extends EventEmitter { // Auth only checks for 'mem' since mem === file auth = 'mem'; let authData; - if (process.env.SCALITY_ACCESS_KEY_ID && - process.env.SCALITY_SECRET_ACCESS_KEY) { + if (process.env.SCALITY_ACCESS_KEY_ID && process.env.SCALITY_SECRET_ACCESS_KEY) { authData = buildAuthDataAccount( - process.env.SCALITY_ACCESS_KEY_ID, - process.env.SCALITY_SECRET_ACCESS_KEY); + process.env.SCALITY_ACCESS_KEY_ID, + process.env.SCALITY_SECRET_ACCESS_KEY, + ); } else { authData = this._getAuthData(); } @@ -1937,7 +1957,7 @@ class Config extends EventEmitter { throw new Error('bad config: invalid auth config file.'); } this.authData = authData; - } else if (auth === 'multiple') { + } else if (auth === 'multiple') { const authData = this._getAuthData(); if (validateAuthConfig(authData)) { throw new Error('bad config: invalid auth config file.'); @@ -1947,18 +1967,18 @@ class Config extends EventEmitter { if (process.env.S3DATA) { const validData = ['mem', 'file', 'scality', 'multiple']; - assert(validData.indexOf(process.env.S3DATA) > -1, - 'bad environment variable: S3DATA environment variable ' + - 'should be one of mem/file/scality/multiple' + assert( + validData.indexOf(process.env.S3DATA) > -1, + 'bad environment variable: S3DATA environment variable ' + 'should be one of mem/file/scality/multiple', ); data = process.env.S3DATA; } if (data === 'scality' || data === 'multiple') { data = 'multiple'; } - assert(this.locationConstraints !== undefined && - this.restEndpoints !== undefined, - 'bad config: locationConstraints and restEndpoints must be set' + assert( + this.locationConstraints !== undefined && this.restEndpoints !== undefined, + 'bad config: locationConstraints and restEndpoints must be set', ); if (process.env.S3METADATA) { @@ -1981,13 +2001,12 @@ class Config extends EventEmitter { // Mongodb backend does not support null keys, so we must enforce null version compatibility // mode. With other backends (esp. metadata), this is used during migration from v0 to v1 // bucket format. - this.nullVersionCompatMode = (metadata === 'mongodb') || - (process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true'); + this.nullVersionCompatMode = metadata === 'mongodb' || process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true'; // Multi-object delete optimizations is only supported for MongoDB at the moment. It relies // on `getObjectsMD()` to return the objects in a single call, which is not supported by // other backends. - this.multiObjectDeleteEnableOptimizations &&= (metadata === 'mongodb'); + this.multiObjectDeleteEnableOptimizations &&= metadata === 'mongodb'; } _sseMigration(config) { @@ -2000,14 +2019,12 @@ class Config extends EventEmitter { this.sseMigration = {}; const { previousKeyType, previousKeyProtocol, previousKeyProvider } = config.sseMigration; if (!previousKeyType) { - assert.fail( - 'NotImplemented: No dynamic KMS key migration. Set sseMigration.previousKeyType'); + assert.fail('NotImplemented: No dynamic KMS key migration. Set sseMigration.previousKeyType'); } // If previousKeyType is provided it's used as static value to migrate the format of the key // without additional dynamic evaluation if the key provider is unknown. - assert(isValidType(previousKeyType), - 'ssenMigration.previousKeyType must be "internal" or "external"'); + assert(isValidType(previousKeyType), 'ssenMigration.previousKeyType must be "internal" or "external"'); this.sseMigration.previousKeyType = previousKeyType; let expectedProtocol; @@ -2018,25 +2035,28 @@ class Config extends EventEmitter { expectedProtocol = [KmsProtocol.scality, KmsProtocol.mem, KmsProtocol.file]; } else if (previousKeyType === KmsType.external) { // No defaults allowed for external provider - assert(previousKeyProtocol, - 'sseMigration.previousKeyProtocol must be defined for external provider'); + assert(previousKeyProtocol, 'sseMigration.previousKeyProtocol must be defined for external provider'); this.sseMigration.previousKeyProtocol = previousKeyProtocol; - assert(previousKeyProvider, - 'sseMigration.previousKeyProvider must be defined for external provider'); + assert(previousKeyProvider, 'sseMigration.previousKeyProvider must be defined for external provider'); this.sseMigration.previousKeyProvider = previousKeyProvider; expectedProtocol = [KmsProtocol.kmip, KmsProtocol.aws_kms]; } - assert(isValidProtocol(previousKeyType, this.sseMigration.previousKeyProtocol), - `sseMigration.previousKeyProtocol must be one of ${expectedProtocol}`); - assert(isValidProvider(previousKeyProvider), - 'sseMigration.previousKeyProvider must be lowercase alphanumeric only'); + assert( + isValidProtocol(previousKeyType, this.sseMigration.previousKeyProtocol), + `sseMigration.previousKeyProtocol must be one of ${expectedProtocol}`, + ); + assert( + isValidProvider(previousKeyProvider), + 'sseMigration.previousKeyProvider must be lowercase alphanumeric only', + ); if (this.sseMigration.previousKeyType === KmsType.external) { if ([KmsProtocol.file, KmsProtocol.mem].includes(this.backends.kms)) { assert.fail( `sseMigration.previousKeyType "external" can't migrate to "internal" KMS provider ${ - this.backends.kms}` + this.backends.kms + }`, ); } // We'd have to compare protocol & providerName @@ -2055,10 +2075,7 @@ class Config extends EventEmitter { } getGcpBucketNames(locationConstraint) { - const { - bucketName, - mpuBucketName, - } = this.locationConstraints[locationConstraint].details; + const { bucketName, mpuBucketName } = this.locationConstraints[locationConstraint].details; return { bucketName, mpuBucketName }; } @@ -2084,9 +2101,10 @@ class Config extends EventEmitter { } setReplicationEndpoints(locationConstraints) { - this.replicationEndpoints = - Object.keys(locationConstraints) - .map(key => ({ site: key, type: locationConstraints[key].type })); + this.replicationEndpoints = Object.keys(locationConstraints).map(key => ({ + site: key, + type: locationConstraints[key].type, + })); } getAzureEndpoint(locationConstraint) { @@ -2103,7 +2121,7 @@ class Config extends EventEmitter { getAzureStorageAccountName(locationConstraint) { const accountName = azureGetStorageAccountName( locationConstraint, - this.locationConstraints[locationConstraint].details + this.locationConstraints[locationConstraint].details, ); if (accountName) { return accountName; @@ -2131,31 +2149,27 @@ class Config extends EventEmitter { } getAzureStorageCredentials(locationConstraint) { - return azureGetLocationCredentials( - locationConstraint, - this.locationConstraints[locationConstraint].details - ); + return azureGetLocationCredentials(locationConstraint, this.locationConstraints[locationConstraint].details); } getPfsDaemonEndpoint(locationConstraint) { - return process.env[`${locationConstraint}_PFSD_ENDPOINT`] || - this.locationConstraints[locationConstraint].details.pfsDaemonEndpoint; + return ( + process.env[`${locationConstraint}_PFSD_ENDPOINT`] || + this.locationConstraints[locationConstraint].details.pfsDaemonEndpoint + ); } isSameAzureAccount(locationConstraintSrc, locationConstraintDest) { if (!locationConstraintDest) { return true; } - const azureSrcAccount = - this.getAzureStorageAccountName(locationConstraintSrc); - const azureDestAccount = - this.getAzureStorageAccountName(locationConstraintDest); + const azureSrcAccount = this.getAzureStorageAccountName(locationConstraintSrc); + const azureDestAccount = this.getAzureStorageAccountName(locationConstraintDest); return azureSrcAccount === azureDestAccount; } isAWSServerSideEncryption(locationConstraint) { - return this.locationConstraints[locationConstraint].details - .serverSideEncryption === true; + return this.locationConstraints[locationConstraint].details.serverSideEncryption === true; } getPublicInstanceId() { @@ -2163,9 +2177,7 @@ class Config extends EventEmitter { } setPublicInstanceId(instanceId) { - this.publicInstanceId = crypto.createHash('sha256') - .update(instanceId) - .digest('hex'); + this.publicInstanceId = crypto.createHash('sha256').update(instanceId).digest('hex'); } isQuotaEnabled() { @@ -2189,5 +2201,4 @@ module.exports = { azureGetStorageAccountName, azureGetLocationCredentials, parseSupportedLifecycleRules, - parseIntegrityChecks, }; diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 1a0272440e..99f083aeb1 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -3,7 +3,6 @@ const crypto = require('crypto'); const { crc32: crtCrc32, crc32c: crtCrc32c } = require('aws-crt').checksums; const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const { errors: ArsenalErrors, errorInstances } = require('arsenal'); -const { config } = require('../../../Config'); const { combinePartCrcs } = require('./crcCombine'); const { supportedSignatureChecksums, unsupportedSignatureChecksums } = require('../../../../constants'); @@ -680,10 +679,6 @@ async function validateMethodChecksumNoChunking(request, body, log) { return arsenalErrorFromChecksumError(contentSHA256Err); } - if (config.integrityChecks[request.apiMethod] === false) { - return null; - } - if (request.apiMethod in checksumedMethods) { return await defaultValidationFunc(request, body, log); } diff --git a/tests/unit/Config.js b/tests/unit/Config.js index ebedbc022a..4ecdbe4bbb 100644 --- a/tests/unit/Config.js +++ b/tests/unit/Config.js @@ -7,13 +7,10 @@ const { azureGetLocationCredentials, locationConstraintAssert, parseSupportedLifecycleRules, - parseIntegrityChecks, ConfigObject, } = require('../../lib/Config'); -const { - LOCATION_NAME_DMF, -} = require('../constants'); +const { LOCATION_NAME_DMF } = require('../constants'); const constants = require('../../constants'); const { ValidLifecycleRules: supportedLifecycleRules } = require('arsenal').models; @@ -25,15 +22,23 @@ describe('Config', () => { const setEnv = (key, value) => { if (key in process.env) { const v = process.env[key]; - envToRestore.push(() => { process.env[key] = v; }); + envToRestore.push(() => { + process.env[key] = v; + }); } else { - envToRestore.push(() => { delete process.env[key]; }); + envToRestore.push(() => { + delete process.env[key]; + }); } process.env[key] = value; }; - beforeEach(() => { envToRestore.length = 0; }); - afterEach(() => { envToRestore.reverse().forEach(cb => cb()); }); + beforeEach(() => { + envToRestore.length = 0; + }); + afterEach(() => { + envToRestore.reverse().forEach(cb => cb()); + }); it('should load default config.json without errors', done => { require('../../lib/Config'); @@ -56,7 +61,7 @@ describe('Config', () => { describe('azureGetStorageAccountName', () => { it('should return the azureStorageAccountName', done => { const accountName = azureGetStorageAccountName('us-west-1', { - azureStorageAccountName: 'someaccount' + azureStorageAccountName: 'someaccount', }); assert.deepStrictEqual(accountName, 'someaccount'); return done(); @@ -66,7 +71,7 @@ describe('Config', () => { setEnv('us-west-1_AZURE_STORAGE_ACCOUNT_NAME', 'other'); setEnv('fr-east-2_AZURE_STORAGE_ACCOUNT_NAME', 'wrong'); const accountName = azureGetStorageAccountName('us-west-1', { - azureStorageAccountName: 'someaccount' + azureStorageAccountName: 'someaccount', }); assert.deepStrictEqual(accountName, 'other'); return done(); @@ -109,7 +114,7 @@ describe('Config', () => { it('should return shared-key credentials with authMethod from details', () => { const creds = azureGetLocationCredentials('us-west-1', { authMode: 'shared-key', - ...locationDetails + ...locationDetails, }); assert.deepStrictEqual(creds, { authMethod: 'shared-key', @@ -150,7 +155,7 @@ describe('Config', () => { it('should return shared-access-signature-token credentials with authMethod from details', () => { const creds = azureGetLocationCredentials('us-west-1', { authMethod: 'shared-access-signature', - ...locationDetails + ...locationDetails, }); assert.deepStrictEqual(creds, { authMethod: 'shared-access-signature', @@ -197,7 +202,7 @@ describe('Config', () => { it('should return client-secret credentials with authMethod from details', () => { const creds = azureGetLocationCredentials('us-west-1', { authMethod: 'client-secret', - ...locationDetails + ...locationDetails, }); assert.deepStrictEqual(creds, { authMethod: 'client-secret', @@ -223,69 +228,54 @@ describe('Config', () => { it('should return account name from config', () => { setEnv('azurebackend_AZURE_STORAGE_ACCOUNT_NAME', ''); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend'), - 'fakeaccountname' - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend'), 'fakeaccountname'); }); it('should return account name from env', () => { setEnv('azurebackend_AZURE_STORAGE_ACCOUNT_NAME', 'foooo'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend'), - 'foooo' - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend'), 'foooo'); }); it('should return account name from shared-access-signature auth', () => { setEnv('S3_LOCATION_FILE', 'tests/locationConfig/locationConfigTests.json'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend3'), - 'fakeaccountname3' - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend3'), 'fakeaccountname3'); }); it('should return account name from client-secret auth', () => { setEnv('S3_LOCATION_FILE', 'tests/locationConfig/locationConfigTests.json'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend4'), - 'fakeaccountname4', - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend4'), 'fakeaccountname4'); }); it('should return account name from endpoint', () => { setEnv('S3_LOCATION_FILE', 'tests/locationConfig/locationConfigTests.json'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azuritebackend'), - 'myfakeaccount', - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azuritebackend'), 'myfakeaccount'); }); }); describe('locationConstraintAssert', () => { const memLocation = { - 'details': {}, - 'isCold': false, - 'isTransient': false, - 'legacyAwsBehavior': false, - 'locationType': 'location-mem-v1', - 'objectId': 'a9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'mem', + details: {}, + isCold: false, + isTransient: false, + legacyAwsBehavior: false, + locationType: 'location-mem-v1', + objectId: 'a9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'mem', }; it('should parse tlp location', () => { const locationConstraints = { 'dmf-1': { - 'details': {}, - 'isCold': true, - 'legacyAwsBehavior': false, - 'locationType': LOCATION_NAME_DMF, - 'objectId': 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'tlp' + details: {}, + isCold: true, + legacyAwsBehavior: false, + locationType: LOCATION_NAME_DMF, + objectId: 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'tlp', }, 'us-east-1': memLocation, }; @@ -295,12 +285,12 @@ describe('Config', () => { it('should fail tlp location is not cold', () => { const locationConstraints = { 'dmf-1': { - 'details': {}, - 'isCold': false, - 'legacyAwsBehavior': false, - 'locationType': LOCATION_NAME_DMF, - 'objectId': 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'tlp' + details: {}, + isCold: false, + legacyAwsBehavior: false, + locationType: LOCATION_NAME_DMF, + objectId: 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'tlp', }, 'us-east-1': memLocation, }; @@ -310,14 +300,14 @@ describe('Config', () => { it('should fail if tlp location has details', () => { const locationConstraints = { 'dmf-1': { - 'details': { - 'endpoint': 'http://localhost:8000', + details: { + endpoint: 'http://localhost:8000', }, - 'isCold': true, - 'legacyAwsBehavior': false, - 'locationType': LOCATION_NAME_DMF, - 'objectId': 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'tlp' + isCold: true, + legacyAwsBehavior: false, + locationType: LOCATION_NAME_DMF, + objectId: 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'tlp', }, 'us-east-1': memLocation, }; @@ -513,8 +503,7 @@ describe('Config', () => { before(() => { oldConfig = process.env.S3_CONFIG_FILE; - process.env.S3_CONFIG_FILE = - 'tests/unit/testConfigs/allOptsConfig/config.json'; + process.env.S3_CONFIG_FILE = 'tests/unit/testConfigs/allOptsConfig/config.json'; }); after(() => { @@ -524,13 +513,10 @@ describe('Config', () => { it('should set up scuba', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.scuba, - { - host: 'localhost', - port: 8100, - }, - ); + assert.deepStrictEqual(config.scuba, { + host: 'localhost', + port: 8100, + }); }); it('should use environment variables for scuba', () => { @@ -539,13 +525,10 @@ describe('Config', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.scuba, - { - host: 'scubahost', - port: 1234, - }, - ); + assert.deepStrictEqual(config.scuba, { + host: 'scubahost', + port: 1234, + }); }); }); @@ -554,8 +537,7 @@ describe('Config', () => { before(() => { oldConfig = process.env.S3_CONFIG_FILE; - process.env.S3_CONFIG_FILE = - 'tests/unit/testConfigs/allOptsConfig/config.json'; + process.env.S3_CONFIG_FILE = 'tests/unit/testConfigs/allOptsConfig/config.json'; }); after(() => { @@ -565,13 +547,10 @@ describe('Config', () => { it('should set up quota', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.quota, - { - maxStaleness: 24 * 60 * 60 * 1000, - enableInflights: false, - }, - ); + assert.deepStrictEqual(config.quota, { + maxStaleness: 24 * 60 * 60 * 1000, + enableInflights: false, + }); }); it('should use environment variables for scuba', () => { @@ -580,13 +559,10 @@ describe('Config', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.quota, - { - maxStaleness: 1234, - enableInflights: true, - }, - ); + assert.deepStrictEqual(config.quota, { + maxStaleness: 1234, + enableInflights: true, + }); }); it('should use the default if the maxStaleness is not a number', () => { @@ -595,13 +571,10 @@ describe('Config', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.quota, - { - maxStaleness: 24 * 60 * 60 * 1000, - enableInflights: true, - }, - ); + assert.deepStrictEqual(config.quota, { + maxStaleness: 24 * 60 * 60 * 1000, + enableInflights: true, + }); }); }); @@ -610,8 +583,7 @@ describe('Config', () => { before(() => { oldConfig = process.env.S3_CONFIG_FILE; - process.env.S3_CONFIG_FILE = - 'tests/unit/testConfigs/allOptsConfig/config.json'; + process.env.S3_CONFIG_FILE = 'tests/unit/testConfigs/allOptsConfig/config.json'; }); after(() => { @@ -621,35 +593,29 @@ describe('Config', () => { it('should set up utapi local cache', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.localCache, - { name: 'zenko', sentinels: [{ host: 'localhost', port: 6379 }] }, - ); - assert.deepStrictEqual( - config.utapi.localCache, - config.localCache, - ); + assert.deepStrictEqual(config.localCache, { + name: 'zenko', + sentinels: [{ host: 'localhost', port: 6379 }], + }); + assert.deepStrictEqual(config.utapi.localCache, config.localCache); }); it('should set up utapi redis', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.utapi.redis, - { - host: 'localhost', - port: 6379, - retry: { - connectBackoff: { - min: 10, - max: 1000, - factor: 1.5, - jitter: 0.1, - deadline: 10000, - }, + assert.deepStrictEqual(config.utapi.redis, { + host: 'localhost', + port: 6379, + retry: { + connectBackoff: { + min: 10, + max: 1000, + factor: 1.5, + jitter: 0.1, + deadline: 10000, }, }, - ); + }); }); }); @@ -674,11 +640,7 @@ describe('Config', () => { }); it('should return the rules provided when they are valid', () => { - const rules = [ - 'Expiration', - 'NoncurrentVersionExpiration', - 'AbortIncompleteMultipartUpload', - ]; + const rules = ['Expiration', 'NoncurrentVersionExpiration', 'AbortIncompleteMultipartUpload']; const parsedRules = parseSupportedLifecycleRules(rules); assert.deepStrictEqual(parsedRules, rules); }); @@ -837,8 +799,7 @@ describe('Config', () => { .withArgs(sinon.match(/\/config\.json$/)) .returns(JSON.stringify({ ...defaultConfig, instanceId: 'test' })); // For all other files, use the original readFileSync - readFileSyncStub - .callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); + readFileSyncStub.callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); // Create a new ConfigObject instance const config = new ConfigObject(); assert.strictEqual(config.instanceId, 'test'); @@ -853,8 +814,7 @@ describe('Config', () => { .withArgs(sinon.match(/\/config\.json$/)) .returns(JSON.stringify({ ...defaultConfig, instanceId: 1234 })); // For all other files, use the original readFileSync - readFileSyncStub - .callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); + readFileSyncStub.callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); // Create a new ConfigObject instance assert.throws(() => new ConfigObject()); }); @@ -894,14 +854,14 @@ describe('Config', () => { const multiObjectDeleteSize = 42; const modifiedConfig = { ...defaultConfig, - apiBodySizeLimits: { 'multiObjectDelete': multiObjectDeleteSize }, + apiBodySizeLimits: { multiObjectDelete: multiObjectDeleteSize }, }; readFileStub.withArgs(sinon.match(/config.json$/)).returns(JSON.stringify(modifiedConfig)); const config = new ConfigObject(); assert.deepStrictEqual(config.apiBodySizeLimits, { - 'multiObjectDelete': multiObjectDeleteSize, // Configured: overwrites default - 'bucketPutPolicy': constants.defaultApiBodySizeLimits['bucketPutPolicy'], // Not configured: default + multiObjectDelete: multiObjectDeleteSize, // Configured: overwrites default + bucketPutPolicy: constants.defaultApiBodySizeLimits['bucketPutPolicy'], // Not configured: default }); }); @@ -914,51 +874,11 @@ describe('Config', () => { assert.throws( () => new ConfigObject(), - /bad config: apiBodySizeLimits for "anApiNotSetInConstants.js" cannot be configured/ + /bad config: apiBodySizeLimits for "anApiNotSetInConstants.js" cannot be configured/, ); }); }); - describe('parse integrity checks', () => { - it('should insert values into integrityCheck object', () => { - const newConfig = { - integrityChecks: { - 'bucketPutACL': false, - 'bucketPutCors': false, - 'bucketPutEncryption': false, - 'bucketPutLifecycle': false, - 'bucketPutNotification': false, - 'bucketPutObjectLock': false, - 'bucketPutPolicy': false, - 'bucketPutReplication': false, - 'bucketPutVersioning': false, - 'bucketPutWebsite': false, - 'bucketPutLogging': false, - 'bucketPutTagging': false, - 'multiObjectDelete': false, - 'objectPutACL': false, - 'objectPutLegalHold': false, - 'objectPutTagging': false, - 'objectPutRetention': false, - 'objectRestore': false, - 'completeMultipartUpload': false, - }, - }; - - const result = parseIntegrityChecks(newConfig); - for (const method in result) { - assert(result[method] == false, method); - } - }); - - it('default method value is true', () => { - const result = parseIntegrityChecks(null); - for (const method in result) { - assert(result[method] == true, method); - } - }); - }); - describe('objectKeyByteLimit', () => { it('should use default objectKeyByteLimit (915) from arsenal constants', () => { const config = new ConfigObject(); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index c56f44d4a8..f456739152 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -17,7 +17,6 @@ const { getCopyObjectChecksumAlgorithm, } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); -const { config } = require('../../../../../lib/Config'); describe('validateChecksumsNoChunking MD5', () => { describe('with valid Content-MD5 header', () => { @@ -329,16 +328,6 @@ describe('validateChecksumsNoChunking CRC32, CRC32C, SHA1, SHA256, CRC64NVME', ( }); describe('validateMethodChecksumNoChunking', () => { - let originalIntegrityChecks; - - beforeEach(() => { - originalIntegrityChecks = { ...config.integrityChecks }; - }); - - afterEach(() => { - config.integrityChecks = originalIntegrityChecks; - }); - describe('when checksum mismatches', () => { Object.keys(checksumedMethods).forEach(method => { it(`should return BadDigest error for ${method} when checksum mismatch`, async () => { @@ -416,28 +405,6 @@ describe('validateMethodChecksumNoChunking', () => { }); }); - describe('when method is disabled in config', () => { - Object.keys(checksumedMethods).forEach(method => { - it(`should return null for ${method} when disabled, even with checksum mismatch`, async () => { - config.integrityChecks[method] = false; - - const body = 'Hello, World!'; - const wrongMd5 = 'wrongchecksum123='; - const request = { - apiMethod: method, - headers: { - 'content-md5': wrongMd5, - }, - }; - const log = new DummyRequestLogger(); - - const result = await validateMethodChecksumNoChunking(request, body, log); - - assert.ifError(result); - }); - }); - }); - describe('when method is not in validation function mapping', () => { it('should return null for unsupported method', async () => { const unsupportedMethod = 'someUnsupportedMethod';