diff --git a/lib/Config.js b/lib/Config.js index 824625950d..a56478c50a 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -212,6 +212,17 @@ function locationConstraintAssert(locationConstraints) { // eslint-disable-next-line no-param-reassign locationConstraints[l].details.pathStyle = false; } + + if (details.supportsVersioning !== undefined) { + assert(typeof details.supportsVersioning === 'boolean', + 'bad config: locationConstraints[region].supportsVersioning' + + 'must be a boolean'); + } else { + // default to true + // eslint-disable-next-line no-param-reassign + locationConstraints[l].details.supportsVersioning = true; + } + if (locationConstraints[l].type === 'azure') { azureLocationConstraintAssert(l, locationConstraints[l]); } @@ -943,6 +954,10 @@ class Config extends EventEmitter { return dataStoreName && dataStoreName.type; } + getLocationConstraint(locationConstraintName) { + return this.locationConstraints[locationConstraintName]; + } + setRestEndpoints(restEndpoints) { restEndpointsAssert(restEndpoints, this.locationConstraints); this.restEndpoints = restEndpoints; diff --git a/lib/api/bucketPutVersioning.js b/lib/api/bucketPutVersioning.js index 2c5a9f9782..ec121f4e4c 100644 --- a/lib/api/bucketPutVersioning.js +++ b/lib/api/bucketPutVersioning.js @@ -60,9 +60,17 @@ function _checkBackendVersioningImplemented(bucket) { const bucketLocation = bucket.getLocationConstraint(); const bucketLocationType = config.getLocationConstraintType(bucketLocation); + // backend types known not to support versioning if (versioningNotImplBackends[bucketLocationType]) { return false; } + + // versioning disabled per-location constraint + const lc = config.getLocationConstraint(bucketLocation); + if (lc.details && !lc.details.supportsVersioning) { + return false; + } + return true; } diff --git a/lib/data/external/AwsClient.js b/lib/data/external/AwsClient.js index acfe9596ae..a6aa79f992 100644 --- a/lib/data/external/AwsClient.js +++ b/lib/data/external/AwsClient.js @@ -21,6 +21,7 @@ class AwsClient { this._bucketMatch = config.bucketMatch; this._dataStoreName = config.dataStoreName; this._serverSideEncryption = config.serverSideEncryption; + this._supportsVersioning = config.supportsVersioning; this._client = new AWS.S3(this._s3Params); } @@ -46,7 +47,7 @@ class AwsClient { `${this.type}: ${err.message}`) ); } - if (!data.VersionId) { + if (!data.VersionId && this._supportsVersioning) { logHelper(log, 'error', 'missing version id for data ' + 'backend object', missingVerIdInternalError, this._dataStoreName, this.clientType); @@ -180,6 +181,12 @@ class AwsClient { awsResp[location] = { error: err, external: true }; return callback(null, awsResp); } + if (!this._supportsVersioning) { + awsResp[location] = { + message: 'Congrats! You own the bucket', + }; + return callback(null, awsResp); + } return this._client.getBucketVersioning({ Bucket: this._awsBucketName }, (err, data) => { @@ -362,7 +369,7 @@ class AwsClient { `${this.type}: ${err.message}`) ); } - if (!completeMpuRes.VersionId) { + if (!completeMpuRes.VersionId && this._supportsVersioning) { logHelper(log, 'error', 'missing version id for data ' + 'backend object', missingVerIdInternalError, this._dataStoreName, this.clientType); @@ -504,7 +511,7 @@ class AwsClient { `${this.type}: ${err.message}`) ); } - if (!copyResult.VersionId) { + if (!copyResult.VersionId && this._supportsVersioning) { logHelper(log, 'error', 'missing version id for data ' + 'backend object', missingVerIdInternalError, this._dataStoreName, this.clientType); diff --git a/lib/data/locationConstraintParser.js b/lib/data/locationConstraintParser.js index 0cea2915de..fafc03ccc0 100644 --- a/lib/data/locationConstraintParser.js +++ b/lib/data/locationConstraintParser.js @@ -99,6 +99,7 @@ function parseLC() { bucketMatch: locationObj.details.bucketMatch, serverSideEncryption: locationObj.details.serverSideEncryption, dataStoreName: location, + supportsVersioning: locationObj.details.supportsVersioning, }; if (locationObj.type === 'gcp') { clientConfig.mpuBucket = locationObj.details.mpuBucketName; diff --git a/lib/management/configuration.js b/lib/management/configuration.js index d94a20b1c2..798d6aed21 100644 --- a/lib/management/configuration.js +++ b/lib/management/configuration.js @@ -1,4 +1,5 @@ const forge = require('node-forge'); +const { URL } = require('url'); const { buildAuthDataAccount } = require('../auth/in_memory/builder'); const _config = require('../Config').config; @@ -95,6 +96,9 @@ function patchConfiguration(instanceId, newConf, log, cb) { Object.keys(newConf.locations || {}).forEach(k => { const l = newConf.locations[k]; const location = {}; + let supportsVersioning = false; + let pathStyle = false; + switch (l.locationType) { case 'location-mem-v1': location.type = 'mem'; @@ -115,11 +119,23 @@ function patchConfiguration(instanceId, newConf, log, cb) { }; } break; - case 'location-do-spaces-v1': + case 'location-scality-ring-s3-v1': + pathStyle = true; // fallthrough case 'location-aws-s3-v1': case 'location-wasabi-v1': + supportsVersioning = true; // fallthrough + case 'location-do-spaces-v1': location.type = 'aws_s3'; if (l.details.secretKey && l.details.secretKey.length > 0) { + let https = true; + let awsEndpoint = l.details.endpoint || + 's3.amazonaws.com'; + if (awsEndpoint.includes('://')) { + const url = new URL(awsEndpoint); + awsEndpoint = url.host; + https = url.scheme === 'https'; + } + location.details = { credentials: { accessKey: l.details.accessKey, @@ -130,8 +146,10 @@ function patchConfiguration(instanceId, newConf, log, cb) { bucketMatch: l.details.bucketMatch, serverSideEncryption: Boolean(l.details.serverSideEncryption), - awsEndpoint: l.details.endpoint || - 's3.amazonaws.com', + awsEndpoint, + supportsVersioning, + pathStyle, + https, }; } break; diff --git a/tests/locationConfig/locationConfigLegacy.json b/tests/locationConfig/locationConfigLegacy.json index 1107f4416e..41d35febfd 100644 --- a/tests/locationConfig/locationConfigLegacy.json +++ b/tests/locationConfig/locationConfigLegacy.json @@ -126,5 +126,27 @@ "bucketMatch": true, "credentialsProfile": "google" } + }, + "withversioning": { + "type": "aws_s3", + "legacyAwsBehavior": true, + "details": { + "awsEndpoint": "s3.amazonaws.com", + "bucketName": "multitester555", + "bucketMatch": true, + "credentialsProfile": "default", + "supportsVersioning": true + } + }, + "withoutversioning": { + "type": "aws_s3", + "legacyAwsBehavior": true, + "details": { + "awsEndpoint": "s3.amazonaws.com", + "bucketName": "multitester555", + "bucketMatch": true, + "credentialsProfile": "default", + "supportsVersioning": false + } } } diff --git a/tests/locationConfig/locationConfigTests.json b/tests/locationConfig/locationConfigTests.json index d656f29caf..3f28174a87 100644 --- a/tests/locationConfig/locationConfigTests.json +++ b/tests/locationConfig/locationConfigTests.json @@ -163,5 +163,27 @@ "bucketMatch": false, "credentialsProfile": "google" } + }, + "withversioning": { + "type": "aws_s3", + "legacyAwsBehavior": true, + "details": { + "awsEndpoint": "s3.amazonaws.com", + "bucketName": "multitester555", + "bucketMatch": true, + "credentialsProfile": "default", + "supportsVersioning": true + } + }, + "withoutversioning": { + "type": "aws_s3", + "legacyAwsBehavior": true, + "details": { + "awsEndpoint": "s3.amazonaws.com", + "bucketName": "multitester555", + "bucketMatch": true, + "credentialsProfile": "default", + "supportsVersioning": false + } } } diff --git a/tests/unit/DummyService.js b/tests/unit/DummyService.js index af3f441285..1d113b68f3 100644 --- a/tests/unit/DummyService.js +++ b/tests/unit/DummyService.js @@ -1,6 +1,18 @@ const uuid = require('uuid/v4'); class DummyService { + constructor(config = {}) { + this.versioning = config.versioning; + } + headBucket(params, callback) { + return callback(); + } + getBucketVersioning(params, callback) { + if (this.versioning) { + return callback(null, { Status: 'Enabled' }); + } + return callback(null, {}); + } headObject(params, callback) { const retObj = { ContentLength: `${1024 * 1024 * 1024}`, @@ -11,10 +23,37 @@ class DummyService { const retObj = { Bucket: params.Bucket, Key: params.Key, - VersionId: uuid().replace(/-/g, ''), ETag: `"${uuid().replace(/-/g, '')}"`, ContentLength: `${1024 * 1024 * 1024}`, }; + if (this.versioning) { + retObj.VersionId = uuid().replace(/-/g, ''); + } + return callback(null, retObj); + } + upload(params, callback) { + this.putObject(params, callback); + } + putObject(params, callback) { + const retObj = { + ETag: `"${uuid().replace(/-/g, '')}"`, + }; + if (this.versioning) { + retObj.VersionId = uuid().replace(/-/g, ''); + } + return callback(null, retObj); + } + copyObject(params, callback) { + const retObj = { + CopyObjectResult: { + ETag: `"${uuid().replace(/-/g, '')}"`, + LastModified: new Date().toISOString(), + }, + VersionId: null, + }; + if (this.versioning) { + retObj.VersionId = uuid().replace(/-/g, ''); + } return callback(null, retObj); } // To-Do: add tests for other methods diff --git a/tests/unit/api/bucketPutVersioning.js b/tests/unit/api/bucketPutVersioning.js new file mode 100644 index 0000000000..ac287ccb57 --- /dev/null +++ b/tests/unit/api/bucketPutVersioning.js @@ -0,0 +1,132 @@ +const assert = require('assert'); + +const { errors } = require('arsenal'); +const { bucketPut } = require('../../../lib/api/bucketPut'); +const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); + +const { cleanup, + DummyRequestLogger, + makeAuthInfo } = require('../helpers'); +const metadata = require('../../../lib/metadata/wrapper'); + +const xmlEnableVersioning = +'' + +'Enabled' + +''; + +const xmlSuspendVersioning = +'' + +'Suspended' + +''; + +const locConstraintVersioned = +'' + +'withversioning' + +''; + +const locConstraintNonVersioned = +'' + +'withoutversioning' + +''; + +const externalVersioningErrorMessage = 'We do not currently support putting ' + +'a versioned object to a location-constraint of type Azure or GCP.'; + +const log = new DummyRequestLogger(); +const bucketName = 'bucketname'; +const authInfo = makeAuthInfo('accessKey1'); + +function _getPutBucketRequest(xml) { + const request = { + bucketName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }; + request.post = xml; + return request; +} + +function _putVersioningRequest(xml) { + const request = { + bucketName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/?versioning', + query: { versioning: '' }, + }; + request.post = xml; + return request; +} + +describe('bucketPutVersioning API', () => { + before(() => cleanup()); + afterEach(() => cleanup()); + + describe('with version enabled location constraint', () => { + beforeEach(done => { + const request = _getPutBucketRequest(locConstraintVersioned); + bucketPut(authInfo, request, log, done); + }); + + const tests = [ + { + msg: 'should successfully enable versioning on location ' + + 'constraint with supportsVersioning set to true', + input: xmlEnableVersioning, + output: { Status: 'Enabled' }, + }, + { + msg: 'should successfully suspend versioning on location ' + + 'constraint with supportsVersioning set to true', + input: xmlSuspendVersioning, + output: { Status: 'Suspended' }, + }, + ]; + tests.forEach(test => it(test.msg, done => { + const request = _putVersioningRequest(test.input); + bucketPutVersioning(authInfo, request, log, err => { + assert.ifError(err, + `Expected success, but got err: ${err}`); + metadata.getBucket(bucketName, log, (err, bucket) => { + assert.ifError(err, + `Expected success, but got err: ${err}`); + assert.deepStrictEqual(bucket._versioningConfiguration, + test.output); + done(); + }); + }); + })); + }); + + describe('with version disabled location constraint', () => { + beforeEach(done => { + const request = _getPutBucketRequest(locConstraintNonVersioned); + bucketPut(authInfo, request, log, done); + }); + + const tests = [ + { + msg: 'should return error if enabling versioning on location ' + + 'constraint with supportsVersioning set to false', + input: xmlEnableVersioning, + output: { error: errors.NotImplemented.customizeDescription( + externalVersioningErrorMessage) }, + }, + { + msg: 'should return error if suspending versioning on ' + + ' location constraint with supportsVersioning set to false', + input: xmlSuspendVersioning, + output: { error: errors.NotImplemented.customizeDescription( + externalVersioningErrorMessage) }, + }, + ]; + tests.forEach(test => it(test.msg, done => { + const putBucketVersioningRequest = + _putVersioningRequest(test.input); + bucketPutVersioning(authInfo, putBucketVersioningRequest, log, + err => { + assert.deepStrictEqual(err, test.output.error); + done(); + }); + })); + }); +}); diff --git a/tests/unit/multipleBackend/ExternalBackendClient.js b/tests/unit/multipleBackend/ExternalBackendClient.js index 3860ecc7d3..bd9c8ea4df 100644 --- a/tests/unit/multipleBackend/ExternalBackendClient.js +++ b/tests/unit/multipleBackend/ExternalBackendClient.js @@ -1,4 +1,5 @@ const assert = require('assert'); + const AwsClient = require('../../../lib/data/external/AwsClient'); const GcpClient = require('../../../lib/data/external/GcpClient'); const DummyService = require('../DummyService'); @@ -28,13 +29,14 @@ const backendClients = [ }, }, ]; +const log = new DummyRequestLogger(); backendClients.forEach(backend => { let testClient; before(() => { testClient = new backend.Class(backend.config); - testClient._client = new DummyService(backend.config); + testClient._client = new DummyService({ versioning: true }); }); describe(`${backend.name} completeMPU:`, () => { @@ -58,8 +60,6 @@ backendClients.forEach(backend => { const key = 'externalBackendTestKey'; const bucketName = 'externalBackendTestBucket'; const uploadId = 'externalBackendTestUploadId'; - const log = new DummyRequestLogger(); - testClient.completeMPU(jsonList, null, key, uploadId, bucketName, log, (err, res) => { assert.strictEqual(typeof res.key, 'string'); diff --git a/tests/unit/multipleBackend/VersioningBackendClient.js b/tests/unit/multipleBackend/VersioningBackendClient.js new file mode 100644 index 0000000000..683b7e6157 --- /dev/null +++ b/tests/unit/multipleBackend/VersioningBackendClient.js @@ -0,0 +1,190 @@ +const assert = require('assert'); +const { errors } = require('arsenal'); + +const AwsClient = require('../../../lib/data/external/AwsClient'); +const DummyService = require('../DummyService'); +const { DummyRequestLogger } = require('../helpers'); + +const missingVerIdInternalError = errors.InternalError.customizeDescription( + 'Invalid state. Please ensure versioning is enabled ' + + 'in AWS for the location constraint and try again.' +); + +const log = new DummyRequestLogger(); +const copyObjectRequest = { + bucketName: 'copyobjecttestbucket', + objectKey: 'copyobjecttestkey', + headers: { + 'x-amz-metadata-directive': 'COPY', + }, +}; + +const sourceLocationConstraint = 'awsbackend'; +const key = 'externalBackendTestKey'; +const bucket = 'externalBackendTestBucket'; +const reqUID = '42'; +const jsonList = { + Part: [ + { PartNumber: [1], ETag: ['testpart0001etag'] }, + { PartNumber: [2], ETag: ['testpart0002etag'] }, + { PartNumber: [3], ETag: ['testpart0003etag'] }, + ], +}; + +const s3Config = { + s3Params: {}, + bucketMatch: true, + bucketName: 'awsTestBucketName', + dataStoreName: 'awsDataStore', + serverSideEncryption: false, + supportsVersioning: true, + type: 'aws', +}; + +const assertSuccess = (err, cb) => { + assert.ifError(err, + `Expected success, but got error ${err}`); + cb(); +}; + +const assertFailure = (err, cb) => { + assert.deepStrictEqual(err, missingVerIdInternalError); + cb(); +}; +const genTests = [ + { + msg: 'should return success if supportsVersioning === true ' + + 'and backend versioning is enabled', + input: { supportsVersioning: true, enableMockVersioning: true }, + callback: assertSuccess, + }, + { + msg: 'should return success if supportsVersioning === false ' + + 'and backend versioning is enabled', + input: { supportsVersioning: false, enableMockVersioning: true }, + callback: assertSuccess, + }, + { + msg: 'should return error if supportsVersioning === true ' + + 'and backend versioning is disabled', + input: { supportsVersioning: true, enableMockVersioning: false }, + callback: assertFailure, + }, + { + msg: 'should return success if supportsVersioning === false ' + + 'and backend versioning is disabled', + input: { supportsVersioning: false, enableMockVersioning: false }, + callback: assertSuccess, + }, +]; + +describe('AwsClient::putObject', () => { + let testClient; + + before(() => { + testClient = new AwsClient(s3Config); + testClient._client = new DummyService({ versioning: true }); + }); + genTests.forEach(test => it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.put('', 0, { bucketName: bucket, objectKey: key }, + reqUID, err => test.callback(err, done)); + })); +}); + +describe('AwsClient::copyObject', () => { + let testClient; + + before(() => { + testClient = new AwsClient(s3Config); + testClient._client = new DummyService({ versioning: true }); + }); + + genTests.forEach(test => it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.copyObject(copyObjectRequest, null, key, + sourceLocationConstraint, null, log, err => test.callback(err, done)); + })); +}); + +describe('AwsClient::completeMPU', () => { + let testClient; + + before(() => { + testClient = new AwsClient(s3Config); + testClient._client = new DummyService({ versioning: true }); + }); + genTests.forEach(test => it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + const uploadId = 'externalBackendTestUploadId'; + testClient.completeMPU(jsonList, null, key, uploadId, + bucket, log, err => test.callback(err, done)); + })); +}); + +describe('AwsClient::healthcheck', () => { + let testClient; + + function assertSuccessVersioned(resp, cb) { + assert.deepStrictEqual(resp, { + versioningStatus: 'Enabled', + message: 'Congrats! You own the bucket', + }); + cb(); + } + function assertSuccessNonVersioned(resp, cb) { + assert.deepStrictEqual(resp, { + message: 'Congrats! You own the bucket', + }); + cb(); + } + function assertFailure(resp, cb) { + assert.strictEqual(!resp.Status || resp.Status === 'Suspended', true); + if (resp.Status) { + assert.strictEqual(resp.message, 'Versioning must be enabled'); + } + assert.strictEqual(resp.external, true); + cb(); + } + + before(() => { + testClient = new AwsClient(s3Config); + testClient._client = new DummyService({ versioning: true }); + }); + + const tests = [ + { + msg: 'should return success if supportsVersioning === true ' + + 'and backend versioning is enabled', + input: { supportsVersioning: true, enableMockVersioning: true }, + callback: assertSuccessVersioned, + }, + { + msg: 'should return success if supportsVersioning === false ' + + 'and backend versioning is enabled', + input: { supportsVersioning: false, enableMockVersioning: true }, + callback: assertSuccessNonVersioned, + }, + { + msg: 'should return error if supportsVersioning === true ' + + ' and backend versioning is disabled', + input: { supportsVersioning: true, enableMockVersioning: false }, + callback: assertFailure, + }, + { + msg: 'should return success if supportsVersioning === false ' + + 'and backend versioning is disabled', + input: { supportsVersioning: false, enableMockVersioning: false }, + callback: assertSuccessNonVersioned, + }, + ]; + tests.forEach(test => it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.healthcheck('backend', + (err, resp) => test.callback(resp.backend, done)); + })); +});