From dc9d5cc93bf6f0b5caa6e752292d331160e0a2e6 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 30 Jul 2026 13:14:00 +0200 Subject: [PATCH 1/4] CLDSRV-960: accept null checksums in prepareStream --- lib/api/apiUtils/object/prepareStream.js | 142 ++++++++++-------- lib/api/apiUtils/object/storeObject.js | 66 ++++---- .../unit/api/apiUtils/object/prepareStream.js | 76 ++++++++++ tests/unit/api/apiUtils/object/storeObject.js | 73 +++++++++ 4 files changed, 266 insertions(+), 91 deletions(-) diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index 9ec1e5ed2a..38f1a07836 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -6,30 +6,81 @@ const { parseContentSHA256, ContentSHA256Type } = require('../integrity/validate const { errors, errorInstances, jsutil } = require('arsenal'); const { unsupportedSignatureChecksums } = require('../../../../constants'); +/** + * Instantiates a ChecksumTransform for the given checksum configuration, or + * returns null if no checksum is requested. + * + * @param {object|null} checksum - { algorithm, isTrailer, expected }, or null + * @param {function} onStreamError - error listener for the transform + * @param {RequestLogger} log - request logger + * @return {ChecksumTransform|null} the transform, or null if no checksum + */ +function createChecksumStream(checksum, onStreamError, log) { + if (!checksum) { + return null; + } + const checksumStream = new ChecksumTransform(checksum.algorithm, checksum.expected, checksum.isTrailer, log); + checksumStream.on('error', onStreamError); + return checksumStream; +} + +/** + * Appends the requested checksum transforms to the pipeline: the secondary + * (only validated) first, then the primary, so that the primary always ends the + * pipeline and its digest covers the whole body. Either may be absent. + * + * @param {stream.Readable} inputStream - stream to append the transforms to + * @param {object|null} primary - primary checksum ({ algorithm, isTrailer, + * expected }), or null to compute no stored checksum + * @param {object|null} secondary - secondary checksum, or null + * @param {function} onStreamError - error listener for the transforms + * @param {RequestLogger} log - request logger + * @return {{ stream: stream.Readable, primaryChecksumStream: + * ChecksumTransform|null, secondaryChecksumStream: ChecksumTransform|null }} + */ +function pipeChecksumStreams(inputStream, primary, secondary, onStreamError, log) { + let stream = inputStream; + const secondaryChecksumStream = createChecksumStream(secondary, onStreamError, log); + if (secondaryChecksumStream) { + stream = stream.pipe(secondaryChecksumStream); + } + const primaryChecksumStream = createChecksumStream(primary, onStreamError, log); + if (primaryChecksumStream) { + stream = stream.pipe(primaryChecksumStream); + } + return { stream, primaryChecksumStream, secondaryChecksumStream }; +} + /** * Prepares the request stream for data storage by wrapping it in the * appropriate transform pipeline based on the x-amz-content-sha256 header. - * The returned stream is always the primary ChecksumTransform (the stored - * checksum). When a secondary checksum is requested it is inserted upstream - * of the primary and exposed via secondaryChecksumStream. + * The primary ChecksumTransform (the stored checksum) is the last transform of + * the pipeline and is returned as primaryChecksumStream; when a secondary + * checksum is requested it is inserted upstream of the primary and exposed via + * secondaryChecksumStream. Callers that need no checksum computed (the + * Backbeat routes, which rely on content-md5) pass no checksums at all, and + * get a pipeline without any ChecksumTransform. * * @param {object} request - incoming HTTP request with headers and body stream * @param {object|null} streamingV4Params - v4 streaming auth params (accessKey, * signatureFromRequest, region, scopeDate, timestamp, credentialScope), or * null/undefined for non-v4 requests - * @param {object} checksums - checksum configuration - * @param {object} checksums.primary - primary checksum + * @param {object|null} checksums - checksum configuration, or null to compute + * no checksum at all + * @param {object|null} checksums.primary - primary checksum * ({ algorithm, isTrailer, expected }) — validated and its digest returned * @param {object|null} checksums.secondary - optional secondary checksum * ({ algorithm, isTrailer, expected }) — only validated; used for MPU parts * @param {RequestLogger} log - request logger * @param {function} errCb - error callback invoked if a stream error occurs - * @return {{ error: Arsenal.Error|null, stream: ChecksumTransform|null, - * secondaryChecksumStream: ChecksumTransform|null }} + * @return {{ error: Arsenal.Error|null, stream: stream.Readable|null, + * primaryChecksumStream: ChecksumTransform|null, + * secondaryChecksumStream: ChecksumTransform|null, + * contentSHA256Stream: ContentSHA256Transform|null }} */ function prepareStream(request, streamingV4Params, checksums, log, errCb) { const xAmzContentSHA256 = request.headers['x-amz-content-sha256']; - const { primary, secondary } = checksums; + const { primary = null, secondary = null } = checksums || {}; switch (xAmzContentSHA256) { case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': { @@ -52,22 +103,11 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { request.pipe(v4Transform); v4Transform.headers = request.headers; - let secondaryChecksumStream = null; - let stream = v4Transform; - if (secondary) { - secondaryChecksumStream = new ChecksumTransform( - secondary.algorithm, - secondary.expected, - secondary.isTrailer, - log, - ); - secondaryChecksumStream.on('error', onStreamError); - stream = v4Transform.pipe(secondaryChecksumStream); - } - - const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); - primaryStream.on('error', onStreamError); - return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; + return { + error: null, + ...pipeChecksumStreams(v4Transform, primary, secondary, onStreamError, log), + contentSHA256Stream: null, + }; } case 'STREAMING-UNSIGNED-PAYLOAD-TRAILER': { const onStreamError = jsutil.once(errCb); @@ -76,30 +116,23 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { request.pipe(trailingChecksumTransform); trailingChecksumTransform.headers = request.headers; - let secondaryChecksumStream = null; - let stream = trailingChecksumTransform; - if (secondary) { - secondaryChecksumStream = new ChecksumTransform( - secondary.algorithm, - secondary.expected, - secondary.isTrailer, - log, - ); - secondaryChecksumStream.on('error', onStreamError); - stream = trailingChecksumTransform.pipe(secondaryChecksumStream); + const checksumStreams = pipeChecksumStreams( + trailingChecksumTransform, + primary, + secondary, + onStreamError, + log, + ); + // The trailer is validated against the secondary checksum when + // there is one, otherwise against the primary. + const trailerChecksumStream = + checksumStreams.secondaryChecksumStream || checksumStreams.primaryChecksumStream; + if (trailerChecksumStream) { trailingChecksumTransform.on('trailer', (name, value) => { - secondaryChecksumStream.setExpectedChecksum(name, value); + trailerChecksumStream.setExpectedChecksum(name, value); }); } - - const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); - primaryStream.on('error', onStreamError); - if (!secondary) { - trailingChecksumTransform.on('trailer', (name, value) => { - primaryStream.setExpectedChecksum(name, value); - }); - } - return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; + return { error: null, ...checksumStreams, contentSHA256Stream: null }; } case 'UNSIGNED-PAYLOAD': // Fallthrough default: { @@ -112,32 +145,17 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { const parsedContentSHA256 = parseContentSHA256(request.headers); const shouldValidateContentSHA256 = parsedContentSHA256.type === ContentSHA256Type.HexSHA256; - const onStreamError = secondary || shouldValidateContentSHA256 ? jsutil.once(errCb) : errCb; + const onStreamError = jsutil.once(errCb); let contentSHA256Stream = null; - let secondaryChecksumStream = null; let stream = request; if (shouldValidateContentSHA256) { contentSHA256Stream = new ContentSHA256Transform(parsedContentSHA256.value, log); contentSHA256Stream.on('error', onStreamError); stream = stream.pipe(contentSHA256Stream); } - if (secondary) { - secondaryChecksumStream = new ChecksumTransform( - secondary.algorithm, - secondary.expected, - secondary.isTrailer, - log, - ); - secondaryChecksumStream.on('error', onStreamError); - stream = stream.pipe(secondaryChecksumStream); - } - - const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); - primaryStream.on('error', onStreamError); return { error: null, - stream: stream.pipe(primaryStream), - secondaryChecksumStream, + ...pipeChecksumStreams(stream, primary, secondary, onStreamError, log), contentSHA256Stream, }; } diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index ee0baa03e8..738bdfac9e 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -12,11 +12,14 @@ const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/vali * @param {object} dataRetrievalInfo - object containing the keys of stored data * @param {number} dataRetrievalInfo.key - key of the stored data * @param {string} dataRetrievalInfo.dataStoreName - the implName of the data - * @param {object} checksumStream - checksum transform stream with digest/algoName properties + * @param {object|null} checksumStream - checksum transform stream with + * digest/algoName properties, or null if no checksum was computed * @param {object} log - request logger instance * @param {function} cb - callback to send error or move to next task * @return {function} - calls callback with arguments: - * error, dataRetrievalInfo, and completedHash (if any) + * error, dataRetrievalInfo, completedHash (if any), and checksum — + * `{ algorithm, value }` from checksumStream, or undefined when no checksum + * was computed */ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStream, log, cb) { const contentMD5 = stream.contentMD5; @@ -38,7 +41,7 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStre return cb(errors.BadDigest); }); } - const checksum = { algorithm: checksumStream.algoName, value: checksumStream.digest }; + const checksum = checksumStream ? { algorithm: checksumStream.algoName, value: checksumStream.digest } : undefined; return cb(null, dataRetrievalInfo, completedHash, checksum); } @@ -54,8 +57,9 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStre * credentialScope (to be used for streaming v4 auth if applicable) * @param {BackendInfo} backendInfo - info to determine which data * backend to use - * @param {object} checksums - checksum configuration - * @param {object} checksums.primary - primary checksum data + * @param {object|null} checksums - checksum configuration, or null to compute + * no checksum at all (callers relying on content-md5 only, e.g. Backbeat) + * @param {object|null} checksums.primary - primary checksum data * @param {object|null} checksums.secondary - secondary checksum data * @param {RequestLogger} log - the current stream logger * @param {function} cb - callback containing result for the next task @@ -69,14 +73,14 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, let onStreamError = cbOnce; const errCb = err => onStreamError(err); - const checksumedStream = prepareStream(stream, streamingV4Params, checksums, log, errCb); - if (checksumedStream.error) { - log.debug('dataStore failed to prepare stream', checksumedStream); - return process.nextTick(() => cbOnce(checksumedStream.error)); + const preparedStream = prepareStream(stream, streamingV4Params, checksums, log, errCb); + if (preparedStream.error) { + log.debug('dataStore failed to prepare stream', preparedStream); + return process.nextTick(() => cbOnce(preparedStream.error)); } return data.put( cipherBundle, - checksumedStream.stream, + preparedStream.stream, size, objectContext, backendInfo, @@ -104,10 +108,10 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, }); }; - // stream is always the primary (end of pipe, stored checksum). - // secondaryChecksumStream and contentSHA256Stream are upstream and - // only validated. - const { secondaryChecksumStream, contentSHA256Stream } = checksumedStream; + // The primary checksum stream (when there is one) is the end of the + // pipe and holds the stored checksum. secondaryChecksumStream and + // contentSHA256Stream are upstream and only validated. + const { primaryChecksumStream, secondaryChecksumStream, contentSHA256Stream } = preparedStream; const doValidate = () => { // Validate the SigV4 payload hash (x-amz-content-sha256) first. @@ -137,22 +141,24 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, } } // Validate the primary (stored) checksum. - const primaryErr = checksumedStream.stream.validateChecksum(); - if (primaryErr) { - log.debug('failed primary checksum validation', { error: primaryErr }); - return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { - if (deleteErr) { - log.error('dataStore failed to delete old data', { error: deleteErr }); - } - return cbOnce(arsenalErrorFromChecksumError(primaryErr)); - }); + if (primaryChecksumStream) { + const primaryErr = primaryChecksumStream.validateChecksum(); + if (primaryErr) { + log.debug('failed primary checksum validation', { error: primaryErr }); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + return cbOnce(arsenalErrorFromChecksumError(primaryErr)); + }); + } } if (!secondaryChecksumStream) { return checkHashMatchMD5( stream, hashedStream, dataRetrievalInfo, - checksumedStream.stream, + primaryChecksumStream, log, cbOnce, ); @@ -165,7 +171,7 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, stream, hashedStream, dataRetrievalInfo, - checksumedStream.stream, + primaryChecksumStream, log, (err, dataInfo, hash, primaryChecksum) => { if (err) { @@ -184,12 +190,14 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, // ChecksumTransform._flush computes the digest asynchronously for // some algorithms (e.g. crc64nvme). writableFinished is true once // _flush has called its callback, guaranteeing this.digest is set. - // stream is the primary (end of pipe) — when it finishes all - // upstream transforms (including the secondary) have flushed. - if (checksumedStream.stream.writableFinished) { + // The last checksum stream of the pipe is awaited: when it finishes + // all upstream transforms have flushed. If there is none, there is + // no digest to wait for. + const lastChecksumStream = primaryChecksumStream || secondaryChecksumStream || contentSHA256Stream; + if (!lastChecksumStream || lastChecksumStream.writableFinished) { return doValidate(); } - checksumedStream.stream.once('finish', doValidate); + lastChecksumStream.once('finish', doValidate); return undefined; }, ); diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index 37f9456bf9..5a582f03ef 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -5,6 +5,8 @@ const { errors } = require('arsenal'); const { prepareStream } = require('../../../../../lib/api/apiUtils/object/prepareStream'); const ChecksumTransform = require('../../../../../lib/auth/streamingV4/ChecksumTransform'); const ContentSHA256Transform = require('../../../../../lib/auth/streamingV4/ContentSHA256Transform'); +const V4Transform = require('../../../../../lib/auth/streamingV4/V4Transform'); +const TrailingChecksumTransform = require('../../../../../lib/auth/streamingV4/trailingChecksumTransform'); const { DummyRequestLogger } = require('../../../helpers'); const DummyRequest = require('../../../DummyRequest'); const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); @@ -45,6 +47,12 @@ describe('prepareStream', () => { assert(result.stream instanceof ChecksumTransform); }); + it('should return the primary ChecksumTransform as the last stream of the pipeline', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.primaryChecksumStream, result.stream); + }); + it('should return { error: BadRequest, stream: null } for unsupported x-amz-content-sha256', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', @@ -247,6 +255,74 @@ describe('prepareStream', () => { }); }); + describe('no checksum requested', () => { + [null, undefined, {}, { primary: null, secondary: null }].forEach(checksums => { + it(`should not create any ChecksumTransform with ${JSON.stringify(checksums)}`, () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, checksums, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream, request); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + assert.strictEqual(result.contentSHA256Stream, null); + }); + }); + + it('should return the request itself when there is no x-amz-content-sha256 header', () => { + const request = makeRequest({}); + const result = prepareStream(request, null, null, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream, request); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should still validate a literal x-amz-content-sha256 payload hash', done => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }, bodyData); + const result = prepareStream(request, null, null, log, done); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + assert(result.contentSHA256Stream instanceof ContentSHA256Transform); + assert.strictEqual(result.stream, result.contentSHA256Stream); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.contentSHA256Stream.validateChecksum(), null); + done(); + }); + result.stream.on('error', done); + }); + + it('should return the V4Transform for STREAMING-AWS4-HMAC-SHA256-PAYLOAD', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof V4Transform); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should return the TrailingChecksumTransform for STREAMING-UNSIGNED-PAYLOAD-TRAILER', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' }); + const result = prepareStream(request, null, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof TrailingChecksumTransform); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should still reject an unsupported x-amz-content-sha256', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', + }); + const result = prepareStream(request, null, null, log, () => {}); + assert.strictEqual(result.error.message, 'BadRequest'); + assert.strictEqual(result.stream, null); + }); + }); + describe('default (no x-amz-content-sha256)', () => { it('should return ChecksumTransform with crc64nvme algorithm when default checksums passed', () => { const request = makeRequest({}); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index 07bbc0b671..9270719c0b 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -290,6 +290,79 @@ describe('dataStore', () => { }); }); + describe('no checksum requested', () => { + function putSucceedsSync(completedHash = null) { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(null, fakeDataRetrievalInfo, { completedHash }); + }); + } + + it('should call data.put with the request itself, without any transform', done => { + putSucceedsSync(); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, 'hello world'); + dataStore({}, null, request, 0, null, {}, null, log, err => { + assert.strictEqual(err, null); + assert.strictEqual(putStub.firstCall.args[1], request); + done(); + }); + }); + + it('should call cb without any checksum', done => { + putSucceedsSync('abc123'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, 'hello world'); + dataStore({}, null, request, 0, null, {}, null, log, (err, dataInfo, completedHash, checksum) => { + assert.strictEqual(err, null); + assert.strictEqual(dataInfo, fakeDataRetrievalInfo); + assert.strictEqual(completedHash, 'abc123'); + assert.strictEqual(checksum, undefined); + done(); + }); + }); + + it('should still call cb with BadDigest and delete stored data when content-md5 does not match', done => { + batchDeleteSucceeds(); + putSucceedsSync('correct-md5'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, 'hello world'); + request.contentMD5 = 'wrong-md5'; + dataStore({}, null, request, 0, null, {}, null, log, err => { + assert.deepStrictEqual(err, errors.BadDigest); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should ignore x-amz-checksum-* headers', done => { + putSucceedsSync(); + // A checksum header that does not match the body: checksums come + // from the caller, headers are never parsed here. + const request = makeStream( + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }, + 'hello world', + ); + dataStore({}, null, request, 0, null, {}, null, log, (err, dataInfo, completedHash, checksum) => { + assert.strictEqual(err, null); + assert.strictEqual(checksum, undefined); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('should still validate x-amz-content-sha256 against the body', done => { + batchDeleteSucceeds(); + putSucceeds(); + const request = makeStream({ authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, null, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + }); + describe('x-amz-content-sha256 body validation', () => { // eslint-disable-next-line max-len it('should call cb with XAmzContentSHA256Mismatch and delete stored data when the hash does not match', done => { From 4c531c8e20aae78fdba7b8885610f670b421906b Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 30 Jul 2026 13:16:04 +0200 Subject: [PATCH 2/4] CLDSRV-960: remove x-amz-checksum- handling in backbeat routes --- lib/routes/routeBackbeat.js | 37 +-- tests/multipleBackend/routes/routeBackbeat.js | 210 +++++++++++++----- tests/unit/routes/routeBackbeat.js | 37 +++ 3 files changed, 195 insertions(+), 89 deletions(-) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index d61f59eebc..18c3ca2b95 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -35,11 +35,6 @@ const locationKeysHaveChanged = require('../api/apiUtils/object/locationKeysHave const { standardMetadataValidateBucketAndObj, metadataGetObject } = require('../metadata/metadataUtils'); const { config } = require('../Config'); const constants = require('../../constants'); -const { - defaultChecksumData, - getChecksumDataFromHeaders, - arsenalErrorFromChecksumError, -} = require('../api/apiUtils/integrity/validateChecksums'); const { BackendInfo } = models; const { pushReplicationMetric } = require('./utilities/pushReplicationMetric'); const writeContinue = require('../utilities/writeContinue'); @@ -497,14 +492,6 @@ function putData(request, response, bucketInfo, objMd, log, callback) { }); return callback(errors.InternalError); } - const headerChecksum = getChecksumDataFromHeaders(request.headers); - if (headerChecksum && headerChecksum.error) { - return callback(arsenalErrorFromChecksumError(headerChecksum)); - } - const checksums = { - primary: headerChecksum || defaultChecksumData, - secondary: null, - }; return dataStore( context, cipherBundle, @@ -512,14 +499,8 @@ function putData(request, response, bucketInfo, objMd, log, callback) { payloadLen, {}, backendInfo, - checksums, + null, log, - // The callback's 4th arg (checksum) is intentionally ignored: any - // x-amz-checksum-* header sent by Backbeat is validated inside - // dataStore by ChecksumTransform. The computed value is not stored - // here because this is a data-only write — metadata is written - // separately by Backbeat, which should propagate the source - // object's checksum. (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { @@ -1119,14 +1100,6 @@ function putObject(request, response, log, callback) { } const payloadLen = parseInt(request.headers['content-length'], 10); const backendInfo = new BackendInfo(config, storageLocation); - const headerChecksum = getChecksumDataFromHeaders(request.headers); - if (headerChecksum && headerChecksum.error) { - return callback(arsenalErrorFromChecksumError(headerChecksum)); - } - const checksums = { - primary: headerChecksum || defaultChecksumData, - secondary: null, - }; return dataStore( context, CIPHER, @@ -1134,14 +1107,8 @@ function putObject(request, response, log, callback) { payloadLen, {}, backendInfo, - checksums, + null, log, - // The callback's 4th arg (checksum) is intentionally ignored: any - // x-amz-checksum-* header sent by Backbeat is validated inside - // dataStore by ChecksumTransform. The computed value is not stored - // here because this is a data-only write to an external backend — - // metadata is managed separately by Backbeat, which should propagate - // the source object's checksum. (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { diff --git a/tests/multipleBackend/routes/routeBackbeat.js b/tests/multipleBackend/routes/routeBackbeat.js index c69a8a0691..40d860d366 100644 --- a/tests/multipleBackend/routes/routeBackbeat.js +++ b/tests/multipleBackend/routes/routeBackbeat.js @@ -3639,14 +3639,24 @@ describe('backbeat routes', () => { }); }); - describe('checksum validation', () => { + describe('checksums', () => { const testDataSha256B64 = crypto.createHash('sha256') .update(testData, 'utf-8').digest('base64'); // A valid-length but wrong sha256 digest (44 base64 chars). const wrongSha256B64 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; - - describe('putData', () => { - it('should return 400 BadDigest when x-amz-checksum-sha256 does not match body', done => { + // Checksum of the source object, as replicated by backbeat in the + // object metadata. + const sourceChecksum = { + checksumAlgorithm: 'sha256', + checksumValue: testDataSha256B64, + checksumType: 'FULL_OBJECT', + }; + + describe('data route integrity check', () => { + // Backbeat only sends content-md5: x-amz-checksum-* headers are not + // parsed at all by the backbeat routes, so a mismatching one must + // not fail the request. + it('should ignore a mismatching x-amz-checksum-sha256 header', done => { makeBackbeatRequest({ method: 'PUT', resourceType: 'data', @@ -3660,38 +3670,15 @@ describe('backbeat routes', () => { }, requestBody: testData, authCredentials: backbeatAuthCredentials, - }, err => { - assert(err, 'expected an error response'); - assert.strictEqual(err.statusCode, 400); - assert.strictEqual(err.code, 'BadDigest'); - done(); - }); - }); - - it('should return 200 when x-amz-checksum-sha256 matches body', done => { - makeBackbeatRequest({ - method: 'PUT', - resourceType: 'data', - bucket: TEST_BUCKET, - objectKey: TEST_KEY, - headers: { - 'x-scal-canonical-id': testMd['owner-id'], - 'content-md5': testDataMd5, - 'content-length': testData.length, - 'x-amz-checksum-sha256': testDataSha256B64, - }, - requestBody: testData, - authCredentials: backbeatAuthCredentials, }, (err, data) => { assert.ifError(err); assert.strictEqual(data.statusCode, 200); done(); }); }); - }); - describe('putObject (multiplebackenddata)', () => { - itIfLocationAws('should return 400 BadDigest when x-amz-checksum-sha256 does not match body', done => { + itIfLocationAws('should ignore a mismatching x-amz-checksum-sha256 header (multiplebackenddata)', + done => { makeBackbeatRequest({ method: 'PUT', resourceType: 'multiplebackenddata', @@ -3708,31 +3695,6 @@ describe('backbeat routes', () => { }, requestBody: testData, authCredentials: backbeatAuthCredentials, - }, err => { - assert(err, 'expected an error response'); - assert.strictEqual(err.statusCode, 400); - assert.strictEqual(err.code, 'BadDigest'); - done(); - }); - }); - - itIfLocationAws('should return 200 when x-amz-checksum-sha256 matches body', done => { - makeBackbeatRequest({ - method: 'PUT', - resourceType: 'multiplebackenddata', - bucket: TEST_BUCKET, - objectKey: TEST_KEY, - queryObj: { operation: 'putobject' }, - headers: { - 'x-scal-canonical-id': testMd['owner-id'], - 'x-scal-storage-type': 'aws_s3', - 'x-scal-storage-class': awsLocation, - 'content-md5': testDataMd5, - 'content-length': testData.length, - 'x-amz-checksum-sha256': testDataSha256B64, - }, - requestBody: testData, - authCredentials: backbeatAuthCredentials, }, (err, data) => { assert.ifError(err); assert.strictEqual(data.statusCode, 200); @@ -3740,5 +3702,145 @@ describe('backbeat routes', () => { }); }); }); + + describe('checksum replication', () => { + // The destination checksum is the one carried in the replicated + // metadata: nothing is recomputed by the data route. + it('should replicate the source object checksum (versioned bucket)', done => { + const objectKey = 'checksum-replication-key'; + async.waterfall([ + next => makeBackbeatRequest({ + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, next), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = getMetadataToPut(response); + newMd.checksum = sourceChecksum; + makeBackbeatRequest({ + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, next); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + s3.send(new HeadObjectCommand({ + Bucket: TEST_BUCKET, + Key: objectKey, + ChecksumMode: 'ENABLED', + })).then(result => { + assert.strictEqual(result.ChecksumSHA256, testDataSha256B64); + assert.strictEqual(result.ChecksumType, 'FULL_OBJECT'); + next(); + }, next); + }, + ], done); + }); + + it('should replicate the source object checksum (non-versioned bucket)', done => { + const objectKey = 'checksum-replication-key-non-versioned'; + async.waterfall([ + next => makeBackbeatRequest({ + method: 'PUT', + bucket: NONVERSIONED_BUCKET, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, next), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = Object.assign({}, nonVersionedTestMd, { + location: JSON.parse(response.body), + checksum: sourceChecksum, + }); + makeBackbeatRequest({ + method: 'PUT', + bucket: NONVERSIONED_BUCKET, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, next); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + s3.send(new HeadObjectCommand({ + Bucket: NONVERSIONED_BUCKET, + Key: objectKey, + ChecksumMode: 'ENABLED', + })).then(result => { + assert.strictEqual(result.ChecksumSHA256, testDataSha256B64); + assert.strictEqual(result.ChecksumType, 'FULL_OBJECT'); + next(); + }, next); + }, + ], done); + }); + + it('should not store a checksum when the replicated metadata has none', done => { + const objectKey = 'checksum-replication-key-none'; + async.waterfall([ + next => makeBackbeatRequest({ + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, next), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + makeBackbeatRequest({ + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(getMetadataToPut(response)), + }, next); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + s3.send(new HeadObjectCommand({ + Bucket: TEST_BUCKET, + Key: objectKey, + ChecksumMode: 'ENABLED', + })).then(result => { + assert.strictEqual(result.ChecksumSHA256, undefined); + assert.strictEqual(result.ChecksumCRC64NVME, undefined); + assert.strictEqual(result.ChecksumType, undefined); + next(); + }, next); + }, + ], done); + }); + }); }); }); diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index 19b4cd1ef9..7afcd8a6ce 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -178,6 +178,43 @@ describe('routeBackbeat', () => { assert.strictEqual(mockResponse.statusCode, 200); assert.deepStrictEqual(mockResponse.body, [{}]); + // Backbeat relies on content-md5: no checksum is computed over the data. + assert.strictEqual(storeObject.dataStore.firstCall.args[6], null); + }); + + it('should reject CRR destination requests (putData) when content-md5 does not match the data', async () => { + // content-md5 is the only integrity check on this route. + mockRequest.method = 'PUT'; + mockRequest.url = '/_/backbeat/data/bucket0/key0'; + mockRequest.headers = { + 'x-scal-canonical-id': 'id', + 'content-md5': '1234', + 'content-length': '0', + 'x-scal-versioning-required': 'true', + }; + mockRequest.destroy = () => {}; + + metadataUtils.standardMetadataValidateBucketAndObj.callsFake((params, denies, log, callback) => { + const bucketInfo = { + getVersioningConfiguration: () => ({ Status: 'Enabled' }), + isVersioningEnabled: () => true, + getLocationConstraint: () => undefined, + }; + const objMd = {}; + callback(null, bucketInfo, objMd); + }); + storeObject.dataStore.callsFake( + (objectContext, cipherBundle, stream, size, streamingV4Params, backendInfo, checksums, log, callback) => { + callback(null, {}, 'a-different-md5'); + }, + ); + + routeBackbeat('127.0.0.1', mockRequest, mockResponse, log); + + void (await endPromise); + + assert.strictEqual(mockResponse.statusCode, 400); + assert.strictEqual(mockResponse.body.code, 'BadDigest'); }); it('should return 409 VersionIdCollisionException when versionId matches master, no microVersionId', async () => { From 153e2fb32a2457d9bb5c1495cc2807a836b5527f Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 4 Aug 2026 16:28:38 +0200 Subject: [PATCH 3/4] CLDSRV-960: validate the primary checksum stream in the Veeam route --- lib/routes/veeam/utils.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/routes/veeam/utils.js b/lib/routes/veeam/utils.js index 9f19e0bab3..cf3fa0f7a3 100644 --- a/lib/routes/veeam/utils.js +++ b/lib/routes/veeam/utils.js @@ -89,9 +89,12 @@ async function receiveData(request, log) { // Checksum transforms only compute digests while streaming: validation // against the expected values (header or trailer) must be done once the // stream is fully consumed. + // `checksums.primary` is always set above, so primaryChecksumStream is the + // end of the pipeline here; validate it explicitly rather than relying on + // `prepared.stream` happening to be that transform. const checksumErr = (prepared.contentSHA256Stream && prepared.contentSHA256Stream.validateChecksum()) || - prepared.stream.validateChecksum(); + prepared.primaryChecksumStream.validateChecksum(); if (checksumErr) { log.debug('failed checksum validation', { error: checksumErr }); throw arsenalErrorFromChecksumError(checksumErr); From a84036788b9413b7c1c11457fa9143395678aae0 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 4 Aug 2026 17:18:58 +0200 Subject: [PATCH 4/4] CLDSRV-960: prettier lint --- lib/routes/routeBackbeat.js | 50 +- tests/multipleBackend/routes/routeBackbeat.js | 7758 ++++++++++------- 2 files changed, 4510 insertions(+), 3298 deletions(-) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 18c3ca2b95..7d1fdc93e2 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -1100,37 +1100,27 @@ function putObject(request, response, log, callback) { } const payloadLen = parseInt(request.headers['content-length'], 10); const backendInfo = new BackendInfo(config, storageLocation); - return dataStore( - context, - CIPHER, - request, - payloadLen, - {}, - backendInfo, - null, - log, - (err, retrievalInfo, md5) => { - if (err) { - log.error('error putting data', { - error: err, - method: 'putObject', - }); - return callback(err); - } - if (contentMD5 !== md5) { - return callback(errors.BadDigest); - } - const responsePayload = constructPutResponse({ - dataStoreName: retrievalInfo.dataStoreName, - dataStoreType: retrievalInfo.dataStoreType, - key: retrievalInfo.key, - size: payloadLen, - dataStoreETag: retrievalInfo.dataStoreETag ? `1:${retrievalInfo.dataStoreETag}` : `1:${md5}`, - dataStoreVersionId: retrievalInfo.dataStoreVersionId, + return dataStore(context, CIPHER, request, payloadLen, {}, backendInfo, null, log, (err, retrievalInfo, md5) => { + if (err) { + log.error('error putting data', { + error: err, + method: 'putObject', }); - return _respond(response, responsePayload, log, callback); - }, - ); + return callback(err); + } + if (contentMD5 !== md5) { + return callback(errors.BadDigest); + } + const responsePayload = constructPutResponse({ + dataStoreName: retrievalInfo.dataStoreName, + dataStoreType: retrievalInfo.dataStoreType, + key: retrievalInfo.key, + size: payloadLen, + dataStoreETag: retrievalInfo.dataStoreETag ? `1:${retrievalInfo.dataStoreETag}` : `1:${md5}`, + dataStoreVersionId: retrievalInfo.dataStoreVersionId, + }); + return _respond(response, responsePayload, log, callback); + }); } function deleteObjectFromExpiration(request, response, userInfo, log, callback) { diff --git a/tests/multipleBackend/routes/routeBackbeat.js b/tests/multipleBackend/routes/routeBackbeat.js index 40d860d366..e372a1c6c0 100644 --- a/tests/multipleBackend/routes/routeBackbeat.js +++ b/tests/multipleBackend/routes/routeBackbeat.js @@ -23,10 +23,7 @@ const versionIdUtils = versioning.VersionID; const { makeid } = require('../../unit/helpers'); const { makeRequest, makeBackbeatRequest } = require('../../functional/raw-node/utils/makeRequest'); const BucketUtility = require('../../functional/aws-node-sdk/lib/utility/bucket-util'); -const { - hasLocation, - describeSkipIfNotMultiple, -} = require('../../functional/aws-node-sdk/lib/utility/test-utils'); +const { hasLocation, describeSkipIfNotMultiple } = require('../../functional/aws-node-sdk/lib/utility/test-utils'); const { awsLocation, awsS3: awsClient, @@ -60,15 +57,12 @@ const testArn = 'aws::iam:123456789012:user/bart'; const testKey = 'testkey'; const testKeyUTF8 = '䆩鈁櫨㟔罳'; const testData = 'testkey data'; -const testDataMd5 = crypto.createHash('md5') - .update(testData, 'utf-8') - .digest('hex'); +const testDataMd5 = crypto.createHash('md5').update(testData, 'utf-8').digest('hex'); const emptyContentsMd5 = 'd41d8cd98f00b204e9800998ecf8427e'; const testMd = { 'md-model-version': 2, 'owner-display-name': 'Bart', - 'owner-id': ('79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be'), + 'owner-id': '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be', 'last-modified': '2017-05-15T20:32:40.032Z', 'content-length': testData.length, 'content-md5': testDataMd5, @@ -77,18 +71,18 @@ const testMd = { 'x-amz-server-side-encryption': '', 'x-amz-server-side-encryption-aws-kms-key-id': '', 'x-amz-server-side-encryption-customer-algorithm': '', - 'location': null, - 'acl': { + location: null, + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], READ: [], READ_ACP: [], }, - 'nullVersionId': '99999999999999999999RG001 ', - 'isDeleteMarker': false, - 'versionId': '98505119639965999999RG001 ', - 'replicationInfo': { + nullVersionId: '99999999999999999999RG001 ', + isDeleteMarker: false, + versionId: '98505119639965999999RG001 ', + replicationInfo: { status: 'COMPLETED', backends: [{ site: 'zenko', status: 'PENDING' }], content: ['DATA', 'METADATA'], @@ -104,8 +98,7 @@ if (process.env.S3_TESTVAL_OWNERCANONICALID) { const nonVersionedTestMd = { 'owner-display-name': 'Bart', - 'owner-id': ('79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be'), + 'owner-id': '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be', 'content-length': testData.length, 'content-md5': testDataMd5, 'x-amz-version-id': 'null', @@ -114,19 +107,19 @@ const nonVersionedTestMd = { 'x-amz-server-side-encryption': '', 'x-amz-server-side-encryption-aws-kms-key-id': '', 'x-amz-server-side-encryption-customer-algorithm': '', - 'acl': { + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], READ: [], READ_ACP: [], }, - 'location': null, - 'isNull': '', - 'nullVersionId': '', - 'isDeleteMarker': false, - 'tags': {}, - 'replicationInfo': { + location: null, + isNull: '', + nullVersionId: '', + isDeleteMarker: false, + tags: {}, + replicationInfo: { status: '', backends: [], content: [], @@ -137,40 +130,49 @@ const nonVersionedTestMd = { dataStoreVersionId: '', isNFS: null, }, - 'dataStoreName': 'us-east-1', + dataStoreName: 'us-east-1', 'last-modified': '2018-12-18T01:22:15.986Z', 'md-model-version': 3, }; function checkObjectData(s3, bucket, objectKey, dataValue, done) { - s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: objectKey, - })).then(async data => { - try { - const body = await data.Body.transformToString(); - assert.strictEqual(body, dataValue); - return done(); - } catch (err) { - return done(err); - } - }).catch(err => done(err)); + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: objectKey, + }), + ) + .then(async data => { + try { + const body = await data.Body.transformToString(); + assert.strictEqual(body, dataValue); + return done(); + } catch (err) { + return done(err); + } + }) + .catch(err => done(err)); } function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) { - return s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: objectKey, - VersionId: versionId, - })).then(async data => { - try { - const body = await data.Body.transformToString(); - assert.strictEqual(body, dataValue); - return done(); - } catch (err) { - return done(err); - } - }).catch(err => done(err)); + return s3 + .send( + new GetObjectCommand({ + Bucket: bucket, + Key: objectKey, + VersionId: versionId, + }), + ) + .then(async data => { + try { + const body = await data.Body.transformToString(); + assert.strictEqual(body, dataValue); + return done(); + } catch (err) { + return done(err); + } + }) + .catch(err => done(err)); } function updateStorageClass(data, storageClass) { @@ -198,47 +200,63 @@ const itSkipS3C = process.env.S3_END_TO_END ? it.skip : it; describeSkipIfNotMultiple('backbeat DELETE routes', () => { itIfLocationAws('abort MPU', done => { const awsKey = 'backbeat-mpu-test'; - async.waterfall([ - next => { - awsClient.send(new CreateMultipartUploadCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(response => next(null, response)).catch(err => next(err)); - }, - (response, next) => { - const { UploadId } = response; - makeBackbeatRequest({ - method: 'DELETE', - bucket: awsBucket, - objectKey: awsKey, - resourceType: 'multiplebackenddata', - queryObj: { operation: 'abortmpu' }, - headers: { - 'x-scal-upload-id': UploadId, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-storage-class': awsLocation, - }, - authCredentials: backbeatAuthCredentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - assert.deepStrictEqual(JSON.parse(response.body), {}); - return next(null, UploadId); - }); - }, (UploadId, next) => { - awsClient.send(new ListMultipartUploadsCommand({ - Bucket: awsBucket, - })).then(response => { - const hasOngoingUpload = - response.Uploads.some(upload => (upload === UploadId)); - assert(!hasOngoingUpload); - return next(); - }).catch(err => next(err)); + async.waterfall( + [ + next => { + awsClient + .send( + new CreateMultipartUploadCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(response => next(null, response)) + .catch(err => next(err)); + }, + (response, next) => { + const { UploadId } = response; + makeBackbeatRequest( + { + method: 'DELETE', + bucket: awsBucket, + objectKey: awsKey, + resourceType: 'multiplebackenddata', + queryObj: { operation: 'abortmpu' }, + headers: { + 'x-scal-upload-id': UploadId, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + }, + authCredentials: backbeatAuthCredentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + assert.deepStrictEqual(JSON.parse(response.body), {}); + return next(null, UploadId); + }, + ); + }, + (UploadId, next) => { + awsClient + .send( + new ListMultipartUploadsCommand({ + Bucket: awsBucket, + }), + ) + .then(response => { + const hasOngoingUpload = response.Uploads.some(upload => upload === UploadId); + assert(!hasOngoingUpload); + return next(); + }) + .catch(err => next(err)); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -246,13 +264,15 @@ function getMetadataToPut(putDataResponse) { const mdToPut = Object.assign({}, testMd); // Reproduce what backbeat does to update target metadata mdToPut.location = JSON.parse(putDataResponse.body); - ['x-amz-server-side-encryption', - 'x-amz-server-side-encryption-aws-kms-key-id', - 'x-amz-server-side-encryption-customer-algorithm'].forEach(headerName => { - if (putDataResponse.headers[headerName]) { - mdToPut[headerName] = putDataResponse.headers[headerName]; - } - }); + [ + 'x-amz-server-side-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id', + 'x-amz-server-side-encryption-customer-algorithm', + ].forEach(headerName => { + if (putDataResponse.headers[headerName]) { + mdToPut[headerName] = putDataResponse.headers[headerName]; + } + }); return mdToPut; } @@ -269,40 +289,50 @@ describe('backbeat routes', () => { before(done => { bucketUtil = new BucketUtility('default', {}); s3 = bucketUtil.s3; - bucketUtil.emptyManyIfExists([TEST_BUCKET, TEST_ENCRYPTED_BUCKET, NONVERSIONED_BUCKET, - VERSION_SUSPENDED_BUCKET]) + bucketUtil + .emptyManyIfExists([TEST_BUCKET, TEST_ENCRYPTED_BUCKET, NONVERSIONED_BUCKET, VERSION_SUSPENDED_BUCKET]) .then(async () => { try { await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: TEST_BUCKET, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new CreateBucketCommand({ - Bucket: NONVERSIONED_BUCKET, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new CreateBucketCommand({ + Bucket: NONVERSIONED_BUCKET, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: VERSION_SUSPENDED_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: VERSION_SUSPENDED_BUCKET, - VersioningConfiguration: { Status: 'Suspended' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: VERSION_SUSPENDED_BUCKET, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: TEST_ENCRYPTED_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: TEST_ENCRYPTED_BUCKET, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new PutBucketEncryptionCommand({ - Bucket: TEST_ENCRYPTED_BUCKET, - ServerSideEncryptionConfiguration: { - Rules: [ - { - ApplyServerSideEncryptionByDefault: { - SSEAlgorithm: 'AES256', + await s3.send( + new PutBucketVersioningCommand({ + Bucket: TEST_ENCRYPTED_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new PutBucketEncryptionCommand({ + Bucket: TEST_ENCRYPTED_BUCKET, + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + }, }, - }, - ], - }, - })); + ], + }, + }), + ); done(); } catch (err) { done(err); @@ -315,14 +345,14 @@ describe('backbeat routes', () => { }); after(async () => { - await bucketUtil.empty(TEST_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET })); - await bucketUtil.empty(TEST_ENCRYPTED_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: TEST_ENCRYPTED_BUCKET })); - await bucketUtil.empty(NONVERSIONED_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: NONVERSIONED_BUCKET })); - await bucketUtil.empty(VERSION_SUSPENDED_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: VERSION_SUSPENDED_BUCKET })); + await bucketUtil.empty(TEST_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET })); + await bucketUtil.empty(TEST_ENCRYPTED_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: TEST_ENCRYPTED_BUCKET })); + await bucketUtil.empty(NONVERSIONED_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: NONVERSIONED_BUCKET })); + await bucketUtil.empty(VERSION_SUSPENDED_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: VERSION_SUSPENDED_BUCKET })); }); describe('null version', () => { @@ -346,2034 +376,2813 @@ describe('backbeat routes', () => { beforeEach(() => { bucket = generateUniqueBucketName(BUCKET_FOR_NULL_VERSION_PREFIX); - return bucketUtil.emptyIfExists(bucket) - .then(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); + return bucketUtil.emptyIfExists(bucket).then(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); }); - afterEach(() => bucketUtil.empty(bucket) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - ); + afterEach(() => bucketUtil.empty(bucket).then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket })))); it('should update metadata of a current null version', done => { let objMD; - async.series({ - putObject: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioningSource: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + async.series( + { + putObject: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + enableVersioningSource: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + }, + (err, results) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + const expectedMd = JSON.parse(objMD); + expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed + if (!isNullVersionCompatMode) { + expectedMd.isNull2 = true; // TODO remove the line once CLDSRV-509 is fixed } - objMD = result; - return next(); - }), - putMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); - - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - const expectedMd = JSON.parse(objMD); - expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed - if (!isNullVersionCompatMode) { - expectedMd.isNull2 = true; // TODO remove the line once CLDSRV-509 is fixed - } - assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd); + assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); + assert.strictEqual(Versions.length, 1); - const [currentVersion] = Versions; - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const [currentVersion] = Versions; + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should update metadata of a non-current null version', done => { let objMD; let expectedVersionId; - return async.series({ - putObjectInitial: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObjectAgain: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + return async.series( + { + putObjectInitial: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - putMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + enableVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + putObjectAgain: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); }, - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + (err, results) => { + if (err) { + return done(err); + } + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 2); - const currentVersion = Versions.find(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + assert.strictEqual(Versions.length, 2); + const currentVersion = Versions.find(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + return done(); + }, + ); }); it('should update metadata of a suspended null version', done => { let objMD; - return async.series({ - suspendVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObject: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + return async.series( + { + suspendVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - putUpdatedMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + putObject: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + enableVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putUpdatedMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); }, - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + (err, results) => { + if (err) { + return done(err); + } + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); + assert.strictEqual(Versions.length, 1); - const [currentVersion] = Versions; - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const [currentVersion] = Versions; + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should update metadata of a suspended null version with internal version id', done => { let objMD; - return async.series({ - suspendVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObject: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObjectTagging: next => { - s3.send(new PutObjectTaggingCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + return async.series( + { + suspendVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - putUpdatedMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + putObject: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + enableVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + putObjectTagging: next => { + s3.send( + new PutObjectTaggingCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putUpdatedMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); }, - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + (err, results) => { + if (err) { + return done(err); + } + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should update metadata of a non-version object', done => { let objMD; - async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + return done(err); } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[3]; - assert(!headObjectRes.VersionId); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[3]; + assert(!headObjectRes.VersionId); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[4]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[4]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should create a new null version if versioning suspended and no version', done => { let objMD; - async.series([ - next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + async.series( + [ + next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => { - s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[5]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => { + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + ], + (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[6]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); + assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + return done(); + }, + ); }); // TODO fix broken on S3C with metadata backend,create 2 null Versions itSkipS3C('should create a new null version if versioning suspended and delete marker null version', done => { let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[5]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[6]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should create a new null version if versioning suspended and version has version id', done => { let expectedVersionId; let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: null, - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: null, + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send(new ListObjectVersionsCommand({ Bucket: bucket })) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + return done(err); } - objMD = result; - return next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ Bucket: bucket })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[7]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[7]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[8]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[8]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 2); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 2); - const currentVersion = Versions.find(v => v.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); + const currentVersion = Versions.find(v => v.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); - const nonCurrentVersion = Versions.find(v => !v.IsLatest); - assertVersionHasNotBeenUpdated(nonCurrentVersion, expectedVersionId); + const nonCurrentVersion = Versions.find(v => !v.IsLatest); + assertVersionHasNotBeenUpdated(nonCurrentVersion, expectedVersionId); - // give some time for the async deletes to complete - return setTimeout(() => checkVersionData(s3, bucket, keyName, expectedVersionId, testData, done), - 1000); - }); + // give some time for the async deletes to complete + return setTimeout( + () => checkVersionData(s3, bucket, keyName, expectedVersionId, testData, done), + 1000, + ); + }, + ); }); it('should update null version with no version id and versioning suspended', done => { let objMD; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[4]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[4]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[5]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + const listObjectVersionsRes = data[5]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + return done(); + }, + ); }); it('should update null version if versioning suspended and null version has a version id', done => { let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[4]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[4]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[5]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); - assert.strictEqual(DeleteMarkers, undefined); + const listObjectVersionsRes = data[5]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); - it('should update null version if versioning suspended and null version has a version id and' + - 'put object afterward', done => { - let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - - const headObjectRes = data[5]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert(!headObjectRes.StorageClass); + it( + 'should update null version if versioning suspended and null version has a version id and' + + 'put object afterward', + done => { + let objMD; + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { + if (err) { + return done(err); + } - const listObjectVersionsRes = data[6]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert(!headObjectRes.StorageClass); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, 'null'); - return done(); - }); - }); + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - it('should update null version if versioning suspended and null version has a version id and' + - 'put version afterward', done => { - let objMD; - let expectedVersionId; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, 'null'); + return done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } + ); + }, + ); - const headObjectRes = data[6]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + it( + 'should update null version if versioning suspended and null version has a version id and' + + 'put version afterward', + done => { + let objMD; + let expectedVersionId; + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { + if (err) { + return done(err); + } - const listObjectVersionsRes = data[7]; - const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 2); + const headObjectRes = data[6]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const [currentVersion] = Versions.filter(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + const listObjectVersionsRes = data[7]; + const { Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 2); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); - }); + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + return done(); + }, + ); + }, + ); it('should update non-current null version if versioning suspended', done => { let expectedVersionId; let objMD; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[6]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[6]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[7]; - const deleteMarkers = listObjectVersionsRes.DeleteMarkers; - assert.strictEqual(deleteMarkers, undefined); - const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 2); + const listObjectVersionsRes = data[7]; + const deleteMarkers = listObjectVersionsRes.DeleteMarkers; + assert.strictEqual(deleteMarkers, undefined); + const { Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 2); - const [currentVersion] = Versions.filter(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); + return done(); + }, + ); }); it('should update current null version if versioning suspended', done => { let objMD; let expectedVersionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - expectedVersionId = result.VersionId; - return next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: expectedVersionId, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + expectedVersionId = result.VersionId; + return next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: expectedVersionId, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + return done(err); } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[7]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); - - const listObjectVersionsRes = data[8]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); - assert.strictEqual(DeleteMarkers, undefined); - - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const headObjectRes = data[7]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[8]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); - it('should update current null version if versioning suspended and put a null version ' + - 'afterwards', done => { - let objMD; - let deletedVersionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - deletedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: deletedVersionId, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } + it( + 'should update current null version if versioning suspended and put a null version ' + 'afterwards', + done => { + let objMD; + let deletedVersionId; + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + deletedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: deletedVersionId, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { + if (err) { + return done(err); + } - const headObjectRes = data[8]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert(!headObjectRes.StorageClass); + const headObjectRes = data[8]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert(!headObjectRes.StorageClass); - const listObjectVersionsRes = data[9]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + const listObjectVersionsRes = data[9]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, 'null'); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, 'null'); - return done(); - }); - }); + return done(); + }, + ); + }, + ); it('should update current null version if versioning suspended and put a version afterwards', done => { let objMD; let deletedVersionId; let expectedVersionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - deletedVersionId = result.VersionId; - return next(); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: deletedVersionId, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + deletedVersionId = result.VersionId; + return next(); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: deletedVersionId, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + expectedVersionId = result.VersionId; + return next(); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - expectedVersionId = result.VersionId; - return next(); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[9]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[9]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[10]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 2); + const listObjectVersionsRes = data[10]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 2); - const [currentVersion] = Versions.filter(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); + return done(); + }, + ); }); }); describe('backbeat PUT routes', () => { - describe('PUT data + metadata should create a new complete object', - () => { - [{ - caption: 'with ascii test key', - key: testKey, encodedKey: testKey, - }, - { - caption: 'with UTF8 key', - key: testKeyUTF8, encodedKey: encodeURI(testKeyUTF8), - }, - { - caption: 'with percents and spaces encoded as \'+\' in key', - key: '50% full or 50% empty', - encodedKey: '50%25+full+or+50%25+empty', - }, - { - caption: 'with legacy API v1', - key: testKey, encodedKey: testKey, - legacyAPI: true, - }, - { - caption: 'with encryption configuration', - key: testKey, encodedKey: testKey, - encryption: true, - }, - { - caption: 'with encryption configuration and legacy API v1', - key: testKey, encodedKey: testKey, - encryption: true, - legacyAPI: true, - }].concat([ - `${testKeyUTF8}/${testKeyUTF8}/%42/mykey`, - 'Pâtisserie=中文-español-English', - 'notes/spring/1.txt', - 'notes/spring/2.txt', - 'notes/spring/march/1.txt', - 'notes/summer/1.txt', - 'notes/summer/2.txt', - 'notes/summer/august/1.txt', - 'notes/year.txt', - 'notes/yore.rs', - 'notes/zaphod/Beeblebrox.txt', - ].map(key => ({ - key, encodedKey: encodeURI(key), - caption: `with key ${key}`, - }))) - .forEach(testCase => { - it(testCase.caption, done => { - async.waterfall([next => { - const queryObj = testCase.legacyAPI ? {} : { v2: '' }; - makeBackbeatRequest({ - method: 'PUT', bucket: testCase.encryption ? - TEST_ENCRYPTED_BUCKET : TEST_BUCKET, - objectKey: testCase.encodedKey, - resourceType: 'data', - queryObj, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + describe('PUT data + metadata should create a new complete object', () => { + [ + { + caption: 'with ascii test key', + key: testKey, + encodedKey: testKey, + }, + { + caption: 'with UTF8 key', + key: testKeyUTF8, + encodedKey: encodeURI(testKeyUTF8), + }, + { + caption: "with percents and spaces encoded as '+' in key", + key: '50% full or 50% empty', + encodedKey: '50%25+full+or+50%25+empty', + }, + { + caption: 'with legacy API v1', + key: testKey, + encodedKey: testKey, + legacyAPI: true, + }, + { + caption: 'with encryption configuration', + key: testKey, + encodedKey: testKey, + encryption: true, + }, + { + caption: 'with encryption configuration and legacy API v1', + key: testKey, + encodedKey: testKey, + encryption: true, + legacyAPI: true, + }, + ] + .concat( + [ + `${testKeyUTF8}/${testKeyUTF8}/%42/mykey`, + 'Pâtisserie=中文-español-English', + 'notes/spring/1.txt', + 'notes/spring/2.txt', + 'notes/spring/march/1.txt', + 'notes/summer/1.txt', + 'notes/summer/2.txt', + 'notes/summer/august/1.txt', + 'notes/year.txt', + 'notes/yore.rs', + 'notes/zaphod/Beeblebrox.txt', + ].map(key => ({ + key, + encodedKey: encodeURI(key), + caption: `with key ${key}`, + })), + ) + .forEach(testCase => { + it(testCase.caption, done => { + async.waterfall( + [ + next => { + const queryObj = testCase.legacyAPI ? {} : { v2: '' }; + makeBackbeatRequest( + { + method: 'PUT', + bucket: testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, + objectKey: testCase.encodedKey, + resourceType: 'data', + queryObj, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = getMetadataToPut(response); + if (testCase.encryption && !testCase.legacyAPI) { + assert.strictEqual(typeof newMd.location[0].cryptoScheme, 'number'); + assert.strictEqual(typeof newMd.location[0].cipheredDataKey, 'string'); + } else { + // if no encryption or legacy API, data should not be encrypted + assert.strictEqual(newMd.location[0].cryptoScheme, undefined); + assert.strictEqual(newMd.location[0].cipheredDataKey, undefined); + } + makeBackbeatRequest( + { + method: 'PUT', + bucket: testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, + objectKey: testCase.encodedKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + checkObjectData( + s3, + testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, + testCase.key, + testData, + next, + ); + }, + ], + err => { + assert.ifError(err); + done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = getMetadataToPut(response); - if (testCase.encryption && !testCase.legacyAPI) { - assert.strictEqual(typeof newMd.location[0].cryptoScheme, 'number'); - assert.strictEqual(typeof newMd.location[0].cipheredDataKey, 'string'); - } else { - // if no encryption or legacy API, data should not be encrypted - assert.strictEqual(newMd.location[0].cryptoScheme, undefined); - assert.strictEqual(newMd.location[0].cipheredDataKey, undefined); - } - makeBackbeatRequest({ - method: 'PUT', bucket: testCase.encryption ? - TEST_ENCRYPTED_BUCKET : TEST_BUCKET, - objectKey: testCase.encodedKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - checkObjectData( - s3, testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, - testCase.key, testData, next); - }], err => { - assert.ifError(err); - done(); + ); }); }); - }); }); it('should PUT metadata for a non-versioned bucket', done => { const bucket = NONVERSIONED_BUCKET; const objectKey = 'non-versioned-key'; - async.waterfall([ - next => - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey, - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'content-md5': testDataMd5, - 'x-scal-canonical-id': testArn, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, (err, response) => { - assert.ifError(err); - const metadata = Object.assign({}, nonVersionedTestMd, { - location: JSON.parse(response.body), - }); - return next(null, metadata); - }), - (metadata, next) => - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(metadata), - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - next(); - }), - next => - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: objectKey, - })).then(result => { - assert.strictEqual(result.StorageClass, 'awsbackend'); - next(); - }).catch(err => { - next(err); - }), - next => checkObjectData(s3, bucket, objectKey, testData, next), - ], done); + async.waterfall( + [ + next => + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + (err, response) => { + assert.ifError(err); + const metadata = Object.assign({}, nonVersionedTestMd, { + location: JSON.parse(response.body), + }); + return next(null, metadata); + }, + ), + (metadata, next) => + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(metadata), + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + next(); + }, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: objectKey, + }), + ) + .then(result => { + assert.strictEqual(result.StorageClass, 'awsbackend'); + next(); + }) + .catch(err => { + next(err); + }), + next => checkObjectData(s3, bucket, objectKey, testData, next), + ], + done, + ); }); - it('PUT metadata with "x-scal-replication-content: METADATA"' + - 'header should replicate metadata only', done => { - async.waterfall([next => { - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_ENCRYPTED_BUCKET, - objectKey: 'test-updatemd-key', - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + it( + 'PUT metadata with "x-scal-replication-content: METADATA"' + 'header should replicate metadata only', + done => { + async.waterfall( + [ + next => { + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_ENCRYPTED_BUCKET, + objectKey: 'test-updatemd-key', + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = getMetadataToPut(response); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_ENCRYPTED_BUCKET, + objectKey: 'test-updatemd-key', + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // Don't update the sent metadata since it is sent by + // backbeat as received from the replication queue, + // without updated data location or encryption info + // (since that info is not known by backbeat) + const newMd = Object.assign({}, testMd); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_ENCRYPTED_BUCKET, + objectKey: 'test-updatemd-key', + resourceType: 'metadata', + headers: { 'x-scal-replication-content': 'METADATA' }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + checkObjectData(s3, TEST_ENCRYPTED_BUCKET, 'test-updatemd-key', testData, next); + }, + ], + err => { + assert.ifError(err); + done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = getMetadataToPut(response); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_ENCRYPTED_BUCKET, - objectKey: 'test-updatemd-key', - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // Don't update the sent metadata since it is sent by - // backbeat as received from the replication queue, - // without updated data location or encryption info - // (since that info is not known by backbeat) - const newMd = Object.assign({}, testMd); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_ENCRYPTED_BUCKET, - objectKey: 'test-updatemd-key', - resourceType: 'metadata', - headers: { 'x-scal-replication-content': 'METADATA' }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - checkObjectData(s3, TEST_ENCRYPTED_BUCKET, 'test-updatemd-key', - testData, next); - }], err => { - assert.ifError(err); - done(); - }); - }); + ); + }, + ); itIfLocationAws('should PUT tags for a non-versioned bucket (awslocation)', function test(done) { this.timeout(10000); const bucket = NONVERSIONED_BUCKET; const awsKey = uuidv4(); - async.waterfall([ - next => - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey: awsKey, - resourceType: 'multiplebackenddata', - queryObj: { operation: 'putobject' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ Key1: 'Value1' }), - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.deepStrictEqual(data.TagSet, [{ - Key: 'Key1', - Value: 'Value1' - }]); - next(null, data); - }).catch(err => { - next(err); - }), - ], done); + async.waterfall( + [ + next => + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: awsKey, + resourceType: 'multiplebackenddata', + queryObj: { operation: 'putobject' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ Key1: 'Value1' }), + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.deepStrictEqual(data.TagSet, [ + { + Key: 'Key1', + Value: 'Value1', + }, + ]); + next(null, data); + }) + .catch(err => { + next(err); + }), + ], + done, + ); }); const testCases = [ @@ -2390,544 +3199,734 @@ describe('backbeat routes', () => { testCases.forEach(({ description, bucket }) => { it(`should PUT metadata and data if ${description} and x-scal-versioning-required is not set`, done => { let objectMd; - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: 'sourcekey', - Body: Buffer.from(testData), - })).then(res => next(null, res)).catch(err => next(err)), - (resp, next) => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: 'sourcekey', - authCredentials: backbeatAuthCredentials, - }, (err, resp) => { - objectMd = JSON.parse(resp.body).Body; - return next(); - }), - next => { - makeBackbeatRequest({ - method: 'PUT', bucket, - objectKey: 'destinationkey', - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - makeBackbeatRequest({ - method: 'PUT', bucket, - objectKey: 'destinationkey', - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: objectMd, - }, next); - }], + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'sourcekey', + Body: Buffer.from(testData), + }), + ) + .then(res => next(null, res)) + .catch(err => next(err)), + (resp, next) => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: 'sourcekey', + authCredentials: backbeatAuthCredentials, + }, + (err, resp) => { + objectMd = JSON.parse(resp.body).Body; + return next(); + }, + ), + next => { + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: 'destinationkey', + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: 'destinationkey', + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: objectMd, + }, + next, + ); + }, + ], err => { assert.ifError(err); done(); - }); + }, + ); }); }); testCases.forEach(({ description, bucket }) => { it(`should refuse PUT data if ${description} and x-scal-versioning-required is true`, done => { - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey: testKey, - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - 'x-scal-versioning-required': 'true', + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: testKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + 'x-scal-versioning-required': 'true', + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, err => { - assert.strictEqual(err.code, 'InvalidBucketState'); - done(); - }); + err => { + assert.strictEqual(err.code, 'InvalidBucketState'); + done(); + }, + ); }); }); testCases.forEach(({ description, bucket }) => { it(`should refuse PUT metadata if ${description} and x-scal-versioning-required is true`, done => { - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + headers: { + 'x-scal-versioning-required': 'true', + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(testMd), }, - headers: { - 'x-scal-versioning-required': 'true', + err => { + assert.strictEqual(err.code, 'InvalidBucketState'); + done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(testMd), - }, err => { - assert.strictEqual(err.code, 'InvalidBucketState'); - done(); - }); - }); - }); - - it('should refuse PUT data if no x-scal-canonical-id header ' + - 'is provided', done => makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, - err => { - assert.strictEqual(err.code, 'BadRequest'); - done(); - })); - - it('should refuse PUT in metadata-only mode if object does not exist', - done => { - async.waterfall([next => { - const newMd = Object.assign({}, testMd); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: 'does-not-exist', - resourceType: 'metadata', - headers: { 'x-scal-replication-content': 'METADATA' }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }], err => { - assert.strictEqual(err.statusCode, 404); - done(); + ); }); }); - it('should remove old object data locations if version is overwritten ' + - 'with same contents', done => { - let oldLocation; - const testKeyOldData = `${testKey}-old-data`; - async.waterfall([next => { - // put object's data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, + it('should refuse PUT data if no x-scal-canonical-id header ' + 'is provided', done => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, objectKey: testKey, resourceType: 'data', + queryObj: { v2: '' }, headers: { 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put object metadata - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - oldLocation = newMd.location; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), }, authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put another object which metadata reference the - // same data locations, we will attempt to retrieve - // this object at the end of the test to confirm that - // its locations have been deleted - const oldDataMd = Object.assign({}, testMd); - oldDataMd.location = oldLocation; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKeyOldData, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(oldDataMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // create new data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + requestBody: testData, + }, + err => { + assert.strictEqual(err.code, 'BadRequest'); + done(); + }, + ), + ); + + it('should refuse PUT in metadata-only mode if object does not exist', done => { + async.waterfall( + [ + next => { + const newMd = Object.assign({}, testMd); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: 'does-not-exist', + resourceType: 'metadata', + headers: { 'x-scal-replication-content': 'METADATA' }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // overwrite the original object version, now - // with references to the new data locations - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + ], + err => { + assert.strictEqual(err.statusCode, 404); + done(); + }, + ); + }); + + it('should remove old object data locations if version is overwritten ' + 'with same contents', done => { + let oldLocation; + const testKeyOldData = `${testKey}-old-data`; + async.waterfall( + [ + next => { + // put object's data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // give some time for the async deletes to complete - setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, testData, next), - 1000); - }, next => { - // check that the object copy referencing the old data - // locations is unreadable, confirming that the old - // data locations have been deleted - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKeyOldData, - })).catch(err => { - assert(err, 'expected error to get object with old data ' + - 'locations, got success'); - next(); - }); - }], err => { - assert.ifError(err); - done(); - }); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put object metadata + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + oldLocation = newMd.location; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put another object which metadata reference the + // same data locations, we will attempt to retrieve + // this object at the end of the test to confirm that + // its locations have been deleted + const oldDataMd = Object.assign({}, testMd); + oldDataMd.location = oldLocation; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKeyOldData, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(oldDataMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // create new data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // overwrite the original object version, now + // with references to the new data locations + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // give some time for the async deletes to complete + setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, testData, next), 1000); + }, + next => { + // check that the object copy referencing the old data + // locations is unreadable, confirming that the old + // data locations have been deleted + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKeyOldData, + }), + ).catch(err => { + assert(err, 'expected error to get object with old data ' + 'locations, got success'); + next(); + }); + }, + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - it('should remove old object data locations if version is overwritten ' + - 'with empty contents', done => { + it('should remove old object data locations if version is overwritten ' + 'with empty contents', done => { let oldLocation; const testKeyOldData = `${testKey}-old-data`; - async.waterfall([next => { - // put object's data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => { + // put object's data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put object metadata - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - oldLocation = newMd.location; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put object metadata + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + oldLocation = newMd.location; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put another object which metadata reference the - // same data locations, we will attempt to retrieve - // this object at the end of the test to confirm that - // its locations have been deleted - const oldDataMd = Object.assign({}, testMd); - oldDataMd.location = oldLocation; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKeyOldData, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put another object which metadata reference the + // same data locations, we will attempt to retrieve + // this object at the end of the test to confirm that + // its locations have been deleted + const oldDataMd = Object.assign({}, testMd); + oldDataMd.location = oldLocation; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKeyOldData, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(oldDataMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(oldDataMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // overwrite the original object version with an empty location - const newMd = Object.assign({}, testMd); - newMd['content-length'] = 0; - newMd['content-md5'] = emptyContentsMd5; - newMd.location = null; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // overwrite the original object version with an empty location + const newMd = Object.assign({}, testMd); + newMd['content-length'] = 0; + newMd['content-md5'] = emptyContentsMd5; + newMd.location = null; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // give some time for the async deletes to complete - setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, '', next), - 1000); - }, next => { - // check that the object copy referencing the old data - // locations is unreadable, confirming that the old - // data locations have been deleted - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKeyOldData, - })).catch(err => { - assert(err, 'expected error to get object with old data ' + - 'locations, got success'); - next(); - }); - }], err => { - assert.ifError(err); - done(); - }); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // give some time for the async deletes to complete + setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, '', next), 1000); + }, + next => { + // check that the object copy referencing the old data + // locations is unreadable, confirming that the old + // data locations have been deleted + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKeyOldData, + }), + ).catch(err => { + assert(err, 'expected error to get object with old data ' + 'locations, got success'); + next(); + }); + }, + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - it('should not remove data locations on replayed metadata PUT', - done => { + it('should not remove data locations on replayed metadata PUT', done => { let serializedNewMd; - async.waterfall([next => { - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => { + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - serializedNewMd = JSON.stringify(newMd); - async.timesSeries(2, (i, putDone) => makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + serializedNewMd = JSON.stringify(newMd); + async.timesSeries( + 2, + (i, putDone) => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: serializedNewMd, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + putDone(err); + }, + ), + () => next(), + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: serializedNewMd, - }, (err, response) => { + next => { + // check that the object is still readable to make + // sure we did not remove the data keys + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + }), + ) + .then(async data => { + const body = await data.Body.transformToString(); + assert.strictEqual(body, testData); + next(); + }) + .catch(err => { + next(err); + }); + }, + ], + err => { assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - putDone(err); - }), () => next()); - }, next => { - // check that the object is still readable to make - // sure we did not remove the data keys - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - })).then(async data => { - const body = await data.Body.transformToString(); - assert.strictEqual(body, testData); - next(); - }).catch(err => { - next(err); - }); - }], err => { - assert.ifError(err); - done(); - }); + done(); + }, + ); }); it('should create a new version when no versionId is passed in query string', done => { let newVersion; - async.waterfall([next => { - // put object's data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => { + // put object's data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put object metadata - const oldMd = Object.assign({}, testMd); - oldMd.location = JSON.parse(response.body); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put object metadata + const oldMd = Object.assign({}, testMd); + oldMd.location = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(oldMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(oldMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const parsedResponse = JSON.parse(response.body); - assert.strictEqual(parsedResponse.versionId, testMd.versionId); - // create new data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const parsedResponse = JSON.parse(response.body); + assert.strictEqual(parsedResponse.versionId, testMd.versionId); + // create new data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // create a new version with the new data locations, - // not passing 'versionId' in the query string - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const parsedResponse = JSON.parse(response.body); - newVersion = parsedResponse.versionId; - assert.notStrictEqual(newVersion, testMd.versionId); - // give some time for the async deletes to complete, - // then check that we can read the latest version - setTimeout(() => s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - })).then(async data => { - const body = await data.Body.transformToString(); - assert.strictEqual(body, testData); - next(); - }).catch(err => { - next(err); - }), 1000); - }, next => { - // check that the previous object version is still readable - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - VersionId: versionIdUtils.encode(testMd.versionId), - })).then(async data => { - const body = await data.Body.transformToString(); - assert.strictEqual(body, testData); - next(); - }).catch(err => { - next(err); - }); - }], err => { - assert.ifError(err); - done(); - }); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // create a new version with the new data locations, + // not passing 'versionId' in the query string + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const parsedResponse = JSON.parse(response.body); + newVersion = parsedResponse.versionId; + assert.notStrictEqual(newVersion, testMd.versionId); + // give some time for the async deletes to complete, + // then check that we can read the latest version + setTimeout( + () => + s3 + .send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + }), + ) + .then(async data => { + const body = await data.Body.transformToString(); + assert.strictEqual(body, testData); + next(); + }) + .catch(err => { + next(err); + }), + 1000, + ); + }, + next => { + // check that the previous object version is still readable + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + VersionId: versionIdUtils.encode(testMd.versionId), + }), + ) + .then(async data => { + const body = await data.Body.transformToString(); + assert.strictEqual(body, testData); + next(); + }) + .catch(err => { + next(err); + }); + }, + ], + err => { + assert.ifError(err); + done(); + }, + ); }); }); describe('backbeat authorization checks', () => { const { accessKeyId: accessKeyLisa, secretAccessKey: secretAccessKeyLisa } = getCredentials('lisa'); - [{ method: 'PUT', resourceType: 'metadata' }, - { method: 'PUT', resourceType: 'data' }].forEach(test => { - const queryObj = test.resourceType === 'data' ? { v2: '' } : {}; - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if no credentials are provided', - done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'AccessDenied'); - done(); - }); - }); - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if wrong credentials are provided', + [ + { method: 'PUT', resourceType: 'metadata' }, + { method: 'PUT', resourceType: 'data' }, + ].forEach(test => { + const queryObj = test.resourceType === 'data' ? { v2: '' } : {}; + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if no credentials are provided', done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - authCredentials: { - accessKey: 'wrong', - secretKey: 'still wrong', + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, }, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'InvalidAccessKeyId'); - done(); - }); - }); - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if the account does not match the ' + - 'backbeat user', + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'AccessDenied'); + done(); + }, + ); + }, + ); + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if wrong credentials are provided', done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - authCredentials: { - accessKey: accessKeyLisa, - secretKey: secretAccessKeyLisa, + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, + authCredentials: { + accessKey: 'wrong', + secretKey: 'still wrong', + }, }, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'AccessDenied'); - done(); - }); - }); - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if backbeat user has wrong secret key', + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'InvalidAccessKeyId'); + done(); + }, + ); + }, + ); + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if the account does not match the ' + + 'backbeat user', done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - authCredentials: { - accessKey: backbeatAuthCredentials.accessKey, - secretKey: 'hastalavista', + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, + authCredentials: { + accessKey: accessKeyLisa, + secretKey: secretAccessKeyLisa, + }, }, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'SignatureDoesNotMatch'); - done(); - }); - }); - }); + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'AccessDenied'); + done(); + }, + ); + }, + ); + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if backbeat user has wrong secret key', + done => { + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, + authCredentials: { + accessKey: backbeatAuthCredentials.accessKey, + secretKey: 'hastalavista', + }, + }, + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'SignatureDoesNotMatch'); + done(); + }, + ); + }, + ); + }); const apiProxy = !!config.backbeat; describe(`when api proxy is ${apiProxy ? '' : 'NOT '}configured`, () => { @@ -2939,271 +3938,325 @@ describe('backbeat routes', () => { it(`GET /_/backbeat/api/... should respond with ${ apiProxy ? 503 : 405 - } on authenticated requests (API server down)`, - done => { - const options = { - authCredentials: { - accessKey: accessKeyLisa, - secretKey: secretAccessKeyLisa, - }, - hostname: ipAddress, - port: 8000, - method: 'GET', - path: '/_/backbeat/api/crr/failed', - jsonResponse: true, - }; - makeRequest(options, err => { - assert(err); - const expected = apiProxy ? 503 : 405; - assert.strictEqual(err.statusCode, expected); - assert.strictEqual(err.code, errors[expected]); - done(); - }); + } on authenticated requests (API server down)`, done => { + const options = { + authCredentials: { + accessKey: accessKeyLisa, + secretKey: secretAccessKeyLisa, + }, + hostname: ipAddress, + port: 8000, + method: 'GET', + path: '/_/backbeat/api/crr/failed', + jsonResponse: true, + }; + makeRequest(options, err => { + assert(err); + const expected = apiProxy ? 503 : 405; + assert.strictEqual(err.statusCode, expected); + assert.strictEqual(err.code, errors[expected]); + done(); }); + }); it(`GET /_/backbeat/api/... should respond with ${ apiProxy ? 403 : 405 - } if the request is unauthenticated`, - done => { - const options = { - hostname: ipAddress, - port: 8000, - method: 'GET', - path: '/_/backbeat/api/crr/failed', - jsonResponse: true, - }; - makeRequest(options, err => { - assert(err); - const expected = apiProxy ? 403 : 405; - assert.strictEqual(err.statusCode, expected); - assert.strictEqual(err.code, errors[expected]); - done(); - }); + } if the request is unauthenticated`, done => { + const options = { + hostname: ipAddress, + port: 8000, + method: 'GET', + path: '/_/backbeat/api/crr/failed', + jsonResponse: true, + }; + makeRequest(options, err => { + assert(err); + const expected = apiProxy ? 403 : 405; + assert.strictEqual(err.statusCode, expected); + assert.strictEqual(err.code, errors[expected]); + done(); }); + }); }); }); describe('GET Metadata route', () => { - beforeEach(done => makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: TEST_KEY, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), - }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(testMd), - }, done)); + beforeEach(done => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(testMd), + }, + done, + ), + ); it('should return metadata blob for a versionId', done => { - makeBackbeatRequest({ - method: 'GET', bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, }, - }, (err, data) => { - assert.ifError(err); - const parsedBody = JSON.parse(JSON.parse(data.body).Body); - assert.strictEqual(data.statusCode, 200); - assert.deepStrictEqual(parsedBody, testMd); - done(); - }); + (err, data) => { + assert.ifError(err); + const parsedBody = JSON.parse(JSON.parse(data.body).Body); + assert.strictEqual(data.statusCode, 200); + assert.deepStrictEqual(parsedBody, testMd); + done(); + }, + ); }); it('should return error if bucket does not exist', done => { - makeBackbeatRequest({ - method: 'GET', bucket: 'blah', - objectKey: TEST_KEY, resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'GET', + bucket: 'blah', + objectKey: TEST_KEY, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, }, - }, (err, data) => { - assert.strictEqual(data.statusCode, 404); - const body = JSON.parse(data.body); - assert.strictEqual(body.code, 'NoSuchBucket'); - // err is parsed data.body + statusCode - assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); - done(); - }); + (err, data) => { + assert.strictEqual(data.statusCode, 404); + const body = JSON.parse(data.body); + assert.strictEqual(body.code, 'NoSuchBucket'); + // err is parsed data.body + statusCode + assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); + done(); + }, + ); }); it('should return error if object does not exist', done => { - makeBackbeatRequest({ - method: 'GET', bucket: TEST_BUCKET, - objectKey: 'blah', resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: 'blah', + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, }, - }, (err, data) => { - assert.strictEqual(data.statusCode, 404); - const body = JSON.parse(data.body); - assert.strictEqual(body.code, 'ObjNotFound'); - // err is parsed data.body + statusCode - assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); - done(); - }); + (err, data) => { + assert.strictEqual(data.statusCode, 404); + const body = JSON.parse(data.body); + assert.strictEqual(body.code, 'ObjNotFound'); + // err is parsed data.body + statusCode + assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); + done(); + }, + ); }); }); describeIfLocationAws('backbeat multipart upload operations (external location)', function test() { this.timeout(10000); - it('should put tags if the source is AWS and tags are ' + - 'provided when initiating the multipart upload', done => { - const awsKey = uuidv4(); - const multipleBackendPath = - `/_/backbeat/multiplebackenddata/${awsBucket}/${awsKey}`; - let uploadId; - let partData; - async.series([ - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: multipleBackendPath, - queryObj: { operation: 'initiatempu' }, - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-tags': JSON.stringify({ 'key1': 'value1' }), - }, - jsonResponse: true, - }, (err, data) => { - if (err) { - return next(err); - } - uploadId = JSON.parse(data.body).uploadId; - return next(); - }), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'PUT', - path: multipleBackendPath, - queryObj: { operation: 'putpart' }, - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-upload-id': uploadId, - 'x-scal-part-number': '1', - 'content-length': testData.length, - }, - requestBody: testData, - jsonResponse: true, - }, (err, data) => { - if (err) { - return next(err); - } - const body = JSON.parse(data.body); - partData = [{ - PartNumber: [body.partNumber], - ETag: [body.ETag], - }]; - return next(); - }), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: multipleBackendPath, - queryObj: { operation: 'completempu' }, - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-upload-id': uploadId, - }, - requestBody: JSON.stringify(partData), - jsonResponse: true, - }, next), - next => - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - }), (err, data) => { - assert.ifError(err); - assert.deepStrictEqual(data.TagSet, [{ - Key: 'key1', - Value: 'value1', - }]); - next(); - }), - ], done); - }); + it( + 'should put tags if the source is AWS and tags are ' + 'provided when initiating the multipart upload', + done => { + const awsKey = uuidv4(); + const multipleBackendPath = `/_/backbeat/multiplebackenddata/${awsBucket}/${awsKey}`; + let uploadId; + let partData; + async.series( + [ + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: multipleBackendPath, + queryObj: { operation: 'initiatempu' }, + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-tags': JSON.stringify({ key1: 'value1' }), + }, + jsonResponse: true, + }, + (err, data) => { + if (err) { + return next(err); + } + uploadId = JSON.parse(data.body).uploadId; + return next(); + }, + ), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'PUT', + path: multipleBackendPath, + queryObj: { operation: 'putpart' }, + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-upload-id': uploadId, + 'x-scal-part-number': '1', + 'content-length': testData.length, + }, + requestBody: testData, + jsonResponse: true, + }, + (err, data) => { + if (err) { + return next(err); + } + const body = JSON.parse(data.body); + partData = [ + { + PartNumber: [body.partNumber], + ETag: [body.ETag], + }, + ]; + return next(); + }, + ), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: multipleBackendPath, + queryObj: { operation: 'completempu' }, + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-upload-id': uploadId, + }, + requestBody: JSON.stringify(partData), + jsonResponse: true, + }, + next, + ), + next => + awsClient.send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + (err, data) => { + assert.ifError(err); + assert.deepStrictEqual(data.TagSet, [ + { + Key: 'key1', + Value: 'value1', + }, + ]); + next(); + }, + ), + ], + done, + ); + }, + ); - it('should put tags if the source is Azure and tags are provided ' + - 'when completing the multipart upload', done => { - const containerName = getAzureContainerName(azureLocation); - const blob = uuidv4(); - const multipleBackendPath = - `/_/backbeat/multiplebackenddata/${containerName}/${blob}`; - const uploadId = uuidv4().replace(/-/g, ''); - let partData; - async.series([ - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'PUT', - path: multipleBackendPath, - queryObj: { operation: 'putpart' }, - headers: { - 'x-scal-storage-class': azureLocation, - 'x-scal-storage-type': 'azure', - 'x-scal-upload-id': uploadId, - 'x-scal-part-number': '1', - 'content-length': testData.length, - }, - requestBody: testData, - jsonResponse: true, - }, (err, data) => { - if (err) { - return next(err); - } - const body = JSON.parse(data.body); - partData = [{ - PartNumber: [body.partNumber], - ETag: [body.ETag], - NumberSubParts: [body.numberSubParts], - }]; - return next(); - }), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: multipleBackendPath, - queryObj: { operation: 'completempu' }, - headers: { - 'x-scal-storage-class': azureLocation, - 'x-scal-storage-type': 'azure', - 'x-scal-upload-id': uploadId, - 'x-scal-tags': JSON.stringify({ 'key1': 'value1' }), - }, - requestBody: JSON.stringify(partData), - jsonResponse: true, - }, next), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(result => { - const tags = JSON.parse(result.metadata.tags); - assert.deepStrictEqual(tags, { key1: 'value1' }); - return next(); - }, next), - ], done); - }); + it( + 'should put tags if the source is Azure and tags are provided ' + 'when completing the multipart upload', + done => { + const containerName = getAzureContainerName(azureLocation); + const blob = uuidv4(); + const multipleBackendPath = `/_/backbeat/multiplebackenddata/${containerName}/${blob}`; + const uploadId = uuidv4().replace(/-/g, ''); + let partData; + async.series( + [ + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'PUT', + path: multipleBackendPath, + queryObj: { operation: 'putpart' }, + headers: { + 'x-scal-storage-class': azureLocation, + 'x-scal-storage-type': 'azure', + 'x-scal-upload-id': uploadId, + 'x-scal-part-number': '1', + 'content-length': testData.length, + }, + requestBody: testData, + jsonResponse: true, + }, + (err, data) => { + if (err) { + return next(err); + } + const body = JSON.parse(data.body); + partData = [ + { + PartNumber: [body.partNumber], + ETag: [body.ETag], + NumberSubParts: [body.numberSubParts], + }, + ]; + return next(); + }, + ), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: multipleBackendPath, + queryObj: { operation: 'completempu' }, + headers: { + 'x-scal-storage-class': azureLocation, + 'x-scal-storage-type': 'azure', + 'x-scal-upload-id': uploadId, + 'x-scal-tags': JSON.stringify({ key1: 'value1' }), + }, + requestBody: JSON.stringify(partData), + jsonResponse: true, + }, + next, + ), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then(result => { + const tags = JSON.parse(result.metadata.tags); + assert.deepStrictEqual(tags, { key1: 'value1' }); + return next(); + }, next), + ], + done, + ); + }, + ); }); describe('Batch Delete Route', function test() { this.timeout(30000); @@ -3212,436 +4265,564 @@ describe('backbeat routes', () => { let location; const testKey = 'batch-delete-test-key'; - async.series([ - done => { - s3.send(new PutObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - Body: Buffer.from('hello'), - })).then(data => { - versionId = data.VersionId; - done(); - }).catch(err => { - done(err); - }); - }, - done => { - makeBackbeatRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId, - }, - }, (err, data) => { - assert.ifError(err); - assert.strictEqual(data.statusCode, 200); - const metadata = JSON.parse( - JSON.parse(data.body).Body); - location = metadata.location; - done(); - }); - }, - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: `/_/backbeat/batchdelete/${TEST_BUCKET}/${testKey}`, - requestBody: - `{"Locations":${JSON.stringify(location)}}`, - jsonResponse: true, - }; - makeRequest(options, done); - }, - done => { - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - })).then(() => { - done(new Error('Expected error')); - }).catch(err => { - // should error out as location shall no longer exist - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 503); - done(); - }); - }, - ], done); + async.series( + [ + done => { + s3.send( + new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + Body: Buffer.from('hello'), + }), + ) + .then(data => { + versionId = data.VersionId; + done(); + }) + .catch(err => { + done(err); + }); + }, + done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId, + }, + }, + (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + const metadata = JSON.parse(JSON.parse(data.body).Body); + location = metadata.location; + done(); + }, + ); + }, + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/batchdelete/${TEST_BUCKET}/${testKey}`, + requestBody: `{"Locations":${JSON.stringify(location)}}`, + jsonResponse: true, + }; + makeRequest(options, done); + }, + done => { + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + }), + ) + .then(() => { + done(new Error('Expected error')); + }) + .catch(err => { + // should error out as location shall no longer exist + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 503); + done(); + }); + }, + ], + done, + ); }); itIfLocationAws('should batch delete a versioned AWS location', done => { let versionId; const awsKey = `${TEST_BUCKET}/batch-delete-test-key-${makeid(8)}`; - async.series([ - done => { - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - Body: Buffer.from('hello'), - })).then(data => { - versionId = data.VersionId; - done(); - }).catch(err => { - done(err); - }); - }, - done => { - const location = [{ - key: awsKey, - size: 5, - dataStoreName: awsLocation, - dataStoreVersionId: versionId, - }]; - const reqBody = `{"Locations":${JSON.stringify(location)}}`; - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: reqBody, - jsonResponse: true, - }; - makeRequest(options, done); - }, - done => { - awsClient.send(new GetObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(() => { - done(new Error('Expected error')); - }).catch(err => { - // should error out as location shall no longer exist - assert(err); - done(); - }); - }, - ], done); + async.series( + [ + done => { + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + Body: Buffer.from('hello'), + }), + ) + .then(data => { + versionId = data.VersionId; + done(); + }) + .catch(err => { + done(err); + }); + }, + done => { + const location = [ + { + key: awsKey, + size: 5, + dataStoreName: awsLocation, + dataStoreVersionId: versionId, + }, + ]; + const reqBody = `{"Locations":${JSON.stringify(location)}}`; + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: reqBody, + jsonResponse: true, + }; + makeRequest(options, done); + }, + done => { + awsClient + .send( + new GetObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(() => { + done(new Error('Expected error')); + }) + .catch(err => { + // should error out as location shall no longer exist + assert(err); + done(); + }); + }, + ], + done, + ); }); it('should fail with error if given malformed JSON', done => { - async.series([ - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: 'NOTJSON', - jsonResponse: true, - }; - makeRequest(options, done); + async.series( + [ + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: 'NOTJSON', + jsonResponse: true, + }; + makeRequest(options, done); + }, + ], + err => { + assert(err); + done(); }, - ], err => { - assert(err); - done(); - }); + ); }); // TODO: unskip test when S3C-9123 is fixed itSkipS3C('should skip batch delete of a non-existent location', done => { - async.series([ - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: - '{"Locations":' + - '[{"key":"abcdef","dataStoreName":"us-east-1"}]}', - jsonResponse: true, - }; - makeRequest(options, done); - }, - ], done); + async.series( + [ + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: '{"Locations":' + '[{"key":"abcdef","dataStoreName":"us-east-1"}]}', + jsonResponse: true, + }; + makeRequest(options, done); + }, + ], + done, + ); }); it('should skip batch delete of empty location array', done => { - async.series([ - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: '{"Locations":[]}', - jsonResponse: true, - }; - makeRequest(options, done); - }, - ], done); + async.series( + [ + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: '{"Locations":[]}', + jsonResponse: true, + }; + makeRequest(options, done); + }, + ], + done, + ); }); - itIfLocationAws('should not put delete tags if the source is not Azure and ' + - 'if-unmodified-since header is not provided', done => { - const awsKey = uuidv4(); - async.series([ - next => { - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), + itIfLocationAws( + 'should not put delete tags if the source is not Azure and ' + 'if-unmodified-since header is not provided', + done => { + const awsKey = uuidv4(); + async.series( + [ + next => { + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - requestBody: JSON.stringify({ - Locations: [{ - key: awsKey, - dataStoreName: awsLocation, - }], - }), - jsonResponse: true, - }, next), - next => { - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.deepStrictEqual(data.TagSet, []); - next(null, data); - }).catch(err => { - next(err); - }); - }, - ], done); - }); - - itIfLocationAws('should not put tags if the source is not Azure and ' + - 'if-unmodified-since condition is not met', done => { - const awsKey = uuidv4(); - async.series([ - next => - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(result => next(null, result)).catch(err => next(err)), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - headers: { - 'if-unmodified-since': - 'Sun, 31 Mar 2019 00:00:00 GMT', - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: awsKey, + dataStoreName: awsLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => { + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.deepStrictEqual(data.TagSet, []); + next(null, data); + }) + .catch(err => { + next(err); + }); }, - requestBody: JSON.stringify({ - Locations: [{ - key: awsKey, - dataStoreName: awsLocation, - }], - }), - jsonResponse: true, - }, next), - next => { - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.deepStrictEqual(data.TagSet, []); - next(); - }).catch(err => { - next(err); - }); - }, - ], done); - }); + ], + done, + ); + }, + ); - itIfLocationAws('should put tags if the source is not Azure and ' + - 'if-unmodified-since condition is met', done => { - const awsKey = uuidv4(); - let lastModified; - async.series([ - next => - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(result => next(null, result)).catch(err => next(err)), - next => - awsClient.send(new HeadObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - lastModified = data.LastModified; - next(null, data); - }).catch(err => next(err)), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: `/_/backbeat/batchdelete/${awsBucket}/${awsKey}`, - headers: { - 'if-unmodified-since': lastModified, - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), + itIfLocationAws( + 'should not put tags if the source is not Azure and ' + 'if-unmodified-since condition is not met', + done => { + const awsKey = uuidv4(); + async.series( + [ + next => + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(result => next(null, result)) + .catch(err => next(err)), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + headers: { + 'if-unmodified-since': 'Sun, 31 Mar 2019 00:00:00 GMT', + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: awsKey, + dataStoreName: awsLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => { + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.deepStrictEqual(data.TagSet, []); + next(); + }) + .catch(err => { + next(err); + }); }, - requestBody: JSON.stringify({ - Locations: [{ - key: awsKey, - dataStoreName: awsLocation, - }], - }), - jsonResponse: true, - }, next), - next => - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.strictEqual(data.TagSet.length, 2); - data.TagSet.forEach(tag => { - const { Key, Value } = tag; - const isValidTag = - Key === 'scal-delete-marker' || - Key === 'scal-delete-service'; - assert(isValidTag); - if (Key === 'scal-delete-marker') { - assert.strictEqual(Value, 'true'); - } - if (Key === 'scal-delete-service') { - assert.strictEqual( - Value, 'lifecycle-transition'); - } - }); - next(null, data); - }).catch(err => { - assert.ifError(err); - next(err); - }), - ], done); - }); + ], + done, + ); + }, + ); - itIfLocationAzure('should not delete the object if the source is Azure and ' + - 'if-unmodified-since condition is not met', done => { - const blob = uuidv4(); - async.series([ - next => - azureClient.getContainerClient(containerName).uploadBlockBlob(blob, 'a', 1) - .then(() => next(), next), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - - method: 'POST', - path: - `/_/backbeat/batchdelete/${containerName}/${blob}`, - headers: { - 'if-unmodified-since': - 'Sun, 31 Mar 2019 00:00:00 GMT', - 'x-scal-storage-class': azureLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), - }, - requestBody: JSON.stringify({ - Locations: [{ - key: blob, - dataStoreName: azureLocation, - }], - }), - jsonResponse: true, - }, err => { - if (err && err.statusCode === 412) { - return next(); - } - return next(err); - }), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(result => { - assert(result); - return next(); - }, next), - ], done); - }); + itIfLocationAws( + 'should put tags if the source is not Azure and ' + 'if-unmodified-since condition is met', + done => { + const awsKey = uuidv4(); + let lastModified; + async.series( + [ + next => + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(result => next(null, result)) + .catch(err => next(err)), + next => + awsClient + .send( + new HeadObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + lastModified = data.LastModified; + next(null, data); + }) + .catch(err => next(err)), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/batchdelete/${awsBucket}/${awsKey}`, + headers: { + 'if-unmodified-since': lastModified, + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: awsKey, + dataStoreName: awsLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.strictEqual(data.TagSet.length, 2); + data.TagSet.forEach(tag => { + const { Key, Value } = tag; + const isValidTag = + Key === 'scal-delete-marker' || Key === 'scal-delete-service'; + assert(isValidTag); + if (Key === 'scal-delete-marker') { + assert.strictEqual(Value, 'true'); + } + if (Key === 'scal-delete-service') { + assert.strictEqual(Value, 'lifecycle-transition'); + } + }); + next(null, data); + }) + .catch(err => { + assert.ifError(err); + next(err); + }), + ], + done, + ); + }, + ); - itIfLocationAzure('should delete the object if the source is Azure and ' + - 'if-unmodified-since condition is met', done => { - const blob = uuidv4(); - let lastModified; - async.series([ - next => - azureClient.getContainerClient(containerName).uploadBlockBlob(blob, 'a', 1) - .then(() => next(), next), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(result => { - lastModified = result.lastModified; - return next(); - }, next), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/batchdelete/${containerName}/${blob}`, - headers: { - 'if-unmodified-since': lastModified, - 'x-scal-storage-class': azureLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), - }, - requestBody: JSON.stringify({ - Locations: [{ - key: blob, - dataStoreName: azureLocation, - }], - }), - jsonResponse: true, - }, next), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(() => assert.fail('Expected error'), err => { - assert.strictEqual(err.statusCode, 404); - return next(); - }), - ], done); - }); + itIfLocationAzure( + 'should not delete the object if the source is Azure and ' + 'if-unmodified-since condition is not met', + done => { + const blob = uuidv4(); + async.series( + [ + next => + azureClient + .getContainerClient(containerName) + .uploadBlockBlob(blob, 'a', 1) + .then(() => next(), next), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + + method: 'POST', + path: `/_/backbeat/batchdelete/${containerName}/${blob}`, + headers: { + 'if-unmodified-since': 'Sun, 31 Mar 2019 00:00:00 GMT', + 'x-scal-storage-class': azureLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: blob, + dataStoreName: azureLocation, + }, + ], + }), + jsonResponse: true, + }, + err => { + if (err && err.statusCode === 412) { + return next(); + } + return next(err); + }, + ), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then(result => { + assert(result); + return next(); + }, next), + ], + done, + ); + }, + ); + + itIfLocationAzure( + 'should delete the object if the source is Azure and ' + 'if-unmodified-since condition is met', + done => { + const blob = uuidv4(); + let lastModified; + async.series( + [ + next => + azureClient + .getContainerClient(containerName) + .uploadBlockBlob(blob, 'a', 1) + .then(() => next(), next), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then(result => { + lastModified = result.lastModified; + return next(); + }, next), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/batchdelete/${containerName}/${blob}`, + headers: { + 'if-unmodified-since': lastModified, + 'x-scal-storage-class': azureLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: blob, + dataStoreName: azureLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then( + () => assert.fail('Expected error'), + err => { + assert.strictEqual(err.statusCode, 404); + return next(); + }, + ), + ], + done, + ); + }, + ); }); describe('checksums', () => { - const testDataSha256B64 = crypto.createHash('sha256') - .update(testData, 'utf-8').digest('base64'); + const testDataSha256B64 = crypto.createHash('sha256').update(testData, 'utf-8').digest('base64'); // A valid-length but wrong sha256 digest (44 base64 chars). const wrongSha256B64 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; // Checksum of the source object, as replicated by backbeat in the @@ -3657,49 +4838,54 @@ describe('backbeat routes', () => { // parsed at all by the backbeat routes, so a mismatching one must // not fail the request. it('should ignore a mismatching x-amz-checksum-sha256 header', done => { - makeBackbeatRequest({ - method: 'PUT', - resourceType: 'data', - bucket: TEST_BUCKET, - objectKey: TEST_KEY, - headers: { - 'x-scal-canonical-id': testMd['owner-id'], - 'content-md5': testDataMd5, - 'content-length': testData.length, - 'x-amz-checksum-sha256': wrongSha256B64, + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'data', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': wrongSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, }, - requestBody: testData, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - assert.ifError(err); - assert.strictEqual(data.statusCode, 200); - done(); - }); + (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + done(); + }, + ); }); - itIfLocationAws('should ignore a mismatching x-amz-checksum-sha256 header (multiplebackenddata)', - done => { - makeBackbeatRequest({ - method: 'PUT', - resourceType: 'multiplebackenddata', - bucket: TEST_BUCKET, - objectKey: TEST_KEY, - queryObj: { operation: 'putobject' }, - headers: { - 'x-scal-canonical-id': testMd['owner-id'], - 'x-scal-storage-type': 'aws_s3', - 'x-scal-storage-class': awsLocation, - 'content-md5': testDataMd5, - 'content-length': testData.length, - 'x-amz-checksum-sha256': wrongSha256B64, + itIfLocationAws('should ignore a mismatching x-amz-checksum-sha256 header (multiplebackenddata)', done => { + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'multiplebackenddata', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + queryObj: { operation: 'putobject' }, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': wrongSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, }, - requestBody: testData, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - assert.ifError(err); - assert.strictEqual(data.statusCode, 200); - done(); - }); + (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + done(); + }, + ); }); }); @@ -3708,138 +4894,174 @@ describe('backbeat routes', () => { // metadata: nothing is recomputed by the data route. it('should replicate the source object checksum (versioned bucket)', done => { const objectKey = 'checksum-replication-key'; - async.waterfall([ - next => makeBackbeatRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey, - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'content-md5': testDataMd5, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = getMetadataToPut(response); + newMd.checksum = sourceChecksum; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, next), - (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = getMetadataToPut(response); - newMd.checksum = sourceChecksum; - makeBackbeatRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, - (response, next) => { - assert.strictEqual(response.statusCode, 200); - s3.send(new HeadObjectCommand({ - Bucket: TEST_BUCKET, - Key: objectKey, - ChecksumMode: 'ENABLED', - })).then(result => { - assert.strictEqual(result.ChecksumSHA256, testDataSha256B64); - assert.strictEqual(result.ChecksumType, 'FULL_OBJECT'); - next(); - }, next); - }, - ], done); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + s3.send( + new HeadObjectCommand({ + Bucket: TEST_BUCKET, + Key: objectKey, + ChecksumMode: 'ENABLED', + }), + ).then(result => { + assert.strictEqual(result.ChecksumSHA256, testDataSha256B64); + assert.strictEqual(result.ChecksumType, 'FULL_OBJECT'); + next(); + }, next); + }, + ], + done, + ); }); it('should replicate the source object checksum (non-versioned bucket)', done => { const objectKey = 'checksum-replication-key-non-versioned'; - async.waterfall([ - next => makeBackbeatRequest({ - method: 'PUT', - bucket: NONVERSIONED_BUCKET, - objectKey, - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'content-md5': testDataMd5, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => + makeBackbeatRequest( + { + method: 'PUT', + bucket: NONVERSIONED_BUCKET, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = Object.assign({}, nonVersionedTestMd, { + location: JSON.parse(response.body), + checksum: sourceChecksum, + }); + makeBackbeatRequest( + { + method: 'PUT', + bucket: NONVERSIONED_BUCKET, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, next), - (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = Object.assign({}, nonVersionedTestMd, { - location: JSON.parse(response.body), - checksum: sourceChecksum, - }); - makeBackbeatRequest({ - method: 'PUT', - bucket: NONVERSIONED_BUCKET, - objectKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, - (response, next) => { - assert.strictEqual(response.statusCode, 200); - s3.send(new HeadObjectCommand({ - Bucket: NONVERSIONED_BUCKET, - Key: objectKey, - ChecksumMode: 'ENABLED', - })).then(result => { - assert.strictEqual(result.ChecksumSHA256, testDataSha256B64); - assert.strictEqual(result.ChecksumType, 'FULL_OBJECT'); - next(); - }, next); - }, - ], done); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + s3.send( + new HeadObjectCommand({ + Bucket: NONVERSIONED_BUCKET, + Key: objectKey, + ChecksumMode: 'ENABLED', + }), + ).then(result => { + assert.strictEqual(result.ChecksumSHA256, testDataSha256B64); + assert.strictEqual(result.ChecksumType, 'FULL_OBJECT'); + next(); + }, next); + }, + ], + done, + ); }); it('should not store a checksum when the replicated metadata has none', done => { const objectKey = 'checksum-replication-key-none'; - async.waterfall([ - next => makeBackbeatRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey, - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'content-md5': testDataMd5, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(getMetadataToPut(response)), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, next), - (response, next) => { - assert.strictEqual(response.statusCode, 200); - makeBackbeatRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(getMetadataToPut(response)), - }, next); - }, - (response, next) => { - assert.strictEqual(response.statusCode, 200); - s3.send(new HeadObjectCommand({ - Bucket: TEST_BUCKET, - Key: objectKey, - ChecksumMode: 'ENABLED', - })).then(result => { - assert.strictEqual(result.ChecksumSHA256, undefined); - assert.strictEqual(result.ChecksumCRC64NVME, undefined); - assert.strictEqual(result.ChecksumType, undefined); - next(); - }, next); - }, - ], done); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + s3.send( + new HeadObjectCommand({ + Bucket: TEST_BUCKET, + Key: objectKey, + ChecksumMode: 'ENABLED', + }), + ).then(result => { + assert.strictEqual(result.ChecksumSHA256, undefined); + assert.strictEqual(result.ChecksumCRC64NVME, undefined); + assert.strictEqual(result.ChecksumType, undefined); + next(); + }, next); + }, + ], + done, + ); }); }); });