From 1a44c52f3065446b110a753349d714e989f861fe Mon Sep 17 00:00:00 2001 From: Alexander Chan Date: Fri, 16 Feb 2018 15:24:56 -0800 Subject: [PATCH] S3C-1115 FT: Adds GCP Functional Tests Adds S3 functional tests for GCP backend: GET/PUT/DEL --- .../test/multipleBackend/delete/deleteGcp.js | 104 +++++++ .../test/multipleBackend/get/getGcp.js | 146 ++++++++++ .../test/multipleBackend/put/putGcp.js | 264 ++++++++++++++++++ 3 files changed, 514 insertions(+) create mode 100644 tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js create mode 100644 tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js create mode 100644 tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js new file mode 100644 index 0000000000..01bee49b18 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js @@ -0,0 +1,104 @@ +const assert = require('assert'); + +const withV4 = require('../../support/withV4'); +const BucketUtility = require('../../../lib/utility/bucket-util'); +const { + describeSkipIfNotMultiple, + gcpLocation, + gcpLocationMismatch, +} = require('../utils'); + +const bucket = 'buckettestmultiplebackenddelete-gcp'; +const gcpObject = `gcpObject-${Date.now()}`; +const emptyObject = `emptyObject-${Date.now()}`; +const bigObject = `bigObject-${Date.now()}`; +const mismatchObject = `mismatchObject-${Date.now()}`; +const body = Buffer.from('I am a body', 'utf8'); +const bigBody = Buffer.alloc(10485760); + +describeSkipIfNotMultiple('Multiple backend delete', () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + + before(() => { + process.stdout.write('Creating bucket\n'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3.createBucketAsync({ Bucket: bucket }) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }).then(() => { + process.stdout.write('Putting object to GCP\n'); + const params = { Bucket: bucket, Key: gcpObject, Body: body, + Metadata: { 'scal-location-constraint': gcpLocation } }; + return s3.putObjectAsync(params); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to GCP\n'); + const params = { Bucket: bucket, Key: emptyObject, + Metadata: { 'scal-location-constraint': gcpLocation } }; + return s3.putObjectAsync(params); + }) + .then(() => { + process.stdout.write('Putting large object to GCP\n'); + const params = { Bucket: bucket, Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': gcpLocation } }; + return s3.putObjectAsync(params); + }) + .then(() => { + process.stdout.write('Putting object to GCP\n'); + const params = { Bucket: bucket, Key: mismatchObject, + Body: body, Metadata: + { 'scal-location-constraint': gcpLocationMismatch } }; + return s3.putObjectAsync(params); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; + }); + }); + after(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket) + .catch(err => { + process.stdout.write(`Error deleting bucket: ${err}\n`); + throw err; + }); + }); + + const deleteTests = [ + { + msg: 'should delete object from GCP', + Bucket: bucket, Key: gcpObject, + }, + { + msg: 'should delete 0-byte object from GCP', + Bucket: bucket, Key: emptyObject, + }, + { + msg: 'should delete large object from GCP', + Bucket: bucket, Key: bigObject, + }, + { + msg: 'should delete object from GCP location with ' + + 'bucketMatch set to false', + Bucket: bucket, Key: mismatchObject, + }, + ]; + deleteTests.forEach(test => { + const { msg, Bucket, Key } = test; + it(msg, done => s3.deleteObject({ Bucket, Key }, err => { + assert.strictEqual(err, null, + `Expected success, got error ${JSON.stringify(err)}`); + s3.getObject({ Bucket, Key }, err => { + assert.strictEqual(err.code, 'NoSuchKey', 'Expected ' + + 'error but got success'); + done(); + }); + })); + }); + }); +}); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js new file mode 100644 index 0000000000..853fdf9dea --- /dev/null +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js @@ -0,0 +1,146 @@ +const assert = require('assert'); +const withV4 = require('../../support/withV4'); +const BucketUtility = require('../../../lib/utility/bucket-util'); +const { + describeSkipIfNotMultiple, + gcpLocation, + gcpLocationMismatch, +} = require('../utils'); + +const bucket = 'buckettestmultiplebackendget-gcp'; +const gcpObject = `gcpobject-${Date.now()}`; +const emptyGcpObject = `emptyObject-${Date.now()}`; +const bigObject = `bigObject-${Date.now()}`; +const mismatchObject = `mismatch-${Date.now()}`; +const body = Buffer.from('I am a body', 'utf8'); +const bigBody = Buffer.alloc(10485760); +const bigBodyLen = bigBody.length; +const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; +const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; +const bigMD5 = 'f1c9645dbc14efddc7d8a322685f26eb'; + +describe('Multiple backend get object', function testSuite() { + this.timeout(30000); + withV4(sigCfg => { + let bucketUtil; + let s3; + before(() => { + process.stdout.write('Creating bucket'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3.createBucketAsync({ Bucket: bucket }) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); + }); + + after(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil.empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + + `${err}\n`); + throw err; + }); + }); + + describeSkipIfNotMultiple('with objects in GCP', () => { + before(() => { + process.stdout.write('Putting object to GCP\n'); + return s3.putObjectAsync({ Bucket: bucket, Key: gcpObject, + Body: body, + Metadata: { + 'scal-location-constraint': gcpLocation }, + }) + .then(() => { + process.stdout.write('Putting 0-byte object to GCP\n'); + return s3.putObjectAsync({ Bucket: bucket, + Key: emptyGcpObject, + Metadata: { + 'scal-location-constraint': gcpLocation } }); + }) + .then(() => { + process.stdout.write('Putting large object to GCP\n'); + return s3.putObjectAsync({ Bucket: bucket, + Key: bigObject, Body: bigBody, + Metadata: { + 'scal-location-constraint': gcpLocation } }); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; + }); + }); + + const getTests = [ + { + msg: 'should get a 0-byte object from GCP', + input: { Bucket: bucket, Key: emptyGcpObject, + range: null, size: null }, + output: { MD5: emptyMD5, contentRange: null }, + }, + { + msg: 'should get an object from GCP', + input: { Bucket: bucket, Key: gcpObject, + range: null, size: null }, + output: { MD5: correctMD5, contentRange: null }, + }, + { + msg: 'should get a large object from GCP', + input: { Bucket: bucket, Key: bigObject, + range: null, size: null }, + output: { MD5: bigMD5, contentRange: null }, + }, + { + msg: 'should get an object using range query from GCP', + input: { Bucket: bucket, Key: bigObject, + range: 'bytes=0-9', size: 10 }, + output: { MD5: bigMD5, + contentRange: `bytes 0-9/${bigBodyLen}` }, + }, + ]; + getTests.forEach(test => { + const { Bucket, Key, range, size } = test.input; + const { MD5, contentRange } = test.output; + it(test.msg, done => { + s3.getObject({ Bucket, Key, Range: range }, + (err, res) => { + assert.equal(err, null, + `Expected success but got error ${err}`); + if (range) { + assert.strictEqual(res.ContentLength, `${size}`); + assert.strictEqual(res.ContentRange, contentRange); + } + assert.strictEqual(res.ETag, `"${MD5}"`); + done(); + }); + }); + }); + }); + + describeSkipIfNotMultiple('with bucketMatch set to false', () => { + beforeEach(done => { + s3.putObject({ Bucket: bucket, Key: mismatchObject, Body: body, + Metadata: { 'scal-location-constraint': gcpLocationMismatch } }, + err => { + assert.equal(err, null, `Err putting object: ${err}`); + done(); + }); + }); + + it('should get an object from GCP', done => { + s3.getObject({ Bucket: bucket, Key: mismatchObject }, + (err, res) => { + assert.equal(err, null, `Error getting object: ${err}`); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + done(); + }); + }); + }); + }); +}); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js new file mode 100644 index 0000000000..43af83466f --- /dev/null +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js @@ -0,0 +1,264 @@ +const assert = require('assert'); + +const withV4 = require('../../support/withV4'); +const BucketUtility = require('../../../lib/utility/bucket-util'); +const { describeSkipIfNotMultiple, gcpClient, gcpBucket, + gcpLocation, fileLocation } = require('../utils'); + +const bucket = 'buckettestmultiplebackendput-gcp'; +const body = Buffer.from('I am a body', 'utf8'); +const bigBody = Buffer.alloc(10485760); +const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; +const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; +const bigS3MD5 = 'f1c9645dbc14efddc7d8a322685f26eb'; +const bigGCPMD5 = 'a7d414b9133d6483d9a1c4e04e856e3b-2'; + +let bucketUtil; +let s3; + +const retryTimeout = 10000; + +function checkGcp(key, gcpMD5, location, callback) { + gcpClient.getObject({ + Bucket: gcpBucket, + Key: key, + }, (err, res) => { + assert.equal(err, null, `Expected success, got error ${err}`); + if (res.Metadata && res.Metadata['scal-etag']) { + assert.strictEqual(res.Metadata['scal-etag'], gcpMD5); + } else { + assert.strictEqual( + res.ETag.substring(1, res.ETag.length - 1), gcpMD5); + } + assert.strictEqual(res.Metadata['scal-location-constraint'], + location); + callback(res); + }); +} + +function checkGcpError(key, expectedError, callback) { + setTimeout(() => { + gcpClient.getObject({ + Bucket: gcpBucket, + Key: key, + }, err => { + assert.notStrictEqual(err, undefined, + 'Expected error but did not find one'); + assert.strictEqual(err.code, expectedError, + `Expected error code ${expectedError} but got ${err.code}`); + callback(); + }); + }, 1000); +} + +function gcpGetCheck(objectKey, s3MD5, gcpMD5, location, callback) { + process.stdout.write('Getting object\n'); + s3.getObject({ Bucket: bucket, Key: objectKey }, + function s3GetCallback(err, res) { + if (err && err.code === 'NetworkingError') { + return setTimeout(() => { + process.stdout.write('Getting object retry\n'); + s3.getObject({ Bucket: bucket, Key: objectKey }, s3GetCallback); + }, retryTimeout); + } + assert.strictEqual(err, null, 'Expected success, got error ' + + `on call to GCP through S3: ${err}`); + assert.strictEqual(res.ETag, `"${s3MD5}"`); + assert.strictEqual(res.Metadata['scal-location-constraint'], + location); + process.stdout.write('Getting object from GCP\n'); + return checkGcp(objectKey, gcpMD5, location, callback); + }); +} + +describeSkipIfNotMultiple('MultipleBackend put object to GCP', function +describeFn() { + this.timeout(250000); + withV4(sigCfg => { + beforeEach(() => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3.createBucketAsync({ Bucket: bucket }) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); + }); + + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil.empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); + }); + + describe('with set location from "x-amz-meta-scal-' + + 'location-constraint" header', function describe() { + if (!process.env.S3_END_TO_END) { + this.retries(2); + } + + const putTests = [ + { + msg: 'should put a 0-byte object to GCP', + input: { Body: null, location: gcpLocation }, + output: { s3MD5: emptyMD5, gcpMD5: emptyMD5 }, + }, + { + msg: 'should put an object to GCP', + input: { Body: body, location: gcpLocation }, + output: { s3MD5: correctMD5, gcpMD5: correctMD5 }, + }, + { + msg: 'should put a large object to GCP', + input: { Body: bigBody, location: gcpLocation }, + output: { s3MD5: bigS3MD5, gcpMD5: bigGCPMD5 }, + }, + ]; + putTests.forEach(test => { + const { location, Body } = test.input; + const { s3MD5, gcpMD5 } = test.output; + it(test.msg, done => { + const key = `somekey-${Date.now()}`; + const params = { Bucket: bucket, Key: key, Body, + Metadata: { 'scal-location-constraint': location }, + }; + return s3.putObject(params, err => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + return gcpGetCheck(key, s3MD5, gcpMD5, location, + () => done()); + }); + }); + }); + }); + + describe('with object rewrites', function describe() { + if (!process.env.S3_END_TO_END) { + this.retries(2); + } + + it('should put objects with same key to GCP ' + + 'then file, and object should only be present in file', done => { + const key = `somekey-${Date.now()}`; + const params = { Bucket: bucket, Key: key, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation } }; + return s3.putObject(params, err => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + params.Metadata = + { 'scal-location-constraint': fileLocation }; + return s3.putObject(params, err => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + return s3.getObject({ Bucket: bucket, Key: key }, + (err, res) => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + assert.strictEqual( + res.Metadata['scal-location-constraint'], + fileLocation); + return checkGcpError(key, 'NoSuchKey', + () => done()); + }); + }); + }); + }); + + it('should put objects with same key to file ' + + 'then GCP, and object should only be present on GCP', done => { + const key = `somekey-${Date.now()}`; + const params = { Bucket: bucket, Key: key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation } }; + return s3.putObject(params, err => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + params.Metadata = { + 'scal-location-constraint': gcpLocation }; + return s3.putObject(params, err => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + return gcpGetCheck(key, correctMD5, correctMD5, + gcpLocation, () => done()); + }); + }); + }); + + it('should put two objects to GCP with same ' + + 'key, and newest object should be returned', done => { + const key = `somekey-${Date.now()}`; + const params = { Bucket: bucket, Key: key, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation, + 'unique-header': 'first object' } }; + return s3.putObject(params, err => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + params.Metadata = { 'scal-location-constraint': gcpLocation, + 'unique-header': 'second object' }; + return s3.putObject(params, err => { + assert.equal(err, null, 'Expected success, ' + + `got error ${err}`); + return gcpGetCheck(key, correctMD5, correctMD5, + gcpLocation, result => { + assert.strictEqual(result.Metadata + ['unique-header'], 'second object'); + done(); + }); + }); + }); + }); + }); + }); +}); + +describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', +() => { + withV4(sigCfg => { + beforeEach(() => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + }); + + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil.empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); + }); + + it('should put an object to GCP with no location header', done => { + process.stdout.write('Creating bucket\n'); + return s3.createBucket({ Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: gcpLocation, + }, + }, err => { + assert.equal(err, null, `Error creating bucket: ${err}`); + process.stdout.write('Putting object\n'); + const key = `somekey-${Date.now()}`; + const params = { Bucket: bucket, Key: key, Body: body }; + return s3.putObject(params, err => { + assert.equal(err, null, + `Expected success, got error ${err}`); + return gcpGetCheck(key, correctMD5, correctMD5, undefined, + () => done()); + }); + }); + }); + }); +});