From 1be83ceacb53ddc5804aa611decbc0a69e0e7276 Mon Sep 17 00:00:00 2001 From: Alexander Chan Date: Wed, 20 Dec 2017 10:15:12 -0800 Subject: [PATCH 1/2] S3C-1115 FT: GCP Object APIs Implements GCP Object PUT/GET/GET/DELETE APIs --- lib/data/external/GCP/GcpService.js | 47 +- lib/data/external/GCP/gcp-2017-11-01.api.json | 458 ++++++++++++++++++ .../raw-node/test/GCP/bucket/get.js | 207 ++++---- .../raw-node/test/GCP/bucket/getVersioning.js | 4 - .../raw-node/test/GCP/bucket/head.js | 7 - .../raw-node/test/GCP/bucket/putVersioning.js | 4 - .../raw-node/test/GCP/object/delete.js | 98 ++++ .../raw-node/test/GCP/object/get.js | 104 ++++ .../raw-node/test/GCP/object/head.js | 103 ++++ .../raw-node/test/GCP/object/put.js | 112 +++++ 10 files changed, 998 insertions(+), 146 deletions(-) create mode 100644 tests/functional/raw-node/test/GCP/object/delete.js create mode 100644 tests/functional/raw-node/test/GCP/object/get.js create mode 100644 tests/functional/raw-node/test/GCP/object/head.js create mode 100644 tests/functional/raw-node/test/GCP/object/put.js diff --git a/lib/data/external/GCP/GcpService.js b/lib/data/external/GCP/GcpService.js index 8b1f9ac4a1..44d890c522 100644 --- a/lib/data/external/GCP/GcpService.js +++ b/lib/data/external/GCP/GcpService.js @@ -1,11 +1,19 @@ const googleAuth = require('google-auto-auth'); const async = require('async'); const AWS = require('aws-sdk'); +const stream = require('stream'); const { errors } = require('arsenal'); const Service = AWS.Service; const GcpSigner = require('./GcpSigner'); +/** + * genAuth - create a google authorizer for generating request tokens + * @param {object} authParams - params that contains the credentials for + * generating the authorizer + * @param {function} callback - callback function to call with the authorizer + * @return {undefined} + */ function genAuth(authParams, callback) { async.tryEach([ function authKeyFile(next) { @@ -32,6 +40,12 @@ const GCP = Service.defineService('gcp', ['2017-11-01'], { _jsonAuth: null, _authParams: null, + /** + * getToken - generate a token for authorizing JSON API requests + * @param {function} callback - callback function to call with the + * generated token + * @return {undefined} + */ getToken(callback) { if (this._jsonAuth) { return this._jsonAuth.getToken(callback); @@ -69,12 +83,17 @@ const GCP = Service.defineService('gcp', ['2017-11-01'], { return this.listObjects(params, callback); }, - // TO-DO: Implemented the following APIs - upload(params, options, callback) { - return callback(errors.NotImplemented - .customizeDescription('GCP: upload not implemented')); + // Object APIs + upload(params, callback) { + if (params.Body instanceof stream) { + return callback(errors.NotImplemented + .customizeDescription( + 'GCP: Upload with stream body not implemented')); + } + return this.putObject(params, callback); }, + // TO-DO: Implement the following APIs // Service API listBuckets(params, callback) { return callback(errors.NotImplemented @@ -148,26 +167,6 @@ const GCP = Service.defineService('gcp', ['2017-11-01'], { }, // Object APIs - headObject(params, callback) { - return callback(errors.NotImplemented - .customizeDescription('GCP: headObject not implemented')); - }, - - putObject(params, callback) { - return callback(errors.NotImplemented - .customizeDescription('GCP: putObject not implemented')); - }, - - getObject(params, callback) { - return callback(errors.NotImplemented - .customizeDescription('GCP: getObject not implemented')); - }, - - deleteObject(params, callback) { - return callback(errors.NotImplemented - .customizeDescription('GCP: deleteObject not implemented')); - }, - deleteObjects(params, callback) { return callback(errors.NotImplemented .customizeDescription('GCP: deleteObjects not implemented')); diff --git a/lib/data/external/GCP/gcp-2017-11-01.api.json b/lib/data/external/GCP/gcp-2017-11-01.api.json index 4aa48b352f..35bbb55c23 100644 --- a/lib/data/external/GCP/gcp-2017-11-01.api.json +++ b/lib/data/external/GCP/gcp-2017-11-01.api.json @@ -159,9 +159,467 @@ "Status": {} } } + }, + "HeadObject": { + "http": { + "method": "HEAD", + "requestUri": "/{Bucket}/{Key+}" + }, + "input": { + "type": "structure", + "required": [ + "Bucket", + "Key" + ], + "members": { + "Date": { + "location": "header", + "locationName": "Date", + "type": "timestamp" + }, + "Bucket": { + "location": "uri", + "locationName": "Bucket" + }, + "IfMatch": { + "location": "header", + "locationName": "If-Match" + }, + "IfModifiedSince": { + "location": "header", + "locationName": "If-Modified-Since", + "type": "timestamp" + }, + "IfNoneMatch": { + "location": "header", + "locationName": "If-None-Match" + }, + "IfUnmodifiedSince": { + "location": "header", + "locationName": "If-Unmodified-Since", + "type": "timestamp" + }, + "Range": { + "location": "header", + "locationName": "Range" + }, + "Key": { + "location": "uri", + "locationName": "Key" + }, + "Range": { + "location": "header", + "locationName": "Range" + }, + "VersionId": { + "location": "querystring", + "locationName": "generation" + }, + "ProjectId": { + "location": "header", + "locationName": "x-goog-project-id" + } + } + }, + "output": { + "type": "structure", + "members": { + "Date": { + "location": "header", + "locationName": "Date", + "type": "timestamp" + }, + "AcceptRanges": { + "location": "header", + "locationName": "accept-ranges" + }, + "Expiration": { + "location": "header", + "locationName": "x-goog-expiration" + }, + "LastModified": { + "location": "header", + "locationName": "Last-Modified", + "type": "timestamp" + }, + "ContentLength": { + "location": "header", + "locationName": "Content-Length", + "type": "long" + }, + "ContentHash": { + "location": "header", + "locationName": "x-goog-hash" + }, + "ETag": { + "location": "header", + "locationName": "ETag" + }, + "VersionId": { + "location": "header", + "locationName": "x-goog-generation" + }, + "MetaVersionId": { + "location": "header", + "locationName": "x-goog-metageneration" + }, + "CacheControl": { + "location": "header", + "locationName": "Cache-Control" + }, + "ContentDisposition": { + "location": "header", + "locationName": "Content-Disposition" + }, + "ContentEncoding": { + "location": "header", + "locationName": "Content-Encoding" + }, + "ContentLanguage": { + "location": "header", + "locationName": "Content-Language" + }, + "ContentType": { + "location": "header", + "locationName": "Content-Type" + }, + "Expires": { + "location": "header", + "locationName": "Expires", + "type": "timestamp" + }, + "Metadata": { + "shape": "MetadataShape", + "location": "headers", + "locationName": "x-goog-meta-" + }, + "StorageClass": { + "location": "headers", + "locationName": "x-goog-storage-class" + } + } + } + }, + "PutObject": { + "http": { + "method": "PUT", + "requestUri": "/{Bucket}/{Key+}" + }, + "input": { + "type": "structure", + "required": [ + "Bucket", + "Key" + ], + "members": { + "Date": { + "location": "header", + "locationName": "Date", + "type": "timestamp" + }, + "ACL": { + "location": "header", + "locationName": "x-goog-acl" + }, + "Body": { + "streaming": true, + "type": "blob" + }, + "Bucket": { + "location": "uri", + "locationName": "Bucket" + }, + "CacheControl": { + "location": "header", + "locationName": "Cache-Control" + }, + "ContentDisposition": { + "location": "header", + "locationName": "Content-Disposition" + }, + "ContentEncoding": { + "location": "header", + "locationName": "Content-Encoding" + }, + "ContentLanguage": { + "location": "header", + "locationName": "Content-Language" + }, + "ContentLength": { + "location": "header", + "locationName": "Content-Length", + "type": "long" + }, + "ContentMD5": { + "location": "header", + "locationName": "Content-MD5" + }, + "ContentType": { + "location": "header", + "locationName": "Content-Type" + }, + "Expires": { + "location": "header", + "locationName": "Expires", + "type": "timestamp" + }, + "Key": { + "location": "uri", + "locationName": "Key" + }, + "Metadata": { + "shape": "MetadataShape", + "location": "headers", + "locationName": "x-goog-meta-" + }, + "ProjectId": { + "location": "header", + "locationName": "x-goog-project-id" + } + }, + "payload": "Body" + }, + "output": { + "type": "structure", + "members": { + "Expiration": { + "location": "header", + "locationName": "x-goog-expiration" + }, + "ETag": { + "location": "header", + "locationName": "ETag" + }, + "ContentHash": { + "location": "header", + "locationName": "x-goog-hash" + }, + "VersionId": { + "location": "header", + "locationName": "x-goog-generation" + }, + "MetaVersionId": { + "location": "header", + "locationName": "x-goog-metageneration" + } + } + } + }, + "GetObject": { + "http": { + "method": "GET", + "requestUri": "/{Bucket}/{Key+}" + }, + "input": { + "type": "structure", + "required": [ + "Bucket", + "Key" + ], + "members": { + "Bucket": { + "location": "uri", + "locationName": "Bucket" + }, + "IfMatch": { + "location": "header", + "locationName": "If-Match" + }, + "IfModifiedSince": { + "location": "header", + "locationName": "If-Modified-Since", + "type": "timestamp" + }, + "IfNoneMatch": { + "location": "header", + "locationName": "If-None-Match" + }, + "IfUnmodifiedSince": { + "location": "header", + "locationName": "If-Unmodified-Since", + "type": "timestamp" + }, + "Key": { + "location": "uri", + "locationName": "Key" + }, + "Range": { + "location": "header", + "locationName": "Range" + }, + "ResponseCacheControl": { + "location": "querystring", + "locationName": "response-cache-control" + }, + "ResponseContentDisposition": { + "location": "querystring", + "locationName": "response-content-disposition" + }, + "ResponseContentEncoding": { + "location": "querystring", + "locationName": "response-content-encoding" + }, + "ResponseContentLanguage": { + "location": "querystring", + "locationName": "response-content-language" + }, + "ResponseContentType": { + "location": "querystring", + "locationName": "response-content-type" + }, + "ResponseExpires": { + "location": "querystring", + "locationName": "response-expires", + "type": "timestamp" + }, + "VersionId": { + "location": "querystring", + "locationName": "generation" + }, + "ProjectId": { + "location": "header", + "locationName": "x-goog-project-id" + } + } + }, + "output": { + "type": "structure", + "members": { + "Body": { + "streaming": true, + "type": "blob" + }, + "AcceptRanges": { + "location": "header", + "locationName": "accept-ranges" + }, + "Expiration": { + "location": "header", + "locationName": "x-goog-expiration" + }, + "LastModified": { + "location": "header", + "locationName": "Last-Modified", + "type": "timestamp" + }, + "ContentLength": { + "location": "header", + "locationName": "Content-Length", + "type": "long" + }, + "ETag": { + "location": "header", + "locationName": "ETag" + }, + "VersionId": { + "location": "header", + "locationName": "x-goog-generation" + }, + "MetaVersionId": { + "location": "header", + "locationName": "x-goog-metageneration" + }, + "CacheControl": { + "location": "header", + "locationName": "Cache-Control" + }, + "ContentDisposition": { + "location": "header", + "locationName": "Content-Disposition" + }, + "ContentEncoding": { + "location": "header", + "locationName": "Content-Encoding" + }, + "ContentLanguage": { + "location": "header", + "locationName": "Content-Language" + }, + "ContentRange": { + "location": "header", + "locationName": "Content-Range" + }, + "ContentType": { + "location": "header", + "locationName": "Content-Type" + }, + "ContentHash": { + "location": "header", + "locationName": "x-goog-hash" + }, + "Expires": { + "location": "header", + "locationName": "Expires", + "type": "timestamp" + }, + "WebsiteRedirectLocation": { + "location": "header", + "locationName": "x-goog-website-redirect-location" + }, + "ServerSideEncryption": { + "location": "header", + "locationName": "x-goog-server-side-encryption" + }, + "Metadata": { + "shape": "MetadataShape", + "location": "headers", + "locationName": "x-goog-meta-" + }, + "StorageClass": { + "location": "header", + "locationName": "x-goog-storage-class" + } + }, + "payload": "Body" + } + }, + "DeleteObject": { + "http": { + "method": "DELETE", + "requestUri": "/{Bucket}/{Key+}" + }, + "input": { + "type": "structure", + "required": [ + "Bucket", + "Key" + ], + "members": { + "Bucket": { + "location": "uri", + "locationName": "Bucket" + }, + "Key": { + "location": "uri", + "locationName": "Key" + }, + "VersionId": { + "location": "querystring", + "locationName": "generation" + }, + "ProjectId": { + "location": "header", + "locationName": "x-goog-project-id" + } + } + }, + "output": { + "type": "structure", + "members": { + "VersionId": { + "location": "header", + "locationName": "x-goog-generation" + } + } + } } }, "shapes": { + "MetadataShape": { + "type": "map", + "key": {}, + "value": {} + }, "OwnerShape": { "locationName": "Owner", "type": "structure", diff --git a/tests/functional/raw-node/test/GCP/bucket/get.js b/tests/functional/raw-node/test/GCP/bucket/get.js index 7c2afbc8ef..05594a7fe6 100644 --- a/tests/functional/raw-node/test/GCP/bucket/get.js +++ b/tests/functional/raw-node/test/GCP/bucket/get.js @@ -5,24 +5,89 @@ const { makeGcpRequest } = require('../../../utils/makeRequest'); const { gcpRequestRetry } = require('../../../utils/gcpUtils'); const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); -const constants = require('../../../../../../constants'); +const { listingHardLimit } = require('../../../../../../constants'); const credentialOne = 'gcpbackend'; +const bucketName = `somebucket-${Date.now()}`; +const smallSize = 20; +const bigSize = listingHardLimit + 1; +const config = getRealAwsConfig(credentialOne); +const gcpClient = new GCP(config); + +function populateBucket(createdObjects, callback) { + process.stdout.write( + `Putting ${createdObjects.length} objects into bucket\n`); + async.mapLimit(createdObjects, 10, + (object, moveOn) => { + makeGcpRequest({ + method: 'PUT', + bucket: bucketName, + objectKey: object, + authCredentials: config.credentials, + }, err => moveOn(err)); + }, err => { + if (err) { + process.stdout + .write(`err putting objects ${err.code}`); + } + return callback(err); + }); +} + +function removeObjects(createdObjects, callback) { + process.stdout.write( + `Deleting ${createdObjects.length} objects from bucket\n`); + async.mapLimit(createdObjects, 10, + (object, moveOn) => { + makeGcpRequest({ + method: 'DELETE', + bucket: bucketName, + objectKey: object, + authCredentials: config.credentials, + }, err => moveOn(err)); + }, err => { + if (err) { + process.stdout + .write(`err deleting objects ${err.code}`); + } + return callback(err); + }); +} describe('GCP: GET Bucket', function testSuite() { this.timeout(180000); - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - describe('without existing bucket', () => { - beforeEach(function beforeFn(done) { - this.currentTest.bucketName = `somebucket-${Date.now()}`; - return done(); + before(done => { + gcpRequestRetry({ + method: 'PUT', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in creating bucket ${err}\n`); + } + return done(err); + }); + }); + + after(done => { + gcpRequestRetry({ + method: 'DELETE', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in deleting bucket ${err}\n`); + } + return done(err); }); + }); - it('should return 404 and NoSuchBucket', function testFn(done) { + describe('without existing bucket', () => { + it('should return 404 and NoSuchBucket', done => { + const badBucketName = `nonexistingbucket-${Date.now()}`; gcpClient.getBucket({ - Bucket: this.test.bucketName, + Bucket: badBucketName, }, err => { assert(err); assert.strictEqual(err.statusCode, 404); @@ -33,101 +98,28 @@ describe('GCP: GET Bucket', function testSuite() { }); describe('with existing bucket', () => { - let numberObjects; - beforeEach(function beforeFn(done) { - this.currentTest.bucketName = `somebucket-${Date.now()}`; - this.currentTest.createdObjects = Array.from( - Array(numberObjects).keys()).map(i => `someObject-${i}`); - process.stdout - .write(`Creating test bucket\n`); - return async.waterfall([ - next => gcpRequestRetry({ - method: 'PUT', - bucket: this.currentTest.bucketName, - authCredentials: config.credentials, - }, 0, err => { - if (err) { - process.stdout.write(`err creating bucket ${err.code}`); - } - return next(err); - }), - next => { - process.stdout.write( - `Putting ${numberObjects} objects into bucket\n`); - async.mapLimit(this.currentTest.createdObjects, 10, - (object, moveOn) => { - makeGcpRequest({ - method: 'PUT', - bucket: this.currentTest.bucketName, - objectKey: object, - authCredentials: config.credentials, - }, err => moveOn(err)); - }, err => { - if (err) { - process.stdout - .write(`err putting objects ${err.code}`); - } - return next(err); - }); - }, - ], err => done(err)); - }); + describe('with less than listingHardLimit number of objects', () => { + const createdObjects = Array.from( + Array(smallSize).keys()).map(i => `someObject-${i}`); - afterEach(function afterFn(done) { - return async.waterfall([ - next => { - process.stdout.write( - `Deleting ${numberObjects} objects from bucket\n`); - async.mapLimit(this.currentTest.createdObjects, 10, - (object, moveOn) => { - makeGcpRequest({ - method: 'DELETE', - bucket: this.currentTest.bucketName, - objectKey: object, - authCredentials: config.credentials, - }, err => moveOn(err)); - }, err => { - if (err) { - process.stdout - .write(`err deleting objects ${err.code}`); - } - return next(err); - }); - }, - next => gcpRequestRetry({ - method: 'DELETE', - bucket: this.currentTest.bucketName, - authCredentials: config.credentials, - }, 0, err => { - if (err) { - process.stdout.write(`err deleting bucket ${err.code}`); - } - return next(err); - }), - ], err => done(err)); - }); + before(done => populateBucket(createdObjects, done)); - describe('with less than listingHardLimit number of objects', () => { - before('Number of objects: 20', () => { - numberObjects = 20; - }); + after(done => removeObjects(createdObjects, done)); - it('should list all 20 created objects', - function testFn(done) { - return gcpClient.listObjects({ - Bucket: this.test.bucketName, + it(`should list all ${smallSize} created objects`, done => { + gcpClient.listObjects({ + Bucket: bucketName, }, (err, res) => { assert.equal(err, null, `Expected success, but got ${err}`); - assert.strictEqual(res.Contents.length, numberObjects); + assert.strictEqual(res.Contents.length, smallSize); return done(); }); }); describe('with MaxKeys at 10', () => { - it('should list MaxKeys number of objects', - function testFn(done) { - return gcpClient.listObjects({ - Bucket: this.test.bucketName, + it('should list MaxKeys number of objects', done => { + gcpClient.listObjects({ + Bucket: bucketName, MaxKeys: 10, }, (err, res) => { assert.equal(err, null, @@ -140,33 +132,34 @@ describe('GCP: GET Bucket', function testSuite() { }); describe('with more than listingHardLimit number of objects', () => { - before('Number of objects: 1001', () => { - numberObjects = constants.listingHardLimit + 1; - }); + const createdObjects = Array.from( + Array(bigSize).keys()).map(i => `someObject-${i}`); + + before(done => populateBucket(createdObjects, done)); + + after(done => removeObjects(createdObjects, done)); - it('should list at max 1000 of objects created', - function testFn(done) { - return gcpClient.listObjects({ - Bucket: this.test.bucketName, + it('should list at max 1000 of objects created', done => { + gcpClient.listObjects({ + Bucket: bucketName, }, (err, res) => { assert.equal(err, null, `Expected success, but got ${err}`); assert.strictEqual(res.Contents.length, - constants.listingHardLimit); + listingHardLimit); return done(); }); }); describe('with MaxKeys at 1001', () => { - it('should list at max 1000, ignoring MaxKeys', - function testFn(done) { - return gcpClient.listObjects({ - Bucket: this.test.bucketName, + it('should list at max 1000, ignoring MaxKeys', done => { + gcpClient.listObjects({ + Bucket: bucketName, MaxKeys: 1001, }, (err, res) => { assert.equal(err, null, `Expected success, but got ${err}`); assert.strictEqual(res.Contents.length, - constants.listingHardLimit); + listingHardLimit); return done(); }); }); diff --git a/tests/functional/raw-node/test/GCP/bucket/getVersioning.js b/tests/functional/raw-node/test/GCP/bucket/getVersioning.js index 5d82a38f57..a02d0750e6 100644 --- a/tests/functional/raw-node/test/GCP/bucket/getVersioning.js +++ b/tests/functional/raw-node/test/GCP/bucket/getVersioning.js @@ -33,8 +33,6 @@ describe('GCP: GET Bucket Versioning', () => { }, 0, err => { if (err) { process.stdout.write(`err in creating bucket ${err}\n`); - } else { - process.stdout.write('Created bucket\n'); } return done(err); }); @@ -48,8 +46,6 @@ describe('GCP: GET Bucket Versioning', () => { }, 0, err => { if (err) { process.stdout.write(`err in deleting bucket ${err}\n`); - } else { - process.stdout.write('Deleted bucket\n'); } return done(err); }); diff --git a/tests/functional/raw-node/test/GCP/bucket/head.js b/tests/functional/raw-node/test/GCP/bucket/head.js index be83d52abd..5cbe743828 100644 --- a/tests/functional/raw-node/test/GCP/bucket/head.js +++ b/tests/functional/raw-node/test/GCP/bucket/head.js @@ -40,11 +40,6 @@ describe('GCP: HEAD Bucket', () => { if (err) { return done(err); } - if (!res) { - // should not happen - return done( - new Error('Error: success response does not result')); - } this.currentTest.bucketObj = { MetaVersionId: res.headers['x-goog-metageneration'], }; @@ -61,8 +56,6 @@ describe('GCP: HEAD Bucket', () => { if (err) { process.stdout .write(`err deleting bucket: ${err.code}\n`); - } else { - process.stdout.write('Deleted bucket\n'); } return done(err); }); diff --git a/tests/functional/raw-node/test/GCP/bucket/putVersioning.js b/tests/functional/raw-node/test/GCP/bucket/putVersioning.js index 0f1313bd23..c9bff55675 100644 --- a/tests/functional/raw-node/test/GCP/bucket/putVersioning.js +++ b/tests/functional/raw-node/test/GCP/bucket/putVersioning.js @@ -35,8 +35,6 @@ describe('GCP: PUT Bucket Versioning', () => { }, 0, err => { if (err) { process.stdout.write(`err in creating bucket ${err}\n`); - } else { - process.stdout.write('Created bucket\n'); } return done(err); }); @@ -50,8 +48,6 @@ describe('GCP: PUT Bucket Versioning', () => { }, 0, err => { if (err) { process.stdout.write(`err in deleting bucket ${err}\n`); - } else { - process.stdout.write('Deleted bucket\n'); } return done(err); }); diff --git a/tests/functional/raw-node/test/GCP/object/delete.js b/tests/functional/raw-node/test/GCP/object/delete.js new file mode 100644 index 0000000000..8f8f67daed --- /dev/null +++ b/tests/functional/raw-node/test/GCP/object/delete.js @@ -0,0 +1,98 @@ +const assert = require('assert'); +const async = require('async'); +const { GCP } = require('../../../../../../lib/data/external/GCP'); +const { makeGcpRequest } = require('../../../utils/makeRequest'); +const { gcpRequestRetry } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); + +const credentialOne = 'gcpbackend'; +const bucketName = `somebucket-${Date.now()}`; +const objectKey = `somekey-${Date.now()}`; +const badObjectKey = `nonexistingkey-${Date.now()}`; + +describe('GCP: DELETE Object', function testSuite() { + this.timeout(30000); + const config = getRealAwsConfig(credentialOne); + const gcpClient = new GCP(config); + + before(done => { + gcpRequestRetry({ + method: 'PUT', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in creating bucket ${err}\n`); + } + return done(err); + }); + }); + + after(done => { + gcpRequestRetry({ + method: 'DELETE', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in deleting bucket ${err}\n`); + } + return done(err); + }); + }); + + describe('with existing object in bucket', () => { + beforeEach(done => { + makeGcpRequest({ + method: 'PUT', + bucket: bucketName, + objectKey, + authCredentials: config.credentials, + }, err => { + if (err) { + process.stdout.write(`err in creating object ${err}\n`); + } + return done(err); + }); + }); + + it('should successfully delete object', done => { + async.waterfall([ + next => gcpClient.deleteObject({ + Bucket: bucketName, + Key: objectKey, + }, err => { + assert.equal(err, null, + `Expected success, got error ${err}`); + return next(); + }), + next => makeGcpRequest({ + method: 'GET', + bucket: bucketName, + objectKey, + authCredentials: config.credentials, + }, err => { + assert(err); + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NoSuchKey'); + return next(); + }), + ], err => done(err)); + }); + }); + + describe('without existing object in bucket', () => { + it('should return 404 and NoSuchKey', done => { + gcpClient.deleteObject({ + Bucket: bucketName, + Key: badObjectKey, + }, err => { + assert(err); + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NoSuchKey'); + return done(); + }); + }); + }); +}); diff --git a/tests/functional/raw-node/test/GCP/object/get.js b/tests/functional/raw-node/test/GCP/object/get.js new file mode 100644 index 0000000000..ff04345e97 --- /dev/null +++ b/tests/functional/raw-node/test/GCP/object/get.js @@ -0,0 +1,104 @@ +const assert = require('assert'); +const { GCP } = require('../../../../../../lib/data/external/GCP'); +const { makeGcpRequest } = require('../../../utils/makeRequest'); +const { gcpRequestRetry } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); + +const credentialOne = 'gcpbackend'; +const bucketName = `somebucket-${Date.now()}`; + +describe('GCP: GET Object', function testSuite() { + this.timeout(30000); + const config = getRealAwsConfig(credentialOne); + const gcpClient = new GCP(config); + + before(done => { + gcpRequestRetry({ + method: 'PUT', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in creating bucket ${err}\n`); + } + return done(err); + }); + }); + + after(done => { + gcpRequestRetry({ + method: 'DELETE', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in deleting bucket ${err}\n`); + } + return done(err); + }); + }); + + describe('with existing object in bucket', () => { + beforeEach(function beforeFn(done) { + this.currentTest.key = `somekey-${Date.now()}`; + makeGcpRequest({ + method: 'PUT', + bucket: bucketName, + objectKey: this.currentTest.key, + authCredentials: config.credentials, + }, (err, res) => { + if (err) { + process.stdout.write(`err in creating object ${err}\n`); + return done(err); + } + this.currentTest.uploadId = + res.headers['x-goog-generation']; + this.currentTest.ETag = res.headers.etag; + return done(); + }); + }); + + afterEach(function afterFn(done) { + makeGcpRequest({ + method: 'DELETE', + bucket: bucketName, + objectKey: this.currentTest.key, + authCredentials: config.credentials, + }, err => { + if (err) { + process.stdout.write(`err in deleting object ${err}\n`); + } + return done(err); + }); + }); + + it('should successfully retrieve object', function testFn(done) { + gcpClient.getObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, res) => { + assert.equal(err, null, + `Expected success, got error ${err}`); + assert.strictEqual(res.ETag, this.test.ETag); + assert.strictEqual(res.VersionId, this.test.uploadId); + return done(); + }); + }); + }); + + describe('without existing object in bucket', () => { + it('should return 404 and NoSuchKey', done => { + const badObjectKey = `nonexistingkey-${Date.now()}`; + gcpClient.getObject({ + Bucket: bucketName, + Key: badObjectKey, + }, err => { + assert(err); + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NoSuchKey'); + return done(); + }); + }); + }); +}); diff --git a/tests/functional/raw-node/test/GCP/object/head.js b/tests/functional/raw-node/test/GCP/object/head.js new file mode 100644 index 0000000000..b4f2326843 --- /dev/null +++ b/tests/functional/raw-node/test/GCP/object/head.js @@ -0,0 +1,103 @@ +const assert = require('assert'); +const { GCP } = require('../../../../../../lib/data/external/GCP'); +const { makeGcpRequest } = require('../../../utils/makeRequest'); +const { gcpRequestRetry } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); + +const credentialOne = 'gcpbackend'; +const bucketName = `somebucket-${Date.now()}`; + +describe('GCP: HEAD Object', function testSuite() { + this.timeout(30000); + const config = getRealAwsConfig(credentialOne); + const gcpClient = new GCP(config); + + before(done => { + gcpRequestRetry({ + method: 'PUT', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in creating bucket ${err}\n`); + } + return done(err); + }); + }); + + after(done => { + gcpRequestRetry({ + method: 'DELETE', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in deleting bucket ${err}\n`); + } + return done(err); + }); + }); + + describe('with existing object in bucket', () => { + beforeEach(function beforeFn(done) { + this.currentTest.key = `somekey-${Date.now()}`; + makeGcpRequest({ + method: 'PUT', + bucket: bucketName, + objectKey: this.currentTest.key, + authCredentials: config.credentials, + }, (err, res) => { + if (err) { + process.stdout.write(`err in creating object ${err}\n`); + return done(err); + } + this.currentTest.uploadId = + res.headers['x-goog-generation']; + this.currentTest.ETag = res.headers.etag; + return done(); + }); + }); + + afterEach(function afterFn(done) { + makeGcpRequest({ + method: 'DELETE', + bucket: bucketName, + objectKey: this.currentTest.key, + authCredentials: config.credentials, + }, err => { + if (err) { + process.stdout.write(`err in deleting object ${err}\n`); + } + return done(err); + }); + }); + + it('should successfully retrieve object', function testFn(done) { + gcpClient.headObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, res) => { + assert.equal(err, null, + `Expected success, got error ${err}`); + assert.strictEqual(res.ETag, this.test.ETag); + assert.strictEqual(res.VersionId, this.test.uploadId); + return done(); + }); + }); + }); + + describe('without existing object in bucket', () => { + it('should return 404', done => { + const badObjectkey = `nonexistingkey-${Date.now()}`; + gcpClient.headObject({ + Bucket: bucketName, + Key: badObjectkey, + }, err => { + assert(err); + assert.strictEqual(err.statusCode, 404); + return done(); + }); + }); + }); +}); diff --git a/tests/functional/raw-node/test/GCP/object/put.js b/tests/functional/raw-node/test/GCP/object/put.js new file mode 100644 index 0000000000..178e5cb57e --- /dev/null +++ b/tests/functional/raw-node/test/GCP/object/put.js @@ -0,0 +1,112 @@ +const assert = require('assert'); +const { GCP } = require('../../../../../../lib/data/external/GCP'); +const { makeGcpRequest } = require('../../../utils/makeRequest'); +const { gcpRequestRetry } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); + +const credentialOne = 'gcpbackend'; +const bucketName = `somebucket-${Date.now()}`; + +describe('GCP: PUT Object', function testSuite() { + this.timeout(30000); + const config = getRealAwsConfig(credentialOne); + const gcpClient = new GCP(config); + + before(done => { + gcpRequestRetry({ + method: 'PUT', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in creating bucket ${err}\n`); + } + return done(err); + }); + }); + + after(done => { + gcpRequestRetry({ + method: 'DELETE', + bucket: bucketName, + authCredentials: config.credentials, + }, 0, err => { + if (err) { + process.stdout.write(`err in deleting bucket ${err}\n`); + } + return done(err); + }); + }); + + afterEach(function afterFn(done) { + makeGcpRequest({ + method: 'DELETE', + bucket: bucketName, + objectKey: this.currentTest.key, + authCredentials: config.credentials, + }, err => { + if (err) { + process.stdout.write(`err in deleting object ${err}\n`); + } + return done(err); + }); + }); + + describe('with existing object in bucket', () => { + beforeEach(function beforeFn(done) { + this.currentTest.key = `somekey-${Date.now()}`; + gcpRequestRetry({ + method: 'PUT', + bucket: this.currentTest.bucketName, + objectKey: this.currentTest.key, + authCredentials: config.credentials, + }, 0, (err, res) => { + if (err) { + process.stdout.write(`err in putting object ${err}\n`); + return done(err); + } + this.currentTest.uploadId = + res.headers['x-goog-generation']; + return done(); + }); + }); + + it('should overwrite object', function testFn(done) { + gcpClient.putObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, res) => { + assert.notStrictEqual(res.VersionId, this.test.uploadId); + return done(); + }); + }); + }); + + describe('without existing object in bucket', () => { + it('should successfully put object', function testFn(done) { + this.test.key = `somekey-${Date.now()}`; + gcpClient.putObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, putRes) => { + assert.equal(err, null, + `Expected success, got error ${err}`); + makeGcpRequest({ + method: 'GET', + bucket: bucketName, + objectKey: this.test.key, + authCredentials: config.credentials, + }, (err, getRes) => { + if (err) { + process.stdout.write(`err in getting bucket ${err}\n`); + return done(err); + } + assert.strictEqual(getRes.headers['x-goog-generation'], + putRes.VersionId); + return done(); + }); + }); + }); + }); +}); From 7c2f210171a2ac37e3e7aa41bf3b4325b43058e3 Mon Sep 17 00:00:00 2001 From: Alexander Chan Date: Tue, 6 Feb 2018 15:25:22 -0800 Subject: [PATCH 2/2] FX: GCP PUT API Test Fixes the use of an undefined variable that leads to creation of buckets that aren't deleted on GCP. --- tests/functional/raw-node/test/GCP/object/put.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/raw-node/test/GCP/object/put.js b/tests/functional/raw-node/test/GCP/object/put.js index 178e5cb57e..2df59af83f 100644 --- a/tests/functional/raw-node/test/GCP/object/put.js +++ b/tests/functional/raw-node/test/GCP/object/put.js @@ -58,7 +58,7 @@ describe('GCP: PUT Object', function testSuite() { this.currentTest.key = `somekey-${Date.now()}`; gcpRequestRetry({ method: 'PUT', - bucket: this.currentTest.bucketName, + bucket: bucketName, objectKey: this.currentTest.key, authCredentials: config.credentials, }, 0, (err, res) => {