Ft/s3 c 1179/gcp s3 mpu

circle.yml

@@ -20,11 +20,16 @@ machine:
     REPORT_TOKEN: report-token-1
   hosts:
     bucketwebsitetester.s3-website-us-east-1.amazonaws.com: 127.0.0.1
+  post:
+    - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+    - echo "deb https://dl.yarnpkg.com/debian/ stable main" |
+      sudo tee /etc/apt/sources.list.d/yarn.list
+    - sudo apt-get update && sudo apt-get install yarn -y
 
 dependencies:
   override:
     - rm -rf node_modules
-    - npm install
+    - yarn install --pure-lockfile
   post:
     - sudo pip install flake8 yamllint
     - sudo pip install s3cmd==1.6.1

constants.js

@@ -123,8 +123,11 @@ const constants = {
     // for external backends, don't call unless at least 1 minute
     // (60,000 milliseconds) since last call
     externalBackendHealthCheckInterval: 60000,
-    versioningNotImplBackends: { azure: true },
-    mpuMDStoredExternallyBackend: { aws_s3: true },
+    versioningNotImplBackends: { azure: true, gcp: true },
+    mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
+    skipBatchDeleteBackends: { azure: true, gcp: true },
+    s3HandledBackends: { azure: true, gcp: true },
+    hasCopyPartBackends: { aws_s3: true, gcp: true },
     /* eslint-enable camelcase */
     mpuMDStoredOnS3Backend: { azure: true },
     azureAccountNameRegex: /^[a-z0-9]{3,24}$/,

lib/Config.js

@@ -876,7 +876,8 @@ class Config extends EventEmitter {
     }
 
     getLocationConstraintType(locationConstraint) {
-        return this.locationConstraints[locationConstraint].type;
+        const dataStoreName = this.locationConstraints[locationConstraint];
+        return dataStoreName && dataStoreName.type;
     }
 
     setRestEndpoints(restEndpoints) {

lib/api/objectPutPart.js

@@ -201,7 +201,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
                         // if data backend handles MPU, skip to end of waterfall
                         return next(skipError, destinationBucket,
                         partInfo.dataStoreETag);
-                    } else if (partInfo && partInfo.dataStoreType === 'azure') {
+                    } else if (partInfo &&
+                        constants.s3HandledBackends[partInfo.dataStoreType]) {
                         return next(null, destinationBucket,
                             objectLocationConstraint, cipherBundle, splitter,
                             partInfo);
@@ -250,7 +251,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
         (destinationBucket, objectLocationConstraint, cipherBundle,
         partKey, prevObjectSize, oldLocations, partInfo, next) => {
             // NOTE: set oldLocations to null so we do not batchDelete for now
-            if (partInfo && partInfo.dataStoreType === 'azure') {
+            if (partInfo &&
+                constants.skipBatchDeleteBackends[partInfo.dataStoreType]) {
                 // skip to storing metadata
                 return next(null, destinationBucket, partInfo,
                   partInfo.dataStoreETag,

lib/data/external/GCP/GcpApis/listParts.js

@@ -18,9 +18,15 @@ function listParts(params, callback) {
         logHelper(logger, 'error', 'error in listParts', error);
         return callback(error);
     }
+    if (params.PartNumberMarker && params.PartNumberMarker < 0) {
+        return callback(errors.InvalidArgument
+            .customizeDescription('The request specified an invalid marker'));
+    }
     const mpuParams = {
         Bucket: params.Bucket,
         Prefix: createMpuKey(params.Key, params.UploadId, 'parts'),
+        Marker: createMpuKey(params.Key, params.UploadId,
+            params.PartNumberMarker, 'parts'),
         MaxKeys: params.MaxParts,
     };
     return this.listObjects(mpuParams, (err, res) => {

lib/data/external/GCP/GcpApis/uploadPart.js

@@ -1,5 +1,5 @@
 const { errors } = require('arsenal');
-const { createMpuKey, logger } = require('../GcpUtils');
+const { getPartNumber, createMpuKey, logger } = require('../GcpUtils');
 const { logHelper } = require('../../utils');
 
 /**
@@ -17,9 +17,16 @@ function uploadPart(params, callback) {
         logHelper(logger, 'error', 'error in uploadPart', error);
         return callback(error);
     }
+    const partNumber = getPartNumber(params.PartNumber);
+    if (!partNumber) {
+        const error = errors.InvalidArgument
+            .customizeDescription('PartNumber is invalid');
+        logHelper(logger, 'debug', 'error in uploadPart', error);
+        return callback(error);
+    }
     const mpuParams = {
         Bucket: params.Bucket,
-        Key: createMpuKey(params.Key, params.UploadId, params.PartNumber),
+        Key: createMpuKey(params.Key, params.UploadId, partNumber),
         Body: params.Body,
         ContentLength: params.ContentLength,
     };

lib/data/external/GCP/GcpApis/uploadPartCopy.js

@@ -1,5 +1,5 @@
 const { errors } = require('arsenal');
-const { createMpuKey, logger } = require('../GcpUtils');
+const { getPartNumber, createMpuKey, logger } = require('../GcpUtils');
 const { logHelper } = require('../../utils');
 
 /**
@@ -19,9 +19,16 @@ function uploadPartCopy(params, callback) {
         logHelper(logger, 'error', 'error in uploadPartCopy', error);
         return callback(error);
     }
+    const partNumber = getPartNumber(params.PartNumber);
+    if (!partNumber) {
+        const error = errors.InvalidArgument
+            .customizeDescription('PartNumber is not a number');
+        logHelper(logger, 'debug', 'error in uploadPartCopy', error);
+        return callback(error);
+    }
     const mpuParams = {
         Bucket: params.Bucket,
-        Key: createMpuKey(params.Key, params.UploadId, params.PartNumber),
+        Key: createMpuKey(params.Key, params.UploadId, partNumber),
         CopySource: params.CopySource,
     };
     return this.copyObject(mpuParams, callback);

lib/data/external/GCP/GcpUtils.js

@@ -32,16 +32,20 @@ function getSourceInfo(CopySource) {
     return { sourceBucket, sourceObject };
 }
 
-function getRandomInt(min, max) {
-    const minVal = Math.ceil(min);
-    const maxVal = Math.floor(max);
-    return Math.floor(Math.random() * (maxVal - minVal)) + minVal;
-}
-
 function getPaddedPartNumber(number) {
     return `000000${number}`.substr(-5);
 }
 
+function getPartNumber(number) {
+    if (isNaN(number)) {
+        return undefined;
+    }
+    if (typeof number === 'string') {
+        return parseInt(number, 10);
+    }
+    return number;
+}
+
 function createMpuKey(key, uploadId, partNumberArg, fileNameArg) {
     let partNumber = partNumberArg;
     let fileName = fileNameArg;
@@ -69,10 +73,8 @@ function createMpuList(params, level, size) {
     // populate and return a parts list for compose
     const retList = [];
     for (let i = 1; i <= size; ++i) {
-        const paddedNumber = getPaddedPartNumber(i);
         retList.push({
-            PartName:
-                `${params.Key}-${params.UploadId}/${level}/${paddedNumber}`,
+            PartName: createMpuKey(params.Key, params.UploadId, i, level),
             PartNumber: i,
         });
     }
@@ -154,14 +156,14 @@ function getPutTagsMetadata(metadata, tagging = '') {
 module.exports = {
     // functions
     eachSlice,
-    getRandomInt,
     createMpuKey,
     createMpuList,
     getSourceInfo,
     processTagSet,
     stripTags,
     retrieveTags,
     getPutTagsMetadata,
+    getPartNumber,
     // util objects
     logger,
 };