Merge pull request #169 from hasonmsft/master

Storage Client Library 1.2.0

BreakingChanges.md

@@ -1,3 +1,7 @@
+Tracking Breaking Changes in 1.2.0
+TABLE
+* Beginning with version 2015-12-11, the Atom feed is no longer supported as a payload format for Table service operations. Version 2015-12-11 and later versions support only JSON for the payload format.
+
 Tracking Breaking Changes in 1.0.0
 BLOB
 * The `blob` property of BlobResult has been renamed to `name` to keep consistent with other services API and the `listBlob` API.

ChangeLog.md

@@ -1,10 +1,27 @@
 Note: This is an Azure Storage only package. The all up Azure node sdk still has the old storage bits in there. In a future release, those storage bits will be removed and an npm dependency to this storage node sdk will 
 be taken. This is a GA release and the changes described below indicate the changes from the Azure node SDK 0.9.8 available here - https://github.com/Azure/azure-sdk-for-node.
 
-2016.06 Version 1.1.0
+2016.07 Version 1.2.0
 
 ALL
+* Fixed the issue that metadata name will be converted to lower-case after retrieving back from the server. **Note** that this fix is only applicable for Node 0.12 or higher version.
+* Added support for EndpointSuffix for all service constructors.
+* Updated storage service version to 2015-12-11. Fore more information, please see - https://msdn.microsoft.com/en-us/library/azure/dd894041.aspx
+* Updated the `request` package to version 2.74.0 to address the security vulnerability - https://nodesecurity.io/advisories/130
+
+BLOB
+* Fixed the issue that the service error message will be written to the destination stream if getting error when downloading the blob to a stream/file.
+* Added `serverEncryped` property to 'BlobResult' class which indicates if the blob data and application metadata are completely encrypted using the specified algorithm on the server.
 
+FILE
+* Fixed the issue that the service error message will be written to the destination stream if getting error when downloading the file to a stream/file.
+
+TABLE
+* The atom feed payload format is not supported anymore for table service APIs.
+
+2016.06 Version 1.1.0
+
+ALL
 * Fixed the issue that using SAS doesn't work against storage emulator.
 * Fixed the issue that the service SAS signature is incorrect when protocol parameter is specified.
 * Fixed the issue that the timeout query string should be in seconds instead of milliseconds.
@@ -27,7 +44,6 @@ FILE
 2016.05 Version 1.0.1
 
 ALL
-
 * Fixed the issue that StorageServiceClient._normalizeError will throw exception on Node below v4 because string.startsWith is not available on Node below v4.
 
 2016.05 Version 1.0.0

lib/common/services/storageserviceclient.js

@@ -304,7 +304,7 @@ StorageServiceClient.prototype._performRequest = function (webResource, body, op
           };
 
           var endResponse;
-          var buildRequest = function (headersOnly) {
+          var buildRequest = function (headersOnly, inputStream) {
             // Build request (if body was set before, request will process immediately, if not it'll wait for the piping to happen
             var requestStream;
 
@@ -323,22 +323,66 @@ StorageServiceClient.prototype._performRequest = function (webResource, body, op
 
               requestStream.on('error', processResponseCallback);
               requestStream.on('response', function (response) {
-                var responseLength = 0;
-                var internalHash = crypto.createHash('md5');
-                requestStream.on('data', function(data) {
-                  responseLength += data.length;
-                  internalHash.update(data);
-                });
-
-                response.on('end', function () {
-                  // Calculate and set MD5 here
-                  if(azureutil.objectIsNull(options.disableContentMD5Validation) || options.disableContentMD5Validation === false) {
-                    response.contentMD5 = internalHash.digest('base64');
-                  }
+                var isValid = WebResource.validResponse(response.statusCode);
+                if (!isValid) {
+                    // When getting invalid response, try to get the error message for future steps to extract the detailed error information
+                    var contentLength = parseInt(response.headers['content-length']);
+                    var errorMessageBuffer;
+                    var index = 0;
+                    if (contentLength !== undefined) {
+                      errorMessageBuffer = new Buffer(contentLength);
+                    }
 
-                  response.length = responseLength;
-                  endResponse = response;
-                });
+                    requestStream.on('data', function (data) {
+                      if (contentLength !== undefined) {
+                        data.copy(errorMessageBuffer, index);
+                        index += data.length;
+                      } else {
+                        if (!errorMessageBuffer) {
+                          errorMessageBuffer = data;
+                        } else {
+                          errorMessageBuffer = Buffer.concat([errorMessageBuffer, data]);
+                        }
+                      }
+                    });
+                    requestStream.on('end', function () {
+                      if (errorMessageBuffer) {
+                        // Strip the UTF8 BOM following the same ways as 'request' module
+                        if (errorMessageBuffer.length > 3 &&
+                          errorMessageBuffer[0] === 239 &&
+                          errorMessageBuffer[1] === 187 &&
+                          errorMessageBuffer[2] === 191) {
+                          response.body = errorMessageBuffer.toString('utf8', 3);
+                        } else {
+                          response.body = errorMessageBuffer.toString('utf8');
+                        }
+                      }
+                      processResponseCallback(null, response);
+                    });
+                } else {
+                  // Only pipe to the destination stream when we get a valid response from service
+                  // Error message should NOT be piped to the destination stream
+                  if (inputStream) {
+                    requestStream.pipe(inputStream);
+                  }
+
+                  var responseLength = 0;
+                  var internalHash = crypto.createHash('md5');
+                  requestStream.on('data', function(data) {
+                    responseLength += data.length;
+                    internalHash.update(data);
+                  });
+
+                  response.on('end', function () {
+                    // Calculate and set MD5 here
+                    if(azureutil.objectIsNull(options.disableContentMD5Validation) || options.disableContentMD5Validation === false) {
+                      response.contentMD5 = internalHash.digest('base64');
+                    }
+
+                    response.length = responseLength;
+                    endResponse = response;
+                  });
+                }
               });
             } else {
               requestStream = requestWithDefaults(finalRequestOptions, processResponseCallback);
@@ -395,7 +439,7 @@ StorageServiceClient.prototype._performRequest = function (webResource, body, op
                 endResponse = null;
               }
             });
-            buildRequest(true).pipe(body.inputStream);
+            buildRequest(true, body.inputStream);
           } else if (body && body.outputStream) {
             var sendUnchunked = function () {
               var size = finalRequestOptions.headers['content-length'] ?
@@ -587,8 +631,27 @@ StorageServiceClient.prototype._buildRequestOptions = function (webResource, bod
 StorageServiceClient.prototype._processResponse = function (webResource, response, options) {
   var self = this;
 
+  function convertRawHeadersToHeaders(rawHeaders) {
+    var headers = {};
+    if(!rawHeaders) {
+      return undefined;
+    }
+
+    for(var i = 0; i < rawHeaders.length; i++) {
+      var headerName;
+      if (rawHeaders[i].indexOf(HeaderConstants.PREFIX_FOR_STORAGE_METADATA) === 0) {
+        headerName = rawHeaders[i];
+      } else {
+        headerName = rawHeaders[i].toLowerCase();
+      }
+      headers[headerName] = rawHeaders[++i];
+    }
+
+    return headers;
+  }
+
   var validResponse = WebResource.validResponse(response.statusCode);
-  var rsp = StorageServiceClient._buildResponse(validResponse, response.body, response.headers, response.statusCode, response.md5);
+  var rsp = StorageServiceClient._buildResponse(validResponse, response.body, convertRawHeadersToHeaders(response.rawHeaders) || response.headers, response.statusCode, response.md5);
   var responseObject;
 
   if (validResponse && webResource.rawResponse) {
@@ -777,14 +840,14 @@ StorageServiceClient._parseResponse = function (response, xml2jsSettings, option
 *
 * @return {StorageServiceSettings}
 */
-StorageServiceClient.getStorageSettings = function (storageAccountOrConnectionString, storageAccessKey, host, sasToken) {
+StorageServiceClient.getStorageSettings = function (storageAccountOrConnectionString, storageAccessKey, host, sasToken, endpointSuffix) {
   var storageServiceSettings;
   if (storageAccountOrConnectionString && !storageAccessKey && !sasToken) {
     // If storageAccountOrConnectionString was passed and no accessKey was passed, assume connection string
     storageServiceSettings = StorageServiceSettings.createFromConnectionString(storageAccountOrConnectionString);
   } else if ((storageAccountOrConnectionString && storageAccessKey) || sasToken || host) {
     // Account and key or credentials or anonymous
-    storageServiceSettings = StorageServiceSettings.createExplicitly(storageAccountOrConnectionString, storageAccessKey, host, sasToken);
+    storageServiceSettings = StorageServiceSettings.createExplicitly(storageAccountOrConnectionString, storageAccessKey, host, sasToken, endpointSuffix);
   } else {
     // Use environment variables
     storageServiceSettings = StorageServiceSettings.createFromEnvironment();
@@ -882,7 +945,7 @@ StorageServiceClient.prototype.parseMetadataHeaders = function (headers) {
       metadata[key] = headers[header];
     }
   }
-
+  
   return metadata;
 };
 

lib/common/services/storageservicesettings.js

@@ -54,6 +54,11 @@ var fileEndpointSetting = ServiceSettings.settingWithFunc(
   Validate.isValidHost
 );
 
+var endpointSuffixSetting = ServiceSettings.settingWithFunc(
+  ConnectionStringKeys.ENDPOINT_SUFFIX_NAME,
+  Validate.isValidHost
+);
+
 var validKeys = [
   ConnectionStringKeys.USE_DEVELOPMENT_STORAGE_NAME,
   ConnectionStringKeys.DEVELOPMENT_STORAGE_PROXY_URI_NAME,
@@ -64,7 +69,8 @@ var validKeys = [
   ConnectionStringKeys.BLOB_ENDPOINT_NAME,
   ConnectionStringKeys.QUEUE_ENDPOINT_NAME,
   ConnectionStringKeys.TABLE_ENDPOINT_NAME,
-  ConnectionStringKeys.FILE_ENDPOINT_NAME
+  ConnectionStringKeys.FILE_ENDPOINT_NAME,
+  ConnectionStringKeys.ENDPOINT_SUFFIX_NAME
 ];
 
 /**
@@ -122,7 +128,7 @@ StorageServiceSettings.createFromConnectionString = function (connectionString)
   }
 };
 
-StorageServiceSettings.createExplicitly = function (storageAccount, storageAccessKey, host, sasToken) {
+StorageServiceSettings.createExplicitly = function (storageAccount, storageAccessKey, host, sasToken, endpointSuffix) {
   var settings = {};
   function addIfNotNullOrEmpty(key, value){
     if(typeof value === 'string' && !util.stringIsEmpty(value)){
@@ -145,6 +151,7 @@ StorageServiceSettings.createExplicitly = function (storageAccount, storageAcces
   addIfNotNullOrEmpty('accountname', storageAccount);
   addIfNotNullOrEmpty('accountkey', storageAccessKey);
   addIfNotNullOrEmpty('sharedaccesssignature', sasToken);
+  addIfNotNullOrEmpty('endpointsuffix', endpointSuffix);
 
   return StorageServiceSettings.createFromSettings(settings);
 };
@@ -163,7 +170,7 @@ StorageServiceSettings.createFromEnvironment = function () {
   var storageAccount = process.env[StorageServiceClientConstants.EnvironmentVariables.AZURE_STORAGE_ACCOUNT];
   var storageAccessKey = process.env[StorageServiceClientConstants.EnvironmentVariables.AZURE_STORAGE_ACCESS_KEY];
   if(storageAccount && storageAccessKey){
-    return StorageServiceSettings.createExplicitly(storageAccount, storageAccessKey, null, null);
+    return StorageServiceSettings.createExplicitly(storageAccount, storageAccessKey, null, null, null);
   }
 
   throw new Error(SR.NO_CREDENTIALS_PROVIDED);
@@ -204,7 +211,8 @@ StorageServiceSettings.createFromSettings = function (settings) {
       blobEndpointSetting,
       queueEndpointSetting,
       tableEndpointSetting,
-      fileEndpointSetting
+      fileEndpointSetting,
+      endpointSuffixSetting
     )
   );
 
@@ -223,7 +231,8 @@ StorageServiceSettings.createFromSettings = function (settings) {
       blobEndpointSetting,
       queueEndpointSetting,
       tableEndpointSetting,
-      fileEndpointSetting
+      fileEndpointSetting,
+      endpointSuffixSetting
     )
   );
 
@@ -241,7 +250,8 @@ StorageServiceSettings.createFromSettings = function (settings) {
       blobEndpointSetting,
       queueEndpointSetting,
       tableEndpointSetting,
-      fileEndpointSetting
+      fileEndpointSetting,
+      endpointSuffixSetting
     )
   );
 
@@ -259,7 +269,8 @@ StorageServiceSettings.createFromSettings = function (settings) {
     ServiceSettings.optional(
       fileEndpointSetting,
       queueEndpointSetting,
-      tableEndpointSetting
+      tableEndpointSetting,
+      endpointSuffixSetting
     )
   );
 
@@ -363,29 +374,34 @@ StorageServiceSettings._createStorageServiceSettings = function (settings) {
     settings
   );
 
+  var endpointSuffix = util.tryGetValueInsensitive(
+    ConnectionStringKeys.ENDPOINT_SUFFIX_NAME,
+    settings
+  );
+
   var blobEndpoint = standardizeHost(
       util.tryGetValueInsensitive(ConnectionStringKeys.BLOB_ENDPOINT_NAME, settings),
       accountName,
       scheme,
-      StorageServiceClientConstants.CLOUD_BLOB_HOST);
+      endpointSuffix ? 'blob.' + endpointSuffix : StorageServiceClientConstants.CLOUD_BLOB_HOST);
 
   var queueEndpoint = standardizeHost(
       util.tryGetValueInsensitive(ConnectionStringKeys.QUEUE_ENDPOINT_NAME, settings),
       accountName,
       scheme,
-      StorageServiceClientConstants.CLOUD_QUEUE_HOST);
+      endpointSuffix ? 'queue.' + endpointSuffix : StorageServiceClientConstants.CLOUD_QUEUE_HOST);
 
   var tableEndpoint = standardizeHost(
       util.tryGetValueInsensitive(ConnectionStringKeys.TABLE_ENDPOINT_NAME, settings),
       accountName,
       scheme,
-      StorageServiceClientConstants.CLOUD_TABLE_HOST);
+      endpointSuffix ? 'table.' + endpointSuffix : StorageServiceClientConstants.CLOUD_TABLE_HOST);
 
   var fileEndpoint = standardizeHost(
       util.tryGetValueInsensitive(ConnectionStringKeys.FILE_ENDPOINT_NAME, settings),
       accountName,
       scheme,
-      StorageServiceClientConstants.CLOUD_FILE_HOST);
+      endpointSuffix ? 'file.' + endpointSuffix : StorageServiceClientConstants.CLOUD_FILE_HOST);
 
 
   return new StorageServiceSettings(

lib/common/util/constants.js

@@ -31,7 +31,7 @@ var Constants = {
   /*
   * Specifies the value to use for UserAgent header.
   */
-  USER_AGENT_PRODUCT_VERSION: '1.1.0',
+  USER_AGENT_PRODUCT_VERSION: '1.2.0',
 
   /**
   * The number of default concurrent requests for parallel operation.
@@ -1559,7 +1559,7 @@ var Constants = {
     * @const
     * @type {string}
     */
-    TARGET_STORAGE_VERSION: '2015-04-05',
+    TARGET_STORAGE_VERSION: '2015-12-11',
 
     /**
     * The UserAgent header.
@@ -1680,6 +1680,14 @@ var Constants = {
     * @type {int}
     */
     BLOB_COMMITTED_BLOCK_COUNT: 'x-ms-blob-committed-block-count',
+
+    /**
+    * If the data and application metadata are completely encrypted using the specified algorithm.
+    *
+    * @const
+    * @type {bool}
+    */
+    SERVER_ENCRYPTED: 'x-ms-server-encrypted',
   },
 
   QueryStringConstants: {
@@ -2140,15 +2148,7 @@ var Constants = {
     }
   },
 
-  CompatibleVersionConstants: {    
-    /**
-    * Constant for the 2015-04-05 version.
-    *
-    * @const
-    * @type {string}
-    */
-    APRIL_2015: '2015-04-05',
-
+  CompatibleVersionConstants: {
     /**
     * Constant for the 2013-08-15 version.
     *
@@ -2338,6 +2338,7 @@ var Constants = {
     QUEUE_ENDPOINT_NAME: 'QueueEndpoint',
     TABLE_ENDPOINT_NAME: 'TableEndpoint',
     SHARED_ACCESS_SIGNATURE_NAME: 'SharedAccessSignature',
+    ENDPOINT_SUFFIX_NAME: 'EndpointSuffix',
     BLOB_BASE_DNS_NAME: 'blob.core.windows.net',
     FILE_BASE_DNS_NAME: 'file.core.windows.net',
     QUEUE_BASE_DNS_NAME: 'queue.core.windows.net',

lib/services/blob/blobservice.js

@@ -93,9 +93,10 @@ var StorageError = errors.StorageError;
 * @param {string|object} [host]                       The host address. To define primary only, pass a string. 
 *                                                     Otherwise 'host.primaryHost' defines the primary host and 'host.secondaryHost' defines the secondary host.
 * @param {string} [sasToken]                          The Shared Access Signature token.
+* @param {string} [endpointSuffix]                    The endpoint suffix.
 */
-function BlobService(storageAccountOrConnectionString, storageAccessKey, host, sasToken) {
-  var storageServiceSettings = StorageServiceClient.getStorageSettings(storageAccountOrConnectionString, storageAccessKey, host, sasToken);
+function BlobService(storageAccountOrConnectionString, storageAccessKey, host, sasToken, endpointSuffix) {
+  var storageServiceSettings = StorageServiceClient.getStorageSettings(storageAccountOrConnectionString, storageAccessKey, host, sasToken, endpointSuffix);
 
   BlobService['super_'].call(this,
     storageServiceSettings._name,