Added browseri specific APIs for uploading files and blobs

Updated sample-blob.html and sample-file.html for new uploading APIs Added TypeScript definitions for new uploading APIs
Azure · Jul 13, 2017 · 4c6aa19 · 4c6aa19
1 parent 7b6a50f
commit 4c6aa19
Show file tree

Hide file tree

Showing 19 changed files with 770 additions and 215 deletions.
diff --git a/.jshintrc b/.jshintrc
@@ -24,5 +24,9 @@
   "trailing": true,
   "undef": true,
   "unused": true,
-  "shadow": true
+  "shadow": true,
+  "globals": {
+    "FileReader": true,
+    "window": true
+  }
 }
diff --git a/browser/ChangeLog.md b/browser/ChangeLog.md
@@ -1,5 +1,16 @@
 Note: This is the change log file for Azure Storage JavaScript Client Library.
 
+2017.07 Version 0.2.2-preview.7
+
+* Added browser specific APIs for blobs and files uploading.
+    * `BlobService.createBlockBlobFromBrowserFile`
+    * `BlobService.createPageBlobFromBrowserFile`
+    * `BlobService.createAppendBlobFromBrowserFile`
+    * `BlobService.appendFromBrowserFile`
+    * `FileService.createFileFromBrowserFile`
+* Updated samples with above new added APIs.
+* Dropped dependency to browserify-fs.
+
 2017.07 Version 0.2.2-preview.6
 
 * Generated browser compatible JavaScript files based on Microsoft Azure Storage SDK for Node.js 2.2.1.

diff --git a/browser/samples/sample-blob.html b/browser/samples/sample-blob.html
@@ -158,61 +158,19 @@ <h3>List Blobs</h3>
 });
 </pre>
             <h3>Upload Blob</h3>
-            <p><code>BlobService</code> provides <code>uploadBlobByStream</code> for uploading a blob from stream.
-               However, currently <code>uploadBlobByStream</code> only accepts a Node.js <code>ReadableStream</code> type which pure JavaScript doesn't provide.
+            <p><code>BlobService</code> provides <code>createBlockBlobFromBrowserFile</code>, <code>createPageBlobFromBrowserFile</code>, <code>createAppendBlobFromBrowserFile</code> and <code>appendFromBrowserFile</code> for uploading or appending a blob from an HTML file in browsers.
             </p>
 
-            <div class="panel panel-primary"> 
-                <div class="panel-body">
-                    <b>Note</b>: After importing <code>azure-storage.common.js</code>, you could require 3 Node.js types: <code>stream</code>, <code>util</code> and <code>buffer</code>, then wrap a ReadableStream based on HTML5 <code>FileReader</code>.
-                </div>
-            </div>
-<pre>
-// Provides a Stream for a file in webpage, inheriting from NodeJS Readable stream.
-var Buffer = require('buffer').Buffer;
-var Stream = require('stream');
-var util = require('util');
-
-function FileStream(file, opt) {
-    Stream.Readable.call(this, opt);
-
-    this.fileReader = new FileReader(file);
-    this.file = file;
-    this.size = file.size;
-    this.chunkSize = 1024 * 1024 * 4; // 4MB
-    this.offset = 0;
-    var _me = this;
-
-    this.fileReader.onloadend = function loaded(event) {
-        var data = event.target.result;
-        var buf = Buffer.from(data);
-        _me.push(buf);
-    }
-}
-util.inherits(FileStream, Stream.Readable);
-FileStream.prototype._read = function() {
-    if (this.offset > this.size) {
-        this.push(null);
-    } else {
-        var end = this.offset + this.chunkSize;
-        var slice = this.file.slice(this.offset, end);
-        this.fileReader.readAsArrayBuffer(slice);
-        this.offset = end;
-    }
-};
-</pre>
             <p>Uploading blob from stream. You can set up the blob name as well as the size of this uploading session. </p>
 <pre>
 // If one file has been selected in the HTML file input element
-var files = document.getElementById('fileinput').files;
-var file = files[0];
-var fileStream = new FileStream(file);
+var file = document.getElementById('fileinput').files[0];
 
 var customBlockSize = file.size > 1024 * 1024 * 32 ? 1024 * 1024 * 4 : 1024 * 512;
 blobService.singleBlobPutThresholdInBytes = customBlockSize;
 
 var finishedOrError = false;
-var speedSummary = blobService.createBlockBlobFromStream('mycontainer', file.name, fileStream, file.size, {blockSize : customBlockSize}, function(error, result, response) {
+var speedSummary = blobService.createBlockBlobFromBrowserFile('mycontainer', file.name, file, {blockSize : customBlockSize}, function(error, result, response) {
     finishedOrError = true;
     if (error) {
         // Upload blob failed
@@ -318,41 +276,6 @@ <h3 id="step6">Step 6: Creating your JavaScript Application based on Azure Stora
         <script src="../bundle/azure-storage.common.js"></script> 
         <script src="../bundle/azure-storage.blob.js"></script>
 
-        <script>
-            // Provides a Stream for a file in webpage, inheriting from NodeJS Readable stream.
-            var Stream = require('stream');
-            var util = require('util');
-            var Buffer = require('buffer').Buffer;
-
-            function FileStream(file, opt) {
-                Stream.Readable.call(this, opt);
-
-                this.fileReader = new FileReader(file);
-                this.file = file;
-                this.size = file.size;
-                this.chunkSize = 1024 * 1024 * 4; // 4MB
-                this.offset = 0;
-                var _me = this;
-
-                this.fileReader.onloadend = function loaded(event) {
-                    var data = event.target.result;
-                    var buf = Buffer.from(data);
-                    _me.push(buf);
-                }
-            }
-            util.inherits(FileStream, Stream.Readable);
-            FileStream.prototype._read = function() {
-                if (this.offset > this.size) {
-                    this.push(null);
-                } else {
-                    var end = this.offset + this.chunkSize;
-                    var slice = this.file.slice(this.offset, end);
-                    this.fileReader.readAsArrayBuffer(slice);
-                    this.offset = end;
-                }
-            };
-        </script>
-
         <script>
             var account = document.getElementById('account').value;
             var sas = document.getElementById('sas').value;
@@ -546,8 +469,6 @@ <h3 id="step6">Step 6: Creating your JavaScript Application based on Azure Stora
                 btn.disabled = true;
                 btn.innerHTML = "Uploading";
 
-                var fileStream = new FileStream(file);
-
                 // Make a smaller block size when uploading small blobs
                 var blockSize = file.size > 1024 * 1024 * 32 ? 1024 * 1024 * 4 : 1024 * 512;
                 var options = {
@@ -557,7 +478,7 @@ <h3 id="step6">Step 6: Creating your JavaScript Application based on Azure Stora
                 blobService.singleBlobPutThresholdInBytes = blockSize;
 
                 var finishedOrError = false;
-                var speedSummary = blobService.createBlockBlobFromStream(container, file.name, fileStream, file.size, options, function(error, result, response) {
+                var speedSummary = blobService.createBlockBlobFromBrowserFile(container, file.name, file, options, function(error, result, response) {
                     finishedOrError = true;
                     btn.disabled = false;
                     btn.innerHTML = "UploadBlob";

diff --git a/browser/samples/sample-file.html b/browser/samples/sample-file.html
@@ -176,57 +176,17 @@ <h3>Delete Directory</h3>
 </pre>
 
             <h3>Upload File</h3>
-            <p><code>FileService</code> provides <code>createFileFromStream</code> for uploading a file from stream.
-               However, currently <code>createFileFromStream</code> only accepts a Node.js <code>ReadableStream</code> type which pure JavaScript doesn't provide.
+            <p><code>FileService</code> provides <code>createFileFromBrowserFile</code> for uploading a file from an HTML file in browsers.
             </p>
-            <div class="panel panel-primary"> 
-                <div class="panel-body">
-                    <b>Note</b>: After importing <code>azure-storage.common.js</code>, you could require 3 Node.js types: <code>stream</code>, <code>util</code> and <code>buffer</code>, then wrap a ReadableStream based on HTML5 <code>FileReader</code>.
-                </div>
-            </div>
-<pre>
-// Provides a Stream for a file in webpage, inheriting from NodeJS Readable stream.
-var Stream = require('stream');
-var util = require('util');
-var Buffer = require('buffer').Buffer;
-
-function FileStream(file, opt) {
-    Stream.Readable.call(this, opt);
-
-    this.fileReader = new FileReader(file);
-    this.file = file;
-    this.size = file.size;
-    this.chunkSize = 1024 * 1024 * 4; // 4MB
-    this.offset = 0;
-    var _me = this;
-
-    this.fileReader.onloadend = function loaded(event) {
-        var data = event.target.result;
-        var buf = Buffer.from(data);
-        _me.push(buf);
-    }
-}
-util.inherits(FileStream, Stream.Readable);
-FileStream.prototype._read = function() {
-    if (this.offset > this.size) {
-        this.push(null);
-    } else {
-        var end = this.offset + this.chunkSize;
-        var slice = this.file.slice(this.offset, end);
-        this.fileReader.readAsArrayBuffer(slice);
-        this.offset = end;
-    }
-};
-</pre>
+
             <p>Uploading file from stream. You can set up the file name as well as the size of this uploading session. </p>
 <pre>
 // If one file has been selected in the HTML file input element
 var files = document.getElementById('fileinput').files;
 var file = files[0];
-var fileStream = new FileStream(file);
 
 var finishedOrError = false;
-var speedSummary = fileService.createFileFromStream('myfileshare', 'mydirectory', file.name, fileStream, file.size, {}, function(error, result, response) {
+var speedSummary = fileService.createFileFromBrowserFile('myfileshare', 'mydirectory', file.name, file, {}, function(error, result, response) {
     finishedOrError = true;
     if (error) {
         // Upload file failed
@@ -316,42 +276,6 @@ <h3 id="step6">Step 6: Creating your JavaScript Application based on Azure Stora
         <script src="../bundle/azure-storage.common.js"></script> 
         <script src="../bundle/azure-storage.file.js"></script>
 
-        <script>
-            // Provides a Stream for a file in webpage, inheriting from NodeJS Readable stream.
-            var Stream = require('stream');
-            var util = require('util');
-            var Buffer = require('buffer').Buffer;
-
-            function FileStream(file, opt) {
-                Stream.Readable.call(this, opt);
-
-                this.fileReader = new FileReader(file);
-                this.file = file;
-                this.size = file.size;
-                this.chunkSize = 1024 * 1024 * 4; // 4MB
-                this.offset = 0;
-                var _me = this;
-
-                this.fileReader.onloadend = function loaded(event) {
-                    var data = event.target.result;
-                    var buf = Buffer.from(data);
-                    _me.push(buf);
-                }
-            }
-            util.inherits(FileStream, Stream.Readable);
-            FileStream.prototype._read = function() {
-                if (this.offset > this.size) {
-                    console.log('FileStream reaches file end');
-                    this.push(null);
-                } else {
-                    var end = this.offset + this.chunkSize;
-                    var slice = this.file.slice(this.offset, end);
-                    this.fileReader.readAsArrayBuffer(slice);
-                    this.offset = end;
-                }
-            };
-        </script>
-
         <script>
             var account = document.getElementById('account').value;
             var sas = document.getElementById('sas').value;
@@ -624,15 +548,14 @@ <h3 id="step6">Step 6: Creating your JavaScript Application based on Azure Stora
                 btn.disabled = true;
                 btn.innerHTML = "Uploading";
                 var finishedOrError = false;
-                var fileStream = new FileStream(file);
                 var options = {
                     contentSettings: {
                         contentType: file.type
                     },
                     storeFileContentMD5 : checkMD5
                 };
 
-                var speedSummary = fileService.createFileFromStream(fileShare, currentPath.join('\\\\'), file.name, fileStream, file.size, options, function(error, result, response) {
+                var speedSummary = fileService.createFileFromBrowserFile(fileShare, currentPath.join('\\\\'), file.name, file, options, function(error, result, response) {
                     finishedOrError = true;
                     btn.disabled = false;
                     btn.innerHTML = "Upload";

diff --git a/lib/common/common.browser.js b/lib/common/common.browser.js
@@ -16,4 +16,6 @@
 
 var azureCommon = require('./common.core');
 
+azureCommon.BrowserFileReadStream = require('./streams/browserfilereadstream');
+
 module.exports = azureCommon;
diff --git a/lib/common/streams/batchoperation.js b/lib/common/streams/batchoperation.js
@@ -29,6 +29,7 @@ var ArgumentError = errors.ArgumentError;
 var DEFAULT_OPERATION_MEMORY_USAGE = Constants.BlobConstants.DEFAULT_WRITE_BLOCK_SIZE_IN_BYTES;
 var DEFAULT_CRITICAL_MEMORY_LIMITATION_32_IN_BYTES = Constants.BlobConstants.DEFAULT_CRITICAL_MEMORY_LIMITATION_32_IN_BYTES;
 var DEFAULT_CRITICAL_MEMORY_LIMITATION_BROWSER_IN_BYTES = Constants.BlobConstants.DEFAULT_CRITICAL_MEMORY_LIMITATION_BROWSER_IN_BYTES;
+var DEFAULT_MINIMUM_MEMORY_USAGE_BROWSER_IN_BYTES = Constants.BlobConstants.DEFAULT_MINIMUM_MEMORY_USAGE_BROWSER_IN_BYTES;
 var DEFAULT_GLOBAL_CONCURRENCY = 5; //Default http connection limitation for nodejs
 
 var SystemTotalMemory = os.totalmem();
@@ -114,14 +115,14 @@ BatchOperation.prototype.IsWorkloadHeavy = function() {
   if(this.enableReuseSocket && !this.callInOrder) {
     sharedRequest = 2;
   }
-  return this._activeOperation >= sharedRequest * this.concurrency ||this._isLowMemory();
+  return this._activeOperation >= sharedRequest * this.concurrency || this._isLowMemory();
 };
 
 /**
 * Get the approximate memory usage for batch operation.
 */
 BatchOperation.prototype._getApproximateMemoryUsage = function() {
-  var currentUsage = azureutil.isBrowser() ? 0 : process.memoryUsage().rss; // Currently, we cannot get memory usage in browsers
+  var currentUsage = azureutil.isBrowser() ? DEFAULT_MINIMUM_MEMORY_USAGE_BROWSER_IN_BYTES : process.memoryUsage().rss; // Currently, we cannot get memory usage in browsers
   var futureUsage = this._queuedOperation * this.operationMemoryUsage;
   return currentUsage + futureUsage;
 };

diff --git a/lib/common/streams/browserfilereadstream.js b/lib/common/streams/browserfilereadstream.js
@@ -0,0 +1,61 @@
+// 
+// Copyright (c) Microsoft and contributors.  All rights reserved.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//   http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// 
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// 
+
+var buffer = require('buffer').Buffer;
+var stream = require('stream');
+var util = require('util');
+
+var Constants = require('../util/constants');
+var bufferSize = Constants.BlobConstants.DEFAULT_WRITE_BLOCK_SIZE_IN_BYTES;
+
+function BrowserFileReadStream(file, options) {
+  stream.Readable.call(this, options);
+
+  if (!options) {
+    options = {};
+  }
+
+  this._fileReader = new FileReader(file); // HTML FileReader
+  this._file = file;
+  this._size = file.size;
+  this._highWaterMark = options.highWaterMark || bufferSize;
+  this._offset = 0;
+  var self = this;
+
+  this._fileReader.onloadend = function (event) {
+    var data = event.target.result;
+    var buf = buffer.from(data);
+    self.push(buf);
+  };
+
+  this._fileReader.onerror = function (error) {
+    self.emit('error', error);
+  };
+}
+util.inherits(BrowserFileReadStream, stream.Readable);
+
+BrowserFileReadStream.prototype._read = function() {
+  if (this._offset >= this._size) {
+    this.push(null);
+  } else {
+    var end = this._offset + this._highWaterMark;
+    var slice = this._file.slice(this._offset, end);
+    this._fileReader.readAsArrayBuffer(slice);
+    this._offset = end;
+  }
+};
+
+module.exports = BrowserFileReadStream;
diff --git a/lib/common/streams/chunkstream.js b/lib/common/streams/chunkstream.js
@@ -17,6 +17,7 @@
 var stream = require('stream');
 var util = require('util');
 
+var azureutil = require('../util/util');
 var Md5Wrapper = require('../md5-wrapper');
 var Constants = require('../util/constants');
 var bufferSize = Constants.BlobConstants.DEFAULT_WRITE_BLOCK_SIZE_IN_BYTES;
@@ -116,7 +117,7 @@ ChunkStream.prototype.error = function () {
 ChunkStream.prototype.destroy = function () {
   this.writable = this.readable = false;
 
-  if (this._allocator && this._allocator.destroy) {
+  if (this._allocator && azureutil.objectIsFunction(this._allocator.destroy)) {
     this._allocator.destroy();
   }
 
@@ -272,7 +273,7 @@ ChunkStream.prototype._popInternalBuffer = function () {
 */
 ChunkStream.prototype._allocateNewBuffer = function() {
   var size = this._highWaterMark;
-  if(this._allocator && this._allocator.getBuffer) {
+  if(this._allocator && azureutil.objectIsFunction(this._allocator.getBuffer)) {
     return this._allocator.getBuffer(size);
   } else {
     var buffer = new Buffer(size);