New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upload 10 files at a time, plus other minor changes #5

Closed
wants to merge 2 commits into
base: master
from
Jump to file or symbol
Failed to load files and symbols.
+81 −83
Diff settings

Always

Just for now

Copy path View file
@@ -1,14 +1,13 @@
{
"boss": true,
"curly": true,
"eqeqeq": true,
"eqnull": true,
"immed": true,
"latedef": true,
"latedef" : false,
"newcap": true,
"noarg": true,
"sub": true,
"undef": true,
"boss": true,
"eqnull": true,
"node": true,
"latedef" : false
"sub": true,
"undef": true
}
Copy path View file
@@ -13,8 +13,8 @@ grunt.loadNpmTasks('grunt-azureblob');
```
# Environment Requirment
+ Azure SDK provides Node.js package for access to Azure Table Storage. By default, this library uses the following environment variables for authentication (set as required as global, user, or with a task). I've had great success with grunt-env to manage the these settings as a task (sample usage shown below). _These environment variables must be set to your appropriate values!_
+ AZURE_STORAGE_ACCOUNT
+ AZURE_STORAGE_ACCESS_KEY
+ AZURE_STORAGE_ACCOUNT
+ AZURE_STORAGE_ACCESS_KEY
## AzureBlob Options and default values
@@ -27,14 +27,13 @@ grunt-azureblob is a multi task that implicity iterates over all of its name sub
containerDelete: false, // deletes container if it exists
containerOptions: {publicAccessLevel: "blob", timeoutIntervalInMs: 10000}, // container options
copySimulation: false, // do everything but physically touch storage blob when true
destPrefix: '', // prefix to use for blob name e.g. 'v0.0.1/'
maskBaseDir: '', // mask off directory portion to map files to root in storage container
metadata: {cacheControl: 'public, max-age=31556926'}, // file metadata properties
gzip: false // gzip files (when true: only js / css will be gzip'd)
gzip: false, // gzip files (when true: only js / css will be gzip'd)
maxNumberOfConcurrentUploads: 10 // Maximum number of concurrent uploads
};
````
## Example gruntfile.js
## Example gruntfile.js
```javascript
module.exports = function(grunt) {
grunt.initConfig({
@@ -54,21 +53,28 @@ module.exports = function(grunt) {
containerDelete: false,
metadata: {cacheControl: 'public, max-age=31556926'}, // max-age 1 year for all entries
gzip: true,
copySimulation: true, // set true: dry-run for what copy would look like in output
destPrefix: '<%= pkg.version %>/'
copySimulation: true // set true: dry-run for what copy would look like in output
},
css :{
options: {
maskBaseDir: '../web/Content/' // strip off this prefix from files
},
src: ['../web/Content/**/*','!../web/Content/themes/**/*'] // copy all files from Content (exclude themes dir)
files: [{
expand: true,
cwd: '../web/Content',
filter: 'isFile',
dest: '<%= pkg.version %>/',
src: ['**/*', '!themes/**/*'] // copy all files from Content (exclude themes dir)
}]
},
js :{
options: {
containerDelete: false,
maskBaseDir: '../web/scripts/'
containerDelete: false
},
src: ['../web/scripts/vendor*.js']
files: [{
expand: true,
cwd: '../web/scripts',
filter: 'isFile',
dest: '<%= pkg.version %>/',
src: ['vendor*.js']
}]
}
}
});
@@ -77,8 +83,8 @@ module.exports = function(grunt) {
grunt.loadNpmTasks('grunt-env'); // https://npmjs.org/package/grunt-env
grunt.loadNpmTasks('grunt-azureblob');
// Default task(s).
grunt.registerTask('blob', ['env:configCDN', 'azureblob']);
grunt.registerTask('blob', ['env:configCDN', 'azureblob']);
grunt.event.on('qunit.spawn', function (url) {
grunt.log.ok("Running test: " + url);
});
@@ -88,7 +94,7 @@ grunt.event.on('qunit.moduleStart', function (name) {
```
## Sample console run (from sample/build/gruntfile.js)
```console
c:\sample>grunt blob
c:\sample>grunt blob
Running "env:configCDN" (env) task
Copy path View file
@@ -10,7 +10,6 @@ module.exports = function(grunt) {
fs = require('fs'),
tmp = require('tmp');
grunt.registerMultiTask('azureblob', 'Copy html assets to azure blob/cdn storage', function() {
// Merge task-specific and/or target-specific options with these defaults.
var options = this.options({
@@ -20,10 +19,9 @@ module.exports = function(grunt) {
containerOptions: {publicAccessLevel: "blob", timeoutIntervalInMs: 10000}, // container options
metadata: {cacheControl: 'public, max-age=31556926'}, // file metadata properties
copySimulation: false,
destPrefix: '', // detination path prefix to use for blob name e.g. 'v0.0.1/'
maskBaseDir: '',
gzip: false // gzip files
}),
gzip: false, // gzip files
maxNumberOfConcurrentUploads: 10 // Maximum number of concurrent uploads
}),
blobService = azure.createBlobService(),
done = this.async(),
self = this;
@@ -49,31 +47,30 @@ module.exports = function(grunt) {
// returns a q promise
function iterateFiles() {
var deferred = Q.defer(),
files = self.filesSrc.filter(fileExists); // filesSrc can include dir's, not just files
files = self.files.filter(fileExistsAndIsWellFormed); // filesSrc can include dir's, not just files
grunt.verbose.writeln(util.format('\tprocess (%s) files',files.length));
grunt.verbose.writeln(util.format('\tprocess (%s) files', files.length));
// Iterate over all specified file groups.
grunt.util.async.forEachSeries(files, copyFile, function(err){
if (err) {
deferred.reject(err);
}
deferred.resolve(files.length);
});
// Iterate over all specified file groups, <options.maxNumberOfConcurrentUploads> files at a time
grunt.util.async.forEachLimit(files, options.maxNumberOfConcurrentUploads, copyFile, function(err){
if (err) {
deferred.reject(err);
}
deferred.resolve(files.length);
});
return deferred.promise;
}
}
// When optioned, delete blob container
// returns q promise
function deleteContainer() {
var deferred = Q.defer();
if (options.containerDelete && !options.copySimulation) {
grunt.log.writeln(util.format('%s - deleting container [%s] ...', self.nameArgs, options.containerName));
blobService.deleteContainer(options.containerName, {timeoutIntervalInMs:25000}, function(err){
/* // ignore errors for now - just move on
/* // ignore errors for now - just move on
if (err) {
grunt.log.writeln(err);
deferred.reject(err);
@@ -97,20 +94,20 @@ module.exports = function(grunt) {
waitMs = 100,
maxTry = 10;
options.containerOptions.timeoutIntervalInMs = options.containerOptions.timeoutIntervalInMs || 15000; // 10sec
options.containerOptions.timeoutIntervalInMs = options.containerOptions.timeoutIntervalInMs || 15000; // 15sec
grunt.log.write(util.format('%s - Create blob containter [%s] ...', self.nameArgs, options.containerName));
if(options.copySimulation){
completed = true;
tryCallback();
} else {
grunt.util.async.whilst(continueAttempts, tryCreate, tryCallback);
}
return deferred.promise;
function continueAttempts() {
return ((count < maxTry) && !completed); // sync truth test before each execution of fn
return ((count < maxTry) && !completed); // sync truth test before each execution of fn
}
function tryCreate(callback) {
count++;
@@ -141,51 +138,44 @@ module.exports = function(grunt) {
deferred.reject(err);
}
}
}
// Iterator called from grunt.util.async.forEachSeries - for each source file in task
function copyFile(source, callback) {
}
var destination = source, // set default destination same as source
// Iterator called from grunt.util.async.forEachLimit - for each source file in task
function copyFile(file, callback) {
var source = file.src,
destination = file.dest,
meta = options.metadata,
srcFile = path.basename(source),
gzip = options.gzip,
fileExt = path.extname(source),
fnCopyToBlob;
// only create gzip copies for css and js files
// only create gzip copies for css and js files
if (fileExt !== '.js' && fileExt !== '.css') {
gzip = false;
}
// ensure trailing slash is present in destination
if (options.maskBaseDir) {
destination = source.replace(options.maskBaseDir,'');
}
if (options.destPrefix && options.destPrefix.length > 0 && options.destPrefix.substr(-1) !== '/') {
options.destPrefix += '/';
}
if (options.destPrefix) {
destination = options.destPrefix + destination;
}
// configure proper azure metadata for file
meta.contentType = mime.lookup(source);
meta.contentTypeHeader = mime.lookup(source);
meta.contentEncoding = gzip ? 'gzip': null;
grunt.log.write(util.format('\tCopy %s => %s/%s ', srcFile, options.containerName, destination));
var logMessage = util.format('\tCopy %s => %s/%s ', srcFile, options.containerName, destination);
if(options.copySimulation){
grunt.log.ok('skip copy ok');
callback();
return;
grunt.log.write(logMessage);
grunt.log.ok('skip copy ok');
callback();
return;
}
fnCopyToBlob = gzip ? compressFileToBlobStorage : copyFileToBlobStorage; // use correct fn to pre-compress
Q.when(fnCopyToBlob(options.containerName, destination, source, meta))
.then(function(){
grunt.log.write(logMessage); // We have to save logging about the file upload until here since
// we are iterating over many files concurrently. Otherwise the
// log output becomes interleaved and therefore unintelligible.
grunt.log.ok();
callback();
}).done();
@@ -197,19 +187,19 @@ module.exports = function(grunt) {
return copyFileToBlobStorage(containerName, destFileName, tmpFile, metadata)
.finally(function(){
fs.unlinkSync(tmpFile);
});
});
});
});
}
function copyFileToBlobStorage(containerName, destFileName, sourceFile, metadata) {
var deferred = Q.defer();
blobService.createBlockBlobFromFile(containerName, destFileName, sourceFile, metadata, function(err) {
if (err) {
grunt.log.error(err);
deferred.reject(err);
} else {
deferred.resolve();
}
if (err) {
grunt.log.error(err);
deferred.reject(err);
} else {
deferred.resolve();
}
});
return deferred.promise;
}
@@ -220,7 +210,7 @@ module.exports = function(grunt) {
fileExt = path.extname(source),
inp,
out;
gzip.on('error', function(err) {
grunt.log.error(err);
grunt.fail.warn('Gziping failed.');
@@ -233,23 +223,26 @@ module.exports = function(grunt) {
}
inp = fs.createReadStream(source);
out = fs.createWriteStream(tempFile);
out.on('close', function() {
deferred.resolve(tempFile); // once file closes, file is writen and avail.
deferred.resolve(tempFile); // once file closes, file is written and available.
});
inp.pipe(gzip).pipe(out);
//inp.pipe(out); // test to just copy file
});
return deferred.promise;
}
function fileExists (dest) {
if (fs.statSync(dest).isFile() && fs.existsSync(dest) ) {
return true;
function fileExistsAndIsWellFormed(file) {
if (file.src.length !== 1) {
grunt.fail.warn('File mapping must contain exactly one source to one destination.');
}
return false;
file.src = file.src[0];
return fs.statSync(file.src).isFile() && fs.existsSync(file.src);
}
});
};
ProTip! Use n and p to navigate between commits in a pull request.