From 101310928798859349ad8ef5fb27c897cf0499c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Sobota?= Date: Tue, 12 Jan 2016 21:06:14 +0100 Subject: [PATCH 1/7] Configurable tasks and specs paths --- src/queue.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/queue.js b/src/queue.js index 367aa96..a80e065 100644 --- a/src/queue.js +++ b/src/queue.js @@ -67,10 +67,8 @@ function Queue() { logger.debug('Queue(): Error during initialization', error); throw new Error(error); } else if (constructorArguments.length === 2) { - self.ref = constructorArguments[0]; self.processingFunction = constructorArguments[1]; } else if (constructorArguments.length === 3) { - self.ref = constructorArguments[0]; var options = constructorArguments[1]; if (!_.isPlainObject(options)) { error = 'Options parameter must be a plain object.'; @@ -122,12 +120,25 @@ function Queue() { logger.debug('Queue(): Error during initialization', error); throw new Error(error); } + + if (_.isPlainObject(constructorArguments[0])) { + if (_.isUndefined(constructorArguments[0].tasksRef) || _.isUndefined(constructorArguments[0].specsRef)) { + error = 'When queueRef is an object then it must specify tasksRef and specsRef.'; + logger.debug('Queue(): Error during initialization', error); + throw new Error(error); + } + self.tasksRef = constructorArguments[0].tasksRef; + self.specsRef = constructorArguments[0].specsRef; + } else { + self.tasksRef = constructorArguments[0].child('tasks'); + self.specsRef = constructorArguments[0].child('specs'); + } self.workers = []; for (var i = 0; i < self.numWorkers; i++) { var processId = (self.specId ? self.specId + ':' : '') + i; self.workers.push(new QueueWorker( - self.ref.child('tasks'), + self.tasksRef, processId, self.sanitize, self.suppressStack, @@ -141,7 +152,7 @@ function Queue() { } self.initialized = true; } else { - self.specChangeListener = self.ref.child('specs').child(self.specId).on( + self.specChangeListener = self.specsRef.child(self.specId).on( 'value', function(taskSpecSnap) { var taskSpec = { @@ -177,7 +188,7 @@ Queue.prototype.shutdown = function() { logger.debug('Queue: Shutting down'); if (!_.isNull(self.specChangeListener)) { - self.ref.child('specs').child(self.specId).off('value', + self.specsRef.child(self.specId).off('value', self.specChangeListener); self.specChangeListener = null; } From 893f2b7fa6c4619c2d491ecd94c19715fddd4681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Sobota?= Date: Tue, 12 Jan 2016 23:26:15 +0100 Subject: [PATCH 2/7] Simplified validation of paths configuration --- src/queue.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/queue.js b/src/queue.js index a80e065..2ac0cf0 100644 --- a/src/queue.js +++ b/src/queue.js @@ -121,14 +121,13 @@ function Queue() { throw new Error(error); } - if (_.isPlainObject(constructorArguments[0])) { - if (_.isUndefined(constructorArguments[0].tasksRef) || _.isUndefined(constructorArguments[0].specsRef)) { - error = 'When queueRef is an object then it must specify tasksRef and specsRef.'; - logger.debug('Queue(): Error during initialization', error); - throw new Error(error); - } + if (_.has(constructorArguments[0], ['tasksRef', 'specsRef'])) { self.tasksRef = constructorArguments[0].tasksRef; self.specsRef = constructorArguments[0].specsRef; + } else if (_.isPlainObject(constructorArguments[0])) { + error = "When ref is an object it must contain both keys 'tasksRef' and 'specsRef'"; + logger.debug('Queue(): Error during initialization', error); + throw new Error(error); } else { self.tasksRef = constructorArguments[0].child('tasks'); self.specsRef = constructorArguments[0].child('specs'); From 62147dab63856edb548a59c2fe29931f5a41c317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Sobota?= Date: Wed, 13 Jan 2016 01:30:59 +0100 Subject: [PATCH 3/7] Updated tests, JSDocs and README --- README.md | 24 ++++++++++++++++++++++++ src/queue.js | 8 ++++++-- test/queue.spec.js | 11 +++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c773f1f..3026a2b 100644 --- a/README.md +++ b/README.md @@ -480,6 +480,30 @@ Since there is no `finished_state` in the `fanout_message` spec, the task will b While this example is a little contrived since you could perform the sanitization and fanout in a single task, creating multiple specs for our tasks allows us to do things like add selective retries to certain tasks more likely to fail, put additional workers on more expensive tasks, or add expressive error states. +## Custom references to tasks and specs + +In order to use custom paths to tasks and specs, specify them explicitly. Instead of a single reference to the queue, use an object containing the keys `tasksRef` and `specsRef`. + +```js +var Queue = require('firebase-queue'), + Firebase = require('firebase'); + +var jobsRef = new Firebase('https://.firebaseio.com/queue/jobs'); +var specsRef = new Firebase('https://.firebaseio.com/queue/specs'); +var queue = new Queue({tasksRef: jobsRef, specsRef: specsRef}, function(data, progress, resolve, reject) { + // Read and process task data + console.log(data); + + // Do some work + progress(50); + + // Finish the task asynchronously + setTimeout(function() { + resolve(); + }, 1000); +}); +``` + ## Wrap Up As you can see, Firebase Queue is a powerful tool that allows you to securely and robustly perform background work on your Firebase data, from sanitization to data fanout and more. We'd love to hear about how you're using Firebase-Queue in your project! Let us know on [Twitter](https://twitter.com/firebase), [Facebook](https://www.facebook.com/Firebase), or [G+](https://plus.google.com/115330003035930967645). If you have any questions, please direct them to our [Google Group](https://groups.google.com/forum/#!forum/firebase-talk) or [support@firebase.com](mailto:support@firebase.com). diff --git a/src/queue.js b/src/queue.js index 2ac0cf0..7b05f23 100644 --- a/src/queue.js +++ b/src/queue.js @@ -24,7 +24,10 @@ var DEFAULT_NUM_WORKERS = 1, /** * @constructor - * @param {Firebase} ref A Firebase reference to the queue. + * @param {(Firebase | Object)} ref A Firebase reference to the queue + * or object containing both keys: + * - tasksRef: {Firebase} A Firebase reference to tasks. + * - specsRef: {Firebase} A Firebase reference to specs. * @param {Object} options (optional) Object containing possible keys: * - specId: {String} the task specification ID for the workers. * - numWorkers: {Number} The number of workers to create for this task. @@ -125,7 +128,8 @@ function Queue() { self.tasksRef = constructorArguments[0].tasksRef; self.specsRef = constructorArguments[0].specsRef; } else if (_.isPlainObject(constructorArguments[0])) { - error = "When ref is an object it must contain both keys 'tasksRef' and 'specsRef'"; + error = 'When ref is an object it must contain both keys ' + + '\'tasksRef\' and \'specsRef\''; logger.debug('Queue(): Error during initialization', error); throw new Error(error); } else { diff --git a/test/queue.spec.js b/test/queue.spec.js index 98ed222..83906d6 100644 --- a/test/queue.spec.js +++ b/test/queue.spec.js @@ -27,6 +27,13 @@ describe('Queue', function() { }).to.throw; }); }); + + _.forEach([{}, { foo: 'bar' }, { tasksRef: th.testRef}, { specsRef: th.testRef }], function(invalidRefConfigurationObject) { + it('should not create a Queue with ref configuration object that contains keys: ' + _.keys(invalidRefConfigurationObject).join(", ") + '.', function() { + expect(function() { + new th.Queue(invalidPathsConfigurationObject); + }).to.throw('When ref is an object it must contain both keys \'tasksRef\' and \'specsRef\''); + }); _.forEach(['', 'foo', NaN, Infinity, true, false, 0, 1, ['foo', 'bar'], { foo: 'bar' }, null, { foo: 'bar' }, { foo: { bar: { baz: true } } }], function(nonFunctionObject) { it('should not create a Queue with a non-function callback: ' + JSON.stringify(nonFunctionObject), function() { @@ -39,6 +46,10 @@ describe('Queue', function() { it('should create a default Queue with just a Firebase reference and a processing callback', function() { new th.Queue(th.testRef, _.noop); }); + + it('should create a default Queue with tasks and specs Firebase references and a processing callback', function() { + new th.Queue({tasksRef: th.testRef, specsRef: th.testRef}, _.noop); + }); _.forEach(['', 'foo', NaN, Infinity, true, false, 0, 1, ['foo', 'bar'], null, _.noop], function(nonPlainObject) { it('should not create a Queue with a Firebase reference, a non-plain object options parameter (' + JSON.stringify(nonPlainObject) + '), and a processingCallback', function() { From 6f26e320e531ce9df11e46adf063a8982e490b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Sobota?= Date: Wed, 13 Jan 2016 01:50:42 +0100 Subject: [PATCH 4/7] Fixed bugs and tests --- src/queue.js | 2 +- test/queue.spec.js | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/queue.js b/src/queue.js index 7b05f23..fe41f05 100644 --- a/src/queue.js +++ b/src/queue.js @@ -124,7 +124,7 @@ function Queue() { throw new Error(error); } - if (_.has(constructorArguments[0], ['tasksRef', 'specsRef'])) { + if (_.has(constructorArguments[0], 'tasksRef') && _.has(constructorArguments[0], 'specsRef')) { self.tasksRef = constructorArguments[0].tasksRef; self.specsRef = constructorArguments[0].specsRef; } else if (_.isPlainObject(constructorArguments[0])) { diff --git a/test/queue.spec.js b/test/queue.spec.js index 83906d6..498c274 100644 --- a/test/queue.spec.js +++ b/test/queue.spec.js @@ -29,10 +29,11 @@ describe('Queue', function() { }); _.forEach([{}, { foo: 'bar' }, { tasksRef: th.testRef}, { specsRef: th.testRef }], function(invalidRefConfigurationObject) { - it('should not create a Queue with ref configuration object that contains keys: ' + _.keys(invalidRefConfigurationObject).join(", ") + '.', function() { - expect(function() { - new th.Queue(invalidPathsConfigurationObject); - }).to.throw('When ref is an object it must contain both keys \'tasksRef\' and \'specsRef\''); + it('should not create a Queue with ref configuration object that contains keys: {' + _.keys(invalidRefConfigurationObject).join(", ") + '}.', function() { + expect(function() { + new th.Queue(invalidRefConfigurationObject, _.noop); + }).to.throw('When ref is an object it must contain both keys \'tasksRef\' and \'specsRef\''); + }); }); _.forEach(['', 'foo', NaN, Infinity, true, false, 0, 1, ['foo', 'bar'], { foo: 'bar' }, null, { foo: 'bar' }, { foo: { bar: { baz: true } } }], function(nonFunctionObject) { From 072fe9f8f5938c1a5b8437cf52cee74130800ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Sobota?= Date: Wed, 13 Jan 2016 01:56:43 +0100 Subject: [PATCH 5/7] Cleanup README example --- README.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3026a2b..60782d5 100644 --- a/README.md +++ b/README.md @@ -488,19 +488,11 @@ In order to use custom paths to tasks and specs, specify them explicitly. Instea var Queue = require('firebase-queue'), Firebase = require('firebase'); -var jobsRef = new Firebase('https://.firebaseio.com/queue/jobs'); -var specsRef = new Firebase('https://.firebaseio.com/queue/specs'); -var queue = new Queue({tasksRef: jobsRef, specsRef: specsRef}, function(data, progress, resolve, reject) { - // Read and process task data - console.log(data); - - // Do some work - progress(50); +var jobsRef = new Firebase('https://.firebaseio.com/jobs'); +var specsRef = new Firebase('https://.firebaseio.com/specs'); - // Finish the task asynchronously - setTimeout(function() { - resolve(); - }, 1000); +var queue = new Queue({ tasksRef: jobsRef, specsRef: specsRef }, function(data, progress, resolve, reject) { + // process task }); ``` From 966764f6ce80ce5c7129fb62422d38f46fa81301 Mon Sep 17 00:00:00 2001 From: Chris Raynor Date: Mon, 21 Mar 2016 15:20:53 -0700 Subject: [PATCH 6/7] Tidying up PR to conform to the styleguide and updating the documentation --- README.md | 4 +++- gulpfile.js | 2 +- src/queue.js | 23 ++++++++++++----------- test/queue.spec.js | 10 +++++----- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0045b53..add7d9a 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,14 @@ Using Firebase Queue, you can create specs for each of these tasks, and then use ## The Queue in Your Firebase Database The queue relies on having a Firebase database reference to coordinate workers e.g. `https://.firebaseio.com/queue`. This queue can be stored at any path in your Firebase database, and you can have multiple queues as well. The queue will respond to tasks pushed onto the `tasks` subtree and optionally read specifications from a `specs` subtree. + ``` queue - specs - tasks ``` +See [Custom references to tasks and specs](#custom-references-to-tasks-and-specs) for defining the locations of these other than the default. ## Queue Workers @@ -488,7 +490,7 @@ While this example is a little contrived since you could perform the sanitizatio ## Custom references to tasks and specs -In order to use custom paths to tasks and specs, specify them explicitly. Instead of a single reference to the queue, use an object containing the keys `tasksRef` and `specsRef`. +It is possible to specify the locations the queue uses for tasks and the specs explicitly instead of using the defaults. To do this, simply pass an object to the Queue constructor in place of the Firebase reference; this object must contain the keys `tasksRef` and `specsRef`, and each value must be a Firebase reference. ```js var Queue = require('firebase-queue'), diff --git a/gulpfile.js b/gulpfile.js index d1ca41c..b6e7243 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -54,7 +54,7 @@ gulp.task('test', function() { gulp.src(paths.tests) .pipe(mocha({ reporter: 'spec', - timeout: 2000 + timeout: 5000 })) .pipe(istanbul.writeReports()) .pipe(exit()); diff --git a/src/queue.js b/src/queue.js index fe41f05..a80f88f 100644 --- a/src/queue.js +++ b/src/queue.js @@ -24,14 +24,14 @@ var DEFAULT_NUM_WORKERS = 1, /** * @constructor - * @param {(Firebase | Object)} ref A Firebase reference to the queue - * or object containing both keys: - * - tasksRef: {Firebase} A Firebase reference to tasks. - * - specsRef: {Firebase} A Firebase reference to specs. + * @param {Firebase|Object} ref A Firebase reference to the queue or an object + * containing both keys: + * - tasksRef: {Firebase} A Firebase reference to the queue tasks location. + * - specsRef: {Firebase} A Firebase reference to the queue specs location. * @param {Object} options (optional) Object containing possible keys: - * - specId: {String} the task specification ID for the workers. - * - numWorkers: {Number} The number of workers to create for this task. - * - sanitize: {Boolean} Whether to sanitize the 'data' passed to the + * - specId: {String} the task specification ID for the workers. + * - numWorkers: {Number} The number of workers to create for this task. + * - sanitize: {Boolean} Whether to sanitize the 'data' passed to the * processing function of internal queue keys. * @param {Function} processingFunction A function that is called each time to * process a task. This function is passed four parameters: @@ -123,13 +123,14 @@ function Queue() { logger.debug('Queue(): Error during initialization', error); throw new Error(error); } - - if (_.has(constructorArguments[0], 'tasksRef') && _.has(constructorArguments[0], 'specsRef')) { + + if (_.has(constructorArguments[0], 'tasksRef') && + _.has(constructorArguments[0], 'specsRef')) { self.tasksRef = constructorArguments[0].tasksRef; self.specsRef = constructorArguments[0].specsRef; } else if (_.isPlainObject(constructorArguments[0])) { - error = 'When ref is an object it must contain both keys ' + - '\'tasksRef\' and \'specsRef\''; + error = 'When ref is an object it must contain both keys \'tasksRef\' ' + + 'and \'specsRef\''; logger.debug('Queue(): Error during initialization', error); throw new Error(error); } else { diff --git a/test/queue.spec.js b/test/queue.spec.js index 498c274..1140d51 100644 --- a/test/queue.spec.js +++ b/test/queue.spec.js @@ -27,9 +27,9 @@ describe('Queue', function() { }).to.throw; }); }); - - _.forEach([{}, { foo: 'bar' }, { tasksRef: th.testRef}, { specsRef: th.testRef }], function(invalidRefConfigurationObject) { - it('should not create a Queue with ref configuration object that contains keys: {' + _.keys(invalidRefConfigurationObject).join(", ") + '}.', function() { + + _.forEach([{}, { foo: 'bar' }, { tasksRef: th.testRef }, { specsRef: th.testRef }], function(invalidRefConfigurationObject) { + it('should not create a Queue with a ref configuration object that contains keys: {' + _.keys(invalidRefConfigurationObject).join(', ') + '}', function() { expect(function() { new th.Queue(invalidRefConfigurationObject, _.noop); }).to.throw('When ref is an object it must contain both keys \'tasksRef\' and \'specsRef\''); @@ -47,9 +47,9 @@ describe('Queue', function() { it('should create a default Queue with just a Firebase reference and a processing callback', function() { new th.Queue(th.testRef, _.noop); }); - + it('should create a default Queue with tasks and specs Firebase references and a processing callback', function() { - new th.Queue({tasksRef: th.testRef, specsRef: th.testRef}, _.noop); + new th.Queue({ tasksRef: th.testRef, specsRef: th.testRef }, _.noop); }); _.forEach(['', 'foo', NaN, Infinity, true, false, 0, 1, ['foo', 'bar'], null, _.noop], function(nonPlainObject) { From b64f4a1a8bb6a848685a2abecc1d52c715fd260c Mon Sep 17 00:00:00 2001 From: Chris Raynor Date: Mon, 21 Mar 2016 15:25:09 -0700 Subject: [PATCH 7/7] Switching security rules in README to JSON instead of JS to display better with syntax highlighting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index add7d9a..4e796ca 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ In this example, there are three categories of users, represented using fields o These don't have to use a custom token, for instance you could use `auth != null` in place of `auth.canAddTasks` if application's users can write directly to the queue. Similarly, `auth.canProcessTasks` and `auth.canAddSpecs` could be `auth.admin === true` if a single trusted server process was used to perform all queue functions. -```js +```json { "rules": { "queue": {