-
Notifications
You must be signed in to change notification settings - Fork 0
/
exponentialBackOff.worker.js
70 lines (61 loc) · 1.67 KB
/
exponentialBackOff.worker.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
// === BEGIN: KUE SETUP ===
var kue = require('kue')
, url = require('url')
, redis = require('kue/node_modules/redis');
var redisUrl = url.parse(process.env.REDISCLOUD_URL);
var jobs = kue.createQueue({
prefix: 'q',
redis: {
port: redisUrl.port,
host: redisUrl.hostname,
auth: redisUrl.auth.split(":")[1],
options: {
// look for more redis options in [node_redis](https://github.com/mranney/node_redis)
}
},
disableSearch: true
});
// === END: KUE SETUP ===
var minInterval = 1;
var interval = minInterval;
var exponent = 2;
var maxInterval = 60;
var setupJob = function() {
var job = jobs.create('crawl', {
url: 'http://example.com'
, token: 'foo'
});
job.on('complete', function(){
console.log("Job complete");
}).on('failed', function(){
res.send("Job failed");
}).on('progress', function(progress){
console.log('job #' + job.id + ' ' + progress + '% complete');
});
job.save();
};
var startProducer = function() {
//var msg = queue.GetMessage();
// maybe you get a message for Q, maybe you don't
var myArray = ['value', null, null];
var msg = myArray[Math.floor(Math.random() * myArray.length)];
if (msg != null)
{
interval = minInterval;
console.log('Interval reset to %d seconds', interval);
// TODO: do something queue.DeleteMessage(msg);
setupJob();
startProducer();
}
else
{
console.log('Sleep for %d seconds', interval);
// timeout after interval
setTimeout(function() {
startProducer();
}, interval*1000);
interval = Math.min(maxInterval, interval * exponent);
console.log('Interval extended to %d seconds', interval);
}
};
startProducer();