Merge pull request #42 from storify/respawn

Respawn

README.md

@@ -79,6 +79,10 @@ The `up` command accepts the following options:
   - Strings like `'10s'` are accepted.
   - Defaults to `'10m'`, or `'500ms'` if `NODE_ENV` is `development`.
 
+- `-k`/`--keepalive`
+
+  - start a new worker after one dies unexpectedly
+
 - `-f`/`--pidfile`
 
   - A filename to write the pid to
@@ -114,6 +118,10 @@ parameters:
   - `workerTimeout`: (`Number`|`String`): see `--timeout` above.
   - `title`: (`String`): see `--title` above.
   - `assumeReady`: (`Boolean`): see Worker readiness below.
+  - `keepAlive`: (`Boolean`): see `--keepalive` above.
+  - `minExpectedLifetime`: (`Number`|`String`): Number of ms a worker is
+    expected to live. Don't auto-respawn if a worker dies earlier. Strings
+    like `'10s'` are accepted. Defaults to `'20s'`.
 
 ## Middleware
 

bin/up

@@ -43,6 +43,7 @@ program
   .option('-n, --number <workers>', 'Number of workers to spawn.'
     , 'development' == process.env.NODE_ENV ? 1 : cpus)
   .option('-t, --timeout [ms]', 'Worker timeout.')
+  .option('-k, --keepalive', 'Restart failed workers.')
 
 /**
  * Capture requires.
@@ -134,6 +135,12 @@ if (null != workerTimeout && isNaN(ms(workerTimeout))) {
     , program.timeout, ms(workerTimeout));
 }
 
+/**
+ * Parse keepalive
+ */
+
+var keepAlive = program.keepalive;
+
 /**
  * Start!
  */
@@ -148,6 +155,7 @@ var httpServer = http.Server().listen(program.port)
       , workerTimeout: workerTimeout
       , requires: requires
       , title: program.title
+      , keepAlive: keepAlive
     })
 
 /**

lib/up.js

@@ -53,6 +53,13 @@ var workerTimeout = 'development' == env ? '500ms' : '10m';
 
 var numWorkers = 'development' == env ? 1 : cpus;
 
+/**
+ * Default minimum expected lifetime of a worker.
+ * If a worker dies younger, we don't respawn even if keepAlive == true.
+ * We want to prevent auto-respawning storms from overloading the system.
+ */
+var minExpectedLifetime = '20s';
+
 /**
  * UpServer factory/constructor.
  *
@@ -75,6 +82,8 @@ function UpServer (server, file, opts) {
     ? opts.workerTimeout : workerTimeout);
   this.requires = opts.requires || [];
   this.assumeReady = opts.assumeReady === undefined ? true : !!opts.assumeReady;
+  this.keepAlive = opts.keepAlive || false;
+  this.minExpectedLifetime = ms(opts.minExpectedLifetime != null ? opts.minExpectedLifetime : minExpectedLifetime);
   if (false !== opts.workerPingInterval) {
     this.workerPingInterval = ms(opts.workerPingInterval || '1m');
   }
@@ -202,7 +211,15 @@ UpServer.prototype.spawnWorker = function (fn) {
         if (~self.workers.indexOf(w)) {
           self.workers.splice(self.workers.indexOf(w), 1);
           self.lastIndex = -1;
-          // @TODO: auto-add workers ?
+          if (self.keepAlive && (self.workers.length + self.spawning.length < self.numWorkers)) {
+            if (new Date().getTime() - w.birthtime < self.minExpectedLifetime) {
+              debug('worker %s found dead at a too young age. won\'t respawn new worker', w.pid);
+            }
+            else {
+              debug('worker %s found dead. spawning 1 new worker', w.pid);
+              self.spawnWorker();
+            }
+          }
         }
         self.emit('terminate', w)
         break;
@@ -273,6 +290,7 @@ function Worker (server) {
   this.proc.on('message', this.onMessage.bind(this));
   this.proc.on('exit', this.onExit.bind(this));
   this.pid = this.proc.pid;
+  this.birthtime = new Date().getTime();
   debug('worker %s created', this.pid);
 }
 

package.json

@@ -20,4 +20,7 @@
       , "express": "*"
       , "superagent": "*"
     }
+  , "scripts": {
+        "test": "./node_modules/mocha/bin/mocha test/*.test.js"
+    }
 }

test/up.js → test/up.test.js

@@ -212,5 +212,23 @@ describe('up', function () {
     testAssumeReady(done, false);
   });
 
+  it('should respawn a worker when it dies', function (done) {
+    var httpServer = http.Server().listen()
+      , opts = { numWorkers: 1, keepAlive: true, minExpectedLifetime: '50' }
+      , srv = up(httpServer, __dirname + '/server', opts)
+      , orgPid = null;
+    srv.once('spawn', function () {
+      expect(srv.workers).to.have.length(1);
+      orgPid = srv.workers[0].pid
+      setTimeout(function () {
+        process.kill(orgPid, 'SIGKILL');
+        setTimeout(function ()  {
+          expect(srv.workers).to.have.length(1);
+          expect(srv.workers[0].pid).to.not.equal(orgPid);
+          done();
+        }, 300)  // give it time to die and respawn
+      }, 75)  // greater than minExpectedLifetime
+    });
+  });
 
 });