added 'rate_limit' config variable to mitigate issues where a failing…

… script is restarted on every cycle of the dispatch loop

config.json

@@ -6,6 +6,8 @@
 	"log": "custodian.log",
 	"pid": "custodian.pid",
 
+	"rate_limit": "60",
+
 	"schedule": {
 		"job1": {"cmd":"test/job1.sh", "args":["last_run"], "when":"every 2s"},
 		"job2": {"cmd":"test/job2.sh", "args":["last_run"], "when":"after job1"},
@@ -17,6 +19,10 @@
 			"cmd":    "test/watch1.sh",
 			"notify": false,
 			"output": "watch1.log"
+		},
+		"watch2": {
+			"cmd":    "test/watch2.sh",
+			"notify": false
 		}
 	}
 }

custodian.js

@@ -14,8 +14,6 @@
  * running, it will be restarted.
  *
  * This code works on NodeJS 0.6.10.
- *
- * TODO: use nodules for hot-loading config?
  */
 
 var util       = require("util");
@@ -130,9 +128,6 @@ function run()
 {
 	// init state
 	init_state();
-	//STATE = {schedule:{}, watch:{}};
-	//for(var x in CONFIG.schedule) STATE.schedule[x] = {running: false, last_run: new Date("1980/01/01 00:00:00")};
-	//for(var x in CONFIG.watch)    STATE.watch[x]    = {pid: 0, last_restart: 0};
 
 	function log(str) {
 		var now = dateFormat(new Date, "yyyy-mm-dd HH:MM:ss");
@@ -164,7 +159,20 @@ function run()
 
 				if(is_running) return;
 
-				if(state.output_fd) fs.closeSync(state.output_fd);
+				if(state.output_fd) {
+					fs.closeSync(state.output_fd);
+					state.output_fd = 0;
+				}
+
+				// if `rate_limit` is set, don't restart the job more than once
+				// every X seconds
+				if(CONFIG.rate_limit) {
+					var now = (new Date).getTime();
+					if(now - state.last_restart < (CONFIG.rate_limit * 1000)) {
+						log(x+" was started less than "+CONFIG.rate_limit+" seconds ago, deferring");
+						return;
+					}
+				}
 
 				log(x+" is not running, restarting");
 				if(cfg.output) {
@@ -190,6 +198,7 @@ function run()
 				}
 				var c = cproc.spawn(cmd, args, opts);
 				state.pid = c.pid;
+				state.last_restart = (new Date()).getTime();
 				log("   pid: "+c.pid);
 
 				if(state.output_fd) {

package.json

@@ -2,9 +2,9 @@
   "author": "Judd Vinet <judd@betsmartmedia.com>",
   "name": "custodian",
   "description": "A different sort of cron",
-  "version": "1.2.5",
+  "version": "1.2.6",
   "main": "custodian.js",
-	"bin": "custodian.js",
+  "bin": "custodian.js",
   "engines": {
     "node": "~0.6.0"
   },
@@ -15,7 +15,7 @@
   "dependencies": {
     "dateformat":  "",
     "daemon":      "0.4.1",
-		"nodemailer":  "0.2.4"
+    "nodemailer":  "0.2.4"
   },
   "devDependencies": {}
 }

test/watch2.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# exit immediately to test rate-limiting
+exit 1
+