Version 0.8.9

Added support for custom maint record types (for running maint on custom days).
Home UI Worker: Max ceiling of 1000 simulated future jobs.
Added `track_manual_jobs` (default false) so manual jobs won't be added to activity log.
Now only setting one record expiration per job, and using a custom record type to delete both the job metadata and job log gzip.
Bumped pixl-server-storage version to 2.0.7 (for custom record types and efficient storage expiration).
Docs: Added recommended AWS settings for retries, clock skew and others to storage migration sample config.

README.md

@@ -2153,6 +2153,12 @@ The contents of the `NewStorage` object should match whatever you'd typically pu
 			"accessKeyId": "YOUR_AMAZON_ACCESS_KEY", 
 			"secretAccessKey": "YOUR_AMAZON_SECRET_KEY", 
 			"region": "us-west-1",
+			"correctClockSkew": true,
+			"maxRetries": 5,
+			"httpOptions": {
+				"connectTimeout": 5000,
+				"timeout": 5000
+			}
 		},
 		"S3": {
 			"keyPrefix": "cronicle",

bin/storage-cli.js

@@ -96,6 +96,17 @@ var storage = new StandaloneStorage(config.Storage, function(err) {
 		process.setuid( config.uid );
 	}
 
+	// custom job data expire handler
+	storage.addRecordType( 'cronicle_job', {
+		'delete': function(key, value, callback) {
+			storage.delete( key, function(err) {
+				storage.delete( key + '/log.txt.gz', function(err) {
+					callback();
+				} ); // delete
+			} ); // delete
+		}
+	} );
+
 	// process command
 	var cmd = commands.shift();
 	verbose("\n");

htdocs/js/home-worker.js

@@ -25,6 +25,7 @@ onmessage = function(e) {
 	var categories = data.categories;
 	var plugins = data.plugins;
 	var events = [];
+	var max_events = 1000;
 
 	var now = normalize_time( time_now(), { sec: 0 } );
 	var max_epoch = now + 86400 + 3600;
@@ -66,6 +67,7 @@ onmessage = function(e) {
 						var actual = hour_start + (min * 60);
 						if ((actual >= min_epoch) && (actual < max_epoch)) {
 							events.push({ epoch: actual, id: item.id });
+							if (events.length >= max_events) { idy = ley; epoch = max_epoch; idx = len; }
 						}
 					} // foreach minute
 				} // individual minutes
@@ -75,6 +77,7 @@ onmessage = function(e) {
 						var actual = hour_start + (idy * 60);
 						if ((actual >= min_epoch) && (actual < max_epoch)) {
 							events.push({ epoch: actual, id: item.id });
+							if (events.length >= max_events) { idy = ley; epoch = max_epoch; idx = len; }
 						}
 					} // foreach minute
 				} // every minute

lib/api/event.js

@@ -392,7 +392,7 @@ module.exports = Class.create({
 						var job = jobs_launched[idx];
 						var stub = { id: job.id, event: job.event };
 						self.logTransaction('job_run', job.event_title, self.getClientInfo(args, stub));
-						self.logActivity('job_run', stub, args);
+						if (self.server.config.get('track_manual_jobs')) self.logActivity('job_run', stub, args);
 						ids.push( job.id );
 					}
 

lib/engine.js

@@ -65,6 +65,11 @@ module.exports = Class.create({
 		this.api = this.server.API;
 		this.usermgr = this.server.User;
 
+		// register custom storage type for dual-metadata-log delete
+		this.storage.addRecordType( 'cronicle_job', {
+			'delete': this.deleteExpiredJobData.bind(this)
+		} );
+
 		// multi-server cluster / failover system
 		this.multi = {
 			cluster: false,
@@ -897,6 +902,25 @@ module.exports = Class.create({
 		}
 	},
 
+	deleteExpiredJobData: function(key, data, callback) {
+		// delete both job data and job log
+		// called from storage maintenance system for 'cronicle_job' record types
+		var self = this;
+		var log_key = key + '/log.txt.gz';
+
+		this.logDebug(6, "Deleting expired job data: " + key);
+		this.storage.delete( key, function(err) {
+			if (err) self.logError('maint', "Failed to delete expired job data: " + key + ": " + err);
+
+			self.logDebug(6, "Deleting expired job log: " + log_key);
+			self.storage.delete( log_key, function(err) {
+				if (err) self.logError('maint', "Failed to delete expired job log: " + log_key + ": " + err);
+
+				callback();
+			} ); // delete
+		} ); // delete
+	},
+
 	_uniqueIDCounter: 0,
 	getUniqueID: function(prefix) {
 		// generate unique id using high-res server time, and a static counter,

lib/job.js

@@ -1231,18 +1231,17 @@ module.exports = Class.create({
 			this.logError('job', "Job failed: " + job.id, job);
 		}
 
-		// set expiration on job log
-		var log_path = 'jobs/' + job.id + '/log.txt.gz';
-		this.storage.expire( log_path, Tools.timeNow(true) + (86400 * (job.log_expire_days || this.server.config.get('job_data_expire_days'))) );
-
 		// add to global activity, event log, and completed events
 		var data = Tools.copyHash(job);
 
+		// add special 'type' property for storage custom maint delete
+		data.type = 'cronicle_job';
+
 		// store job in its own record
 		this.storage.enqueue( function(task, callback) {
 			self.storage.put( 'jobs/' + job.id, data, callback );
 		});
-		this.storage.expire( 'jobs/' + job.id, Tools.timeNow(true) + (86400 * this.server.config.get('job_data_expire_days')) );
+		this.storage.expire( 'jobs/' + job.id, Tools.timeNow(true) + (86400 * (job.log_expire_days || this.server.config.get('job_data_expire_days'))) );
 
 		// create stub containing a small subset of the job data, for lists
 		var stub = {

lib/test.js

@@ -36,6 +36,7 @@ config.job_startup_grace = 1;
 config.Storage.Filesystem.base_dir = "data/unittest";
 config.web_hook_config_keys = ["base_app_url", "something_custom"];
 config.something_custom = "nonstandard property";
+config.track_manual_jobs = true;
 
 // chdir to the proper server root dir
 process.chdir( require('path').dirname( __dirname ) );

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "Cronicle",
-	"version": "0.8.8",
+	"version": "0.8.9",
 	"description": "A simple, distributed task scheduler and runner with a web based UI.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/Cronicle",
@@ -51,7 +51,7 @@
 		"pixl-mail": "^1.0.8",
 		"pixl-perf": "^1.0.5",
 		"pixl-server": "^1.0.13",
-		"pixl-server-storage": "^2.0.1",
+		"pixl-server-storage": "^2.0.7",
 		"pixl-server-web": "^1.1.2",
 		"pixl-server-api": "^1.0.1",
 		"pixl-server-user": "^1.0.8"

sample_conf/config.json

@@ -24,6 +24,7 @@
 	"udp_broadcast_port": 3014,
 	"scheduler_startup_grace": 10,
 	"universal_web_hook": "",
+	"track_manual_jobs": false,
 
 	"server_comm_use_hostnames": 0,
 	"web_socket_use_hostnames": 0,