Permalink
Browse files

add restart limiting

  • Loading branch information...
1 parent c0bfbbe commit a0d347be95605dd118aefd35be42dd86f9b2ba93 @tj committed Nov 10, 2012
Showing with 139 additions and 21 deletions.
  1. +47 −11 deps/ms/ms.c
  2. +92 −10 src/mon.c
View
@@ -10,6 +10,15 @@
#include <stdio.h>
#include "ms.h"
+// microseconds
+
+#define US_SEC 1000000
+#define US_MIN 60 * US_SEC
+#define US_HOUR 60 * US_MIN
+#define US_DAY 24 * US_HOUR
+#define US_WEEK 7 * US_HOUR
+#define US_YEAR 52 * US_WEEK
+
// milliseconds
#define MS_SEC 1000
@@ -30,10 +39,12 @@ string_to_microseconds(const char *str) {
long long val = strtoll(str, NULL, 10);
if (!val) return -1;
switch (str[len - 1]) {
- case 's': return 'm' == str[len - 2] ? val * 1000 : val * 1000000;
- case 'm': return val * 60000000;
- case 'h': return val * 3600000000;
- case 'd': return val * 86400000000;
+ case 's': return 'm' == str[len - 2] ? val * 1000 : val * US_SEC;
+ case 'm': return val * US_MIN;
+ case 'h': return val * US_HOUR;
+ case 'd': return val * US_DAY;
+ case 'w': return val * US_WEEK;
+ case 'y': return val * US_YEAR;
default: return val;
}
}
@@ -50,14 +61,27 @@ string_to_milliseconds(const char *str) {
if (!val) return -1;
switch (str[len - 1]) {
case 's': return 'm' == str[len - 2] ? val : val * 1000;
- case 'm': return val * 60 * 1000;
- case 'h': return val * 60 * 60 * 1000;
- case 'd': return val * 24 * 60 * 60 * 1000;
+ case 'm': return val * MS_MIN;
+ case 'h': return val * MS_HOUR;
+ case 'd': return val * MS_DAY;
+ case 'w': return val * MS_WEEK;
+ case 'y': return val * MS_YEAR;
default: return val;
}
}
/*
+ * Convert the given `str` representation to seconds.
+ */
+
+long long
+string_to_seconds(const char *str) {
+ long long ret = string_to_milliseconds(str);
+ if (-1 == ret) return ret;
+ return ret / 1000;
+}
+
+/*
* Convert the given `ms` to a string. This
* value must be `free()`d by the developer.
*/
@@ -143,7 +167,6 @@ test_string_to_microseconds() {
assert(string_to_microseconds("1m") == 60000000);
assert(string_to_microseconds("1h") == 3600000000);
assert(string_to_microseconds("2d") == 2 * 24 * 3600000000);
- assert(strtous("1ms") == 1000);
}
void
@@ -159,12 +182,25 @@ test_string_to_milliseconds() {
assert(string_to_milliseconds("1m") == 60 * 1000);
assert(string_to_milliseconds("1h") == 60 * 60 * 1000);
assert(string_to_milliseconds("1d") == 24 * 60 * 60 * 1000);
- assert(strtoms("50s") == 50000);
+}
+
+void
+test_string_to_seconds() {
+ assert(string_to_seconds("") == -1);
+ assert(string_to_seconds("s") == -1);
+ assert(string_to_seconds("hey") == -1);
+ assert(string_to_seconds("5000") == 5);
+ assert(string_to_seconds("1ms") == 0);
+ assert(string_to_seconds("5ms") == 0);
+ assert(string_to_seconds("1s") == 1);
+ assert(string_to_seconds("5s") == 5);
+ assert(string_to_seconds("1m") == 60);
+ assert(string_to_seconds("1h") == 60 * 60);
+ assert(string_to_seconds("1d") == 24 * 60 * 60);
}
void
test_milliseconds_to_string() {
- equal("500ms", mstostr(500));
equal("500ms", milliseconds_to_string(500));
equal("5s", milliseconds_to_string(5000));
equal("2s", milliseconds_to_string(2500));
@@ -178,7 +214,6 @@ test_milliseconds_to_string() {
void
test_milliseconds_to_long_string() {
- equal("less than one second", mstolstr(500));
equal("less than one second", milliseconds_to_long_string(500));
equal("5 seconds", milliseconds_to_long_string(5000));
equal("2 seconds", milliseconds_to_long_string(2500));
@@ -195,6 +230,7 @@ int
main(){
test_string_to_microseconds();
test_string_to_milliseconds();
+ test_string_to_seconds();
test_milliseconds_to_string();
test_milliseconds_to_long_string();
printf("\n \e[32m\u2713 \e[90mok\e[0m\n\n");
View
102 src/mon.c
@@ -12,6 +12,7 @@
#include <string.h>
#include <fcntl.h>
#include <signal.h>
+#include <stdint.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/wait.h>
@@ -46,8 +47,12 @@ typedef struct {
const char *mon_pidfile;
const char *logfile;
const char *on_error;
+ int64_t last_restart_at;
+ int64_t clock;
int daemon;
int sleepsec;
+ int max_attempts;
+ int attempts;
} monitor_t;
/*
@@ -93,6 +98,18 @@ graceful_exit(int sig) {
}
/*
+ * Return a timestamp in milliseconds.
+ */
+
+int64_t
+timestamp() {
+ struct timeval tv;
+ int ret = gettimeofday(&tv, NULL);
+ if (-1 == ret) return -1;
+ return (int64_t) ((int64_t) tv.tv_sec * 1000 + (int64_t) tv.tv_usec / 1000);
+}
+
+/*
* Write `pid` to `file`.
*/
@@ -198,6 +215,55 @@ daemonize() {
}
/*
+ * Invoke the --on-error command.
+ */
+
+void
+exec_error_command(monitor_t *monitor) {
+ log("on error \"%s\"", monitor->on_error);
+ int status = system(monitor->on_error);
+ if (status) {
+ log("exit(%d)", status);
+ log("shutting down");
+ exit(status);
+ }
+}
+
+/*
+ * Return the ms since the last restart.
+ */
+
+int64_t
+ms_since_last_restart(monitor_t *monitor) {
+ if (0 == monitor->last_restart_at) return 0;
+ int64_t now = timestamp();
+ return now - monitor->last_restart_at;
+}
+
+/*
+ * Check if the maximum restarts within 60 seconds
+ * have been exceeded and return 1, 0 otherwise.
+ */
+
+int
+attempts_exceeded(monitor_t *monitor, int64_t ms) {
+ monitor->attempts++;
+ monitor->clock -= ms;
+
+ // reset
+ if (monitor->clock <= 0) {
+ monitor->clock = 60000;
+ monitor->attempts = 0;
+ return 0;
+ }
+
+ // all good
+ if (monitor->attempts < monitor->max_attempts) return 0;
+
+ return 1;
+}
+
+/*
* Monitor the given `cmd`.
*/
@@ -246,18 +312,19 @@ exec: {
goto error;
}
- // alerts
+ // restart
error: {
- if (monitor->on_error) {
- log("on error \"%s\"", monitor->on_error);
- int status = system(monitor->on_error);
- if (status) {
- log("exit(%d)", status);
- log("shutting down");
- exit(status);
- }
+ if (monitor->on_error) exec_error_command(monitor);
+ int64_t ms = ms_since_last_restart(monitor);
+ monitor->last_restart_at = timestamp();
+ log("last restart %s ago", milliseconds_to_long_string(ms));
+ log("%d attempts remaining", monitor->max_attempts - monitor->attempts);
+ if (attempts_exceeded(monitor, ms)) {
+ char *time = milliseconds_to_long_string(60000 - monitor->clock);
+ log("%d restarts within %s", monitor->max_attempts, time);
+ log("bailing");
+ exit(2);
}
-
goto exec;
}
}
@@ -347,6 +414,16 @@ on_error(command_t *self) {
}
/*
+ * --attempts <n>
+ */
+
+static void
+on_attempts(command_t *self) {
+ monitor_t *monitor = (monitor_t *) self->data;
+ monitor->attempts = atoi(self->arg);
+}
+
+/*
* [options] <cmd>
*/
@@ -359,6 +436,10 @@ main(int argc, char **argv){
monitor.logfile = "mon.log";
monitor.daemon = 0;
monitor.sleepsec = 1;
+ monitor.max_attempts = 10;
+ monitor.attempts = 0;
+ monitor.last_restart_at = 0;
+ monitor.clock = 60000;
command_t program;
command_init(&program, "mon", VERSION);
@@ -372,6 +453,7 @@ main(int argc, char **argv){
command_option(&program, "-P", "--prefix <str>", "add a log prefix", on_prefix);
command_option(&program, "-d", "--daemonize", "daemonize the program", on_daemonize);
command_option(&program, "-e", "--on-error <cmd>", "execute <cmd> on errors", on_error);
+ command_option(&program, "-a", "--attempts <n>", "retry attempts within 60 seconds [10]", on_attempts);
command_parse(&program, argc, argv);
// command required

0 comments on commit a0d347b

Please sign in to comment.