Implement staged kill: first sigterm, then sigkill

Closes #67
rfjakob · Jul 22, 2018 · bcd1ad4 · bcd1ad4
1 parent 5f796f9
commit bcd1ad4
Show file tree

Hide file tree

Showing 8 changed files with 149 additions and 101 deletions.
diff --git a/MANPAGE.md b/MANPAGE.md
@@ -25,21 +25,27 @@ If there is a failure when trying to kill a process, **earlyoom** sleeps for
 
 # OPTIONS
 
-#### -m PERCENT
-set available memory minimum to PERCENT of total (default 10 %)
+#### -m PERCENT[,KILL_PERCENT]
+set available memory minimum to PERCENT of total (default 10 %).
+Send sigkill if at or below KILL_PERCENT (default 1/2 PERCENT), otherwise sigterm.
 
-#### -s PERCENT
+Use the same value for PERCENT and KILL_PERCENT if you always want to use sigkill.
+
+#### -s PERCENT[,KILL_PERCENT]
 set free swap minimum to PERCENT of total (default 10 %).
+Send sigkill if at or below KILL_PERCENT (default 1/2 PERCENT), otherwise sigterm.
 
 You can use `-s 100` to have earlyoom effectively ignore swap usage:
 Processes are killed once available memory drops below the configured
 minimum, no matter how much swap is free.
 
-#### -M SIZE
-set available memory minimum to SIZE KiB
+#### -M SIZE[,KILL_SIZE]
+set available memory minimum to SIZE KiB.
+Send sigkill if at or below KILL_SIZE (default 1/2 SIZE), otherwise sigterm.
 
-#### -S SIZE
-set free swap minimum to SIZE KiB
+#### -S SIZE[,KILL_SIZE]
+set free swap minimum to SIZE KiB.
+Send sigkill if at or below KILL_SIZE (default 1/2 SIZE), otherwise sigterm.
 
 #### -k
 removed in earlyoom v1.2, ignored for compatibility

diff --git a/README.md b/README.md
@@ -217,6 +217,8 @@ Changelog
   * Gracefully handle the case of swap being added or removed after earlyoom was started
     ([issue 62](https://github.com/rfjakob/earlyoom/issues/62),
     [commit](https://github.com/rfjakob/earlyoom/commit/88e58903fec70b105aebba39cd584add5e1d1532))
+  * Implement staged kill: first sigterm, then sigkill, with configurable limits
+    ([issue #67](https://github.com/rfjakob/earlyoom/issues/67))
 * v1.1, 2018-07-07
   * Fix possible shell code injection through GUI notifications
     ([commit](https://github.com/rfjakob/earlyoom/commit/ab79aa3895077676f50120f15e2bb22915446db9))

diff --git a/kill.c b/kill.c
@@ -222,8 +222,8 @@ void userspace_kill(poll_loop_args_t args, int sig)
 
     // sig == 0 is used as a self-test during startup. Don't notifiy the user.
     if (sig != 0) {
-        fprintf(stderr, "Killing process: %s, pid: %d, badness: %d, VmRSS: %lu MiB\n",
-            victim_name, victim_pid, victim_badness, victim_vm_rss / 1024);
+        warn("Killing process '%s' with signal %d, pid: %d, badness: %d, VmRSS: %lu MiB\n",
+            victim_name, sig, victim_pid, victim_badness, victim_vm_rss / 1024);
     }
 
     int res = kill(victim_pid, sig);

diff --git a/kill.h b/kill.h
@@ -9,8 +9,10 @@ typedef struct {
     DIR* procdir;
     /* if the available memory AND swap goes below these percentages,
      * we start killing processes */
-    int mem_min_percent;
-    int swap_min_percent;
+    int mem_term_percent;
+    int mem_kill_percent;
+    int swap_term_percent;
+    int swap_kill_percent;
     /* ignore /proc/PID/oom_score_adj? */
     bool ignore_oom_score_adj;
     /* notifcation command to launch when killing something. NULL = no-op. */

diff --git a/main.c b/main.c
@@ -13,6 +13,7 @@
 #include <sys/mman.h>
 #include <sys/resource.h>
 #include <unistd.h>
+#include <signal.h>
 
 #include "kill.h"
 #include "meminfo.h"
@@ -26,7 +27,7 @@ enum {
 };
 
 static int set_oom_score_adj(int);
-static void print_mem_stats(FILE* procdir, const meminfo_t m);
+static void print_mem_stats(bool lowmem, const meminfo_t m);
 static void poll_loop(const poll_loop_args_t args);
 
 int enable_debug = 0;
@@ -35,10 +36,13 @@ long page_size = 0;
 int main(int argc, char* argv[])
 {
     poll_loop_args_t args = {
+        .mem_term_percent = 10,
+        .swap_term_percent = 10,
+        .mem_kill_percent = 5,
+        .swap_kill_percent = 5,
         .report_interval_ms = 1000,
         /* omitted fields are set to zero */
     };
-    long mem_min_kib = 0, swap_min_kib = 0; /* Same thing in KiB */
     int set_my_priority = 0;
     char* prefer_cmds = NULL;
     char* avoid_cmds = NULL;
@@ -71,38 +75,41 @@ int main(int argc, char* argv[])
         { "help", no_argument, NULL, 'h' },
         { 0, 0, NULL, 0 } /* end-of-array marker */
     };
+    bool have_m = 0, have_M = 0, have_s = 0, have_S = 0;
 
     while ((c = getopt_long(argc, argv, short_opt, long_opt, NULL)) != -1) {
         float report_interval_f = 0;
+        term_kill_tuple_t tuple;
+
         switch (c) {
         case -1: /* no more arguments */
         case 0: /* long option toggles */
             break;
         case 'm':
-            args.mem_min_percent = strtol(optarg, NULL, 10);
-            // Using "-m 100" makes no sense
-            if (args.mem_min_percent <= 0 || args.mem_min_percent >= 100) {
-                fatal(15, "-m: invalid percentage '%s'\n", optarg);
-            }
+            // Use 99 as upper limit. Passing "-m 100" makes no sense.
+            tuple = parse_term_kill_tuple("-m", optarg, 99, 15);
+            args.mem_term_percent = tuple.term;
+            args.mem_kill_percent = tuple.kill;
+            have_m = 1;
             break;
         case 's':
-            args.swap_min_percent = strtol(optarg, NULL, 10);
             // Using "-s 100" is a valid way to ignore swap usage
-            if (args.swap_min_percent <= 0 || args.swap_min_percent > 100) {
-                fatal(16, "-s: invalid percentage: '%s'\n", optarg);
-            }
+            tuple = parse_term_kill_tuple("-s", optarg, 100, 16);
+            args.swap_term_percent = tuple.term;
+            args.swap_kill_percent = tuple.kill;
+            have_s = 1;
             break;
         case 'M':
-            mem_min_kib = strtol(optarg, NULL, 10);
-            if (mem_min_kib <= 0) {
-                fatal(15, "-M: invalid KiB value '%s'\n", optarg);
-            }
+            tuple = parse_term_kill_tuple("-M", optarg, m.MemTotalKiB * 100 / 99, 15);
+            args.mem_term_percent = 100 * tuple.term / m.MemTotalKiB;
+            args.mem_kill_percent = 100 * tuple.kill / m.MemTotalKiB;
+            have_M = 1;
             break;
         case 'S':
-            swap_min_kib = strtol(optarg, NULL, 10);
-            if (swap_min_kib <= 0) {
-                fatal(16, "-S: invalid KiB value: '%s'\n", optarg);
-            }
+            tuple = parse_term_kill_tuple("-S", optarg, m.SwapTotalKiB * 100 / 99, 16);
+            args.swap_term_percent = 100 * tuple.term / m.SwapTotalKiB;
+            args.swap_kill_percent = 100 * tuple.kill / m.SwapTotalKiB;
+            have_S = 1;
             break;
         case 'k':
             fprintf(stderr, "Option -k is ignored since earlyoom v1.2\n");
@@ -170,57 +177,26 @@ int main(int argc, char* argv[])
     if (optind < argc) {
         fatal(13, "extra argument not understood: '%s'\n", argv[optind]);
     }
-
-    if (args.mem_min_percent && mem_min_kib) {
-        fatal(2, "can't use -m with -M\n");
+    if (have_m && have_M) {
+        fatal(2, "can't use both -m and -M\n");
     }
-
-    if (args.swap_min_percent && swap_min_kib) {
-        fatal(2, "can't use -s with -S\n");
+    if (have_s && have_S) {
+        fatal(2, "can't use both -s and -S\n");
     }
-
     if (prefer_cmds) {
         args.prefer_regex = &_prefer_regex;
         if (regcomp(args.prefer_regex, prefer_cmds, REG_EXTENDED | REG_NOSUB) != 0) {
             fatal(6, "could not compile regexp '%s'\n", prefer_cmds);
         }
         fprintf(stderr, "Prefering to kill process names that match regex '%s'\n", prefer_cmds);
     }
-
     if (avoid_cmds) {
         args.avoid_regex = &_avoid_regex;
         if (regcomp(args.avoid_regex, avoid_cmds, REG_EXTENDED | REG_NOSUB) != 0) {
             fatal(6, "could not compile regexp '%s'\n", avoid_cmds);
         }
         fprintf(stderr, "Avoiding to kill process names that match regex '%s'\n", avoid_cmds);
     }
-
-    if (mem_min_kib) {
-        if (mem_min_kib >= m.MemTotalKiB) {
-            fatal(15,
-                "-M: the value you passed (%ld kiB) is at or above total memory (%ld kiB)\n",
-                mem_min_kib, m.MemTotalKiB);
-        }
-        args.mem_min_percent = 100 * mem_min_kib / m.MemTotalKiB;
-    } else {
-        if (!args.mem_min_percent) {
-            args.mem_min_percent = 10;
-        }
-    }
-
-    if (swap_min_kib) {
-        if (swap_min_kib > m.SwapTotalKiB) {
-            fatal(16,
-                "-S: the value you passed (%ld kiB) is above total swap (%ld kiB)\n",
-                swap_min_kib, m.SwapTotalKiB);
-        }
-        args.swap_min_percent = 100 * swap_min_kib / m.SwapTotalKiB;
-    } else {
-        if (!args.swap_min_percent) {
-            args.swap_min_percent = 10;
-        }
-    }
-
     if (set_my_priority) {
         bool fail = 0;
         if (setpriority(PRIO_PROCESS, 0, -20) != 0) {
@@ -237,10 +213,10 @@ int main(int argc, char* argv[])
     }
 
     // Print memory limits
-    fprintf(stderr, "mem  total: %4d MiB, min: %2d %%\n",
-        m.MemTotalMiB, args.mem_min_percent);
-    fprintf(stderr, "swap total: %4d MiB, min: %2d %%\n",
-        m.SwapTotalMiB, args.swap_min_percent);
+    fprintf(stderr, "mem  total: %4d MiB, sending sigterm at %2d %%, sigkill at %2d %%\n",
+        m.MemTotalMiB, args.mem_term_percent, args.mem_kill_percent);
+    fprintf(stderr, "swap total: %4d MiB, sending sigterm at %2d %%, sigkill at %2d %%\n",
+        m.SwapTotalMiB, args.swap_term_percent, args.swap_kill_percent);
 
     /* Dry-run oom kill to make sure stack grows to maximum size before
      * calling mlockall()
@@ -259,10 +235,13 @@ int main(int argc, char* argv[])
  *   mem avail: 5259 MiB (67 %), swap free: 0 MiB (0 %)"
  * to the fd passed in out_fd.
  */
-static void print_mem_stats(FILE* out_fd, const meminfo_t m)
+static void print_mem_stats(bool lowmem, const meminfo_t m)
 {
-    fprintf(out_fd,
-        "mem avail: %4d of %4d MiB (%2d %%), swap free: %4d of %4d MiB (%2d %%)\n",
+    int(*out_func)(const char* fmt, ...) = &printf;
+    if(lowmem) {
+        out_func=&warn;
+    }
+    out_func("mem avail: %4d of %4d MiB (%2d %%), swap free: %4d of %4d MiB (%2d %%)\n",
         m.MemAvailableMiB,
         m.MemTotalMiB,
         m.MemAvailablePercent,
@@ -306,11 +285,11 @@ static int sleep_time_ms(const poll_loop_args_t* args, const meminfo_t* m)
     const int min_sleep = 100;
     const int max_sleep = 1000;
 
-    int mem_headroom_kib = (m->MemAvailablePercent - args->mem_min_percent) * 10 * m->MemTotalMiB;
+    int mem_headroom_kib = (m->MemAvailablePercent - args->mem_term_percent) * 10 * m->MemTotalMiB;
     if (mem_headroom_kib < 0) {
         mem_headroom_kib = 0;
     }
-    int swap_headroom_kib = (m->SwapFreePercent - args->swap_min_percent) * 10 * m->SwapTotalMiB;
+    int swap_headroom_kib = (m->SwapFreePercent - args->swap_term_percent) * 10 * m->SwapTotalMiB;
     if (swap_headroom_kib < 0) {
         swap_headroom_kib = 0;
     }
@@ -334,18 +313,19 @@ static void poll_loop(const poll_loop_args_t args)
     while (1) {
         m = parse_meminfo();
 
-        if (m.MemAvailablePercent <= args.mem_min_percent && m.SwapFreePercent <= args.swap_min_percent) {
-            fprintf(stderr,
-                "Low memory! mem avail: %d of %d MiB (%d) %% <= min %d %%, swap free: %d of %d MiB (%d %%) <= min %d %%\n",
-                m.MemAvailableMiB,
-                m.MemTotalMiB,
-                m.MemAvailablePercent,
-                args.mem_min_percent,
-                m.SwapFreeMiB,
-                m.SwapTotalMiB,
-                m.SwapFreePercent,
-                args.swap_min_percent);
-            userspace_kill(args, 9);
+        if (m.MemAvailablePercent <= args.mem_term_percent && m.SwapFreePercent <= args.swap_term_percent) {
+            int sig = 0;
+            if(m.MemAvailablePercent <= args.mem_kill_percent && m.SwapFreePercent <= args.swap_kill_percent) {
+                warn("Low memory! At or below sigkill limits (mem: %d %%, swap: %d %%)\n",
+                    args.mem_kill_percent, args.swap_kill_percent);
+                sig = SIGKILL;
+            } else {
+                warn("Low Memory! At or below sigterm limits (mem: %d %%, swap: %d %%)\n",
+                    args.mem_term_percent, args.swap_term_percent);
+                sig = SIGTERM;
+            }
+            print_mem_stats(1, m);
+            userspace_kill(args, sig);
             // With swap enabled, the kernel seems to need more than 100ms to free the memory
             // of the killed process. This means that earlyoom would immediately kill another
             // process. Sleep a little extra to give the kernel time to free the memory.
@@ -356,7 +336,7 @@ static void poll_loop(const poll_loop_args_t args)
                 report_countdown_ms -= cooldown_ms;
             }
         } else if (args.report_interval_ms && report_countdown_ms <= 0) {
-            print_mem_stats(stdout, m);
+            print_mem_stats(0, m);
             report_countdown_ms = args.report_interval_ms;
         }
         int sleep_ms = sleep_time_ms(&args, &m);

diff --git a/msg.c b/msg.c
@@ -5,6 +5,8 @@
 #include <stdlib.h>
 #include <unistd.h>
 
+#include "msg.h"
+
 // Print message to stderr and exit with "code".
 // Example: fatal(6, "could not compile regexp '%s'\n", regex_str);
 void fatal(int code, char* fmt, ...)
@@ -25,8 +27,9 @@ void fatal(int code, char* fmt, ...)
 }
 
 // Print a yellow warning message to stderr.
-void warn(char* fmt, ...)
+int warn(const char* fmt, ...)
 {
+    int ret = 0;
     char* yellow = "";
     char* reset = "";
     if (isatty(fileno(stderr))) {
@@ -37,6 +40,45 @@ void warn(char* fmt, ...)
     snprintf(fmt2, sizeof(fmt2), "%s%s%s", yellow, fmt, reset);
     va_list args;
     va_start(args, fmt); // yes fmt, NOT fmt2!
-    vfprintf(stderr, fmt2, args);
+    ret = vfprintf(stderr, fmt2, args);
     va_end(args);
+    return ret;
+}
+
+term_kill_tuple_t parse_term_kill_tuple(char* opt, char* optarg, long upper_limit, int exitcode)
+{
+    term_kill_tuple_t tuple = { 0 };
+    int n;
+
+    n = sscanf(optarg, "%ld,%ld", &tuple.term, &tuple.kill);
+    if (n == 0) {
+        fatal(exitcode, "%s: could not parse '%s'\n", opt, optarg);
+    }
+    if (tuple.term == 0) {
+        fatal(exitcode, "%s: zero sigterm value in '%s'\n", opt, optarg);
+    }
+    if (tuple.term < 0) {
+        fatal(exitcode, "%s: negative sigterm value in '%s'\n", opt, optarg);
+    }
+    if (tuple.term > upper_limit) {
+        fatal(exitcode, "%s: sigterm value in '%s' exceeds limit %ld\n",
+            opt, optarg, upper_limit);
+    }
+    // User passed only "term" value
+    if (n == 1) {
+        tuple.kill = tuple.term / 2;
+        return tuple;
+    }
+    // User passed "term,kill" values
+    if (tuple.kill == 0) {
+        fatal(exitcode, "%s: zero sigkill value in '%s'\n", opt, optarg);
+    }
+    if (tuple.kill < 0) {
+        fatal(exitcode, "%s: negative sigkill value in '%s'\n", opt, optarg);
+    }
+    if (tuple.kill > tuple.term) {
+        fatal(exitcode, "%s: sigkill value exceeds sigterm value in '%s'\n",
+            opt, optarg);
+    }
+    return tuple;
 }
diff --git a/msg.h b/msg.h
@@ -3,6 +3,13 @@
 #define MSG_H
 
 void fatal(int code, char* fmt, ...);
-void warn(char* fmt, ...);
+int warn(const char* fmt, ...);
+
+typedef struct {
+    long term;
+    long kill;
+} term_kill_tuple_t;
+
+term_kill_tuple_t parse_term_kill_tuple(char* opt, char* optarg, long upper_limit, int exitcode);
 
 #endif