Reimplemented kill-timeout in lithos_knot

docs/changelog.rst

@@ -3,6 +3,18 @@ Lithos Changes By Release
 =========================
 
 
+.. _changelog 0.18.4:
+
+v0.18.4
+=======
+
+* Bugfix: only send SIGTERM to the process once when upgrading or stopping it
+  (this prevents certain issues with the applications themselves)
+* Bugfix: use don't reset kill timeout on SIGQUIT of lithos_tree
+* Bugfix: correctly wait for kill timeout for retired children (not in the
+  config any more)
+
+
 .. _changelog 0.18.3:
 
 v0.18.3

src/bin/lithos_knot/main.rs

@@ -55,32 +55,39 @@ mod secrets;
 
 struct SignalIter<'a> {
     trap: &'a mut Trap,
-    interrupt: bool,
+    deadline: Option<Instant>,
 }
 
 impl<'a> SignalIter<'a> {
     fn new(trap: &mut Trap) -> SignalIter {
         SignalIter {
             trap: trap,
-            interrupt: false,
+            deadline: None,
         }
     }
+    fn set_deadline(&mut self, time: Instant) {
+        self.deadline = Some(time);
+    }
     fn interrupt(&mut self) {
-        self.interrupt = true;
+        self.deadline = Some(Instant::now());
     }
 }
 
 impl<'a> Iterator for SignalIter<'a> {
     type Item = Signal;
     fn next(&mut self) -> Option<Signal> {
-        if self.interrupt {
-            return self.trap.wait(Instant::now());
+        if let Some(dline) = self.deadline {
+            return self.trap.wait(dline);
         } else {
             return self.trap.next();
         }
     }
 }
 
+fn duration(inp: f32) -> Duration {
+    Duration::from_millis((inp * 1000.) as u64)
+}
+
 fn run(options: &Options) -> Result<i32, String>
 {
     let master: MasterConfig = try!(parse_config(&options.master_config,
@@ -312,6 +319,8 @@ fn run(options: &Options) -> Result<i32, String>
     let mut exit_code = 2;
     loop {
         let start = Instant::now();
+        let mut killed = false;
+        let mut dead = false;
 
         if !local.interactive {
             if let Some(ref path) = local.stdout_stderr_file {
@@ -361,11 +370,18 @@ fn run(options: &Options) -> Result<i32, String>
                     debug!("Received SIGTERM signal, propagating");
                     should_exit = true;
                     exit_code = 0;
-                    child.signal(SIGTERM).ok();
+                    if !killed {
+                        if let Ok(()) = child.signal(SIGTERM) {
+                            killed = true;
+                        }
+                        iter.set_deadline(
+                            Instant::now() + duration(container.kill_timeout));
+                    }
                 }
                 SIGCHLD => {
                     for (pid, status) in reap_zombies() {
                         if pid == child.pid() {
+                            dead = true;
                             if status.signal() == Some(SIGTERM as i32) ||
                                 status.code().map(|c| {
                                     if container.normal_exit_codes.is_empty() {
@@ -395,6 +411,22 @@ fn run(options: &Options) -> Result<i32, String>
                 _ => unreachable!(),
             }
         }
+        if !dead {
+            let uptime = Instant::now() - start;
+            error!("Process {:?} \
+                did not respond to SIGTERM in {}s, uptime {}s. \
+                Killing container so hanging process will die.",
+                options.name, container.kill_timeout, uptime.as_secs());
+            stderr_file.write_all(
+                format!("{}: ----- \
+                    Process {:?} did not respond to SIGTERM in {}, \
+                    uptime {}s. Killing.. -----\n",
+                    format_rfc3339_seconds(SystemTime::now()),
+                    options.name, container.kill_timeout, uptime.as_secs(),
+                ).as_bytes()
+            ).ok();
+            return Ok(3);
+        }
 
         if should_exit {
             break;

src/bin/lithos_tree/args.rs

@@ -13,8 +13,6 @@ pub enum Child {
     Error,
 }
 
-fn discard<E>(_: E) { }
-
 pub fn read(pid: Pid, global_config: &Path) -> Child {
     use self::Child::*;
     let start = Instant::now();

src/bin/lithos_tree/main.rs

@@ -26,7 +26,6 @@ use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::time::{SystemTime, Instant, Duration};
 use std::process::exit;
-use std::thread::sleep;
 use std::collections::{HashMap, BTreeMap, HashSet};
 use std::os::unix::io::{RawFd, AsRawFd};