osvcd enhancements

* close_fds=True on all osvcd Popen(), so that the socket fds are not
inherited by children. Most notably services spawning a dockerd end up
blocking the listeners <addr>:<port>
* display the nodes loadavg in "daemon status"
* remove the "@" node prefix. uselessly consuming horizontal space
* allow the "clear" action on frozen services
* more verbose error log in the listener bind error codepath

lib/node.py

@@ -2691,17 +2691,16 @@ def get_nodes():
         else:
             daemon_node = rcEnv.nodename
 
-        def load_svc_header():
+        def load_header(title):
             line = [
-                colorize("Services", color.GRAY),
+                colorize(title, color.GRAY),
                 "",
                 "",
                 "",
             ]
             for nodename in nodenames:
-                line.append(colorize("@" + nodename, color.GRAY))
+                line.append(colorize(nodename, color.GRAY))
             out.append(line)
-            return True
 
         def load_svc(svcname, data):
             if svcname not in self.services:
@@ -2745,17 +2744,6 @@ def load_svc(svcname, data):
                     line.append(colorize("?", color.RED))
             out.append(line)
 
-        def load_threads_header():
-            line = [
-                colorize("Threads", color.GRAY),
-                "",
-                "",
-                "",
-            ]
-            for nodename in nodenames:
-                line.append(colorize("@" + nodename, color.GRAY))
-            out.append(line)
-
         def load_hb(key, _data):
             if _data["state"] == "running":
                 state = colorize(_data["state"], color.GREEN)
@@ -2858,8 +2846,20 @@ def load_threads():
                 else:
                     load_thread(key, data[key])
 
+        def load_metrics():
+            for metric in ("1m", "5m", "15m"):
+                line = [
+                    colorize(" "+metric, color.BOLD),
+                    "",
+                    "",
+                    "|",
+                ]
+                for nodename in nodenames:
+                    line.append(str(data["monitor"]["nodes"][nodename].get("load", {}).get(metric, "")))
+                out.append(line)
+
         # init the services hash
-        for node in data.get("monitor", {}).get("nodes", []):
+        for node in nodenames:
             for svcname, _data in data["monitor"]["nodes"][node]["services"]["status"].items():
                 if svcname not in services:
                     services[svcname] = Storage({
@@ -2875,10 +2875,13 @@ def load_threads():
             services[svcname].avail = _data["avail"]
 
         # load data in lists
-        load_threads_header()
+        load_header("Threads")
         load_threads()
         out.append([])
-        load_svc_header()
+        load_header("Metrics")
+        load_metrics()
+        out.append([])
+        load_header("Services")
         for svcname in sorted(list(services.keys())):
             load_svc(svcname, services[svcname])
 

lib/osvcd.py

@@ -866,7 +866,7 @@ def run(self):
             self.sock.listen(5)
             self.sock.settimeout(self.sock_tmo)
         except socket.error as exc:
-            self.log.error("init error: %s", str(exc))
+            self.log.error("bind %s:%d error: %s", self.addr, self.port, str(exc))
             return
 
         self.log.info("listening on %s:%s", self.addr, self.port)
@@ -950,7 +950,7 @@ def _handle_client(self, conn, addr):
         self.stats.sessions.clients[addr[0]].auth_validated += 1
         if data is None:
             cmd = [rcEnv.paths.nodemgr, 'dequeue_actions']
-            p = Popen(cmd, stdout=None, stderr=None, stdin=None)
+            p = Popen(cmd, stdout=None, stderr=None, stdin=None, close_fds=True)
             p.communicate()
         else:
             result = self.router(nodename, data)
@@ -1082,7 +1082,7 @@ def do(self):
     def run_scheduler(self):
         self.log.info("run schedulers")
         cmd = [rcEnv.paths.nodemgr, 'schedulers']
-        proc = Popen(cmd, stdout=None, stderr=None, stdin=None)
+        proc = Popen(cmd, stdout=None, stderr=None, stdin=None, close_fds=True)
         self.procs.append(proc)
 
 #
@@ -1181,7 +1181,7 @@ def service_command(self, svcname, cmd):
     def _service_command(self, svcname, cmd):
         cmd = [rcEnv.paths.svcmgr, '-s', svcname] + cmd
         self.log.info("execute: %s", " ".join(cmd))
-        p = Popen(cmd, stdout=None, stderr=None, stdin=None)
+        p = Popen(cmd, stdout=None, stderr=None, stdin=None, close_fds=True)
         p.communicate()
         if p.returncode != 0:
             self.set_service_monitor(svcname, "start failed")
@@ -1418,7 +1418,7 @@ def get_last_svc_status_mtime(self, svcname):
     def service_status_fallback(self, svcname):
         self.log.info("slow path service status eval: %s", svcname)
         cmd = [rcEnv.paths.svcmgr, "-s", svcname, "json", "status"]
-        proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=True)
         out, err = proc.communicate()
         return json.loads(bdecode(out))
 
@@ -1483,6 +1483,7 @@ def update_hb_data(self):
             self.log.error("failed to refresh local cluster data: invalid json")
 
     def status(self):
+        self.update_hb_data()
         data = OsvcThread.status(self)
         with CLUSTER_DATA_LOCK:
             data.nodes = dict(CLUSTER_DATA)

lib/svc.py

@@ -65,6 +65,7 @@ def signal_handler(*args):
 ACTIONS_ALLOW_ON_FROZEN = [
     "autopush",
     "delete",
+    "clear",
     "docker",
     "disable",
     "edit_config",