Fix some logrotate issues.

- The logs for the root perpd process weren't being reopened during log
  rotation.
- If API Umbrella was running under a different user (not
  "api-umbrella"), then the logrotate process would lead to files being
  created which couldn't be written to, which could eventually lock up
  the Ruby web processes (they would seem to lock up once the last file
  they were being written to got moved to a gzip file).

  This issues impacted the vagrant development environment (where we run
  everything as the vagrant users) if things were running for more than
  a couple days.

build/package/files/etc/logrotate.d/api-umbrella

@@ -3,7 +3,6 @@
 /opt/api-umbrella/var/log/*/current /opt/api-umbrella/var/log/nginx/*.log /opt/api-umbrella/var/log/rsyslog/*.log {
   daily
   rotate 90
-  create 644 api-umbrella api-umbrella
   missingok
   compress
   delaycompress
@@ -18,7 +17,6 @@
 /opt/api-umbrella/var/log/nginx/*.log.gz {
   daily
   rotate 90
-  create 644 api-umbrella api-umbrella
   missingok
   nocompress
   notifempty
@@ -33,7 +31,6 @@
 /opt/api-umbrella/var/log/rsyslog/*.log.gz {
   daily
   rotate 21
-  create 644 api-umbrella api-umbrella
   missingok
   nocompress
   notifempty
@@ -48,7 +45,6 @@
 /opt/api-umbrella/var/log/elasticsearch/*.log /opt/api-umbrella/var/log/trafficserver/*.blog /opt/api-umbrella/var/log/trafficserver/*.log /opt/api-umbrella/var/log/trafficserver/*.out {
   daily
   rotate 90
-  create 644 api-umbrella api-umbrella
   missingok
   compress
   delaycompress

build/package_dependencies.sh

@@ -39,14 +39,17 @@ if [ -f /etc/redhat-release ]; then
 
     # ElasticSearch
     java-1.8.0-openjdk-headless
-    # For getopt, should no longer be necessary in ElasticSearch 2:
-    # https://github.com/elastic/elasticsearch/pull/12165
-    $util_linux_package
     which
 
     # init.d script helpers
     initscripts
 
+    # For kill used in stop/reopen-logs commands.
+    $util_linux_package
+
+    # For pstree used in reopen-logs command.
+    psmisc
+
     # For pkill/pgrep used for legacy status/stop commands.
     $procps_package
   )
@@ -142,6 +145,12 @@ elif [ -f /etc/debian_version ]; then
     sysvinit-utils
     lsb-base
 
+    # For kill used in stop/reopen-logs commands.
+    procps
+
+    # For pstree used in reopen-logs command.
+    psmisc
+
     # For pkill/pgrep used for legacy status/stop commands.
     procps
   )

src/api-umbrella/cli/reopen_logs.lua

@@ -3,19 +3,27 @@ local read_config = require "api-umbrella.cli.read_config"
 local run_command = require "api-umbrella.utils.run_command"
 local status = require "api-umbrella.cli.status"
 
-local function reopen_perp_logs(perp_base)
-  local _, output, err = run_command("perpls -g -b " .. perp_base)
+local function reopen_perp_logs(parent_pid)
+  -- Use pstree and parse the output to find all the log processes under the
+  -- root process.
+  --
+  -- We use this instead of perpctl for finding the log processes, since
+  -- perpctl doesn't seem to have a way to send signals to the root perpd's log
+  -- process (just the services underneath perpd). Since we also want to be
+  -- sure to reopen perpd's logs, we need to use this approach.
+  local _, output, err = run_command("pstree -p -A " .. parent_pid)
   if err then
     print("Failed to reopen logs for perp\n" .. err)
     os.exit(1)
   end
 
+  local log_process_name = "svlogd"
   for line in string.gmatch(output, "[^\r\n]+") do
-    local service_status, service = string.match(line, "^%[(.) .-%]%s+(%S+)")
-    if service_status == "+" then
-      local _, _, reload_err = run_command("perpctl -L -b " .. perp_base .. " hup " .. service)
+    local log_pid = string.match(line, log_process_name .. "%((%d+)%)")
+    if log_pid then
+      local _, _, reload_err = run_command("kill -s HUP " .. log_pid)
       if reload_err then
-        print("Failed to reopen logs for " .. service .. "\n" .. reload_err)
+        print("Failed to reopen logs for " .. log_pid .. "\n" .. reload_err)
         os.exit(1)
       end
     end
@@ -39,7 +47,7 @@ local function reopen_rsyslog(perp_base)
 end
 
 return function()
-  local running = status()
+  local running, pid = status()
   if not running then
     print("api-umbrella is stopped")
     os.exit(1)
@@ -48,7 +56,7 @@ return function()
   local config = read_config()
   local perp_base = path.join(config["etc_dir"], "perp")
 
-  reopen_perp_logs(perp_base)
+  reopen_perp_logs(pid)
 
   if config["_service_router_enabled?"] then
     reopen_nginx(perp_base)