Hack rage - capture pstacks and old logs

Summary:
When the user hits the bug button in their editor, it sends a "telemetry/rage" LSP request to clientLsp.ml. Currently this gathers up the .log and .monitor_log file.

This diff brings two changes:

* It now gathers up the .log.old and monitor_log.old files as well (so we can see why the previous instance crashed!)

* It now gathers PSTACKS for the server process and the monitor process, so if they've hanged, then we can see why.

Discussion: I had first considered obtaining all logs from the /tmp/hh_server/logs and /monitor_logs directories. But this would have combined logs from ALL different instances of hh_server on this box, rather than just the hh_server for this particular project root. I preferred to keep it local.

I had first considered doing "ps -aux | grep hh_server" to find all processes. But once again that would have obtained pstacks from ALL different instances of hh_server on this box. And the PidLogs.ml file does after all contain the logs I want.

I had first considered getting PSTACKS for *all* PIDs mentioned by PidLogs.ml, including all the workers. But there were ~30 of them, and they were all doing boring stuff, and they crowded out the logs, so it didn't seem worth it.

Reviewed By: arxanas

Differential Revision: D6882227

fbshipit-source-id: f8e58afd3c61469bede3ab052050e6de73663d78

hphp/hack/src/client/clientLsp.ml

@@ -506,42 +506,51 @@ let do_shutdown (state: state) : state =
 
 let do_rage (state: state) : Rage.result =
   let open Rage in
-  let logItems = match get_root_opt () with
-    | None -> []
-    | Some root ->
-      let monitor_log_link = ServerFiles.monitor_log_link root in
-      let log_link = ServerFiles.log_link root in
-      [
-        {
-          title = Some log_link;
-          data = Sys_utils.cat log_link;
-        };
-        {
-          title = Some monitor_log_link;
-          data = Sys_utils.cat monitor_log_link;
-        }
-      ]
-  in
-  let clientItems = [
-    {
-      title = None;
-      data = "LSP adapter state: " ^ (state_to_string state);
-    };
-  ]
-  in
-  let serverItems = match state with
-    | Main_loop menv ->
-      let open Main_env in
-      let items = rpc menv.conn ServerCommandTypes.RAGE in
-      let hack_to_lsp item = {
-        title = item.ServerRageTypes.title;
-        data = item.ServerRageTypes.data;
-        } in
-      List.map items ~f:hack_to_lsp
-    | _ ->
-      []
+  let items: rageItem list ref = ref [] in
+  let add item = items := item :: !items in
+  let add_data data = add { title = None; data; } in
+  let add_fn fn =  if Sys.file_exists fn then add { title = Some fn; data = Sys_utils.cat fn; } in
+  let add_stack (pid, reason) =
+    let pid = string_of_int pid in
+    let stack = try Sys_utils.exec_read_lines ~reverse:true ("pstack " ^ pid)
+    with _ -> begin
+      try Sys_utils.exec_read_lines ~reverse:true ("gstack " ^ pid)
+      with e -> ["unable to pstack - " ^ (Printexc.to_string e)]
+    end in
+    add_data (Printf.sprintf "PSTACK %s (%s) - %s\n\n" pid reason (String.concat "\n" stack))
   in
-  clientItems @ logItems @ serverItems
+  (* logfiles *)
+  begin match get_root_opt () with
+    | Some root -> begin
+        add_fn (ServerFiles.log_link root);
+        add_fn ((ServerFiles.log_link root) ^ ".old");
+        add_fn (ServerFiles.monitor_log_link root);
+        add_fn ((ServerFiles.monitor_log_link root) ^ ".old");
+        try
+          let pids = PidLog.get_pids (ServerFiles.pids_file root) in
+          let is_interesting (_, reason) = not (String_utils.string_starts_with reason "slave") in
+          List.filter pids ~f:is_interesting |> List.iter ~f:add_stack
+        with e ->
+          let message = Printexc.to_string e in
+          let stack = Printexc.get_backtrace () in
+          add_data (Printf.sprintf "Failed to get PIDs: %s - %s" message stack)
+      end
+    | None -> ()
+  end;
+  (* client *)
+  add_data ("LSP adapter state: " ^ (state_to_string state) ^ "\n");
+  (* server *)
+  begin match state with
+    | Main_loop menv -> begin
+        let open Main_env in
+        let items = rpc menv.conn ServerCommandTypes.RAGE in
+        let add i = add { title = i.ServerRageTypes.title; data = i.ServerRageTypes.data; } in
+        List.iter items ~f:add
+      end
+    | _ -> ()
+  end;
+  (* that's it! *)
+  !items
 
 
 let do_didOpen (conn: server_conn) (params: DidOpen.params) : unit =