From ca50bc68ad624046a796ccd3635a38f8e341fb9d Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 14:39:18 -0500
Subject: [PATCH 01/23] debian,rpm: /var/lib/ceph/crash

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit e37e6407b9cbac8561e4f79cb9284820b2c508ff)
---
 ceph.spec.in          | 1 +
 debian/ceph-base.dirs | 1 +
 2 files changed, 2 insertions(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index 19092061a8896..4bd38de91f77b 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -1000,6 +1000,7 @@ mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-mds
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-rgw
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-mgr
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-rbd
+mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/crash
 
 %if 0%{?suse_version}
 # create __pycache__ directories and their contents
diff --git a/debian/ceph-base.dirs b/debian/ceph-base.dirs
index 2ae6860022e03..262e6f6a508e2 100644
--- a/debian/ceph-base.dirs
+++ b/debian/ceph-base.dirs
@@ -4,3 +4,4 @@ var/lib/ceph/bootstrap-osd
 var/lib/ceph/bootstrap-rgw
 var/lib/ceph/bootstrap-rbd
 var/lib/ceph/tmp
+var/lib/ceph/crash

From 613eaaa7a2f86cfd6634a2ff6623a6de55d642a2 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 15:57:40 -0500
Subject: [PATCH 02/23] log: do not discard recent after dumping it

We want to call dump_recent multiple times without discarding those
log events.  Just iterate the list; don't discard it!

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 76802af0f8ee2d1a4d9b8b4fee23b9fdfbdf1304)
---
 src/log/Log.cc | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/src/log/Log.cc b/src/log/Log.cc
index 9240751dd1228..018060711fe70 100644
--- a/src/log/Log.cc
+++ b/src/log/Log.cc
@@ -298,8 +298,31 @@ void Log::flush()
 
 void Log::_flush(EntryQueue *t, EntryQueue *requeue, bool crash)
 {
-  Entry *e;
-  while ((e = t->dequeue()) != NULL) {
+  Entry *e = nullptr;
+  long len = 0;
+  if (crash) {
+    len = t->m_len;
+  }
+  if (!requeue) {
+    e = t->m_head;
+    if (!e) {
+      return;
+    }
+  }
+  while (true) {
+    if (requeue) {
+      e = t->dequeue();
+      if (!e) {
+	break;
+      }
+      requeue->enqueue(e);
+    } else {
+      e = e->m_next;
+      if (!e) {
+	break;
+      }
+    }
+
     unsigned sub = e->m_subsys;
 
     bool should_log = crash || m_subs->get_log_level(sub) >= e->m_prio;
@@ -323,8 +346,9 @@ void Log::_flush(EntryQueue *t, EntryQueue *requeue, bool crash)
         buf = buf0;
       }
 
-      if (crash)
+      if (crash) {
 	buflen += snprintf(buf, buf_size, "%6d> ", -t->m_len);
+      }
       buflen += append_time(e->m_stamp, buf + buflen, buf_size - buflen);
       buflen += snprintf(buf + buflen, buf_size-buflen, " %lx %2d ",
 			(unsigned long)e->m_thread, e->m_prio);
@@ -361,7 +385,6 @@ void Log::_flush(EntryQueue *t, EntryQueue *requeue, bool crash)
       m_graylog->log_entry(e);
     }
 
-    requeue->enqueue(e);
   }
 }
 
@@ -401,9 +424,8 @@ void Log::dump_recent()
   pthread_mutex_unlock(&m_queue_mutex);
   _flush(&t, &m_recent, false);
 
-  EntryQueue old;
   _log_message("--- begin dump of recent events ---", true);
-  _flush(&m_recent, &old, true);
+  _flush(&m_recent, nullptr, true);
 
   char buf[4096];
   _log_message("--- logging levels ---", true);

From 8b3562ad65ff645f5bf0c3aa6376ce025559e054 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 17:54:55 -0500
Subject: [PATCH 03/23] common/BackTrace: add dump()

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 4998b5b584ac57d67ca5ba28d3ec3a3e4fe4486f)
---
 src/common/BackTrace.cc | 66 +++++++++++++++++++++++++++++++++++++++++
 src/common/BackTrace.h  |  6 ++++
 2 files changed, 72 insertions(+)

diff --git a/src/common/BackTrace.cc b/src/common/BackTrace.cc
index c179d1dac1252..90b83df356c1c 100644
--- a/src/common/BackTrace.cc
+++ b/src/common/BackTrace.cc
@@ -1,9 +1,13 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
 #include <ostream>
 #include <cxxabi.h>
 #include <string.h>
 
 #include "BackTrace.h"
 #include "common/version.h"
+#include "common/Formatter.h"
 
 #define _STR(x) #x
 #define STRINGIFY(x) _STR(x)
@@ -70,4 +74,66 @@ void BackTrace::print(std::ostream& out) const
   }
 }
 
+void BackTrace::dump(Formatter *f) const
+{
+  f->open_array_section("backtrace");
+  for (size_t i = skip; i < size; i++) {
+    //      out << " " << (i-skip+1) << ": " << strings[i] << std::endl;
+
+    size_t sz = 1024; // just a guess, template names will go much wider
+    char *function = (char *)malloc(sz);
+    if (!function)
+      return;
+    char *begin = 0, *end = 0;
+
+    // find the parentheses and address offset surrounding the mangled name
+#ifdef __FreeBSD__
+    static constexpr char OPEN = '<';
+#else
+    static constexpr char OPEN = '(';
+#endif
+    for (char *j = strings[i]; *j; ++j) {
+      if (*j == OPEN)
+	begin = j+1;
+      else if (*j == '+')
+	end = j;
+    }
+    if (begin && end) {
+      int len = end - begin;
+      char *foo = (char *)malloc(len+1);
+      if (!foo) {
+	free(function);
+	return;
+      }
+      memcpy(foo, begin, len);
+      foo[len] = 0;
+
+      int status;
+      char *ret = nullptr;
+      // only demangle a C++ mangled name
+      if (foo[0] == '_' && foo[1] == 'Z')
+	ret = abi::__cxa_demangle(foo, function, &sz, &status);
+      if (ret) {
+	// return value may be a realloc() of the input
+	function = ret;
+      }
+      else {
+	// demangling failed, just pretend it's a C function with no args
+	strncpy(function, foo, sz);
+	strncat(function, "()", sz);
+	function[sz-1] = 0;
+      }
+      f->dump_stream("frame") << OPEN << function << end;
+      //fprintf(out, "    %s:%s\n", stack.strings[i], function);
+      free(foo);
+    } else {
+      // didn't find the mangled name, just print the whole line
+      //out << " " << (i-skip+1) << ": " << strings[i] << std::endl;
+      f->dump_string("frame", strings[i]);
+    }
+    free(function);
+  }
+  f->close_section();
+}
+
 }
diff --git a/src/common/BackTrace.h b/src/common/BackTrace.h
index 372788e6b64e6..5cb73d47bd6c4 100644
--- a/src/common/BackTrace.h
+++ b/src/common/BackTrace.h
@@ -1,3 +1,6 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
 #ifndef CEPH_BACKTRACE_H
 #define CEPH_BACKTRACE_H
 
@@ -10,6 +13,8 @@
 
 namespace ceph {
 
+class Formatter;
+
 struct BackTrace {
   const static int max = 100;
 
@@ -36,6 +41,7 @@ struct BackTrace {
   const BackTrace& operator=(const BackTrace& other);
 
   void print(std::ostream& out) const;
+  void dump(Formatter *f) const;
 };
 
 inline std::ostream& operator<<(std::ostream& out, const BackTrace& bt) {

From 583f5920ea840269101a896455ae047de2a4c591 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 14:40:10 -0500
Subject: [PATCH 04/23] common: add crash_dir option

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 72139f0d23a9173448298c60fd3d3f1c48a97eeb)
---
 src/common/legacy_config_opts.h | 1 +
 src/common/options.cc           | 4 ++++
 src/vstart.sh                   | 1 +
 3 files changed, 6 insertions(+)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index 82e81d24a977c..f375e210ce90f 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -34,6 +34,7 @@ OPTION(chdir, OPT_STR)
 OPTION(restapi_log_level, OPT_STR) 	// default set by Python code
 OPTION(restapi_base_url, OPT_STR)	// "
 OPTION(fatal_signal_handlers, OPT_BOOL)
+OPTION(crash_dir, OPT_STR)
 SAFE_OPTION(erasure_code_dir, OPT_STR) // default location for erasure-code plugins
 
 OPTION(log_file, OPT_STR) // default changed by common_preinit()
diff --git a/src/common/options.cc b/src/common/options.cc
index 878f84b71f51a..9e567b6f438a2 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -537,6 +537,10 @@ std::vector<Option> get_global_options() {
     .add_service({"mon", "mgr", "osd", "mds"})
     .add_tag("service"),
 
+    Option("crash_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
+    .set_default("/var/lib/ceph/crash")
+    .set_description("Directory where crash reports are archived"),
+
     // restapi
     Option("restapi_log_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_description("default set by python code"),
diff --git a/src/vstart.sh b/src/vstart.sh
index 97d17bbbdd436..84e84e4c19ae0 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -458,6 +458,7 @@ prepare_conf() {
         plugin dir = $CEPH_LIB
         filestore fd cache size = 32
         run dir = $CEPH_OUT_DIR
+	crash dir = $CEPH_OUT_DIR
         enable experimental unrecoverable data corrupting features = *
 	osd_crush_chooseleaf_type = 0
 $extra_conf

From 3e8a2831028f2091b37a486127f8123f47286821 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 17:37:10 -0500
Subject: [PATCH 05/23] common/assert: get rid of duplicate log dump from
 assert handler

When we abort() we get the log dump.  Only do it from the assert handler
if the fatal_signal_handlers are disabled for some reason.

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 3a7b9e47bd89c24adb82e068020dada270bb08ee)
---
 src/common/assert.cc      | 24 ++++++++++--------------
 src/global/global_init.cc |  6 +++---
 2 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/src/common/assert.cc b/src/common/assert.cc
index 0bc04a385a8b2..4501c8b38a1c1 100644
--- a/src/common/assert.cc
+++ b/src/common/assert.cc
@@ -51,16 +51,14 @@ namespace ceph {
     oss << BackTrace(1);
     dout_emergency(oss.str());
 
-    dout_emergency(" NOTE: a copy of the executable, or `objdump -rdS <executable>` "
-		   "is needed to interpret this.\n");
-
     if (g_assert_context) {
       lderr(g_assert_context) << buf << std::endl;
-      *_dout << oss.str();
-      *_dout << " NOTE: a copy of the executable, or `objdump -rdS <executable>` "
-	     << "is needed to interpret this.\n" << dendl;
+      *_dout << oss.str() << dendl;
 
-      g_assert_context->_log->dump_recent();
+      // dump recent only if the abort signal handler won't do it for us
+      if (!g_assert_context->_conf->fatal_signal_handlers) {
+	g_assert_context->_log->dump_recent();
+      }
     }
 
     abort();
@@ -126,16 +124,14 @@ namespace ceph {
     oss << *bt;
     dout_emergency(oss.str());
 
-    dout_emergency(" NOTE: a copy of the executable, or `objdump -rdS <executable>` "
-		   "is needed to interpret this.\n");
-
     if (g_assert_context) {
       lderr(g_assert_context) << buf << std::endl;
-      *_dout << oss.str();
-      *_dout << " NOTE: a copy of the executable, or `objdump -rdS <executable>` "
-	     << "is needed to interpret this.\n" << dendl;
+      *_dout << oss.str() << dendl;
 
-      g_assert_context->_log->dump_recent();
+      // dump recent only if the abort signal handler won't do it for us
+      if (!g_assert_context->_conf->fatal_signal_handlers) {
+	g_assert_context->_log->dump_recent();
+      }
     }
 
     abort();
diff --git a/src/global/global_init.cc b/src/global/global_init.cc
index 8fd9247b0421a..41ea214a38f90 100644
--- a/src/global/global_init.cc
+++ b/src/global/global_init.cc
@@ -191,8 +191,10 @@ global_init(const std::map<std::string,std::string> *defaults,
   int siglist[] = { SIGPIPE, 0 };
   block_signals(siglist, NULL);
 
-  if (g_conf->fatal_signal_handlers)
+  if (g_conf->fatal_signal_handlers) {
     install_standard_sighandlers();
+  }
+  register_assert_context(g_ceph_context);
 
   if (g_conf->log_flush_on_exit)
     g_ceph_context->_log->set_flush_on_exit();
@@ -315,8 +317,6 @@ global_init(const std::map<std::string,std::string> *defaults,
     }
   }
 
-  register_assert_context(g_ceph_context);
-
   // call all observers now.  this has the side-effect of configuring
   // and opening the log file immediately.
   g_conf->call_all_observers();

From db9a2916431046b491eb23f870c56f6126650f67 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 21:53:32 -0500
Subject: [PATCH 06/23] common/assert: record assert info in g_assert_* globals

These will then be available for others who are interested.

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit b0e59ce2f51b15e6f22b217fb28971746b350806)
---
 src/common/assert.cc         | 16 ++++++++++++++++
 src/global/global_context.cc |  7 +++++++
 src/global/global_context.h  |  7 +++++++
 3 files changed, 30 insertions(+)

diff --git a/src/common/assert.cc b/src/common/assert.cc
index 4501c8b38a1c1..45d44d12ea1aa 100644
--- a/src/common/assert.cc
+++ b/src/common/assert.cc
@@ -35,6 +35,14 @@ namespace ceph {
   void __ceph_assert_fail(const char *assertion, const char *file, int line,
 			  const char *func)
   {
+    g_assert_condition = assertion;
+    g_assert_file = file;
+    g_assert_line = line;
+    g_assert_func = func;
+    g_assert_thread = (unsigned long long)pthread_self();
+    pthread_getname_np(pthread_self(), g_assert_thread_name,
+		       sizeof(g_assert_thread_name));
+
     ostringstream tss;
     tss << ceph_clock_now();
 
@@ -75,6 +83,14 @@ namespace ceph {
     ostringstream tss;
     tss << ceph_clock_now();
 
+    g_assert_condition = assertion;
+    g_assert_file = file;
+    g_assert_line = line;
+    g_assert_func = func;
+    g_assert_thread = (unsigned long long)pthread_self();
+    pthread_getname_np(pthread_self(), g_assert_thread_name,
+		       sizeof(g_assert_thread_name));
+
     class BufAppender {
     public:
       BufAppender(char* buf, int size) : bufptr(buf), remaining(size) {
diff --git a/src/global/global_context.cc b/src/global/global_context.cc
index 05f76321ca3d9..9e511d44da95f 100644
--- a/src/global/global_context.cc
+++ b/src/global/global_context.cc
@@ -20,3 +20,10 @@
  */
 CephContext *g_ceph_context = NULL;
 md_config_t *g_conf = NULL;
+
+const char *g_assert_file = 0;
+int g_assert_line = 0;
+const char *g_assert_func = 0;
+const char *g_assert_condition = 0;
+unsigned long long g_assert_thread = 0;
+char g_assert_thread_name[4096];
diff --git a/src/global/global_context.h b/src/global/global_context.h
index 29e5a973eb51d..9ac4388d08a26 100644
--- a/src/global/global_context.h
+++ b/src/global/global_context.h
@@ -21,4 +21,11 @@ struct md_config_t;
 extern CephContext *g_ceph_context;
 extern md_config_t *g_conf;
 
+extern const char *g_assert_file;
+extern int g_assert_line;
+extern const char *g_assert_func;
+extern const char *g_assert_condition;
+extern unsigned long long g_assert_thread;
+extern char g_assert_thread_name[4096];
+
 #endif

From 044632b3362f188ed7da0d718a3606cb4f4ee0ee Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 17:29:11 -0500
Subject: [PATCH 07/23] common/ceph_context: add "assert" and "abort" asok
 commands

Require 'debug_asok_assert_abort = true'

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 14699d2997f8b1e6e65cc2c24b33e2fcaf9cee4d)
---
 src/common/ceph_context.cc      | 8 ++++++++
 src/common/legacy_config_opts.h | 1 +
 src/common/options.cc           | 4 ++++
 src/vstart.sh                   | 1 +
 4 files changed, 14 insertions(+)

diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc
index 5fee784edae9f..014741b276fe6 100644
--- a/src/common/ceph_context.cc
+++ b/src/common/ceph_context.cc
@@ -391,6 +391,12 @@ void CephContext::do_command(std::string_view command, const cmdmap_t& cmdmap,
   }
   lgeneric_dout(this, 1) << "do_command '" << command << "' '"
 			 << ss.str() << dendl;
+  if (command == "assert" && _conf->debug_asok_assert_abort) {
+    assert(0 == "assert");
+  }
+  if (command == "abort" && _conf->debug_asok_assert_abort) {
+    abort();
+  }
   if (command == "perfcounters_dump" || command == "1" ||
       command == "perf dump") {
     std::string logger;
@@ -582,6 +588,8 @@ CephContext::CephContext(uint32_t module_type_,
   _plugin_registry = new PluginRegistry(this);
 
   _admin_hook = new CephContextHook(this);
+  _admin_socket->register_command("assert", "assert", _admin_hook, "");
+  _admin_socket->register_command("abort", "abort", _admin_hook, "");
   _admin_socket->register_command("perfcounters_dump", "perfcounters_dump", _admin_hook, "");
   _admin_socket->register_command("1", "1", _admin_hook, "");
   _admin_socket->register_command("perf dump", "perf dump name=logger,type=CephString,req=false name=counter,type=CephString,req=false", _admin_hook, "dump perfcounters value");
diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index f375e210ce90f..c05e7d10a95c2 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1513,6 +1513,7 @@ OPTION(rgw_torrent_sha_unit, OPT_INT)    // torrent field piece length 512K
 OPTION(event_tracing, OPT_BOOL) // true if LTTng-UST tracepoints should be enabled
 
 OPTION(debug_deliberately_leak_memory, OPT_BOOL)
+OPTION(debug_asok_assert_abort, OPT_BOOL)
 
 OPTION(rgw_swift_custom_header, OPT_STR) // option to enable swift custom headers
 
diff --git a/src/common/options.cc b/src/common/options.cc
index 9e567b6f438a2..22adf6a7a368a 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4864,6 +4864,10 @@ std::vector<Option> get_global_options() {
     Option("debug_asserts_on_shutdown", Option::TYPE_BOOL,Option::LEVEL_DEV)
     .set_default(false)
     .set_description("Enable certain asserts to check for refcounting bugs on shutdown; see http://tracker.ceph.com/issues/21738"),
+
+    Option("debug_asok_assert_abort", Option::TYPE_BOOL, Option::LEVEL_DEV)
+    .set_default(false)
+    .set_description("allow commands 'assert' and 'abort' via asok for testing crash dumps etc"),
   });
 }
 
diff --git a/src/vstart.sh b/src/vstart.sh
index 84e84e4c19ae0..9b4af13d86cb7 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -461,6 +461,7 @@ prepare_conf() {
 	crash dir = $CEPH_OUT_DIR
         enable experimental unrecoverable data corrupting features = *
 	osd_crush_chooseleaf_type = 0
+	debug asok assert abort = true
 $extra_conf
 EOF
 	if [ "$lockdep" -eq 1 ] ; then

From 7a24c04800f5b1cff9fe8940e8d7a7bc99f809f6 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Fri, 15 Jun 2018 15:54:02 -0500
Subject: [PATCH 08/23] global/signal_handler: write crash dumps to
 /var/lib/ceph/crash/$uuid/

Include two files:

- meta, a JSON blob with everything interesting we can think of
- log, the dump_recent log events

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 90a46dcf40e2e9467b9ce278a54f82efed5838bd)
---
 src/global/signal_handler.cc | 152 +++++++++++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)

diff --git a/src/global/signal_handler.cc b/src/global/signal_handler.cc
index ae860e214560c..bdcc06f215523 100644
--- a/src/global/signal_handler.cc
+++ b/src/global/signal_handler.cc
@@ -12,11 +12,17 @@
  *
  */
 
+#include <sys/utsname.h>
+
 #include "include/compat.h"
 #include "pthread.h"
 
 #include "common/BackTrace.h"
 #include "common/debug.h"
+#include "common/safe_io.h"
+#include "common/version.h"
+
+#include "include/uuid.h"
 #include "global/pidfile.h"
 #include "global/signal_handler.h"
 
@@ -88,6 +94,49 @@ static void reraise_fatal(int signum)
   exit(1);
 }
 
+
+// /etc/os-release looks like
+//
+//   NAME=Fedora
+//   VERSION="28 (Server Edition)"
+//   ID=fedora
+//   VERSION_ID=28
+//
+// or
+//
+//   NAME="Ubuntu"
+//   VERSION="16.04.3 LTS (Xenial Xerus)"
+//   ID=ubuntu
+//   ID_LIKE=debian
+//
+// get_from_os_release("FOO=bar\nTHIS=\"that\"\n", "FOO=", ...) will
+// write "bar\0" to out buffer, which is assumed to be as large as the input
+// file.
+static int parse_from_os_release(
+  const char *file, const char *key,
+  char *out)
+{
+  const char *p = strstr(file, key);
+  if (!p) {
+    return -1;
+  }
+  const char *start = p + strlen(key);
+  const char *end = strchr(start, '\n');
+  if (!end) {
+    return -1;
+  }
+  if (*start == '"' && *(end - 1) == '"') {
+    ++start;
+    --end;
+  }
+  if (start >= end) {
+    return -1;
+  }
+  memcpy(out, start, end - start);
+  out[end - start] = 0;
+  return 0;
+}
+
 static void handle_fatal_signal(int signum)
 {
   // This code may itself trigger a SIGSEGV if the heap is corrupt. In that
@@ -119,6 +168,101 @@ static void handle_fatal_signal(int signum)
   bt.print(oss);
   dout_emergency(oss.str());
 
+  char base[PATH_MAX] = { 0 };
+  if (g_ceph_context &&
+      g_ceph_context->_conf->crash_dir.size()) {
+    // -- crash dump --
+    // id
+    ostringstream idss;
+    utime_t now = ceph_clock_now();
+    now.gmtime(idss);
+    uuid_d uuid;
+    uuid.generate_random();
+    idss << "_" << uuid;
+    string id = idss.str();
+    std::replace(id.begin(), id.end(), ' ', '_');
+
+    snprintf(base, sizeof(base), "%s/%s",
+	     g_ceph_context->_conf->crash_dir.c_str(),
+	     id.c_str());
+    int r = ::mkdir(base, 0700);
+    if (r >= 0) {
+      char fn[PATH_MAX*2];
+      snprintf(fn, sizeof(fn)-1, "%s/meta", base);
+      int fd = ::open(fn, O_CREAT|O_WRONLY, 0600);
+      if (fd >= 0) {
+	JSONFormatter jf(true);
+	jf.open_object_section("crash");
+	jf.dump_string("crash_id", id);
+	now.gmtime(jf.dump_stream("timestamp"));
+	jf.dump_string("entity_name", g_ceph_context->_conf->name.to_str());
+	jf.dump_string("ceph_version", ceph_version_to_str());
+
+	struct utsname u;
+	r = uname(&u);
+	if (r >= 0) {
+	  jf.dump_string("utsname_hostname", u.nodename);
+	  jf.dump_string("utsname_sysname", u.sysname);
+	  jf.dump_string("utsname_release", u.release);
+	  jf.dump_string("utsname_version", u.version);
+	  jf.dump_string("utsname_machine", u.machine);
+	}
+
+	// os-releaes
+	int in = ::open("/etc/os-release", O_RDONLY);
+	if (in >= 0) {
+	  char buf[4096];
+	  r = safe_read(in, buf, sizeof(buf)-1);
+	  if (r >= 0) {
+	    buf[r] = 0;
+	    char v[4096];
+	    if (parse_from_os_release(buf, "NAME=", v) >= 0) {
+	      jf.dump_string("os_name", v);
+	    }
+	    if (parse_from_os_release(buf, "ID=", v) >= 0) {
+	      jf.dump_string("os_id", v);
+	    }
+	    if (parse_from_os_release(buf, "VERSION_ID=", v) >= 0) {
+	      jf.dump_string("os_version_id", v);
+	    }
+	    if (parse_from_os_release(buf, "VERSION=", v) >= 0) {
+	      jf.dump_string("os_version", v);
+	    }
+	  }
+	  ::close(in);
+	}
+
+	// assert?
+	if (g_assert_condition) {
+	  jf.dump_string("assert_condition", g_assert_condition);
+	}
+	if (g_assert_func) {
+	  jf.dump_string("assert_func", g_assert_func);
+	}
+	if (g_assert_file) {
+	  jf.dump_string("assert_file", g_assert_file);
+	}
+	if (g_assert_line) {
+	  jf.dump_unsigned("assert_file", g_assert_line);
+	}
+	if (g_assert_thread_name[0]) {
+	  jf.dump_string("assert_thread_name", g_assert_thread_name);
+	}
+
+	// backtrace
+	bt.dump(&jf);
+
+	jf.close_section();
+	ostringstream oss;
+	jf.flush(oss);
+	string s = oss.str();
+	r = safe_write(fd, s.c_str(), s.size());
+	(void)r;
+	::close(fd);
+      }
+    }
+  }
+
   // avoid recursion back into logging code if that is where
   // we got the SEGV.
   if (g_ceph_context &&
@@ -133,6 +277,14 @@ static void handle_fatal_signal(int signum)
 	   << dendl;
 
     g_ceph_context->_log->dump_recent();
+
+    if (base[0]) {
+      char fn[PATH_MAX*2];
+      snprintf(fn, sizeof(fn)-1, "%s/log", base);
+      g_ceph_context->_log->set_log_file(fn);
+      g_ceph_context->_log->reopen_log_file();
+      g_ceph_context->_log->dump_recent();
+    }
   }
 
   reraise_fatal(signum);

From 1c3472cf3e9525fa3ae959bec9c209d25af961f5 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Sat, 16 Jun 2018 07:55:12 -0500
Subject: [PATCH 09/23] global/signal_handler: one less frame of context

    "backtrace": [
        "(()+0x942e6e) [0x55859889ae6e]",
        "(()+0x11fc0) [0x7f955b1aafc0]",
        "(gsignal()+0x10b) [0x7f9559e91f2b]",
        "(abort()+0x12b) [0x7f9559e7c561]",
	...

Drop one frame; it's not really helpful.  We could probably drop up through
gsignal(), but 1 seems safer across platforms.

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 7566c2eee2be3f035295d3ab59285fbc89a2df89)
---
 src/global/signal_handler.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/global/signal_handler.cc b/src/global/signal_handler.cc
index bdcc06f215523..5af31c28df236 100644
--- a/src/global/signal_handler.cc
+++ b/src/global/signal_handler.cc
@@ -163,7 +163,7 @@ static void handle_fatal_signal(int signum)
   // TODO: don't use an ostringstream here. It could call malloc(), which we
   // don't want inside a signal handler.
   // Also fix the backtrace code not to allocate memory.
-  BackTrace bt(0);
+  BackTrace bt(1);
   ostringstream oss;
   bt.print(oss);
   dout_emergency(oss.str());

From ed27df6f59acc61b2546737dd590779c349376d1 Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Tue, 19 Jun 2018 20:36:08 -0700
Subject: [PATCH 10/23] mgr, pybind/mgr: pass inbuf (ceph -i <file>) to modules

Modules may wish to receive bulk data; allow it

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 140761f8a2df10c45eb02041fad5e8237bf57c20)
---
 src/mgr/ActivePyModule.cc           | 6 +++++-
 src/mgr/ActivePyModule.h            | 1 +
 src/mgr/ActivePyModules.cc          | 3 ++-
 src/mgr/ActivePyModules.h           | 1 +
 src/mgr/DaemonServer.cc             | 3 ++-
 src/mgr/PyModuleRegistry.cc         | 3 ++-
 src/mgr/PyModuleRegistry.h          | 1 +
 src/pybind/mgr/balancer/module.py   | 2 +-
 src/pybind/mgr/dashboard/module.py  | 2 +-
 src/pybind/mgr/hello/module.py      | 2 +-
 src/pybind/mgr/influx/module.py     | 2 +-
 src/pybind/mgr/iostat/module.py     | 2 +-
 src/pybind/mgr/mgr_module.py        | 3 ++-
 src/pybind/mgr/prometheus/module.py | 2 +-
 src/pybind/mgr/restful/module.py    | 2 +-
 src/pybind/mgr/selftest/module.py   | 2 +-
 src/pybind/mgr/smart/module.py      | 2 +-
 src/pybind/mgr/status/module.py     | 2 +-
 src/pybind/mgr/telegraf/module.py   | 2 +-
 src/pybind/mgr/telemetry/module.py  | 2 +-
 src/pybind/mgr/zabbix/module.py     | 2 +-
 21 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/mgr/ActivePyModule.cc b/src/mgr/ActivePyModule.cc
index 2f52dd622620a..42ad021d02d1b 100644
--- a/src/mgr/ActivePyModule.cc
+++ b/src/mgr/ActivePyModule.cc
@@ -104,6 +104,7 @@ void ActivePyModule::notify_clog(const LogEntry &log_entry)
 
 int ActivePyModule::handle_command(
   const cmdmap_t &cmdmap,
+  const bufferlist &inbuf,
   std::stringstream *ds,
   std::stringstream *ss)
 {
@@ -122,9 +123,12 @@ int ActivePyModule::handle_command(
   PyFormatter f;
   cmdmap_dump(cmdmap, &f);
   PyObject *py_cmd = f.get();
+  string instr;
+  inbuf.copy(0, inbuf.length(), instr);
 
   auto pResult = PyObject_CallMethod(pClassInstance,
-      const_cast<char*>("handle_command"), const_cast<char*>("(O)"), py_cmd);
+      const_cast<char*>("handle_command"), const_cast<char*>("s#O"),
+      instr.c_str(), instr.length(), py_cmd);
 
   Py_DECREF(py_cmd);
 
diff --git a/src/mgr/ActivePyModule.h b/src/mgr/ActivePyModule.h
index 85d50807a053d..ebb1ba182be02 100644
--- a/src/mgr/ActivePyModule.h
+++ b/src/mgr/ActivePyModule.h
@@ -54,6 +54,7 @@ class ActivePyModule : public PyModuleRunner
 
   int handle_command(
     const cmdmap_t &cmdmap,
+    const bufferlist &inbuf,
     std::stringstream *ds,
     std::stringstream *ss);
 
diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index fdf6652fed9e1..8c1c91bb92892 100644
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -766,6 +766,7 @@ void ActivePyModules::set_health_checks(const std::string& module_name,
 int ActivePyModules::handle_command(
   std::string const &module_name,
   const cmdmap_t &cmdmap,
+  const bufferlist &inbuf,
   std::stringstream *ds,
   std::stringstream *ss)
 {
@@ -777,7 +778,7 @@ int ActivePyModules::handle_command(
   }
 
   lock.Unlock();
-  return mod_iter->second->handle_command(cmdmap, ds, ss);
+  return mod_iter->second->handle_command(cmdmap, inbuf, ds, ss);
 }
 
 void ActivePyModules::get_health_checks(health_check_map_t *checks)
diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h
index c10a087c4e2cc..322f2bf10f0d7 100644
--- a/src/mgr/ActivePyModules.h
+++ b/src/mgr/ActivePyModules.h
@@ -96,6 +96,7 @@ class ActivePyModules
   int handle_command(
     const std::string &module_name,
     const cmdmap_t &cmdmap,
+    const bufferlist &inbuf,
     std::stringstream *ds,
     std::stringstream *ss);
 
diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index 94a1495f5f66e..c9aeba07d7211 100644
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -1730,7 +1730,8 @@ bool DaemonServer::handle_command(MCommand *m)
     }
 
     std::stringstream ds;
-    int r = py_modules.handle_command(handler_name, cmdctx->cmdmap, &ds, &ss);
+    bufferlist inbl = cmdctx->m->get_data();
+    int r = py_modules.handle_command(handler_name, cmdctx->cmdmap, inbl, &ds, &ss);
     cmdctx->odata.append(ds);
     cmdctx->reply(r, ss);
   }));
diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc
index 3ceb4d0e93934..84531ee837791 100644
--- a/src/mgr/PyModuleRegistry.cc
+++ b/src/mgr/PyModuleRegistry.cc
@@ -287,11 +287,12 @@ std::set<std::string> PyModuleRegistry::probe_modules() const
 int PyModuleRegistry::handle_command(
   std::string const &module_name,
   const cmdmap_t &cmdmap,
+  const bufferlist &inbuf,
   std::stringstream *ds,
   std::stringstream *ss)
 {
   if (active_modules) {
-    return active_modules->handle_command(module_name, cmdmap, ds, ss);
+    return active_modules->handle_command(module_name, cmdmap, inbuf, ds, ss);
   } else {
     // We do not expect to be called before active modules is up, but
     // it's straightfoward to handle this case so let's do it.
diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h
index 5fb0148e7102e..3050b865958b8 100644
--- a/src/mgr/PyModuleRegistry.h
+++ b/src/mgr/PyModuleRegistry.h
@@ -140,6 +140,7 @@ class PyModuleRegistry
   int handle_command(
     std::string const &module_name,
     const cmdmap_t &cmdmap,
+    const bufferlist &inbuf,
     std::stringstream *ds,
     std::stringstream *ss);
 
diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py
index 8f0ea5fc84a59..7326a885c009c 100644
--- a/src/pybind/mgr/balancer/module.py
+++ b/src/pybind/mgr/balancer/module.py
@@ -287,7 +287,7 @@ def __init__(self, *args, **kwargs):
         super(Module, self).__init__(*args, **kwargs)
         self.event = Event()
 
-    def handle_command(self, command):
+    def handle_command(self, inbuf, command):
         self.log.warn("Handling command: '%s'" % str(command))
         if command['prefix'] == 'balancer status':
             s = {
diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
index a3934e93a8b29..967b45aa29b98 100644
--- a/src/pybind/mgr/dashboard/module.py
+++ b/src/pybind/mgr/dashboard/module.py
@@ -305,7 +305,7 @@ def shutdown(self):
         logger.info('Stopping engine...')
         self.shutdown_event.set()
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         res = handle_option_command(cmd)
         if res[0] != -errno.ENOSYS:
             return res
diff --git a/src/pybind/mgr/hello/module.py b/src/pybind/mgr/hello/module.py
index ce8590d9faf8d..37f9e2df2f039 100644
--- a/src/pybind/mgr/hello/module.py
+++ b/src/pybind/mgr/hello/module.py
@@ -18,7 +18,7 @@ class Hello(MgrModule):
         },
     ]
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         self.log.info("hello_world_info")
         self.log.debug("hello_world_debug")
         self.log.error("hello_world_error")
diff --git a/src/pybind/mgr/influx/module.py b/src/pybind/mgr/influx/module.py
index 379668a81a803..c8ea5cf23ee15 100644
--- a/src/pybind/mgr/influx/module.py
+++ b/src/pybind/mgr/influx/module.py
@@ -330,7 +330,7 @@ def shutdown(self):
         self.run = False
         self.event.set()
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         if cmd['prefix'] == 'influx config-show':
             return 0, json.dumps(self.config), ''
         elif cmd['prefix'] == 'influx config-set':
diff --git a/src/pybind/mgr/iostat/module.py b/src/pybind/mgr/iostat/module.py
index e9f7c3eb92b95..9133fafae2e70 100644
--- a/src/pybind/mgr/iostat/module.py
+++ b/src/pybind/mgr/iostat/module.py
@@ -22,7 +22,7 @@ def __init__(self, *args, **kwargs):
         super(Module, self).__init__(*args, **kwargs)
 
 
-    def handle_command(self, command):
+    def handle_command(self, inbuf, command):
         rd = 0
         wr = 0
         total = 0
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index f1e5c127f5e38..e54455ed59bb3 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -517,7 +517,7 @@ def set_health_checks(self, checks):
         """
         self._ceph_set_health_checks(checks)
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         """
         Called by ceph-mgr to request the plugin to handle one
         of the commands that it declared in self.COMMANDS
@@ -526,6 +526,7 @@ def handle_command(self, cmd):
         output string.  The output buffer is for data results,
         the output string is for informative text.
 
+        :param string inbuf: content of any "-i <file>" supplied to ceph cli
         :param dict cmd: from Ceph's cmdmap_t
 
         :return: 3-tuple of (int, str, str)
diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py
index ad5838b382d0b..c807157d007eb 100644
--- a/src/pybind/mgr/prometheus/module.py
+++ b/src/pybind/mgr/prometheus/module.py
@@ -628,7 +628,7 @@ def get_file_sd_config(self):
         ]
         return 0, json.dumps(ret), ""
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         if cmd['prefix'] == 'prometheus self-test':
             self.collect()
             self.get_file_sd_config()
diff --git a/src/pybind/mgr/restful/module.py b/src/pybind/mgr/restful/module.py
index a6c9c638fe247..8c20e4c043d0f 100644
--- a/src/pybind/mgr/restful/module.py
+++ b/src/pybind/mgr/restful/module.py
@@ -400,7 +400,7 @@ def create_self_signed_cert(self):
         )
 
 
-    def handle_command(self, command):
+    def handle_command(self, inbuf, command):
         self.log.warn("Handling command: '%s'" % str(command))
         if command['prefix'] == "restful create-key":
             if command['key_name'] in self.keys:
diff --git a/src/pybind/mgr/selftest/module.py b/src/pybind/mgr/selftest/module.py
index dfc57ae5c2bfd..8ba36f622f65d 100644
--- a/src/pybind/mgr/selftest/module.py
+++ b/src/pybind/mgr/selftest/module.py
@@ -67,7 +67,7 @@ def __init__(self, *args, **kwargs):
         self._event = threading.Event()
         self._workload = None
 
-    def handle_command(self, command):
+    def handle_command(self, inbuf, command):
         if command['prefix'] == 'mgr self-test run':
             self._self_test()
             return 0, '', 'Self-test succeeded'
diff --git a/src/pybind/mgr/smart/module.py b/src/pybind/mgr/smart/module.py
index 8b2d54c176776..3df9356ea1b4a 100644
--- a/src/pybind/mgr/smart/module.py
+++ b/src/pybind/mgr/smart/module.py
@@ -17,7 +17,7 @@ class Module(MgrModule):
         },
     ]
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         self.log.error("handle_command")
 
         if cmd['prefix'] == 'osd smart get':
diff --git a/src/pybind/mgr/status/module.py b/src/pybind/mgr/status/module.py
index 0467cf2b0669f..88c91289f9fa4 100644
--- a/src/pybind/mgr/status/module.py
+++ b/src/pybind/mgr/status/module.py
@@ -304,7 +304,7 @@ def handle_osd_status(self, cmd):
 
         return 0, osd_table.get_string(), ""
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         self.log.error("handle_command")
 
         if cmd['prefix'] == "fs status":
diff --git a/src/pybind/mgr/telegraf/module.py b/src/pybind/mgr/telegraf/module.py
index 179e67ff2fa00..bf4399d58b5db 100644
--- a/src/pybind/mgr/telegraf/module.py
+++ b/src/pybind/mgr/telegraf/module.py
@@ -271,7 +271,7 @@ def shutdown(self):
         self.run = False
         self.event.set()
 
-    def handle_command(self, cmd):
+    def handle_command(self, inbuf, cmd):
         if cmd['prefix'] == 'telegraf config-show':
             return 0, json.dumps(self.config), ''
         elif cmd['prefix'] == 'telegraf config-set':
diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py
index f83ddfe26c213..c3e24f150e047 100644
--- a/src/pybind/mgr/telemetry/module.py
+++ b/src/pybind/mgr/telemetry/module.py
@@ -287,7 +287,7 @@ def send(self, report):
 
         requests.put(url=self.config['url'], json=report, proxies=proxies)
 
-    def handle_command(self, command):
+    def handle_command(self, inbuf, command):
         if command['prefix'] == 'telemetry config-show':
             return 0, json.dumps(self.config), ''
         elif command['prefix'] == 'telemetry config-set':
diff --git a/src/pybind/mgr/zabbix/module.py b/src/pybind/mgr/zabbix/module.py
index df319f7b05ca0..43199fe3ddf77 100644
--- a/src/pybind/mgr/zabbix/module.py
+++ b/src/pybind/mgr/zabbix/module.py
@@ -279,7 +279,7 @@ def send(self):
 
         return False
 
-    def handle_command(self, command):
+    def handle_command(self, inbuf, command):
         if command['prefix'] == 'zabbix config-show':
             return 0, json.dumps(self.config), ''
         elif command['prefix'] == 'zabbix config-set':

From 65d00d68c3354756a68901d88dc7ab4a2289d59a Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Thu, 28 Jun 2018 15:36:25 -0700
Subject: [PATCH 11/23] qa/tasks/{ceph_manager.py,vstart_runner.py}: allow
 kwargs in raw_*

Allow passing kwargs (like stdin=) to the local and teuthology
clusters when running tests

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 7fc8714a27450823e49a86f9da4bcaa73bf7b8da)
---
 qa/tasks/ceph_manager.py  | 9 ++++-----
 qa/tasks/vstart_runner.py | 9 +++++----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 758c7e1484173..e507782f13619 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -1135,7 +1135,7 @@ def raw_cluster_cmd(self, *args):
             )
         return proc.stdout.getvalue()
 
-    def raw_cluster_cmd_result(self, *args):
+    def raw_cluster_cmd_result(self, *args, **kwargs):
         """
         Start ceph on a cluster.  Return success or failure information.
         """
@@ -1152,10 +1152,9 @@ def raw_cluster_cmd_result(self, *args):
             self.cluster,
         ]
         ceph_args.extend(args)
-        proc = self.controller.run(
-            args=ceph_args,
-            check_status=False,
-            )
+        kwargs['args'] = ceph_args
+        kwargs['check_status'] = False
+        proc = self.controller.run(**kwargs)
         return proc.exitstatus
 
     def run_ceph_w(self):
diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index 3c8134ec46335..0462514b1bbe7 100644
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -560,19 +560,20 @@ def run_ceph_w(self):
         proc = self.controller.run([os.path.join(BIN_PREFIX, "ceph"), "-w"], wait=False, stdout=StringIO())
         return proc
 
-    def raw_cluster_cmd(self, *args):
+    def raw_cluster_cmd(self, *args, **kwargs):
         """
         args like ["osd", "dump"}
         return stdout string
         """
-        proc = self.controller.run([os.path.join(BIN_PREFIX, "ceph")] + list(args))
+        proc = self.controller.run([os.path.join(BIN_PREFIX, "ceph")] + list(args), **kwargs)
         return proc.stdout.getvalue()
 
-    def raw_cluster_cmd_result(self, *args):
+    def raw_cluster_cmd_result(self, *args, **kwargs):
         """
         like raw_cluster_cmd but don't check status, just return rc
         """
-        proc = self.controller.run([os.path.join(BIN_PREFIX, "ceph")] + list(args), check_status=False)
+        kwargs['check_status'] = False
+        proc = self.controller.run([os.path.join(BIN_PREFIX, "ceph")] + list(args), **kwargs)
         return proc.exitstatus
 
     def admin_socket(self, daemon_type, daemon_id, command, check_status=True):

From a3baaab0538a9ef319e2f3caa65f3d7845622286 Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Tue, 19 Jun 2018 13:46:36 -0700
Subject: [PATCH 12/23] mgr/pybind/crash: handle crashdumps

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 29209a3ea8a6cf378654f7b9b6bf923b2de23218)
---
 src/pybind/mgr/crash/__init__.py |   1 +
 src/pybind/mgr/crash/module.py   | 188 +++++++++++++++++++++++++++++++
 2 files changed, 189 insertions(+)
 create mode 100644 src/pybind/mgr/crash/__init__.py
 create mode 100644 src/pybind/mgr/crash/module.py

diff --git a/src/pybind/mgr/crash/__init__.py b/src/pybind/mgr/crash/__init__.py
new file mode 100644
index 0000000000000..8f210ac9247ea
--- /dev/null
+++ b/src/pybind/mgr/crash/__init__.py
@@ -0,0 +1 @@
+from .module import Module
diff --git a/src/pybind/mgr/crash/module.py b/src/pybind/mgr/crash/module.py
new file mode 100644
index 0000000000000..668ef62ea8464
--- /dev/null
+++ b/src/pybind/mgr/crash/module.py
@@ -0,0 +1,188 @@
+from mgr_module import MgrModule
+import datetime
+import errno
+import json
+
+
+DATEFMT = '%Y-%m-%d %H:%M:%S.%f'
+
+
+class Module(MgrModule):
+
+    def __init__(self, *args, **kwargs):
+        super(Module, self).__init__(*args, **kwargs)
+
+    def handle_command(self, inbuf, command):
+        for cmd in self.COMMANDS:
+            if cmd['cmd'].startswith(command['prefix']):
+                handler = cmd['handler']
+                break
+        if handler is None:
+            return errno.EINVAL, '', 'unknown command %s' % command['prefix']
+
+        return handler(self, command, inbuf)
+
+    @staticmethod
+    def validate_crash_metadata(inbuf):
+        # raise any exceptions to caller
+        metadata = json.loads(inbuf)
+        if 'crash_id' not in metadata:
+            raise AttributeError("missing 'crash_id' field")
+        return metadata
+
+    @staticmethod
+    def time_from_string(timestr):
+        # drop the 'Z' timezone indication, it's always UTC
+        timestr = timestr.rstrip('Z')
+        return datetime.datetime.strptime(timestr, DATEFMT)
+
+    # command handlers
+
+    def do_info(self, cmd, inbuf):
+        crashid = cmd['id']
+        key = 'crash/%s' % crashid
+        val = self.get_store(key)
+        if not val:
+            return errno.EINVAL, '', 'crash info: %s not found' % crashid
+        return 0, val, ''
+
+    def do_post(self, cmd, inbuf):
+        try:
+            metadata = self.validate_crash_metadata(inbuf)
+        except Exception as e:
+            return errno.EINVAL, '', 'malformed crash metadata: %s' % e
+
+        crashid = metadata['crash_id']
+        key = 'crash/%s' % crashid
+        # repeated stores of same item are ignored silently
+        if not self.get_store(key):
+            self.set_store(key, inbuf)
+        return 0, '', ''
+
+    def do_ls(self, cmd, inbuf):
+        keys = []
+        for key in self.get_store_prefix('crash/').iterkeys():
+            keys.append(key.replace('crash/', ''))
+        return 0, '\n'.join(keys), ''
+
+    def do_rm(self, cmd, inbuf):
+        crashid = cmd['id']
+        key = 'crash/%s' % crashid
+        self.set_store(key, None)       # removes key
+        return 0, '', ''
+
+    def do_prune(self, cmd, inbuf):
+        now = datetime.datetime.utcnow()
+
+        keep = cmd['keep']
+        try:
+            keep = int(keep)
+        except ValueError:
+            return errno.EINVAL, '', 'keep argument must be integer'
+
+        keeptime = datetime.timedelta(days=keep)
+
+        for key, meta in self.get_store_prefix('crash/').iteritems():
+            meta = json.loads(meta)
+            stamp = self.time_from_string(meta['timestamp'])
+            if stamp <= now - keeptime:
+                self.set_store(key, None)
+
+        return 0, '', ''
+
+    def do_stat(self, cmd, inbuf):
+        # age in days for reporting, ordered smallest first
+        bins = [1, 3, 7]
+        retlines = list()
+
+        def binstr(bindict):
+            binlines = list()
+            count = len(bindict['idlist'])
+            if count:
+                binlines.append(
+                    '%d older than %s days old:' % (count, bindict['age'])
+                )
+                for crashid in bindict['idlist']:
+                    binlines.append(crashid)
+            return '\n'.join(binlines)
+
+        total = 0
+        now = datetime.datetime.utcnow()
+        for i, age in enumerate(bins):
+            agelimit = now - datetime.timedelta(days=age)
+            bins[i] = {
+                'age': age,
+                'agelimit': agelimit,
+                'idlist': list()
+            }
+
+        for key, meta in self.get_store_prefix('crash/').iteritems():
+            total += 1
+            meta = json.loads(meta)
+            stamp = self.time_from_string(meta['timestamp'])
+            crashid = meta['crash_id']
+            for i, bindict in enumerate(bins):
+                if stamp <= bindict['agelimit']:
+                    bindict['idlist'].append(crashid)
+                    # don't count this one again
+                    continue
+
+        retlines.append('%d crashes recorded' % total)
+
+        for bindict in bins:
+            retlines.append(binstr(bindict))
+        return 0, '\n'.join(retlines), ''
+
+    def do_self_test(self, cmd, inbuf):
+        # test time conversion
+        timestr = '2018-06-22 20:35:38.058818Z'
+        dt = self.time_from_string(timestr)
+        if dt != datetime.datetime(2018, 6, 22, 20, 35, 38, 58818):
+            return errno.EINVAL, '', 'time_from_string() failed'
+
+        return 0, '', 'self-test succeeded'
+
+    COMMANDS = [
+        {
+            'cmd': 'crash info name=id,type=CephString',
+            'desc': 'show crash dump metadata',
+            'perm': 'r',
+            'handler': do_info,
+        },
+        {
+            'cmd': 'crash ls',
+            'desc': 'Show saved crash dumps',
+            'perm': 'r',
+            'handler': do_ls,
+        },
+        {
+            'cmd': 'crash post',
+            'desc': 'Add a crash dump (use -i <jsonfile>)',
+            'perm': 'rw',
+            'handler': do_post,
+        },
+        {
+            'cmd': 'crash prune name=keep,type=CephString',
+            'desc': 'Remove crashes older than <keep> days',
+            'perm': 'rw',
+            'handler': do_prune,
+        },
+        {
+            'cmd': 'crash rm name=id,type=CephString',
+            'desc': 'Remove a saved crash <id>',
+            'perm': 'rw',
+            'handler': do_rm,
+        },
+        {
+            'cmd': 'crash self-test',
+            'desc': 'Run a self test of the crash module',
+            'perm': 'r',
+            'handler': do_self_test,
+        },
+        {
+            'cmd': 'crash stat',
+            'desc': 'Summarize recorded crashes',
+            'perm': 'r',
+            'handler': do_stat,
+        },
+    ]

From 263b1a9309195fbea66670427a8f0052235477bb Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Tue, 26 Jun 2018 19:54:56 -0700
Subject: [PATCH 13/23] qa/tasks/mgr: add test_crash, call from
 test_module_selftest

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 8145598f59c65b0e6481c0ee634f194d2bf912f1)
---
 qa/tasks/mgr/test_crash.py           | 108 +++++++++++++++++++++++++++
 qa/tasks/mgr/test_module_selftest.py |   3 +
 2 files changed, 111 insertions(+)
 create mode 100644 qa/tasks/mgr/test_crash.py

diff --git a/qa/tasks/mgr/test_crash.py b/qa/tasks/mgr/test_crash.py
new file mode 100644
index 0000000000000..0c751d1c98d31
--- /dev/null
+++ b/qa/tasks/mgr/test_crash.py
@@ -0,0 +1,108 @@
+
+
+from mgr_test_case import MgrTestCase
+
+import json
+import logging
+import datetime
+
+log = logging.getLogger(__name__)
+UUID = 'd5775432-0742-44a3-a435-45095e32e6b1'
+DATEFMT = '%Y-%m-%d %H:%M:%S.%f'
+
+
+class TestCrash(MgrTestCase):
+
+    def setUp(self):
+        self.setup_mgrs()
+        self._load_module('crash')
+
+        # Whip up some crash data
+        self.crashes = dict()
+        now = datetime.datetime.utcnow()
+
+        for i in (0, 1, 3, 4, 8):
+            timestamp = now - datetime.timedelta(days=i)
+            timestamp = timestamp.strftime(DATEFMT) + 'Z'
+            crash_id = '_'.join((timestamp, UUID)).replace(' ', '_')
+            self.crashes[crash_id] = {
+                'crash_id': crash_id, 'timestamp': timestamp,
+            }
+
+            self.assertEqual(
+                0,
+                self.mgr_cluster.mon_manager.raw_cluster_cmd_result(
+                    'crash', 'post', '-i', '-',
+                    stdin=json.dumps(self.crashes[crash_id]),
+                )
+            )
+
+        retstr = self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'crash', 'ls',
+        )
+        log.warning("setUp: crash ls returns %s" % retstr)
+
+        self.oldest_crashid = crash_id
+
+    def tearDown(self):
+        for crash in self.crashes.itervalues():
+            self.mgr_cluster.mon_manager.raw_cluster_cmd_result(
+                'crash', 'rm', crash['crash_id']
+            )
+
+    def test_info(self):
+        for crash in self.crashes.itervalues():
+            log.warning('test_info: crash %s' % crash)
+            retstr = self.mgr_cluster.mon_manager.raw_cluster_cmd(
+                'crash', 'ls'
+            )
+            log.warning('ls output: %s' % retstr)
+            retstr = self.mgr_cluster.mon_manager.raw_cluster_cmd(
+                'crash', 'info', crash['crash_id'],
+            )
+            log.warning('crash info output: %s' % retstr)
+            crashinfo = json.loads(retstr)
+            self.assertIn('crash_id', crashinfo)
+            self.assertIn('timestamp', crashinfo)
+
+    def test_ls(self):
+        retstr = self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'crash', 'ls',
+        )
+        for crash in self.crashes.itervalues():
+            self.assertIn(crash['crash_id'], retstr)
+
+    def test_rm(self):
+        crashid = self.crashes.keys()[0]
+        self.assertEqual(
+            0,
+            self.mgr_cluster.mon_manager.raw_cluster_cmd_result(
+                'crash', 'rm', crashid,
+            )
+        )
+
+        retstr = self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'crash', 'ls',
+        )
+        self.assertNotIn(crashid, retstr)
+
+    def test_stat(self):
+        retstr = self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'crash', 'stat',
+        )
+        self.assertIn('5 crashes recorded', retstr)
+        self.assertIn('4 older than 1 days old:', retstr)
+        self.assertIn('3 older than 3 days old:', retstr)
+        self.assertIn('1 older than 7 days old:', retstr)
+
+    def test_prune(self):
+        self.assertEqual(
+            0,
+            self.mgr_cluster.mon_manager.raw_cluster_cmd_result(
+                'crash', 'prune', '5'
+            )
+        )
+        retstr = self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'crash', 'ls',
+        )
+        self.assertNotIn(self.oldest_crashid, retstr)
diff --git a/qa/tasks/mgr/test_module_selftest.py b/qa/tasks/mgr/test_module_selftest.py
index f3920eb09e83c..d1bc04b6ec333 100644
--- a/qa/tasks/mgr/test_module_selftest.py
+++ b/qa/tasks/mgr/test_module_selftest.py
@@ -58,6 +58,9 @@ def test_selftest_run(self):
     def test_telemetry(self):
         self._selftest_plugin("telemetry")
 
+    def test_crash(self):
+        self._selftest_plugin("crash")
+
     def test_selftest_config_update(self):
         """
         That configuration updates are seen by running mgr modules

From f0a0ed8efe34c6a8df312eaafadf4ea14789f4dd Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Wed, 27 Jun 2018 19:50:57 -0700
Subject: [PATCH 14/23] qa/suites/rados/mgr: Add test_crash

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 85ab978567a0281d41ac130c0443a11fd22cd19d)
---
 qa/suites/rados/mgr/tasks/crash.yaml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 qa/suites/rados/mgr/tasks/crash.yaml

diff --git a/qa/suites/rados/mgr/tasks/crash.yaml b/qa/suites/rados/mgr/tasks/crash.yaml
new file mode 100644
index 0000000000000..77183c74f320e
--- /dev/null
+++ b/qa/suites/rados/mgr/tasks/crash.yaml
@@ -0,0 +1,16 @@
+
+tasks:
+  - install:
+  - ceph:
+      # tests may leave mgrs broken, so don't try and call into them
+      # to invoke e.g. pg dump during teardown.
+      wait-for-scrub: false
+      log-whitelist:
+        - overall HEALTH_
+        - \(MGR_DOWN\)
+        - \(PG_
+        - replacing it with standby
+        - No standby daemons available
+  - cephfs_test_runner:
+      modules:
+        - tasks.mgr.test_crash

From 19ab472f6a389918ceefc2fb182368453474a8da Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Wed, 27 Jun 2018 15:14:08 -0700
Subject: [PATCH 15/23] doc/mgr: add doc for crash mgr module

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit eb1103e94d0cf6754d077eb64b7f7b94bd684e98)
---
 doc/mgr/crash.rst | 60 +++++++++++++++++++++++++++++++++++++++++++++++
 doc/mgr/index.rst |  1 +
 2 files changed, 61 insertions(+)
 create mode 100644 doc/mgr/crash.rst

diff --git a/doc/mgr/crash.rst b/doc/mgr/crash.rst
new file mode 100644
index 0000000000000..5639f32ebb2b9
--- /dev/null
+++ b/doc/mgr/crash.rst
@@ -0,0 +1,60 @@
+Crash plugin
+============
+The crash plugin collects information about daemon crashdumps and stores
+it in the Ceph cluster for later analysis.
+
+Daemon crashdumps are dumped in /var/lib/ceph/crash by default; this can
+be configured with the option 'crash dir'.  Crash directories are named by
+time and date and a randomly-generated UUID, and contain a metadata file
+'meta' and a recent log file, with a "crash_id" that is the same.
+This plugin allows the metadata about those dumps to be persisted in
+the monitors' storage.
+
+Enabling
+--------
+
+The *crash* module is enabled with::
+
+  ceph mgr module enable crash
+
+Commands
+--------
+::
+
+  ceph crash post -i <metafile>
+
+Save a crash dump.  The metadata file is a JSON blob stored in the crash
+dir as ``meta``.  As usual, the ceph command can be invoked with ``-i -``,
+and will read from stdin.
+
+::
+
+  ceph rm <crashid>
+
+Remove a specific crash dump.
+
+::
+
+  ceph crash ls
+
+List the timestamp/uuid crashids for all saved crash info.
+
+::
+
+  ceph crash stat
+
+Show a summary of saved crash info grouped by age.
+
+::
+
+  ceph crash info <crashid>
+
+Show all details of a saved crash.
+
+::
+
+   ceph crash prune <keep>
+
+Remove saved crashes older than 'keep' days.  <keep> must be an integer.
+
+
diff --git a/doc/mgr/index.rst b/doc/mgr/index.rst
index d8440cb8e6a3d..86b7889fd4e02 100644
--- a/doc/mgr/index.rst
+++ b/doc/mgr/index.rst
@@ -38,3 +38,4 @@ sensible.
     Telemetry plugin <telemetry>
     Telegraf plugin <telegraf>
     Iostat plugin <iostat>
+    Crash plugin <crash>

From 71fff0918c05942e806f6a97e834438dc0c350d0 Mon Sep 17 00:00:00 2001
From: Noah Watkins <nwatkins@redhat.com>
Date: Wed, 18 Jul 2018 13:49:52 -0700
Subject: [PATCH 16/23] mgr/crash: add timestamp filter helper

Signed-off-by: Noah Watkins <nwatkins@redhat.com>
(cherry picked from commit 4dda7cb5ddef81d7f053ed9deb0c287fa7c58880)
---
 src/pybind/mgr/crash/module.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/crash/module.py b/src/pybind/mgr/crash/module.py
index 668ef62ea8464..34584aaa0c0d6 100644
--- a/src/pybind/mgr/crash/module.py
+++ b/src/pybind/mgr/crash/module.py
@@ -36,6 +36,21 @@ def time_from_string(timestr):
         timestr = timestr.rstrip('Z')
         return datetime.datetime.strptime(timestr, DATEFMT)
 
+    def timestamp_filter(self, f):
+        """
+        Filter crash reports by timestamp.
+
+        :param f: f(time) return true to keep crash report
+        :returns: crash reports for which f(time) returns true
+        """
+        def inner((_, meta)):
+            meta = json.loads(meta)
+            time = self.time_from_string(meta["timestamp"])
+            return f(time)
+        matches = filter(inner, six.iteritems(
+            self.get_store_prefix("crash/")))
+        return map(lambda (k, m): (k, json.loads(m)), matches)
+
     # command handlers
 
     def do_info(self, cmd, inbuf):
@@ -80,13 +95,10 @@ def do_prune(self, cmd, inbuf):
         except ValueError:
             return errno.EINVAL, '', 'keep argument must be integer'
 
-        keeptime = datetime.timedelta(days=keep)
+        cutoff = now - datetime.timedelta(days=keep)
 
-        for key, meta in self.get_store_prefix('crash/').iteritems():
-            meta = json.loads(meta)
-            stamp = self.time_from_string(meta['timestamp'])
-            if stamp <= now - keeptime:
-                self.set_store(key, None)
+        for key, _ in self.timestamp_filter(lambda ts: ts <= cutoff):
+            self.set_store(key, None)
 
         return 0, '', ''
 

From b8a31fb842963eca1ef0857c01a9c1d5167f44bf Mon Sep 17 00:00:00 2001
From: Noah Watkins <nwatkins@redhat.com>
Date: Wed, 18 Jul 2018 14:26:47 -0700
Subject: [PATCH 17/23] mgr/crash: json report of recent crashes

reports crashes per entity type with hour granularity. primarily for
consumption by the insights module.

Signed-off-by: Noah Watkins <nwatkins@redhat.com>
(cherry picked from commit caa121b5f07273f5da0fee6625122488d865c5a8)
---
 src/pybind/mgr/crash/module.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/src/pybind/mgr/crash/module.py b/src/pybind/mgr/crash/module.py
index 34584aaa0c0d6..a4f6ecd79bcda 100644
--- a/src/pybind/mgr/crash/module.py
+++ b/src/pybind/mgr/crash/module.py
@@ -2,6 +2,8 @@
 import datetime
 import errno
 import json
+import six
+from collections import defaultdict
 
 
 DATEFMT = '%Y-%m-%d %H:%M:%S.%f'
@@ -145,6 +147,31 @@ def binstr(bindict):
             retlines.append(binstr(bindict))
         return 0, '\n'.join(retlines), ''
 
+    def do_json_report(self, cmd, inbuf):
+        """
+        Return a machine readable summary of recent crashes.
+        """
+        try:
+            hours = int(cmd['hours'])
+        except ValueError:
+            return errno.EINVAL, '', '<hours> argument must be integer'
+
+        report = defaultdict(lambda: 0)
+        cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=hours)
+        for _, meta in self.timestamp_filter(lambda ts: ts >= cutoff):
+            try:
+                etype = meta["entity_name"]
+                etype, _ = etype.split(".")
+                etype = "unknown" if not etype else etype
+            except KeyError:
+                etype = "unknown"
+            except (ValueError, AttributeError):
+                etype = str(etype)
+                pass
+            report[etype] += 1
+
+        return 0, '', json.dumps(report)
+
     def do_self_test(self, cmd, inbuf):
         # test time conversion
         timestr = '2018-06-22 20:35:38.058818Z'
@@ -197,4 +224,10 @@ def do_self_test(self, cmd, inbuf):
             'perm': 'r',
             'handler': do_stat,
         },
+        {
+            'cmd': 'crash json_report name=hours,type=CephString',
+            'desc': 'Crashes in the last <hours> hours',
+            'perm': 'r',
+            'handler': do_json_report,
+        },
     ]

From 5bb28ba7240043cb9aefe006de512802acf6a042 Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Thu, 2 Aug 2018 20:20:41 -0700
Subject: [PATCH 18/23] qa/suites/rados, qa/workunits/rados: Add suite/workunit
 for ceph-crash

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 298a1d92d2aba1ae6c8e6c87744b1261b5ac57e1)
---
 qa/suites/rados/singleton/all/test-crash.yaml | 14 ++++++++
 qa/workunits/rados/test_crash.sh              | 33 +++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 qa/suites/rados/singleton/all/test-crash.yaml
 create mode 100755 qa/workunits/rados/test_crash.sh

diff --git a/qa/suites/rados/singleton/all/test-crash.yaml b/qa/suites/rados/singleton/all/test-crash.yaml
new file mode 100644
index 0000000000000..6dbffb48c31b3
--- /dev/null
+++ b/qa/suites/rados/singleton/all/test-crash.yaml
@@ -0,0 +1,14 @@
+roles:
+  - [client.0, mon.a, mgr.x, osd.0, osd.1, osd.2]
+
+tasks:
+  - install:
+  - ceph:
+      log-whitelist:
+        - Reduced data availability
+        - OSD_.*DOWN
+  - workunit:
+      clients:
+         client.0:
+           - rados/test_crash.sh
+  - ceph.restart: [osd.*]
diff --git a/qa/workunits/rados/test_crash.sh b/qa/workunits/rados/test_crash.sh
new file mode 100755
index 0000000000000..6e7aaaaba6348
--- /dev/null
+++ b/qa/workunits/rados/test_crash.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -x
+
+# run on a single-node three-OSD cluster
+
+sudo killall -ABRT ceph-osd
+sleep 5
+
+# kill caused coredumps; find them and delete them, carefully, so as
+# not to disturb other coredumps, or else teuthology will see them
+# and assume test failure.  sudos are because the core files are
+# root/600
+for f in $(find $TESTDIR/archive/coredump -type f); do
+	gdb_output=$(echo "quit" | sudo gdb /usr/bin/ceph-osd $f)
+	if expr match "$gdb_output" ".*generated.*ceph-osd.*" && \
+	   ( \
+
+	   	expr match "$gdb_output" ".*terminated.*signal 6.*" || \
+	   	expr match "$gdb_output" ".*terminated.*signal SIGABRT.*" \
+	   )
+	then
+		sudo rm $f
+	fi
+done
+
+# let daemon find crashdumps on startup
+sudo systemctl restart ceph-crash
+sleep 30
+
+# must be 3 crashdumps registered and moved to crash/posted
+[ $(ceph crash ls | wc -l) = 3 ]  || exit 1
+[ $(sudo find /var/lib/ceph/crash/posted/ -name meta | wc -l) = 3 ] || exit 1

From e842d309205ac9e5b9ca5d31d0f6ef5951c3ca16 Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Wed, 8 Aug 2018 15:11:44 -0700
Subject: [PATCH 19/23] common/options: enable mgr 'crash' module by default

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 34e2853804ea8a4020e7a750e1b882042f1951b7)
---
 src/common/options.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index 22adf6a7a368a..09dfaece1b0a3 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4737,7 +4737,7 @@ std::vector<Option> get_global_options() {
     .set_description("Filesystem path to manager modules."),
 
     Option("mgr_initial_modules", Option::TYPE_STR, Option::LEVEL_BASIC)
-    .set_default("restful status balancer iostat")
+    .set_default("restful status balancer iostat crash")
     .set_flag(Option::FLAG_NO_MON_UPDATE)
     .set_flag(Option::FLAG_CLUSTER_CREATE)
     .add_service("mon")

From 1e15b2ea9cf2d082e16abb7bf7715d139948eaaa Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Mon, 23 Jul 2018 22:05:01 -0700
Subject: [PATCH 20/23] add ceph-crash service

ceph-crash runs from systemd and watches /var/lib/ceph/crash
for crashdumps, posting them to the mgrs using the mgr's
crash plugin

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit da20184a166dd29babb4b70afcf562ea08e89598)
---
 ceph.spec.in                  | 17 ++++---
 debian/ceph-base.dirs         |  1 +
 debian/ceph-base.install      |  2 +
 src/CMakeLists.txt            |  4 ++
 src/ceph-crash.in             | 83 +++++++++++++++++++++++++++++++++++
 systemd/50-ceph.preset        |  1 +
 systemd/CMakeLists.txt        |  1 +
 systemd/ceph-crash.service.in | 13 ++++++
 8 files changed, 116 insertions(+), 6 deletions(-)
 create mode 100755 src/ceph-crash.in
 create mode 100644 systemd/ceph-crash.service.in

diff --git a/ceph.spec.in b/ceph.spec.in
index 4bd38de91f77b..3b43e3496342b 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -994,13 +994,14 @@ mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/mon
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/osd
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/mds
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/mgr
+mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/crash
+mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/crash/posted
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/radosgw
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-osd
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-mds
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-rgw
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-mgr
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-rbd
-mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/crash
 
 %if 0%{?suse_version}
 # create __pycache__ directories and their contents
@@ -1016,6 +1017,7 @@ rm -rf %{buildroot}
 %files
 
 %files base
+%{_bindir}/ceph-crash
 %{_bindir}/crushtool
 %{_bindir}/monmaptool
 %{_bindir}/osdmaptool
@@ -1034,6 +1036,7 @@ rm -rf %{buildroot}
 %{_libdir}/ceph/erasure-code/libec_*.so*
 %dir %{_libdir}/ceph/compressor
 %{_libdir}/ceph/compressor/libceph_*.so*
+%{_unitdir}/ceph-crash.service
 %ifarch x86_64
 %dir %{_libdir}/ceph/crypto
 %{_libdir}/ceph/crypto/libceph_*.so*
@@ -1079,6 +1082,8 @@ rm -rf %{buildroot}
 %{_mandir}/man8/monmaptool.8*
 %{_mandir}/man8/ceph-kvstore-tool.8*
 #set up placeholder directories
+%attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/crash
+%attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/crash/posted
 %attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/tmp
 %attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/bootstrap-osd
 %attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/bootstrap-mds
@@ -1091,22 +1096,22 @@ rm -rf %{buildroot}
 %if 0%{?suse_version}
 %fillup_only
 if [ $1 -eq 1 ] ; then
-/usr/bin/systemctl preset ceph-disk@\*.service ceph.target >/dev/null 2>&1 || :
+/usr/bin/systemctl preset ceph-disk@\*.service ceph.target ceph-crash.service >/dev/null 2>&1 || :
 fi
 %endif
 %if 0%{?fedora} || 0%{?rhel}
-%systemd_post ceph-disk@\*.service ceph.target
+%systemd_post ceph-disk@\*.service ceph.target ceph-crash.service
 %endif
 if [ $1 -eq 1 ] ; then
-/usr/bin/systemctl start ceph.target >/dev/null 2>&1 || :
+/usr/bin/systemctl start ceph.target ceph-crash.service >/dev/null 2>&1 || :
 fi
 
 %preun base
 %if 0%{?suse_version}
-%service_del_preun ceph-disk@\*.service ceph.target
+%service_del_preun ceph-disk@\*.service ceph.target ceph-crash.service
 %endif
 %if 0%{?fedora} || 0%{?rhel}
-%systemd_preun ceph-disk@\*.service ceph.target
+%systemd_preun ceph-disk@\*.service ceph.target ceph-crash.service
 %endif
 
 %postun base
diff --git a/debian/ceph-base.dirs b/debian/ceph-base.dirs
index 262e6f6a508e2..6f580230a65f4 100644
--- a/debian/ceph-base.dirs
+++ b/debian/ceph-base.dirs
@@ -5,3 +5,4 @@ var/lib/ceph/bootstrap-rgw
 var/lib/ceph/bootstrap-rbd
 var/lib/ceph/tmp
 var/lib/ceph/crash
+var/lib/ceph/crash/posted
diff --git a/debian/ceph-base.install b/debian/ceph-base.install
index 65d1511d75bb3..1f5619cf8241f 100644
--- a/debian/ceph-base.install
+++ b/debian/ceph-base.install
@@ -1,4 +1,6 @@
 etc/init.d/ceph
+lib/systemd/system/ceph-crash.service
+usr/bin/ceph-crash
 usr/bin/ceph-debugpack
 usr/bin/ceph-detect-init
 usr/bin/ceph-run
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0f08c2dae4e3a..c6e16944ab96f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -934,6 +934,9 @@ configure_file(${CMAKE_SOURCE_DIR}/src/init-ceph.in
 configure_file(ceph-post-file.in
   ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ceph-post-file @ONLY)
 
+configure_file(ceph-crash.in
+  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ceph-crash @ONLY)
+
 if(WITH_TESTS)
   install(PROGRAMS
     ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ceph-debugpack
@@ -944,6 +947,7 @@ endif()
 install(PROGRAMS
   ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ceph
   ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ceph-post-file
+  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ceph-crash
   ${CMAKE_SOURCE_DIR}/src/ceph-run
   ${CMAKE_SOURCE_DIR}/src/ceph-clsinfo
   DESTINATION bin)
diff --git a/src/ceph-crash.in b/src/ceph-crash.in
new file mode 100755
index 0000000000000..b43cd782c3fe3
--- /dev/null
+++ b/src/ceph-crash.in
@@ -0,0 +1,83 @@
+#!@PYTHON_EXECUTABLE@
+# -*- mode:python -*-
+# vim: ts=4 sw=4 smarttab expandtab
+
+import argparse
+import logging
+import os
+import subprocess
+import sys
+import time
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-p', '--path', default='/var/lib/ceph/crash',
+        help='base path to monitor for crash dumps')
+    parser.add_argument(
+        '-d', '--delay', default=10.0, type=float,
+        help='minutes to delay between scans (0 to exit after one)',
+    )
+    return parser.parse_args()
+
+
+def post_crash(path):
+    pr = subprocess.Popen(
+        args=['timeout', '30', 'ceph', 'crash', 'post', '-i', '-'],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    f = open(os.path.join(path, 'meta'), 'r')
+    stdout, stderr = pr.communicate(input=f.read())
+    rc = pr.wait()
+    f.close()
+    if rc != 0:
+        log.warning('post %s failed: %s' % (path, stderr))
+    return rc
+
+
+def scrape_path(path):
+    for p in os.listdir(path):
+        crashpath = os.path.join(path, p)
+        metapath = os.path.join(crashpath, 'meta')
+        donepath = os.path.join(crashpath, 'done')
+        if os.path.isfile(metapath):
+            if not os.path.isfile(donepath):
+                # hang out just for a bit; either we interrupted the dump
+                # or the daemon crashed before finishing it
+                time.sleep(1)
+                if not os.path.isfile(donepath):
+                    return
+            # ok, we can process this one
+            rc = post_crash(crashpath)
+            if rc == 0:
+                os.rename(crashpath, os.path.join(path, 'posted/', p))
+                log.debug(
+                    "posted %s and renamed %s -> %s " %
+                    (metapath, p, os.path.join('posted/', p))
+                )
+
+
+def main():
+    args = parse_args()
+    postdir = os.path.join(args.path, 'posted')
+
+    while not os.path.isdir(postdir):
+        log.error("%s does not exist; please create" % postdir)
+        time.sleep(30)
+
+    log.info("monitoring path %s, delay %ds" % (args.path, args.delay * 60.0))
+    while True:
+        scrape_path(args.path)
+        if args.delay == 0:
+            sys.exit(0)
+        time.sleep(args.delay * 60)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/systemd/50-ceph.preset b/systemd/50-ceph.preset
index 34c0801f9f2ba..da3ee0b7b0bcf 100644
--- a/systemd/50-ceph.preset
+++ b/systemd/50-ceph.preset
@@ -4,3 +4,4 @@ enable ceph-mgr.target
 enable ceph-mon.target
 enable ceph-osd.target
 enable ceph-radosgw.target
+enable ceph-crash.service
diff --git a/systemd/CMakeLists.txt b/systemd/CMakeLists.txt
index 37fb1d47aba48..51aa29f1f5d62 100644
--- a/systemd/CMakeLists.txt
+++ b/systemd/CMakeLists.txt
@@ -4,6 +4,7 @@ set(CEPH_SYSTEMD_ENV_DIR "/etc/sysconfig"
   CACHE PATH "Location for systemd service environmental variable settings files")
 set(SYSTEMD_ENV_FILE "${CEPH_SYSTEMD_ENV_DIR}/ceph")
 foreach(service
+    ceph-crash
     ceph-fuse@
     ceph-mds@
     ceph-mgr@
diff --git a/systemd/ceph-crash.service.in b/systemd/ceph-crash.service.in
new file mode 100644
index 0000000000000..8304dd61220bb
--- /dev/null
+++ b/systemd/ceph-crash.service.in
@@ -0,0 +1,13 @@
+[Unit]
+Description=Ceph crash dump collector
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/ceph-crash
+Restart=always
+RestartSec=10
+StartLimitInterval=10min
+StartLimitBurst=10
+
+[Install]
+WantedBy=ceph.target

From 1934d437c75bb53fcbb92fa84d7b33c017277db8 Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Tue, 10 Jul 2018 21:43:00 -0700
Subject: [PATCH 21/23] global/signal_handler: add 'done' file to signal
 crashdump is ready

for an asynchronous crash scraper to use for synchronization

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 1bb5c63b05f244100a2830bc97442da0dc131d0c)
---
 src/global/signal_handler.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/global/signal_handler.cc b/src/global/signal_handler.cc
index 5af31c28df236..6922fe34df604 100644
--- a/src/global/signal_handler.cc
+++ b/src/global/signal_handler.cc
@@ -260,6 +260,8 @@ static void handle_fatal_signal(int signum)
 	(void)r;
 	::close(fd);
       }
+      snprintf(fn, sizeof(fn)-1, "%s/done", base);
+      ::creat(fn, 0444);
     }
   }
 

From b7dc916e22a30310f45f2cd80af0d7a236b2a062 Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Fri, 24 Aug 2018 20:52:50 -0700
Subject: [PATCH 22/23] global/signal_handler.cc: report assert_file as correct
 name

Boilerplate error led to two 'assert_line' items in the dump

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit f6a4897ecb455d3dde97cef1cb7133a28d7098f5)
---
 src/global/signal_handler.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/global/signal_handler.cc b/src/global/signal_handler.cc
index 6922fe34df604..e04a80112de9b 100644
--- a/src/global/signal_handler.cc
+++ b/src/global/signal_handler.cc
@@ -243,7 +243,7 @@ static void handle_fatal_signal(int signum)
 	  jf.dump_string("assert_file", g_assert_file);
 	}
 	if (g_assert_line) {
-	  jf.dump_unsigned("assert_file", g_assert_line);
+	  jf.dump_unsigned("assert_line", g_assert_line);
 	}
 	if (g_assert_thread_name[0]) {
 	  jf.dump_string("assert_thread_name", g_assert_thread_name);

From 1feaa08a1510385985c44d2e928ae1eb97479491 Mon Sep 17 00:00:00 2001
From: Dan Mick <dan.mick@redhat.com>
Date: Fri, 24 Aug 2018 20:56:29 -0700
Subject: [PATCH 23/23] mgr/telemetry: add crashdump info to report

Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit 2c17bc98bf788430f8ce5c459ab23567827dec67)
---
 src/pybind/mgr/telemetry/module.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py
index c3e24f150e047..ff64b8276d2fc 100644
--- a/src/pybind/mgr/telemetry/module.py
+++ b/src/pybind/mgr/telemetry/module.py
@@ -209,6 +209,19 @@ def gather_mon_metadata(self, mon_map):
 
         return metadata
 
+    def gather_crashinfo(self):
+        crashdict = dict()
+        errno, crashids, err = self.remote('crash', 'do_ls', '', '')
+        if errno:
+            return ''
+        for crashid in crashids.split():
+            cmd = {'id': crashid}
+            errno, crashinfo, err = self.remote('crash', 'do_info', cmd, '')
+            if errno:
+                continue
+            crashdict[crashid] = json.loads(crashinfo)
+        return crashdict
+
     def compile_report(self):
         report = {'leaderboard': False, 'report_version': 1}
 
@@ -275,6 +288,8 @@ def compile_report(self):
         for key, value in service_map['services'].items():
             report['services'][key] += 1
 
+        report['crashes'] = self.gather_crashinfo()
+
         return report
 
     def send(self, report):