Skip to content
Permalink
Browse files

3136 windows agent: improved crash reporting

With crash debug enabled the agent will now report the windows exception code on crash
as well as the address causing the crash and the exact git revision he agent was built
from.
On 64bit systems it will also print a full backtrace including register contents.

Technical details:
Our official builds are built with mingw-w64, so we can't use windows api to resolve
debugging symbols in our own binaryi. These frames will only include addresses, while
api calls are resolved.

However, the agent is now built with dwarf-2 debugging symbols (outdated but better
tool support) and stripped of symbols in a separate build step. With the correct
unstripped exe the adresses can still be resolved using addr2line:
addr2line -e check_mk_agent-64.unstripped.exe <address>
  • Loading branch information
Sebastian Herbord
Sebastian Herbord committed May 24, 2016
1 parent dd012e6 commit a476038710272c50904438446691d1c71164583e
@@ -0,0 +1,22 @@
Title: windows agent: improved crash reporting
Level: 1
Component: checks
Compatible: compat
Version: 1.2.9i1
Date: 1464079421
Class: feature

With crash debug enabled the agent will now report the windows exception code on crash
as well as the address causing the crash and the exact git revision he agent was built
from.
On 64bit systems it will also print a full backtrace including register contents.

Technical details:
Our official builds are built with mingw-w64, so we can't use windows api to resolve
debugging symbols in our own binaryi. These frames will only include addresses, while
api calls are resolved.

However, the agent is now built with dwarf-2 debugging symbols (outdated but better
tool support) and stripped of symbols in a separate build step. With the correct
unstripped exe the adresses can still be resolved using addr2line:
addr2line -e check_mk_agent-64.unstripped.exe <address>
@@ -83,6 +83,7 @@
* 3606 saprouter_cert: new check which monitors the age of validity of a SAP router certificate...
* 3135 skype, skype.xmpp_proxy, skype.conferencing, skype.sip_stack: added furter metrics and adjusted default levels
* 3609 apc_rackpdu_power: now upper levels for electrical current are confiugurable
* 3136 windows agent: improved crash reporting...
* 3073 FIX: windows agent: relative paths to mrpe scripts are now treated as relative to the agent installation directory...
* 3061 FIX: mk_jolokia: Fixed debugging of the agent plugin
* 3074 FIX: windows agent: fixed incorrect values for 32-bit performance counters
@@ -11,3 +11,4 @@ state
log
.ycm_extra_conf.py
.depend
check_mk_agent*.unstripped.exe
@@ -52,4 +52,6 @@ EventLog.h (hide)
EventLog.cc (hide)
ExternalCmd.h (hide)
ExternalCmd.cc (hide)
crashhandling.cc (hide)
crashhandling.h (hide)
.depend (hide)
@@ -27,7 +27,8 @@ SHELL = /bin/bash
VERSION = 1.4.0i1
VERSION_DUMMY = @@@@@CHECK_MK_VERSION@@@@@
WINDRES = i686-w64-mingw32-windres
COPTIONS = -s -O2 -Wall -Wformat=2 -Werror -static -static-libgcc -static-libstdc++ -fno-rtti -std=c++11 -DCHECK_MK_VERSION='"$(VERSION_DUMMY)"' -D__USE_MINGW_ANSI_STDIO
GIT_REV = $(shell git describe --abbrev=4 --dirty --always --tags)
COPTIONS = -O2 -gdwarf-2 -Wall -Wformat=2 -Werror -static -static-libgcc -static-libstdc++ -fno-rtti -std=c++11 -DCHECK_MK_VERSION='"$(VERSION_DUMMY)"' -D__USE_MINGW_ANSI_STDIO -DVCS_REV=\"$(GIT_REV)\"
# for debug builds
#COPTIONS = -O0 -Wall -static -static-libgcc -static-libstdc++ -fno-rtti -std=c++11 -gdwarf-2 -DCHECK_MK_VERSION='"$(VERSION_DUMMY)"' -D__USE_MINGW_ANSI_STDIO
MKDIR = mkdir -p
@@ -91,7 +92,7 @@ directories:
# number. Instead a placeholder is being used as version.a
# -----------------------------------------------------------------------------

SRCS = check_mk_agent.cc SettingsCollector.cc Configuration.cc Environment.cc stringutil.cc wmiHelper.cc ListenSocket.cc PerfCounter.cc PerfCounterCommon.cc Thread.cc OutputProxy.cc logging.cc Crypto.cc OHMMonitor.cc EventLog.cc ExternalCmd.cc EventLogVista.cc IEventLog.cc
SRCS = check_mk_agent.cc SettingsCollector.cc Configuration.cc Environment.cc stringutil.cc wmiHelper.cc ListenSocket.cc PerfCounter.cc PerfCounterCommon.cc Thread.cc OutputProxy.cc logging.cc Crypto.cc OHMMonitor.cc EventLog.cc ExternalCmd.cc EventLogVista.cc IEventLog.cc crashhandling.cc
OBJS = $(addprefix obj/,$(SRCS:.cc=.o))
OBJS64 = $(addprefix obj64/,$(SRCS:.cc=.o))

@@ -110,16 +111,22 @@ obj/%.o: %.cc
obj64/%.o: %.cc
$(CXX_64) -c $(COPTIONS) -o $@ $<


check_mk_agent.unversioned.exe: $(OBJS)
check_mk_agent.unstripped.exe: $(OBJS)
$(MAKE) inc_build_version
$(MAKE) check_mk_agent.res
$(CXX) $(COPTIONS) -o $@ $^ check_mk_agent.res -lwsock32 -lws2_32 -lole32 -loleaut32 -lwbemuuid -lpsapi -lshlwapi
$(CXX) $(COPTIONS) -o $@ $^ check_mk_agent.res -lwsock32 -lws2_32 -lole32 -loleaut32 -lwbemuuid -lpsapi -lshlwapi -limagehlp

check_mk_agent-64.unversioned.exe: $(OBJS64)
check_mk_agent-64.unstripped.exe: $(OBJS64)
$(MAKE) inc_build_version
$(MAKE) check_mk_agent-64.res
$(CXX_64) $(COPTIONS) -o $@ $^ check_mk_agent-64.res -lwsock32 -lws2_32 -lole32 -loleaut32 -lwbemuuid -lpsapi -lshlwapi
$(CXX_64) $(COPTIONS) -o $@ $^ check_mk_agent-64.res -lwsock32 -lws2_32 -lole32 -loleaut32 -lwbemuuid -lpsapi -lshlwapi -limagehlp


check_mk_agent.unversioned.exe: check_mk_agent.unstripped.exe
strip -o $@ $^

check_mk_agent-64.unversioned.exe: check_mk_agent-64.unstripped.exe
strip -o $@ $^


# -----------------------------------------------------------------------------
@@ -1 +1 @@
1677
1727
Binary file not shown.
Binary file not shown.
@@ -74,6 +74,7 @@
#include "OutputProxy.h"
#include "PerfCounter.h"
#include "Thread.h"
#include "crashhandling.h"
#include "dynamic_func.h"
#include "logging.h"
#include "stringutil.h"
@@ -680,7 +681,8 @@ void output_eventlog(OutputProxy &out, LPCWSTR logname, uint64_t &first_record,

out.output("[[[%ls]]]\n", logname);
int worst_state = 0;
// record_number is the last event we read, so we want to seek past it
// record_number is the last event we read, so we want to seek past
// it
first_record = log->seek(first_record + 1);

uint64_t last_record = first_record;
@@ -974,8 +976,7 @@ void section_ps_wmi(OutputProxy &out) {
}
crash_log(
"Data types are different than expected, please report this and "
"include "
"the following: %ls",
"include the following: %ls",
types.c_str());
abort();
}
@@ -3836,6 +3837,8 @@ void RunImmediate(const char *mode, int argc, char **argv) {
int main(int argc, char **argv) {
wsa_startup();

SetUnhandledExceptionFilter(exception_handler);

SetConsoleCtrlHandler((PHANDLER_ROUTINE)ctrl_handler, TRUE);

if ((argc > 2) && (strcmp(argv[1], "file") && strcmp(argv[1], "unpack")))
Binary file not shown.
BIN +30.5 KB (100%) agents/windows/check_mk_agent.msi
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,155 @@
// +------------------------------------------------------------------+
// | ____ _ _ __ __ _ __ |
// | / ___| |__ ___ ___| | __ | \/ | |/ / |
// | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / |
// | | |___| | | | __/ (__| < | | | | . \ |
// | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ |
// | |
// | Copyright Mathias Kettner 2016 mk@mathias-kettner.de |
// +------------------------------------------------------------------+
//
// This file is part of Check_MK.
// The official homepage is at http://mathias-kettner.de/check_mk.
//
// check_mk is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by
// the Free Software Foundation in version 2. check_mk is distributed
// in the hope that it will be useful, but WITHOUT ANY WARRANTY; with-
// out even the implied warranty of MERCHANTABILITY or FITNESS FOR A
// PARTICULAR PURPOSE. See the GNU General Public License for more de-
// ails. You should have received a copy of the GNU General Public
// License along with GNU Make; see the file COPYING. If not, write
// to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
// Boston, MA 02110-1301 USA.

#include "crashhandling.h"
#include <dbghelp.h>
#include <windows.h>
#include <winnt.h>
#include "logging.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>

#ifdef __x86_64

static void dump_registers(CONTEXT *c) {
crash_log("rax %016" PRIx64 " rbx %016" PRIx64 " rcx %016" PRIx64
" rdx %016" PRIx64,
c->Rax, c->Rbx, c->Rcx, c->Rdx);
crash_log("rsp %016" PRIx64 " rbp %016" PRIx64 " rsi %016" PRIx64
" rdi %016" PRIx64,
c->Rsp, c->Rbp, c->Rsi, c->Rdi);
crash_log("r8 %016" PRIx64 " r9 %016" PRIx64 " r10 %016" PRIx64
" r11 %016" PRIx64,
c->R8, c->R9, c->R10, c->R11);
crash_log("r12 %016" PRIx64 " r13 %016" PRIx64 " r14 %016" PRIx64
" r15 %016" PRIx64,
c->R12, c->R13, c->R14, c->R15);
}

/**
* converts instruction pointer to "filename (line)"
**/
static std::string resolve(ULONG64 rip) {
std::string result;

HANDLE process = ::GetCurrentProcess();
DWORD64 symbol_offset = 0;

{ // Get file / line of source code.
IMAGEHLP_LINE64 line_str = {0};
line_str.SizeOfStruct = sizeof(IMAGEHLP_LINE64);

if (::SymGetLineFromAddr64(process, (DWORD64)rip,
(DWORD *)&symbol_offset, &line_str)) {
result = line_str.FileName;
result += "(";
result += std::to_string((uint64_t)line_str.LineNumber).c_str();
result += "): ";
}
}

{ // get symbol name
struct {
union {
SYMBOL_INFO symbol;
char buf[sizeof(SYMBOL_INFO) + 1024];
} u;
} image_symbol = {0};

image_symbol.u.symbol.SizeOfStruct = sizeof(SYMBOL_INFO);
image_symbol.u.symbol.Name[0] = 0;
image_symbol.u.symbol.MaxNameLen =
sizeof(image_symbol) - sizeof(SYMBOL_INFO);

// Successor of SymGetSymFromAddr64.
if (::SymFromAddr(process, (DWORD64)rip, &symbol_offset,
&image_symbol.u.symbol)) {
result += image_symbol.u.symbol.Name;
}
}

return result;
}

// display backtrace. with mingw this will resolve only symbols from
// windows dlls, not our own code. we can use addr2line on the unstripped
// exe to resolve those.
static void log_backtrace(PVOID exc_address) {
CONTEXT context;
context.ContextFlags = CONTEXT_ALL;
::RtlCaptureContext(&context);

// the backtrace includes all the stack frames from the exception handler
// itself. Only start outputting with the frame the exception occured in
int exc_frame = -1;

for (int i = 0;; ++i) {
ULONG64 rip = context.Rip;
ULONG64 image_base;
PRUNTIME_FUNCTION entry =
::RtlLookupFunctionEntry(rip, &image_base, nullptr);

if (entry == nullptr) break;

if (rip == reinterpret_cast<ULONG64>(exc_address)) {
exc_frame = i;
}

if (exc_frame != -1) {
crash_log("#%d %016" PRIx64 " %s", i - exc_frame, rip,
resolve(rip).c_str());
dump_registers(&context);
}

PVOID handler_data;
ULONG64 establisher_frame;
::RtlVirtualUnwind(0, image_base, rip, entry, &context, &handler_data,
&establisher_frame, nullptr);
}
}

#endif

LONG WINAPI exception_handler(LPEXCEPTION_POINTERS ptrs) {
crash_log("windows exception 0x%" PRIx32 " from address 0x%p (revision %s)",
static_cast<unsigned int>(ptrs->ExceptionRecord->ExceptionCode),
ptrs->ExceptionRecord->ExceptionAddress, VCS_REV);

#ifdef __x86_64

HANDLE proc = ::GetCurrentProcess();
::SymInitialize(proc, nullptr, TRUE);

::SymSetOptions(SymGetOptions() | SYMOPT_DEFERRED_LOADS |
SYMOPT_NO_IMAGE_SEARCH);

log_backtrace(ptrs->ExceptionRecord->ExceptionAddress);

::SymCleanup(proc);
#else // __x86_64
// on x86 the backtrace can't be implemented in the same way
#endif // __x86_64

return EXCEPTION_CONTINUE_SEARCH;
}
@@ -0,0 +1,32 @@
// +------------------------------------------------------------------+
// | ____ _ _ __ __ _ __ |
// | / ___| |__ ___ ___| | __ | \/ | |/ / |
// | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / |
// | | |___| | | | __/ (__| < | | | | . \ |
// | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ |
// | |
// | Copyright Mathias Kettner 2016 mk@mathias-kettner.de |
// +------------------------------------------------------------------+
//
// This file is part of Check_MK.
// The official homepage is at http://mathias-kettner.de/check_mk.
//
// check_mk is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by
// the Free Software Foundation in version 2. check_mk is distributed
// in the hope that it will be useful, but WITHOUT ANY WARRANTY; with-
// out even the implied warranty of MERCHANTABILITY or FITNESS FOR A
// PARTICULAR PURPOSE. See the GNU General Public License for more de-
// ails. You should have received a copy of the GNU General Public
// License along with GNU Make; see the file COPYING. If not, write
// to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
// Boston, MA 02110-1301 USA.

#ifndef crash_handling_h
#define crash_handling_h

#include <windows.h>

LONG WINAPI exception_handler(LPEXCEPTION_POINTERS ptrs);

#endif // crash_handling_h
Binary file not shown.
BIN +7.96 KB (100%) agents/windows/install_agent.exe
Binary file not shown.

0 comments on commit a476038

Please sign in to comment.
You can’t perform that action at this time.