kernel: Add core dump facility

* Add function core_dump_write_core_file(). It writes a core file for
  the current thread's team. The file format is similar to that of
  other OSs (i.e. ELF with PT_LOAD segments and a PT_NOTE segment), but
  most of the notes are Haiku specific (infos for team, areas, images,
  threads). More data will probably need to be added.
* Add team flag TEAM_FLAG_DUMP_CORE, thread flag
  THREAD_FLAGS_TRAP_FOR_CORE_DUMP, and Team property coreDumpCondition,
  a condition variable available while a core dump is progress. A
  thread that finds its flag THREAD_FLAGS_TRAP_FOR_CORE_DUMP set before
  exiting the kernel to userland calls core_dump_trap_thread(), which
  blocks on the condition variable until the core dump has finished. We
  need the team's threads to stop so we can get their CPU state (and
  have a generally unchanging team state while writing the core file).
* Add user debugger message B_DEBUG_WRITE_CORE_FILE. It causes
  core_dump_write_core_file() to be called for the team.
* Dumping core as an immediate effect of a terminal signal has not been
  implemented yet, but that should be fairly straight forward.

headers/os/kernel/debugger.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2005, Ingo Weinhold, bonefish@users.sf.net.
+ * Copyright 2005-2016 Haiku, Inc. All rights reserved.
  * Distributed under the terms of the MIT License.
  */
 #ifndef _DEBUGGER_H
@@ -163,7 +163,9 @@ typedef enum {
 										// install_team_debugger()
 
 	B_DEBUG_START_PROFILER,				// start/stop sampling
-	B_DEBUG_STOP_PROFILER				//
+	B_DEBUG_STOP_PROFILER,				//
+
+	B_DEBUG_WRITE_CORE_FILE				// write a core file
 } debug_nub_message;
 
 // messages sent to the debugger
@@ -412,6 +414,20 @@ typedef struct {
 	thread_id			thread;			// thread to profile
 } debug_nub_stop_profiler;
 
+// B_DEBUG_WRITE_CORE_FILE
+
+typedef struct {
+	port_id				reply_port;		// port to send the reply to
+	char				path[B_PATH_NAME_LENGTH];
+										// path of the core file; must not exist
+										// yet; must be absolute
+} debug_nub_write_core_file;
+
+typedef struct {
+	status_t	error;					// B_OK on success
+} debug_nub_write_core_file_reply;
+
+
 // reply is debug_profiler_update
 
 // union of all messages structures sent to the debug nub thread
@@ -433,6 +449,7 @@ typedef union {
 	debug_nub_get_signal_handler	get_signal_handler;
 	debug_nub_start_profiler		start_profiler;
 	debug_nub_stop_profiler			stop_profiler;
+	debug_nub_write_core_file		write_core_file;
 } debug_nub_message_data;
 
 

headers/os/kernel/elf.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2015 Haiku, Inc. All rights reserved.
+ * Copyright 2002-2016 Haiku, Inc. All rights reserved.
  * Distributed under the terms of the MIT License.
  */
 #ifndef _ELF_H
@@ -153,6 +153,9 @@ typedef struct {
 #define ELFDATA2LSB	1	/* little endian */
 #define ELFDATA2MSB	2	/* big endian */
 
+/* ELF version (EI_VERSION) */
+#define EV_NONE		0	/* invalid */
+#define EV_CURRENT	1	/* current version */
 
 /*** section header ***/
 
@@ -578,6 +581,125 @@ typedef struct {
 #define VER_FLG_WEAK	0x2		/* weak version identifier */
 
 
+/*** core files ***/
+
+/* note section header */
+
+typedef struct {
+	Elf32_Word n_namesz;		/* length of the note's name */
+	Elf32_Word n_descsz;		/* length of the note's descriptor */
+	Elf32_Word n_type;			/* note type */
+} Elf32_Nhdr;
+
+typedef struct {
+	Elf64_Word n_namesz;		/* length of the note's name */
+	Elf64_Word n_descsz;		/* length of the note's descriptor */
+	Elf64_Word n_type;			/* note type */
+} Elf64_Nhdr;
+
+/* values for note name */
+#define ELF_NOTE_CORE		"CORE"
+#define ELF_NOTE_HAIKU		"Haiku"
+
+/* values for note type (n_type) */
+/* ELF_NOTE_CORE/... */
+#define NT_FILE				0x46494c45 /* mapped files */
+
+/* ELF_NOTE_HAIKU/... */
+#define NT_TEAM				0x7465616d 	/* team */
+#define NT_AREAS			0x61726561 	/* areas */
+#define NT_IMAGES			0x696d6167 	/* images */
+#define NT_THREADS			0x74687264 	/* threads */
+
+/* NT_TEAM: Elf32_Note_Team; char[] args */
+typedef struct {
+	int32		nt_id;				/* team ID */
+	int32		nt_uid;				/* team owner ID */
+	int32		nt_gid;				/* team group ID */
+} Elf32_Note_Team;
+
+typedef Elf32_Note_Team Elf64_Note_Team;
+
+/* NT_AREAS: uint32 count; Elf32_Note_Area_Entry[count]; char[] names */
+typedef struct {
+	int32		na_id;				/* area ID */
+	uint32		na_lock;			/* lock type (B_NO_LOCK, ...) */
+	uint32		na_protection;		/* protection (B_READ_AREA | ...) */
+	uint32		na_base;			/* area base address */
+	uint32		na_size;			/* area size */
+	uint32		na_ram_size;		/* physical memory used */
+} Elf32_Note_Area_Entry;
+
+/* NT_AREAS: uint64 count; Elf64_Note_Area_Entry[count]; char[] names */
+typedef struct {
+	int32		na_id;				/* area ID */
+	uint32		na_lock;			/* lock type (B_NO_LOCK, ...) */
+	uint32		na_protection;		/* protection (B_READ_AREA | ...) */
+	uint32		na_pad1;
+	uint64		na_base;			/* area base address */
+	uint64		na_size;			/* area size */
+	uint64		na_ram_size;		/* physical memory used */
+} Elf64_Note_Area_Entry;
+
+/* NT_IMAGES: uint32 count; Elf32_Note_Image_Entry[count]; char[] names */
+typedef struct {
+	int32		ni_id;				/* image ID */
+	int32		ni_type;			/* image type (B_APP_IMAGE, ...) */
+	uint32		ni_init_routine;	/* address of init function */
+	uint32		ni_term_routine;	/* address of termination function */
+	int32		ni_device;			/* device ID of mapped file */
+	int64		ni_node;			/* node ID of mapped file */
+	uint32		ni_text_base;		/* base address of text segment */
+	uint32		ni_text_size;		/* size of text segment */
+	uint32		ni_data_base;		/* base address of data segment */
+	uint32		ni_data_size;		/* size of data segment */
+} Elf32_Note_Image_Entry;
+
+/* NT_IMAGES: uint64 count; Elf64_Note_Image_Entry[count]; char[] names */
+typedef struct {
+	int32		ni_id;				/* image ID */
+	int32		ni_type;			/* image type (B_APP_IMAGE, ...) */
+	uint64		ni_init_routine;	/* address of init function */
+	uint64		ni_term_routine;	/* address of termination function */
+	uint32		ni_pad1;
+	int32		ni_device;			/* device ID of mapped file */
+	int64		ni_node;			/* node ID of mapped file */
+	uint64		ni_text_base;		/* base address of text segment */
+	uint64		ni_text_size;		/* size of text segment */
+	uint64		ni_data_base;		/* base address of data segment */
+	uint64		ni_data_size;		/* size of data segment */
+} Elf64_Note_Image_Entry;
+
+/* NT_THREADS:
+ * uint32 count;
+ * uint32 cpuStateSize;
+ * {Elf32_Note_Thread_Entry, uint8[cpuStateSize] cpuState}[count];
+ * char[] names
+ */
+typedef struct {
+	int32		nth_id;				/* thread ID */
+	int32		nth_state;			/* thread state (B_THREAD_RUNNING, ...) */
+	int32		nth_priority;		/* thread priority */
+	uint32		nth_stack_base;		/* thread stack base address */
+	uint32		nth_stack_end;		/* thread stack end address */
+} Elf32_Note_Thread_Entry;
+
+/* NT_THREADS:
+ * uint64 count;
+ * uint64 cpuStateSize;
+ * {Elf64_Note_Thread_Entry, uint8[cpuStateSize] cpuState}[count];
+ * char[] names
+ */
+typedef struct {
+	int32		nth_id;				/* thread ID */
+	int32		nth_state;			/* thread state (B_THREAD_RUNNING, ...) */
+	int32		nth_priority;		/* thread priority */
+	uint32		nth_pad1;
+	uint64		nth_stack_base;		/* thread stack base address */
+	uint64		nth_stack_end;		/* thread stack end address */
+} Elf64_Note_Thread_Entry;
+
+
 /*** inline functions ***/
 
 #ifdef __cplusplus

headers/private/kernel/core_dump.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2016, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ *
+ * Core dump support.
+ */
+#ifndef _KERNEL_CORE_DUMP_H
+#define _KERNEL_CORE_DUMP_H
+
+
+#include <SupportDefs.h>
+
+
+status_t	core_dump_write_core_file(const char* path, bool killTeam);
+void		core_dump_trap_thread();
+
+
+#endif	// _KERNEL_CORE_DUMP_H

headers/private/kernel/thread_types.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2011, Haiku, Inc.
+ * Copyright 2004-2016, Haiku, Inc.
  * Distributed under the terms of the MIT License.
  *
  * Thread definition and structures
@@ -44,6 +44,9 @@ enum team_state {
 };
 
 #define	TEAM_FLAG_EXEC_DONE	0x01
+	// team has executed exec*()
+#define	TEAM_FLAG_DUMP_CORE	0x02
+	// a core dump is in progress
 
 typedef enum job_control_state {
 	JOB_CONTROL_STATE_NONE,
@@ -392,6 +395,11 @@ struct Team : TeamThreadIteratorEntry<team_id>, KernelReferenceable,
 									Thread* lockedThread = NULL) const;
 			bigtime_t			UserCPUTime() const;
 
+			ConditionVariable*	CoreDumpCondition() const
+									{ return fCoreDumpCondition; }
+			void				SetCoreDumpCondition(
+									ConditionVariable* condition)
+									{ fCoreDumpCondition = condition; }
 private:
 								Team(team_id id, bool kernel);
 
@@ -412,6 +420,9 @@ struct Team : TeamThreadIteratorEntry<team_id>, KernelReferenceable,
 									// protected by scheduler lock
 			TeamUserTimeUserTimerList fUserTimeUserTimers;
 			int32				fUserDefinedTimerCount;	// accessed atomically
+
+			ConditionVariable*	fCoreDumpCondition;
+									// protected by fLock
 };
 
 
@@ -821,6 +832,9 @@ using BKernel::ProcessGroupList;
 	// the thread is currently in a syscall; set/reset only for certain
 	// functions (e.g. ioctl()) to allow inner functions to discriminate
 	// whether e.g. parameters were passed from userland or kernel
+#define	THREAD_FLAGS_TRAP_FOR_CORE_DUMP		0x1000
+	// core dump in progress; the thread shall not exit the kernel to userland,
+	// but shall invoke core_dump_trap_thread() instead.
 
 
 #endif	/* _KERNEL_THREAD_TYPES_H */

headers/private/system/elf_private.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2015 Haiku, Inc. All rights reserved.
+ * Copyright 2002-2016 Haiku, Inc. All rights reserved.
  * Distributed under the terms of the MIT License.
  *
  * Copyright 2001 Travis Geiselbrecht. All rights reserved.
@@ -37,6 +37,11 @@ DEFINE_ELF_TYPE(Verdef, elf_verdef);
 DEFINE_ELF_TYPE(Verdaux, elf_verdaux);
 DEFINE_ELF_TYPE(Verneed, elf_verneed);
 DEFINE_ELF_TYPE(Vernaux, elf_vernaux);
+DEFINE_ELF_TYPE(Nhdr, elf_nhdr);
+DEFINE_ELF_TYPE(Note_Team, elf_note_team);
+DEFINE_ELF_TYPE(Note_Area_Entry, elf_note_area_entry);
+DEFINE_ELF_TYPE(Note_Image_Entry, elf_note_image_entry);
+DEFINE_ELF_TYPE(Note_Thread_Entry, elf_note_thread_entry);
 
 #undef DEFINE_ELF_TYPE
 #undef _ELF_TYPE

src/system/kernel/arch/x86/32/interrupts.S

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2011, The Haiku Team. All rights reserved.
+ * Copyright 2002-2016, The Haiku Team. All rights reserved.
  * Distributed under the terms of the MIT License.
  *
  * Copyright 2001, Travis Geiselbrecht. All rights reserved.
@@ -571,7 +571,8 @@ STATIC_FUNCTION(int_bottom_user):
 	jne		1f
 
 	testl	$(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
-			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \
+			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%edi)
 	jnz		kernel_exit_work
 1:
@@ -654,6 +655,7 @@ STATIC_FUNCTION(handle_syscall):
 
 	testl	$(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
 			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP \
 			| THREAD_FLAGS_64_BIT_SYSCALL_RETURN \
 			| THREAD_FLAGS_RESTART_SYSCALL | THREAD_FLAGS_SYSCALL_RESTARTED) \
 			, THREAD_flags(%edi)
@@ -714,9 +716,10 @@ FUNCTION_END(handle_syscall)
 
   bad_syscall_number:
   STATIC_FUNCTION(kernel_exit_work):
-	// if no signals are pending and the thread shall not be debugged, we can
-	// use the quick kernel exit function
-	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD) \
+	// if no signals are pending and the thread shall not be debugged or stopped
+	// for a core dump, we can use the quick kernel exit function
+	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%edi)
 	jnz		kernel_exit_handle_signals
 	cli								// disable interrupts
@@ -828,7 +831,8 @@ FUNCTION(x86_return_to_userland):
 	// check, if any kernel exit work has to be done
 	movl	%gs:0, %edi
 	testl	$(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
-			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \
+			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%edi)
 	jnz		kernel_exit_work
 

src/system/kernel/arch/x86/64/interrupts.S

@@ -272,7 +272,8 @@ STATIC_FUNCTION(int_bottom_user):
 	// If there are no signals pending or we're not debugging, we can avoid
 	// most of the work here, just need to update the kernel time.
 	testl	$(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
-			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \
+			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%r12)
 	jnz		.Lkernel_exit_work
 
@@ -294,7 +295,8 @@ STATIC_FUNCTION(int_bottom_user):
 	// Slow path for return to userland.
 
 	// Do we need to handle signals?
-	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD) \
+	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%r12)
 	jnz		.Lkernel_exit_handle_signals
 	cli
@@ -421,7 +423,7 @@ FUNCTION(x86_64_syscall_entry):
 2:
 	testl	$(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
 			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
-			| THREAD_FLAGS_RESTART_SYSCALL) \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP | THREAD_FLAGS_RESTART_SYSCALL) \
 			, THREAD_flags(%r12)
 	jnz		.Lpost_syscall_work
 
@@ -482,7 +484,8 @@ FUNCTION(x86_64_syscall_entry):
 	addq	$48, %rsp
 1:
 	// Do we need to handle signals?
-	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD) \
+	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%r12)
 	jnz		.Lpost_syscall_handle_signals
 	cli
@@ -578,7 +581,8 @@ FUNCTION(x86_return_to_userland):
 	// Perform kernel exit work.
 	movq	%gs:0, %r12
 	testl	$(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
-			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \
+			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%r12)
 	jnz		.Luserland_return_work
 
@@ -593,7 +597,8 @@ FUNCTION(x86_return_to_userland):
 	// Slow path for return to userland.
 
 	// Do we need to handle signals?
-	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD) \
+	testl	$(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
+			| THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
 			, THREAD_flags(%r12)
 	jnz		.Luserland_return_handle_signals
 	cli