From 6ee6dc77307bb2092ea6d0a15b47beb86f8955eb Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Wed, 22 Oct 2025 16:01:53 +0100 Subject: [PATCH 1/2] Adds a function to create a core dump on demand Signed-off-by: Simon Davies --- docs/how-to-debug-a-hyperlight-guest.md | 35 ++++++++++++++-- .../src/sandbox/initialized_multi_use.rs | 41 +++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index 0cf2e0753..2dc5b2256 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -207,13 +207,12 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or | └───────────────────────────────────────────────────────────────────────────────────────────────┘ ``` -## Dumping the guest state to an ELF core dump when an unhandled crash occurs +## Dumping the guest state to an ELF core dump -When a guest crashes because of an unknown VmExit or unhandled exception, the vCPU state is dumped to an `ELF` core dump file. +When a guest crashes because of an unknown VmExit or unhandled exception, the vCPU state can be optionally dumped to an `ELF` core dump file. This can be used to inspect the state of the guest at the time of the crash. -To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, enable the `crashdump` -feature and run. +To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, enable the `crashdump` feature and run. The feature enables the creation of core dump files for both debug and release builds of Hyperlight hosts. By default, Hyperlight places the core dumps in the temporary directory (platform specific). To change this, use the `HYPERLIGHT_CORE_DUMP_DIR` environment variable to specify a directory. @@ -227,6 +226,34 @@ To selectively disable this feature for a specific sandbox, you can set the `gue cfg.set_guest_core_dump(false); // Disable core dump for this sandbox ``` +### Creating a dump on demand + +You can also create a core dump of the current state of the guest on demand by calling the `generate_crashdump` method on the `InitializedMultiUseSandbox` instance. This can be useful for debugging issues in the guest that do not cause crashes (e.g., a guest function that does not return). + +This is only available when the `crashdump` feature is enabled and then only if the sandbox +is also configured to allow core dumps (which is the default behavior). + +# Examples + +Attach to your running process with gdb and call this function: + +```shell +sudo gdb -p +(gdb) info threads +# find the thread that is running the guest function you want to debug +(gdb) thread +# switch to the frame where you have access to your MultiUseSandbox instance +(gdb) backtrace +(gdb) frame +# get the pointer to your MultiUseSandbox instance +# Get the sandbox pointer +(gdb) print sandbox +# Call the crashdump function with the pointer + # Call the crashdump function +call sandbox.generate_crashdump() +``` +The crashdump should be available in crash dump directory (see `HYPERLIGHT_CORE_DUMP_DIR` env var). + ### Inspecting the core dump After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`. diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index bbb23638b..87b52603c 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -475,6 +475,47 @@ impl MultiUseSandbox { pub fn interrupt_handle(&self) -> Arc { self.vm.interrupt_handle() } + /// Generate a crash dump of the current state of the VM underlying this sandbox. + /// + /// Creates an ELF core dump file that can be used for debugging. The dump + /// captures the current state of the sandbox including registers, memory regions, + /// and other execution context. + /// + /// The location of the core dump file is determined by the `HYPERLIGHT_CORE_DUMP_DIR` + /// environment variable. If not set, it defaults to the system's temporary directory. + /// + /// This is only available when the `crashdump` feature is enabled and then only if the sandbox + /// is also configured to allow core dumps (which is the default behavior). + /// + /// This can be useful for generating a crash dump from gdb when trying to debug issues in the + /// guest that dont cause crashes (e.g. a guest function that does not return) + /// + /// # Examples + /// + /// Attach to your running process with gdb and call this function: + /// + /// ```shell + /// sudo gdb -p + /// (gdb) info threads + /// # find the thread that is running the guest function you want to debug + /// (gdb) thread + /// # switch to the frame where you have access to your MultiUseSandbox instance + /// (gdb) backtrace + /// (gdb) frame + /// # get the pointer to your MultiUseSandbox instance + /// # Get the sandbox pointer + /// (gdb) print sandbox + /// # Call the crashdump function + /// call sandbox.generate_crashdump() + /// ``` + /// The crashdump should be available in crash dump directory (see `HYPERLIGHT_CORE_DUMP_DIR` env var). + /// + #[cfg(crashdump)] + #[instrument(err(Debug), skip_all, parent = Span::current())] + + pub fn generate_crashdump(&self) -> Result<()> { + crate::hypervisor::crashdump::generate_crashdump(self.vm.as_ref()) + } } impl Callable for MultiUseSandbox { From cba2dcb6d9478360194933bce583626f326e87a5 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Thu, 23 Oct 2025 15:00:29 +0100 Subject: [PATCH 2/2] Add gdb script and update docs Signed-off-by: Simon Davies --- docs/how-to-debug-a-hyperlight-guest.md | 11 +++-- .../scripts/dump_all_sandboxes.gdb | 41 +++++++++++++++++++ .../src/sandbox/initialized_multi_use.rs | 6 +-- 3 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 src/hyperlight_host/scripts/dump_all_sandboxes.gdb diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index 2dc5b2256..f4b45e996 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -226,14 +226,14 @@ To selectively disable this feature for a specific sandbox, you can set the `gue cfg.set_guest_core_dump(false); // Disable core dump for this sandbox ``` -### Creating a dump on demand +## Creating a dump on demand You can also create a core dump of the current state of the guest on demand by calling the `generate_crashdump` method on the `InitializedMultiUseSandbox` instance. This can be useful for debugging issues in the guest that do not cause crashes (e.g., a guest function that does not return). This is only available when the `crashdump` feature is enabled and then only if the sandbox is also configured to allow core dumps (which is the default behavior). -# Examples +### Example Attach to your running process with gdb and call this function: @@ -252,7 +252,12 @@ sudo gdb -p # Call the crashdump function call sandbox.generate_crashdump() ``` -The crashdump should be available in crash dump directory (see `HYPERLIGHT_CORE_DUMP_DIR` env var). +The crashdump should be available `/tmp` or in the crash dump directory (see `HYPERLIGHT_CORE_DUMP_DIR` env var). To make this process easier, you can also create a gdb script that automates these steps. You can find an example script [here](../scripts/dump_all_sandboxes.gdb). This script will try and generate a crashdump for every active thread except thread 1 , it assumes that the variable sandbox exists in frame 15 on every thread. You can edit it to fit your needs. Then use it like this: + +```shell +(gdb) source scripts/dump_all_sandboxes.gdb +(gdb) dump_all_sandboxes +``` ### Inspecting the core dump diff --git a/src/hyperlight_host/scripts/dump_all_sandboxes.gdb b/src/hyperlight_host/scripts/dump_all_sandboxes.gdb new file mode 100644 index 000000000..f2ef79c5c --- /dev/null +++ b/src/hyperlight_host/scripts/dump_all_sandboxes.gdb @@ -0,0 +1,41 @@ +define dump_all_sandboxes + set pagination off + + # Get the total number of threads + info threads + + # Loop through all threads (adjust max if you have more than 200 threads) + set $thread_num = 2 + while $thread_num <= 200 + # Try to switch to this thread + thread $thread_num + + # Check if thread switch succeeded (GDB sets $_thread to current thread) + if $_thread == $thread_num + echo \n=== Thread + p $thread_num + echo ===\n + + # Go to frame 15 + frame 15 + + + set $sb = &sandbox + call sandbox.generate_crashdump() + + set $thread_num = $thread_num + 1 + else + # No more threads, exit loop + set $thread_num = 201 + end + end + + echo \nDone dumping all sandboxes\n + set pagination on +end + +document dump_all_sandboxes +Dump crashdumps for sandboxes on all threads (except thread 1). +Assumes sandbox is in frame 15 on each thread. +Usage: dump_all_sandboxes +end \ No newline at end of file diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index 87b52603c..874e4758f 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -493,7 +493,7 @@ impl MultiUseSandbox { /// # Examples /// /// Attach to your running process with gdb and call this function: - /// + /// /// ```shell /// sudo gdb -p /// (gdb) info threads @@ -505,11 +505,11 @@ impl MultiUseSandbox { /// # get the pointer to your MultiUseSandbox instance /// # Get the sandbox pointer /// (gdb) print sandbox - /// # Call the crashdump function + /// # Call the crashdump function /// call sandbox.generate_crashdump() /// ``` /// The crashdump should be available in crash dump directory (see `HYPERLIGHT_CORE_DUMP_DIR` env var). - /// + /// #[cfg(crashdump)] #[instrument(err(Debug), skip_all, parent = Span::current())]