From 6a92ea517bba60e21bc66f8783fd6bf8f637e74f Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Sun, 22 Jul 2018 16:22:54 +0200 Subject: [PATCH 1/3] Add guest debugging support Signed-off-by: Alexandro Sanchez Bach --- API.md | 1 + core/cpu.c | 4 +- core/hax.c | 1 + core/include/cpu.h | 6 + core/include/hax_core_interface.h | 1 + core/include/vcpu.h | 8 +- core/include/vm.h | 3 +- core/vcpu.c | 104 +++++++++++++++++- .../com_intel_hax/com_intel_hax_ui.c | 6 + include/darwin/hax_interface_mac.h | 3 + include/hax_interface.h | 17 +++ windows/hax_entry.c | 13 ++- windows/hax_entry.h | 4 + 13 files changed, 160 insertions(+), 11 deletions(-) diff --git a/API.md b/API.md index 2d3d3170..b3f6f283 100644 --- a/API.md +++ b/API.md @@ -95,6 +95,7 @@ itself as well as the host environment. #define HAX_CAP_64BIT_RAMBLOCK (1 << 3) #define HAX_CAP_64BIT_SETRAM (1 << 4) #define HAX_CAP_TUNNEL_PAGE (1 << 5) + #define HAX_CAP_DEBUG (1 << 7) ``` * (Output) `wstatus`: The first set of capability flags reported to the caller. The following bits may be set, while others are reserved: diff --git a/core/cpu.c b/core/cpu.c index 3529e414..b111a29a 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -347,12 +347,12 @@ vmx_result_t cpu_vmx_run(struct vcpu_t *vcpu, struct hax_tunnel *htun) /* Must ensure the IRQ is disabled before setting CR2 */ set_cr2(vcpu->state->_cr2); - load_guest_msr(vcpu); + vcpu_load_guest_state(vcpu); result = asm_vmxrun(vcpu->state, vcpu->launched); vcpu->is_running = 0; - save_guest_msr(vcpu); + vcpu_save_guest_state(vcpu); vcpu_load_host_state(vcpu); #ifdef DEBUG_HOST_STATE diff --git a/core/hax.c b/core/hax.c index 29354c38..e3d0cf89 100644 --- a/core/hax.c +++ b/core/hax.c @@ -348,6 +348,7 @@ int hax_get_capability(void *buf, int bufLeng, int *outLength) #endif cap->winfo |= HAX_CAP_TUNNEL_PAGE; cap->winfo |= HAX_CAP_RAM_PROTECTION; + cap->winfo |= HAX_CAP_DEBUG; if (cpu_data->vmx_info._ept_cap) { cap->winfo |= HAX_CAP_EPT; } diff --git a/core/include/cpu.h b/core/include/cpu.h index 214636ca..7fc9eaeb 100644 --- a/core/include/cpu.h +++ b/core/include/cpu.h @@ -77,6 +77,12 @@ struct hstate { uint64 fake_gs; system_desc_t host_gdtr; system_desc_t host_idtr; + // Debug registers + uint64 dr0; + uint64 dr1; + uint64 dr2; + uint64 dr3; + uint64 dr6; }; struct hstate_compare { diff --git a/core/include/hax_core_interface.h b/core/include/hax_core_interface.h index b2e6cf16..b86fa544 100644 --- a/core/include/hax_core_interface.h +++ b/core/include/hax_core_interface.h @@ -45,6 +45,7 @@ int vcpu_put_fpu(struct vcpu_t *vcpu, struct fx_layout *fl); int vcpu_get_fpu(struct vcpu_t *vcpu, struct fx_layout *fl); int vcpu_set_regs(struct vcpu_t *vcpu, struct vcpu_state_t *vs); int vcpu_get_regs(struct vcpu_t *vcpu, struct vcpu_state_t *vs); +void vcpu_debug(struct vcpu_t *vcpu, struct hax_debug_t *debug); void * get_vcpu_host(struct vcpu_t *vcpu); int set_vcpu_host(struct vcpu_t *vcpu, void *vcpu_host); diff --git a/core/include/vcpu.h b/core/include/vcpu.h index c08466fc..7ad36092 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -209,6 +209,9 @@ struct vcpu_t { uint64 cr_pat; uint64 cpuid_features_flag_mask; + /* Debugging */ + uint32 debug_control; + /* Interrupt stuff */ uint32 intr_pending[8]; uint32 nr_pending_intrs; @@ -229,10 +232,10 @@ struct vcpu_t { struct vcpu_t *vcpu_create(struct vm_t *vm, void *vm_host, int vcpu_id); int vcpu_execute(struct vcpu_t *vcpu); +void vcpu_load_guest_state(struct vcpu_t *vcpu); +void vcpu_save_guest_state(struct vcpu_t *vcpu); void vcpu_load_host_state(struct vcpu_t *vcpu); void vcpu_save_host_state(struct vcpu_t *vcpu); -void load_guest_msr(struct vcpu_t *vcpu); -void save_guest_msr(struct vcpu_t *vcpu); int vtlb_active(struct vcpu_t *vcpu); int vcpu_vmexit_handler(struct vcpu_t *vcpu, exit_reason_t exit_reason, @@ -248,6 +251,7 @@ int vcpu_get_fpu(struct vcpu_t *vcpu, struct fx_layout *fl); int vcpu_put_fpu(struct vcpu_t *vcpu, struct fx_layout *fl); int vcpu_get_msr(struct vcpu_t *vcpu, uint64 entry, uint64 *val); int vcpu_put_msr(struct vcpu_t *vcpu, uint64 entry, uint64 val); +void vcpu_debug(struct vcpu_t *vcpu, struct hax_debug_t *debug); /* The declaration for OS wrapper code */ int hax_vcpu_destroy_host(struct vcpu_t *cvcpu, void *vcpu_host); diff --git a/core/include/vm.h b/core/include/vm.h index dc77d31c..f94c72e2 100644 --- a/core/include/vm.h +++ b/core/include/vm.h @@ -110,7 +110,8 @@ enum exit_status { HAX_EXIT_STATECHANGE, HAX_EXIT_PAUSED, HAX_EXIT_FAST_MMIO, - HAX_EXIT_PAGEFAULT + HAX_EXIT_PAGEFAULT, + HAX_EXIT_DEBUG }; enum run_flag { diff --git a/core/vcpu.c b/core/vcpu.c index c5191c36..e2fc4f15 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -1154,17 +1154,35 @@ void vcpu_save_host_state(struct vcpu_t *vcpu) save_host_msr(vcpu); hstate->hcr2 = get_cr2(); + hstate->dr0 = get_dr0(); + hstate->dr1 = get_dr1(); + hstate->dr2 = get_dr2(); + hstate->dr3 = get_dr3(); + hstate->dr6 = get_dr6(); vcpu_enter_fpu_state(vcpu); // CR0 should be written after host fpu state is saved vmwrite(vcpu, HOST_CR0, get_cr0()); } +static void vcpu_update_exception_bitmap(struct vcpu_t *vcpu) +{ + uint32 exc_bitmap; + + exc_bitmap = (1u << VECTOR_MC); + if (vcpu->debug_control & (HAX_DEBUG_USE_HW_BP | HAX_DEBUG_STEP)) { + exc_bitmap |= (1u << VECTOR_DB); + } + if (vcpu->debug_control & HAX_DEBUG_USE_SW_BP) { + exc_bitmap |= (1u << VECTOR_BP); + } + vmwrite(vcpu, VMX_EXCEPTION_BITMAP, exc_bitmap); +} + static void fill_common_vmcs(struct vcpu_t *vcpu) { uint32 pin_ctls; uint32 pcpu_ctls; uint32 scpu_ctls; - uint32 exc_bitmap; uint32 exit_ctls = 0; uint32 entry_ctls; uint32 vmcs_err = 0; @@ -1198,8 +1216,6 @@ static void fill_common_vmcs(struct vcpu_t *vcpu) } } - exc_bitmap = 1u << VECTOR_MC; - #ifdef __x86_64__ exit_ctls = EXIT_CONTROL_HOST_ADDR_SPACE_SIZE | EXIT_CONTROL_LOAD_EFER | EXIT_CONTROL_SAVE_DEBUG_CONTROLS; @@ -1272,7 +1288,7 @@ static void fill_common_vmcs(struct vcpu_t *vcpu) WRITE_CONTROLS(vcpu, VMX_SECONDARY_PROCESSOR_CONTROLS, scpu_ctls); } - vmwrite(vcpu, VMX_EXCEPTION_BITMAP, exc_bitmap); + vcpu_update_exception_bitmap(vcpu); WRITE_CONTROLS(vcpu, VMX_EXIT_CONTROLS, exit_ctls); @@ -1368,10 +1384,39 @@ void vcpu_load_host_state(struct vcpu_t *vcpu) load_host_msr(vcpu); set_cr2(hstate->hcr2); + set_dr0(hstate->dr0); + set_dr1(hstate->dr1); + set_dr2(hstate->dr2); + set_dr3(hstate->dr3); + set_dr6(hstate->dr6); vcpu_exit_fpu_state(vcpu); } +void vcpu_save_guest_state(struct vcpu_t *vcpu) +{ + struct vcpu_state_t *state = vcpu->state; + + save_guest_msr(vcpu); + state->_dr0 = get_dr0(); + state->_dr1 = get_dr1(); + state->_dr2 = get_dr2(); + state->_dr3 = get_dr3(); + state->_dr6 = get_dr6(); +} + +void vcpu_load_guest_state(struct vcpu_t *vcpu) +{ + struct vcpu_state_t *state = vcpu->state; + + load_guest_msr(vcpu); + set_dr0(state->_dr0); + set_dr1(state->_dr1); + set_dr2(state->_dr2); + set_dr3(state->_dr3); + set_dr6(state->_dr6); +} + /* * Copies bits 0, 1, 2, ..., (|size| * 8 - 1) of |src| to the same positions * in the 64-bit buffer pointed to by |pdst|, and clears bits (|size| * 8) @@ -2213,6 +2258,20 @@ static int exit_exc_nmi(struct vcpu_t *vcpu, struct hax_tunnel *htun) dump_vmcs(vcpu); break; } + case VECTOR_DB: { + htun->_exit_status = HAX_EXIT_DEBUG; + htun->debug.rip = vcpu->state->_rip; + htun->debug.dr6 = vmx(vcpu, exit_qualification).raw; + htun->debug.dr7 = vmread(vcpu, GUEST_DR7); + return HAX_EXIT; + } + case VECTOR_BP: { + htun->_exit_status = HAX_EXIT_DEBUG; + htun->debug.rip = vcpu->state->_rip; + htun->debug.dr6 = 0; + htun->debug.dr7 = 0; + return HAX_EXIT; + } } if (exit_intr_info.vector == VECTOR_PF) { @@ -2797,6 +2856,10 @@ static int exit_cr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) static int exit_dr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) { + // TODO: DR-exiting flag is disabled so this function is never executed. + // We should conditionally enable "DR exiting" whenever + // HAX_DEBUG_USE_HW_BP is enabled, to prevent the guest from + // tampering with debug drN registers. uint64 *dr; struct vcpu_state_t *state = vcpu->state; @@ -2856,7 +2919,7 @@ static int exit_dr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) if (vmx(vcpu, exit_qualification.dr.direction)) { // MOV DR -> GPR state->_regs[vmx(vcpu, exit_qualification).dr.gpr] = *dr; - } else { + } else if (!(vcpu->debug_control & HAX_DEBUG_USE_HW_BP)) { // MOV DR <- GPR *dr = state->_regs[vmx(vcpu, exit_qualification).dr.gpr]; } @@ -3799,6 +3862,37 @@ int vcpu_set_msr(struct vcpu_t *vcpu, uint64 entry, uint64 val) return handle_msr_write(vcpu, entry, val); } +void vcpu_debug(struct vcpu_t *vcpu, struct hax_debug_t *debug) +{ + if (debug->control & HAX_DEBUG_ENABLE) { + vcpu->debug_control = debug->control; + // Hardware breakpoints + if (debug->control & HAX_DEBUG_USE_HW_BP) { + vcpu->state->_dr0 = debug->dr[0]; + vcpu->state->_dr1 = debug->dr[1]; + vcpu->state->_dr2 = debug->dr[2]; + vcpu->state->_dr3 = debug->dr[3]; + vmwrite(vcpu, GUEST_DR7, debug->dr[7]); + } else { + vmwrite(vcpu, GUEST_DR7, 0); + } + // Single-stepping + if (debug->control & HAX_DEBUG_STEP) { + vmwrite(vcpu, GUEST_RFLAGS, + vmread(vcpu, GUEST_RFLAGS) | EFLAGS_TF); + } else { + vmwrite(vcpu, GUEST_RFLAGS, + vmread(vcpu, GUEST_RFLAGS) & ~EFLAGS_TF); + } + } else { + vcpu->debug_control = 0; + vmwrite(vcpu, GUEST_DR7, 0); + vmwrite(vcpu, GUEST_RFLAGS, + vmread(vcpu, GUEST_RFLAGS) & ~EFLAGS_TF); + } + vcpu_update_exception_bitmap(vcpu); +}; + static void vcpu_dump(struct vcpu_t *vcpu, uint32 mask, const char *caption) { vcpu_vmread_all(vcpu); diff --git a/darwin/hax_driver/com_intel_hax/com_intel_hax_ui.c b/darwin/hax_driver/com_intel_hax/com_intel_hax_ui.c index 30ce44fe..edd2faa6 100644 --- a/darwin/hax_driver/com_intel_hax/com_intel_hax_ui.c +++ b/darwin/hax_driver/com_intel_hax/com_intel_hax_ui.c @@ -229,6 +229,12 @@ static int hax_vcpu_ioctl(dev_t dev, ulong cmd, caddr_t data, int flag, vcpu_interrupt(mvcpu2cvcpu(vcpu), vector); break; } + case HAX_IOCTL_VCPU_DEBUG: { + struct hax_debug_t *hax_debug; + hax_debug = (struct hax_debug_t *)data; + vcpu_debug(cvcpu, hax_debug); + break; + } default: { int pid; char task_name[17]; diff --git a/include/darwin/hax_interface_mac.h b/include/darwin/hax_interface_mac.h index 7a235df5..ee7d1c7c 100644 --- a/include/darwin/hax_interface_mac.h +++ b/include/darwin/hax_interface_mac.h @@ -68,6 +68,9 @@ /* API 2.0 */ #define HAX_VM_IOCTL_NOTIFY_QEMU_VERSION _IOW(0, 0x84, struct hax_qemu_version) +/* API 5.0 */ +#define HAX_IOCTL_VCPU_DEBUG _IOW(0, 0xc9, struct hax_debug_t) + #define HAX_KERNEL64_CS 0x80 #define HAX_KERNEL32_CS 0x08 #ifdef __i386__ diff --git a/include/hax_interface.h b/include/hax_interface.h index 7aba2319..6ffb6744 100644 --- a/include/hax_interface.h +++ b/include/hax_interface.h @@ -143,6 +143,11 @@ struct hax_tunnel { struct { paddr_t dummy; } state; + struct { + uint64_t rip; + uint64_t dr6; + uint64_t dr7; + } debug; }; uint64_t apic_base; } PACKED; @@ -182,6 +187,7 @@ struct hax_module_version { #define HAX_CAP_64BIT_SETRAM (1 << 4) #define HAX_CAP_TUNNEL_PAGE (1 << 5) #define HAX_CAP_RAM_PROTECTION (1 << 6) +#define HAX_CAP_DEBUG (1 << 7) struct hax_capabilityinfo { /* @@ -269,4 +275,15 @@ struct hax_qemu_version { uint32_t least_version; } PACKED; +#define HAX_DEBUG_ENABLE (1 << 0) +#define HAX_DEBUG_STEP (1 << 1) +#define HAX_DEBUG_USE_SW_BP (1 << 2) +#define HAX_DEBUG_USE_HW_BP (1 << 3) + +struct hax_debug_t { + uint32_t control; + uint32_t reserved; + uint64_t dr[8]; +} PACKED; + #endif // HAX_INTERFACE_H_ diff --git a/windows/hax_entry.c b/windows/hax_entry.c index 601ac19a..9cfb7b58 100644 --- a/windows/hax_entry.c +++ b/windows/hax_entry.c @@ -36,6 +36,10 @@ #include "hax_win.h" +// vcpu.h +int vcpu_takeoff(struct vcpu_t *vcpu); +void vcpu_debug(struct vcpu_t *vcpu, struct hax_debug_t *debug); + #define NT_DEVICE_NAME L"\\Device\\HAX" #define DOS_DEVICE_NAME L"\\DosDevices\\HAX" @@ -244,7 +248,6 @@ NTSTATUS hax_get_versioninfo(void *buf, int buflength, int *ret_bl) return STATUS_SUCCESS; } -int vcpu_takeoff(struct vcpu_t *vcpu); NTSTATUS HaxVcpuControl(PDEVICE_OBJECT DeviceObject, struct hax_vcpu_windows *ext, PIRP Irp) { @@ -419,6 +422,14 @@ NTSTATUS HaxVcpuControl(PDEVICE_OBJECT DeviceObject, vcpu_takeoff(cvcpu); break; } + case HAX_IOCTL_VCPU_DEBUG: { + if (inBufLength < sizeof(struct hax_debug_t)) { + ret = STATUS_INVALID_PARAMETER; + goto done; + } + vcpu_debug(cvcpu, (struct hax_debug_t*)inBuf); + break; + } default: hax_error("Unknow vcpu ioctl %lx\n", irpSp->Parameters.DeviceIoControl.IoControlCode); diff --git a/windows/hax_entry.h b/windows/hax_entry.h index f7278047..90f2df9a 100644 --- a/windows/hax_entry.h +++ b/windows/hax_entry.h @@ -163,4 +163,8 @@ extern PDRIVER_OBJECT HaxDriverObject; #define HAX_VM_IOCTL_NOTIFY_QEMU_VERSION \ CTL_CODE(HAX_DEVICE_TYPE, 0x910, METHOD_BUFFERED, FILE_ANY_ACCESS) +/* API version 4.0 */ +#define HAX_IOCTL_VCPU_DEBUG \ + CTL_CODE(HAX_DEVICE_TYPE, 0x916, METHOD_BUFFERED, FILE_ANY_ACCESS) + #endif // HAX_WINDOWS_HAX_ENTRY_H_ From e37664c0660199d25e784a97121106eeba08f6df Mon Sep 17 00:00:00 2001 From: Yu Ning Date: Mon, 17 Sep 2018 11:30:11 +0800 Subject: [PATCH 2/3] Undo unnecessary API version bump --- include/darwin/hax_interface_mac.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/darwin/hax_interface_mac.h b/include/darwin/hax_interface_mac.h index ee7d1c7c..43a9719d 100644 --- a/include/darwin/hax_interface_mac.h +++ b/include/darwin/hax_interface_mac.h @@ -68,7 +68,6 @@ /* API 2.0 */ #define HAX_VM_IOCTL_NOTIFY_QEMU_VERSION _IOW(0, 0x84, struct hax_qemu_version) -/* API 5.0 */ #define HAX_IOCTL_VCPU_DEBUG _IOW(0, 0xc9, struct hax_debug_t) #define HAX_KERNEL64_CS 0x80 From 8bbc47d202766dfe439405f31a40d837ad9258ed Mon Sep 17 00:00:00 2001 From: Yu Ning Date: Mon, 17 Sep 2018 11:33:29 +0800 Subject: [PATCH 3/3] Remove API version comment The new DEBUG ioctl is identified by a feature flag, so it's not tied to a specific HAXM API version. --- windows/hax_entry.h | 1 - 1 file changed, 1 deletion(-) diff --git a/windows/hax_entry.h b/windows/hax_entry.h index 90f2df9a..d7d0a1de 100644 --- a/windows/hax_entry.h +++ b/windows/hax_entry.h @@ -163,7 +163,6 @@ extern PDRIVER_OBJECT HaxDriverObject; #define HAX_VM_IOCTL_NOTIFY_QEMU_VERSION \ CTL_CODE(HAX_DEVICE_TYPE, 0x910, METHOD_BUFFERED, FILE_ANY_ACCESS) -/* API version 4.0 */ #define HAX_IOCTL_VCPU_DEBUG \ CTL_CODE(HAX_DEVICE_TYPE, 0x916, METHOD_BUFFERED, FILE_ANY_ACCESS)