Auto merge of #67502 - Mark-Simulacrum:opt-catch, r=Mark-Simulacrum

Optimize catch_unwind to match C++ try/catch

This refactors the implementation of catching unwinds to allow LLVM to inline the "try" closure directly into the happy path, avoiding indirection. This means that the catch_unwind implementation is (after this PR) zero-cost unless a panic is thrown.

https://rust.godbolt.org/z/cZcUSB is an example of the current codegen in a simple case. Notably, the codegen is *exactly the same* if `-Cpanic=abort` is passed, which is clearly not great.

This PR, on the other hand, generates the following assembly:

```asm
# -Cpanic=unwind:
	push   rbx
	mov    ebx,0x2a
	call   QWORD PTR [rip+0x1c53c]        # <happy>
	mov    eax,ebx
	pop    rbx
	ret
	mov    rdi,rax
	call   QWORD PTR [rip+0x1c537]        # cleanup function call
	call   QWORD PTR [rip+0x1c539]        # <unfortunate>
	mov    ebx,0xd
	mov    eax,ebx
	pop    rbx
	ret

# -Cpanic=abort:
	push   rax
	call   QWORD PTR [rip+0x20a1]        # <happy>
	mov    eax,0x2a
	pop    rcx
	ret
```

Fixes #64224, and resolves #64222.

src/doc/unstable-book/src/language-features/lang-items.md

@@ -52,7 +52,6 @@ fn main(_argc: isize, _argv: *const *const u8) -> isize {
 
 #[lang = "eh_personality"] extern fn rust_eh_personality() {}
 #[lang = "panic_impl"] extern fn rust_begin_panic(info: &PanicInfo) -> ! { unsafe { intrinsics::abort() } }
-#[lang = "eh_unwind_resume"] extern fn rust_eh_unwind_resume() {}
 #[no_mangle] pub extern fn rust_eh_register_frames () {}
 #[no_mangle] pub extern fn rust_eh_unregister_frames () {}
 ```
@@ -67,7 +66,7 @@ Other features provided by lang items include:
   marked with lang items; those specific four are `eq`, `ord`,
   `deref`, and `add` respectively.
 - stack unwinding and general failure; the `eh_personality`,
-  `eh_unwind_resume`, `fail` and `fail_bounds_checks` lang items.
+  `panic` and `panic_bounds_checks` lang items.
 - the traits in `std::marker` used to indicate types of
   various kinds; lang items `send`, `sync` and `copy`.
 - the marker types and variance indicators found in
@@ -130,12 +129,6 @@ fn start(_argc: isize, _argv: *const *const u8) -> isize {
 pub extern fn rust_eh_personality() {
 }
 
-// This function may be needed based on the compilation target.
-#[lang = "eh_unwind_resume"]
-#[no_mangle]
-pub extern fn rust_eh_unwind_resume() {
-}
-
 #[lang = "panic_impl"]
 #[no_mangle]
 pub extern fn rust_begin_panic(info: &PanicInfo) -> ! {
@@ -173,12 +166,6 @@ pub extern fn main(_argc: i32, _argv: *const *const u8) -> i32 {
 pub extern fn rust_eh_personality() {
 }
 
-// This function may be needed based on the compilation target.
-#[lang = "eh_unwind_resume"]
-#[no_mangle]
-pub extern fn rust_eh_unwind_resume() {
-}
-
 #[lang = "panic_impl"]
 #[no_mangle]
 pub extern fn rust_begin_panic(info: &PanicInfo) -> ! {
@@ -211,10 +198,8 @@ compiler. When a panic happens, this controls the message that's displayed on
 the screen. While the language item's name is `panic_impl`, the symbol name is
 `rust_begin_panic`.
 
-A third function, `rust_eh_unwind_resume`, is also needed if the `custom_unwind_resume`
-flag is set in the options of the compilation target. It allows customizing the
-process of resuming unwind at the end of the landing pads. The language item's name
-is `eh_unwind_resume`.
+Finally, a `eh_catch_typeinfo` static is needed for certain targets which
+implement Rust panics on top of C++ exceptions.
 
 ## List of all language items
 
@@ -247,8 +232,6 @@ the source code.
   - `eh_personality`: `libpanic_unwind/emcc.rs` (EMCC)
   - `eh_personality`: `libpanic_unwind/gcc.rs` (GNU)
   - `eh_personality`: `libpanic_unwind/seh.rs` (SEH)
-  - `eh_unwind_resume`: `libpanic_unwind/gcc.rs` (GCC)
-  - `eh_catch_typeinfo`: `libpanic_unwind/seh.rs` (SEH)
   - `eh_catch_typeinfo`: `libpanic_unwind/emcc.rs` (EMCC)
   - `panic`: `libcore/panicking.rs`
   - `panic_bounds_check`: `libcore/panicking.rs`

src/libcore/intrinsics.rs

@@ -1871,14 +1871,16 @@ extern "rust-intrinsic" {
     #[rustc_const_unstable(feature = "const_discriminant", issue = "69821")]
     pub fn discriminant_value<T>(v: &T) -> u64;
 
-    /// Rust's "try catch" construct which invokes the function pointer `f` with
-    /// the data pointer `data`.
+    /// Rust's "try catch" construct which invokes the function pointer `try_fn`
+    /// with the data pointer `data`.
     ///
-    /// The third pointer is a target-specific data pointer which is filled in
-    /// with the specifics of the exception that occurred. For examples on Unix
-    /// platforms this is a `*mut *mut T` which is filled in by the compiler and
-    /// on MSVC it's `*mut [usize; 2]`. For more information see the compiler's
+    /// The third argument is a function called if a panic occurs. This function
+    /// takes the data pointer and a pointer to the target-specific exception
+    /// object that was caught. For more information see the compiler's
     /// source as well as std's catch implementation.
+    #[cfg(not(bootstrap))]
+    pub fn r#try(try_fn: fn(*mut u8), data: *mut u8, catch_fn: fn(*mut u8, *mut u8)) -> i32;
+    #[cfg(bootstrap)]
     pub fn r#try(f: fn(*mut u8), data: *mut u8, local_ptr: *mut u8) -> i32;
 
     /// Emits a `!nontemporal` store according to LLVM (see their docs).

src/libpanic_abort/lib.rs

@@ -18,17 +18,11 @@
 #![feature(staged_api)]
 #![feature(rustc_attrs)]
 
-// Rust's "try" function, but if we're aborting on panics we just call the
-// function as there's nothing else we need to do here.
+use core::any::Any;
+
 #[rustc_std_internal_symbol]
-pub unsafe extern "C" fn __rust_maybe_catch_panic(
-    f: fn(*mut u8),
-    data: *mut u8,
-    _data_ptr: *mut usize,
-    _vtable_ptr: *mut usize,
-) -> u32 {
-    f(data);
-    0
+pub unsafe extern "C" fn __rust_panic_cleanup(_: *mut u8) -> *mut (dyn Any + Send + 'static) {
+    unreachable!()
 }
 
 // "Leak" the payload and shim to the relevant abort on the platform in
@@ -92,7 +86,7 @@ pub unsafe extern "C" fn __rust_start_panic(_payload: usize) -> u32 {
 // binaries, but it should never be called as we don't link in an unwinding
 // runtime at all.
 pub mod personalities {
-    #[no_mangle]
+    #[rustc_std_internal_symbol]
     #[cfg(not(any(
         all(target_arch = "wasm32", not(target_os = "emscripten"),),
         all(target_os = "windows", target_env = "gnu", target_arch = "x86_64",),
@@ -101,7 +95,7 @@ pub mod personalities {
 
     // On x86_64-pc-windows-gnu we use our own personality function that needs
     // to return `ExceptionContinueSearch` as we're passing on all our frames.
-    #[no_mangle]
+    #[rustc_std_internal_symbol]
     #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86_64"))]
     pub extern "C" fn rust_eh_personality(
         _record: usize,
@@ -117,16 +111,16 @@ pub mod personalities {
     //
     // Note that we don't execute landing pads, so this is never called, so it's
     // body is empty.
-    #[no_mangle]
-    #[cfg(all(target_os = "windows", target_env = "gnu"))]
+    #[rustc_std_internal_symbol]
+    #[cfg(all(bootstrap, target_os = "windows", target_env = "gnu"))]
     pub extern "C" fn rust_eh_unwind_resume() {}
 
     // These two are called by our startup objects on i686-pc-windows-gnu, but
     // they don't need to do anything so the bodies are nops.
-    #[no_mangle]
+    #[rustc_std_internal_symbol]
     #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))]
     pub extern "C" fn rust_eh_register_frames() {}
-    #[no_mangle]
+    #[rustc_std_internal_symbol]
     #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))]
     pub extern "C" fn rust_eh_unregister_frames() {}
 }

src/libpanic_unwind/dummy.rs

@@ -6,10 +6,6 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::intrinsics;
 
-pub fn payload() -> *mut u8 {
-    core::ptr::null_mut()
-}
-
 pub unsafe fn cleanup(_ptr: *mut u8) -> Box<dyn Any + Send> {
     intrinsics::abort()
 }

src/libpanic_unwind/emcc.rs

@@ -48,15 +48,11 @@ static EXCEPTION_TYPE_INFO: TypeInfo = TypeInfo {
     name: b"rust_panic\0".as_ptr(),
 };
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 struct Exception {
     // This needs to be an Option because the object's lifetime follows C++
     // semantics: when catch_unwind moves the Box out of the exception it must
     // still leave the exception object in a valid state because its destructor
-    // is still going to be called by __cxa_end_catch..
+    // is still going to be called by __cxa_end_catch.
     data: Option<Box<dyn Any + Send>>,
 }
 
@@ -98,7 +94,6 @@ extern "C" fn exception_cleanup(ptr: *mut libc::c_void) -> DestructorRet {
 }
 
 #[lang = "eh_personality"]
-#[no_mangle]
 unsafe extern "C" fn rust_eh_personality(
     version: c_int,
     actions: uw::_Unwind_Action,

src/libpanic_unwind/gcc.rs

@@ -35,20 +35,11 @@
 //!
 //! Once stack has been unwound down to the handler frame level, unwinding stops
 //! and the last personality routine transfers control to the catch block.
-//!
-//! ## `eh_personality` and `eh_unwind_resume`
-//!
-//! These language items are used by the compiler when generating unwind info.
-//! The first one is the personality routine described above. The second one
-//! allows compilation target to customize the process of resuming unwind at the
-//! end of the landing pads. `eh_unwind_resume` is used only if
-//! `custom_unwind_resume` flag in the target options is set.
 
 #![allow(private_no_mangle_fns)]
 
 use alloc::boxed::Box;
 use core::any::Any;
-use core::ptr;
 
 use crate::dwarf::eh::{self, EHAction, EHContext};
 use libc::{c_int, uintptr_t};
@@ -83,10 +74,6 @@ pub unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
     }
 }
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 pub unsafe fn cleanup(ptr: *mut u8) -> Box<dyn Any + Send> {
     let exception = Box::from_raw(ptr as *mut Exception);
     exception.cause
@@ -143,7 +130,6 @@ cfg_if::cfg_if! {
         //
         // iOS uses the default routine instead since it uses SjLj unwinding.
         #[lang = "eh_personality"]
-        #[no_mangle]
         unsafe extern "C" fn rust_eh_personality(state: uw::_Unwind_State,
                                                  exception_object: *mut uw::_Unwind_Exception,
                                                  context: *mut uw::_Unwind_Context)
@@ -277,7 +263,6 @@ cfg_if::cfg_if! {
                 // On x86_64 MinGW targets, the unwinding mechanism is SEH however the unwind
                 // handler data (aka LSDA) uses GCC-compatible encoding.
                 #[lang = "eh_personality"]
-                #[no_mangle]
                 #[allow(nonstandard_style)]
                 unsafe extern "C" fn rust_eh_personality(exceptionRecord: *mut uw::EXCEPTION_RECORD,
                         establisherFrame: uw::LPVOID,
@@ -293,7 +278,6 @@ cfg_if::cfg_if! {
             } else {
                 // The personality routine for most of our targets.
                 #[lang = "eh_personality"]
-                #[no_mangle]
                 unsafe extern "C" fn rust_eh_personality(version: c_int,
                         actions: uw::_Unwind_Action,
                         exception_class: uw::_Unwind_Exception_Class,
@@ -329,8 +313,8 @@ unsafe fn find_eh_action(
     eh::find_eh_action(lsda, &eh_context, foreign_exception)
 }
 
-// See docs in the `unwind` module.
 #[cfg(all(
+    bootstrap,
     target_os = "windows",
     any(target_arch = "x86", target_arch = "x86_64"),
     target_env = "gnu"
@@ -364,12 +348,12 @@ pub mod eh_frame_registry {
         fn __deregister_frame_info(eh_frame_begin: *const u8, object: *mut u8);
     }
 
-    #[no_mangle]
+    #[rustc_std_internal_symbol]
     pub unsafe extern "C" fn rust_eh_register_frames(eh_frame_begin: *const u8, object: *mut u8) {
         __register_frame_info(eh_frame_begin, object);
     }
 
-    #[no_mangle]
+    #[rustc_std_internal_symbol]
     pub unsafe extern "C" fn rust_eh_unregister_frames(eh_frame_begin: *const u8, object: *mut u8) {
         __deregister_frame_info(eh_frame_begin, object);
     }

src/libpanic_unwind/hermit.rs

@@ -6,10 +6,6 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::ptr;
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 pub unsafe fn cleanup(_ptr: *mut u8) -> Box<dyn Any + Send> {
     extern "C" {
         pub fn __rust_abort() -> !;

src/libpanic_unwind/lib.rs

@@ -22,19 +22,18 @@
 #![feature(libc)]
 #![feature(nll)]
 #![feature(panic_unwind)]
-#![feature(raw)]
 #![feature(staged_api)]
 #![feature(std_internals)]
 #![feature(unwind_attributes)]
 #![feature(abi_thiscall)]
+#![feature(rustc_attrs)]
+#![feature(raw)]
 #![panic_runtime]
 #![feature(panic_runtime)]
 
 use alloc::boxed::Box;
-use core::intrinsics;
-use core::mem;
+use core::any::Any;
 use core::panic::BoxMeUp;
-use core::raw;
 
 cfg_if::cfg_if! {
     if #[cfg(target_os = "emscripten")] {
@@ -69,33 +68,14 @@ extern "C" {
 
 mod dwarf;
 
-// Entry point for catching an exception, implemented using the `try` intrinsic
-// in the compiler.
-//
-// The interaction between the `payload` function and the compiler is pretty
-// hairy and tightly coupled, for more information see the compiler's
-// implementation of this.
-#[no_mangle]
-pub unsafe extern "C" fn __rust_maybe_catch_panic(
-    f: fn(*mut u8),
-    data: *mut u8,
-    data_ptr: *mut usize,
-    vtable_ptr: *mut usize,
-) -> u32 {
-    let mut payload = imp::payload();
-    if intrinsics::r#try(f, data, &mut payload as *mut _ as *mut _) == 0 {
-        0
-    } else {
-        let obj = mem::transmute::<_, raw::TraitObject>(imp::cleanup(payload));
-        *data_ptr = obj.data as usize;
-        *vtable_ptr = obj.vtable as usize;
-        1
-    }
+#[rustc_std_internal_symbol]
+pub unsafe extern "C" fn __rust_panic_cleanup(payload: *mut u8) -> *mut (dyn Any + Send + 'static) {
+    Box::into_raw(imp::cleanup(payload))
 }
 
 // Entry point for raising an exception, just delegates to the platform-specific
 // implementation.
-#[no_mangle]
+#[rustc_std_internal_symbol]
 #[unwind(allowed)]
 pub unsafe extern "C" fn __rust_start_panic(payload: usize) -> u32 {
     let payload = payload as *mut &mut dyn BoxMeUp;