add extern "rust-tail" calling convention

compiler/rustc_abi/src/canon_abi.rs

@@ -28,6 +28,7 @@ pub enum CanonAbi {
     Rust,
     RustCold,
     RustPreserveNone,
+    RustTail,
 
     /// An ABI that rustc does not know how to call or define.
     Custom,
@@ -59,7 +60,10 @@ pub enum CanonAbi {
 impl CanonAbi {
     pub fn is_rustic_abi(self) -> bool {
         match self {
-            CanonAbi::Rust | CanonAbi::RustCold | CanonAbi::RustPreserveNone => true,
+            CanonAbi::Rust
+            | CanonAbi::RustCold
+            | CanonAbi::RustPreserveNone
+            | CanonAbi::RustTail => true,
             CanonAbi::C
             | CanonAbi::Custom
             | CanonAbi::Swift
@@ -81,6 +85,7 @@ impl fmt::Display for CanonAbi {
             CanonAbi::Rust => ExternAbi::Rust,
             CanonAbi::RustCold => ExternAbi::RustCold,
             CanonAbi::RustPreserveNone => ExternAbi::RustPreserveNone,
+            CanonAbi::RustTail => ExternAbi::RustTail,
             CanonAbi::Custom => ExternAbi::Custom,
             CanonAbi::Swift => ExternAbi::Swift,
             CanonAbi::Arm(arm_call) => match arm_call {

compiler/rustc_abi/src/extern_abi.rs

@@ -49,6 +49,11 @@ pub enum ExternAbi {
     /// forcing callers to save all registers.
     RustPreserveNone,
 
+    /// Ensures that calls in tail position can always be optimized into a jump.
+    ///
+    /// This ABI is not stable, and relies on LLVM implementation details.
+    RustTail,
+
     /// Unstable impl detail that directly uses Rust types to describe the ABI to LLVM.
     /// Even normally-compatible Rust types can become ABI-incompatible with this ABI!
     Unadjusted,
@@ -199,6 +204,7 @@ abi_impls! {
             RustCold =><= "rust-cold",
             RustInvalid =><= "rust-invalid",
             RustPreserveNone =><= "rust-preserve-none",
+            RustTail =><= "rust-tail",
             Stdcall { unwind: false } =><= "stdcall",
             Stdcall { unwind: true } =><= "stdcall-unwind",
             System { unwind: false } =><= "system",
@@ -280,7 +286,7 @@ impl ExternAbi {
     /// - are subject to change between compiler versions
     pub fn is_rustic_abi(self) -> bool {
         use ExternAbi::*;
-        matches!(self, Rust | RustCall | RustCold | RustPreserveNone)
+        matches!(self, Rust | RustCall | RustCold | RustPreserveNone | RustTail)
     }
 
     /// Returns whether the ABI supports C variadics. This only controls whether we allow *imports*
@@ -354,6 +360,7 @@ impl ExternAbi {
             | Self::SysV64 { .. }
             | Self::Win64 { .. }
             | Self::RustPreserveNone
+            | Self::RustTail
             | Self::Swift => true,
         }
     }

compiler/rustc_ast_lowering/src/stability.rs

@@ -100,6 +100,9 @@ pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {
             feature: sym::rust_preserve_none_cc,
             explain: GateReason::Experimental,
         }),
+        ExternAbi::RustTail => {
+            Err(UnstableAbi { abi, feature: sym::rust_tail_cc, explain: GateReason::Experimental })
+        }
         ExternAbi::RustInvalid => {
             Err(UnstableAbi { abi, feature: sym::rustc_attrs, explain: GateReason::ImplDetail })
         }

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -424,6 +424,7 @@ impl<'a> AstValidator<'a> {
                     | CanonAbi::Rust
                     | CanonAbi::RustCold
                     | CanonAbi::RustPreserveNone
+                    | CanonAbi::RustTail
                     | CanonAbi::Swift
                     | CanonAbi::Arm(_)
                     | CanonAbi::X86(_) => { /* nothing to check */ }

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -55,8 +55,9 @@ pub(crate) fn conv_to_call_conv(
     match c {
         CanonAbi::Rust | CanonAbi::RustCold | CanonAbi::C => default_call_conv,
 
-        // Cranelift doesn't currently have anything for this.
-        CanonAbi::RustPreserveNone => default_call_conv,
+        CanonAbi::RustPreserveNone | CanonAbi::RustTail => {
+            sess.dcx().fatal("call conv {c:?} is LLVM-specific")
+        }
 
         // Functions with this calling convention can only be called from assembly, but it is
         // possible to declare an `extern "custom"` block, so the backend still needs a calling

compiler/rustc_codegen_gcc/src/abi.rs

@@ -238,8 +238,14 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
 pub fn conv_to_fn_attribute<'gcc>(conv: CanonAbi, arch: &Arch) -> Option<FnAttribute<'gcc>> {
     let attribute = match conv {
         CanonAbi::C | CanonAbi::Rust => return None,
-        // gcc/gccjit does not have anything for this.
-        CanonAbi::RustPreserveNone => return None,
+        CanonAbi::RustPreserveNone => {
+            // This calling convention is LLVM-specific and unspecified.
+            panic!("gcc/gccjit backend does not support RustPreserveNone calling convention")
+        }
+        CanonAbi::RustTail => {
+            // This calling convention is LLVM-specific and unspecified.
+            panic!("gcc/gccjit backend does not support RustTail calling convention")
+        }
         CanonAbi::RustCold => FnAttribute::Cold,
         // Functions with this calling convention can only be called from assembly, but it is
         // possible to declare an `extern "custom"` block, so the backend still needs a calling

compiler/rustc_codegen_llvm/src/abi.rs

@@ -718,6 +718,10 @@ pub(crate) fn to_llvm_calling_convention(sess: &Session, abi: CanonAbi) -> llvm:
             Arch::X86_64 | Arch::AArch64 => llvm::PreserveNone,
             _ => llvm::CCallConv,
         },
+        CanonAbi::RustTail => match &sess.target.arch {
+            Arch::X86_64 | Arch::AArch64 => llvm::Tail,
+            _ => sess.dcx().fatal("extern \"rust-tail\" is only supported on x86_644 and aarch64"),
+        },
         // Functions with this calling convention can only be called from assembly, but it is
         // possible to declare an `extern "custom"` block, so the backend still needs a calling
         // convention for declaring foreign functions.

compiler/rustc_feature/src/unstable.rs

@@ -709,6 +709,8 @@ declare_features! (
     (unstable, rust_cold_cc, "1.63.0", Some(97544)),
     /// Allows `extern "rust-preserve-none"`.
     (unstable, rust_preserve_none_cc, "1.95.0", Some(151401)),
+    /// Allows `extern "rust-tail-cc"`.
+    (unstable, rust_tail_cc, "CURRENT_RUSTC_VERSION", Some(123)), // FIXME needs an issue
     /// Target features on s390x.
     (unstable, s390x_target_feature, "1.82.0", Some(150259)),
     /// Allows the use of the `sanitize` attribute.

compiler/rustc_hir_typeck/src/callee.rs

@@ -202,6 +202,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             | CanonAbi::Rust
             | CanonAbi::RustCold
             | CanonAbi::RustPreserveNone
+            | CanonAbi::RustTail
             | CanonAbi::Swift
             | CanonAbi::Arm(_)
             | CanonAbi::X86(_) => {}

compiler/rustc_middle/src/ty/layout.rs

@@ -1292,7 +1292,9 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi)
         | RustInvalid
         | Swift
         | Unadjusted => false,
-        Rust | RustCall | RustCold | RustPreserveNone => tcx.sess.panic_strategy().unwinds(),
+        Rust | RustCall | RustCold | RustPreserveNone | RustTail => {
+            tcx.sess.panic_strategy().unwinds()
+        }
     }
 }
 

compiler/rustc_public/src/abi.rs

@@ -455,6 +455,7 @@ pub enum CallConvention {
     PreserveMost,
     PreserveAll,
     PreserveNone,
+    Tail,
 
     Custom,
 

compiler/rustc_public/src/ty.rs

@@ -1167,6 +1167,7 @@ pub enum Abi {
     RiscvInterruptM,
     RiscvInterruptS,
     RustPreserveNone,
+    RustTail,
     RustInvalid,
     Custom,
     Swift,

compiler/rustc_public/src/unstable/convert/internal.rs

@@ -618,6 +618,7 @@ impl RustcInternal for Abi {
             Abi::RiscvInterruptM => rustc_abi::ExternAbi::RiscvInterruptM,
             Abi::RiscvInterruptS => rustc_abi::ExternAbi::RiscvInterruptS,
             Abi::RustPreserveNone => rustc_abi::ExternAbi::RustPreserveNone,
+            Abi::RustTail => rustc_abi::ExternAbi::RustTail,
             Abi::Custom => rustc_abi::ExternAbi::Custom,
             Abi::Swift => rustc_abi::ExternAbi::Swift,
         }

compiler/rustc_public/src/unstable/convert/stable/abi.rs

@@ -125,6 +125,7 @@ impl<'tcx> Stable<'tcx> for CanonAbi {
             CanonAbi::Rust => CallConvention::Rust,
             CanonAbi::RustCold => CallConvention::Cold,
             CanonAbi::RustPreserveNone => CallConvention::PreserveNone,
+            CanonAbi::RustTail => CallConvention::Tail,
             CanonAbi::Custom => CallConvention::Custom,
             CanonAbi::Swift => CallConvention::Swift,
             CanonAbi::Arm(arm_call) => match arm_call {

compiler/rustc_public/src/unstable/convert/stable/ty.rs

@@ -1046,6 +1046,7 @@ impl<'tcx> Stable<'tcx> for rustc_abi::ExternAbi {
             ExternAbi::Unadjusted => Abi::Unadjusted,
             ExternAbi::RustCold => Abi::RustCold,
             ExternAbi::RustPreserveNone => Abi::RustPreserveNone,
+            ExternAbi::RustTail => Abi::RustTail,
             ExternAbi::RustInvalid => Abi::RustInvalid,
             ExternAbi::RiscvInterruptM => Abi::RiscvInterruptM,
             ExternAbi::RiscvInterruptS => Abi::RiscvInterruptS,

compiler/rustc_span/src/symbol.rs

@@ -1704,6 +1704,7 @@ symbols! {
         rust_logo,
         rust_out,
         rust_preserve_none_cc,
+        rust_tail_cc,
         rustc,
         rustc_abi,
         // FIXME(#82232, #143834): temporary name to mitigate `#[align]` nameres ambiguity

compiler/rustc_target/src/spec/abi_map.rs

@@ -100,6 +100,7 @@ impl AbiMap {
             (ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust,
             (ExternAbi::RustCold, _) => CanonAbi::RustCold,
             (ExternAbi::RustPreserveNone, _) => CanonAbi::RustPreserveNone,
+            (ExternAbi::RustTail, _) => CanonAbi::RustTail,
 
             (ExternAbi::Custom, _) => CanonAbi::Custom,
 

tests/codegen-llvm/tailcc.rs

@@ -0,0 +1,32 @@
+//@ add-minicore
+//@ revisions: X86 AARCH64 UNSUPPORTED
+//@ [X86] compile-flags: -C no-prepopulate-passes --target=x86_64-unknown-linux-gnu
+//@ [X86] needs-llvm-components: x86
+//@ [AARCH64] compile-flags: -C no-prepopulate-passes --target=aarch64-unknown-linux-gnu
+//@ [AARCH64] needs-llvm-components: aarch64
+//@ [UNSUPPORTED] compile-flags: -C no-prepopulate-passes --target=powerpc64le-unknown-linux-gnu
+//@ [UNSUPPORTED] needs-llvm-components: powerpc
+
+#![crate_type = "lib"]
+#![feature(no_core, rust_tail_cc, explicit_tail_calls)]
+#![no_core]
+
+extern crate minicore;
+
+// X86: define{{( dso_local)?}} tailcc void @peach(i16
+// AARCH64: define{{( dso_local)?}} tailcc void @peach(i16
+// UNSUPPORTED: define{{( dso_local)?}} void @peach(i16
+#[no_mangle]
+#[inline(never)]
+pub extern "rust-tail" fn peach(x: u16) {
+    loop {}
+}
+
+// X86: call tailcc void @peach(i16
+// AARCH64: call tailcc void @peach(i16
+// UNSUPPORTED: call void @peach(i16
+pub fn quince(x: u16) {
+    if let 12345u16 = x {
+        peach(54321);
+    }
+}

tests/ui/abi/rust-tail-cc.rs

@@ -0,0 +1,75 @@
+//@ run-pass
+//@ needs-unwind
+
+#![feature(rust_tail_cc)]
+
+struct CrateOf<'a> {
+    mcintosh: f64,
+    golden_delicious: u64,
+    jonagold: Option<&'a u64>,
+    rome: [u64; 12],
+}
+
+#[inline(never)]
+extern "rust-tail" fn oven_explosion() {
+    panic!("bad time");
+}
+
+#[inline(never)]
+fn bite_into(yummy: u64) -> u64 {
+    let did_it_actually = std::panic::catch_unwind(move || oven_explosion());
+    assert!(did_it_actually.is_err());
+    yummy - 25
+}
+
+#[inline(never)]
+extern "rust-tail" fn lotsa_apples(
+    honeycrisp: u64,
+    gala: u32,
+    fuji: f64,
+    granny_smith: &[u64],
+    pink_lady: (),
+    and_a: CrateOf<'static>,
+    cosmic_crisp: u64,
+    ambrosia: f64,
+    winesap: &[u64],
+) -> (u64, f64, u64, u64) {
+    assert_eq!(honeycrisp, 220);
+    assert_eq!(gala, 140);
+    assert_eq!(fuji, 210.54201234);
+    assert_eq!(granny_smith, &[180, 210]);
+    assert_eq!(pink_lady, ());
+    assert_eq!(and_a.mcintosh, 150.0);
+    assert_eq!(and_a.golden_delicious, 185);
+    assert_eq!(and_a.jonagold, None); // my scales can't weight these gargantuans.
+    assert_eq!(and_a.rome, [180, 182, 184, 186, 188, 190, 192, 194, 196, 198, 200, 202]);
+    assert_eq!(cosmic_crisp, 270);
+    assert_eq!(ambrosia, 193.1);
+    assert_eq!(winesap, &[]);
+    (
+        and_a.rome.iter().sum(),
+        fuji + ambrosia,
+        cosmic_crisp - honeycrisp,
+        bite_into(and_a.golden_delicious),
+    )
+}
+
+fn main() {
+    let pie = lotsa_apples(
+        220,
+        140,
+        210.54201234,
+        &[180, 210],
+        (),
+        CrateOf {
+            mcintosh: 150.0,
+            golden_delicious: 185,
+            jonagold: None,
+            rome: [180, 182, 184, 186, 188, 190, 192, 194, 196, 198, 200, 202],
+        },
+        270,
+        193.1,
+        &[],
+    );
+    assert_eq!(pie, (2292, 403.64201234, 50, 160));
+}

tests/ui/feature-gates/feature-gate-rust-tail-cc.rs

@@ -0,0 +1,21 @@
+#![crate_type = "lib"]
+
+extern "rust-tail" fn apple() {} //~ ERROR "rust-tail" ABI is experimental
+
+trait T {
+    extern "rust-tail" fn banana(); //~ ERROR "rust-tail" ABI is experimental
+    extern "rust-tail" fn citrus() {} //~ ERROR "rust-tail" ABI is experimental
+}
+
+struct S;
+impl T for S {
+    extern "rust-tail" fn banana() {} //~ ERROR "rust-tail" ABI is experimental
+}
+
+impl S {
+    extern "rust-tail" fn durian() {} //~ ERROR "rust-tail" ABI is experimental
+}
+
+type Fig = extern "rust-tail" fn(); //~ ERROR "rust-tail" ABI is experimental
+
+extern "rust-tail" {} //~ ERROR "rust-tail" ABI is experimental