From 0f4925e436abba349e8f70502c6911c62c6a7276 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Mon, 19 Feb 2024 18:57:27 +0100 Subject: [PATCH] Make intrinsic fallback bodies cross-crate inlineable This change was prompted by the stage1 compiler spending 4% of its time when compiling the polymorphic-recursion MIR opt test in `unlikely`. Intrinsic fallback bodies like `unlikely` should always be inlined, it's very silly if they are not. To do this, we enable the fallback bodies to be cross-crate inlineable. Not that this matters for our workloads since the compiler never actually _uses_ the "fallback bodies", it just uses whatever was cfg(bootstrap)ped, so I've also added `#[inline]` to those. --- compiler/rustc_mir_transform/src/cross_crate_inline.rs | 9 +++++++++ compiler/rustc_mir_transform/src/inline.rs | 8 ++++++++ library/core/src/intrinsics.rs | 6 ++++++ .../unstable-book/src/language-features/intrinsics.md | 2 ++ 4 files changed, 25 insertions(+) diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 5f01b841867ea..483fd753e7077 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -9,6 +9,7 @@ use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::config::InliningThreshold; use rustc_session::config::OptLevel; +use rustc_span::sym; pub fn provide(providers: &mut Providers) { providers.cross_crate_inlinable = cross_crate_inlinable; @@ -34,6 +35,14 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { return true; } + if tcx.has_attr(def_id, sym::rustc_intrinsic) { + // Intrinsic fallback bodies are always cross-crate inlineable. + // To ensure that the MIR inliner doesn't cluelessly try to inline fallback + // bodies even when the backend would implement something better, we stop + // the MIR inliner from ever inlining an intrinsic. + return true; + } + // Obey source annotations first; this is important because it means we can use // #[inline(never)] to force code generation. match codegen_fn_attrs.inline { diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 956d855ab81a2..2009539d4d084 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -13,6 +13,7 @@ use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; use rustc_session::config::OptLevel; use rustc_span::source_map::Spanned; +use rustc_span::sym; use rustc_target::abi::FieldIdx; use rustc_target::spec::abi::Abi; @@ -170,6 +171,13 @@ impl<'tcx> Inliner<'tcx> { let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id()); self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?; + // Intrinsic fallback bodies are automatically made cross-crate inlineable, + // but at this stage we don't know whether codegen knows the intrinsic, + // so just conservatively don't inline it. + if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) { + return Err("Callee is an intrinsic, do not inline fallback bodies"); + } + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index ce1876d5a2f2f..64d3db777c3fa 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -953,6 +953,7 @@ extern "rust-intrinsic" { #[rustc_nounwind] #[unstable(feature = "core_intrinsics", issue = "none")] #[cfg_attr(not(bootstrap), rustc_intrinsic)] +#[cfg_attr(bootstrap, inline)] pub const unsafe fn assume(b: bool) { if !b { // SAFETY: the caller must guarantee the argument is never `false` @@ -975,6 +976,7 @@ pub const unsafe fn assume(b: bool) { #[unstable(feature = "core_intrinsics", issue = "none")] #[cfg_attr(not(bootstrap), rustc_intrinsic)] #[rustc_nounwind] +#[cfg_attr(bootstrap, inline)] pub const fn likely(b: bool) -> bool { b } @@ -994,6 +996,7 @@ pub const fn likely(b: bool) -> bool { #[unstable(feature = "core_intrinsics", issue = "none")] #[cfg_attr(not(bootstrap), rustc_intrinsic)] #[rustc_nounwind] +#[cfg_attr(bootstrap, inline)] pub const fn unlikely(b: bool) -> bool { b } @@ -2556,6 +2559,7 @@ extern "rust-intrinsic" { #[rustc_nounwind] #[unstable(feature = "core_intrinsics", issue = "none")] #[cfg_attr(not(bootstrap), rustc_intrinsic)] +#[cfg_attr(bootstrap, inline)] pub const fn is_val_statically_known(_arg: T) -> bool { false } @@ -2592,6 +2596,7 @@ pub(crate) const fn debug_assertions() -> bool { #[unstable(feature = "core_intrinsics", issue = "none")] #[rustc_nounwind] #[cfg_attr(not(bootstrap), rustc_intrinsic)] +#[cfg_attr(bootstrap, inline)] pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 { // const eval overrides this function, but runtime code should always just return null pointers. crate::ptr::null_mut() @@ -2611,6 +2616,7 @@ pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 { #[unstable(feature = "core_intrinsics", issue = "none")] #[rustc_nounwind] #[cfg_attr(not(bootstrap), rustc_intrinsic)] +#[cfg_attr(bootstrap, inline)] pub const unsafe fn const_deallocate(_ptr: *mut u8, _size: usize, _align: usize) {} // Some functions are defined here because they accidentally got made diff --git a/src/doc/unstable-book/src/language-features/intrinsics.md b/src/doc/unstable-book/src/language-features/intrinsics.md index 9d07ae6fc67e1..1a8c1c0b36aff 100644 --- a/src/doc/unstable-book/src/language-features/intrinsics.md +++ b/src/doc/unstable-book/src/language-features/intrinsics.md @@ -14,6 +14,8 @@ the intrinsic directly when you can. Many intrinsics can be written in pure rust, albeit inefficiently or without supporting some features that only exist on some backends. Backends can simply not implement those intrinsics without causing any code miscompilations or failures to compile. +All intrinsic fallback bodies are automatically made cross-crate inlineable (like `#[inline]`) +by the codegen backend, but not the MIR inliner. ```rust #![feature(rustc_attrs, effects)]