From 2012b7344fc5d810729e53693be20eaa6b9174b7 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 15 Oct 2025 09:55:38 +0000 Subject: [PATCH 1/3] Remove -Zoom=panic There are major questions remaining about the reentrancy that this allows. It doesn't have any users on github outside of a single project that uses it in a panic=abort project to show backtraces. It can still be emulated through #[alloc_error_handler] or set_alloc_error_hook depending on if you use the standard library or not. And finally it makes it harder to do various improvements to the allocator shim. --- .../rustc_codegen_cranelift/src/allocator.rs | 39 +--------------- compiler/rustc_codegen_gcc/src/allocator.rs | 39 +--------------- compiler/rustc_codegen_llvm/src/allocator.rs | 44 +------------------ .../src/back/symbol_export.rs | 3 +- compiler/rustc_interface/src/tests.rs | 9 ++-- compiler/rustc_session/src/config.rs | 30 ++----------- compiler/rustc_session/src/options.rs | 12 ----- library/alloc/src/alloc.rs | 19 ++------ library/std/src/alloc.rs | 27 ++++-------- src/tools/miri/src/shims/foreign_items.rs | 9 +--- .../tests/panic/alloc_error_handler_panic.rs | 18 -------- .../panic/alloc_error_handler_panic.stderr | 6 --- ...c-unwind.rs => alloc_error_hook-unwind.rs} | 8 ++-- 13 files changed, 32 insertions(+), 231 deletions(-) delete mode 100644 src/tools/miri/tests/panic/alloc_error_handler_panic.rs delete mode 100644 src/tools/miri/tests/panic/alloc_error_handler_panic.stderr rename tests/ui/panics/{oom-panic-unwind.rs => alloc_error_hook-unwind.rs} (84%) diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs index 67b89114356b5..4a9b0c0952ff3 100644 --- a/compiler/rustc_codegen_cranelift/src/allocator.rs +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -6,7 +6,6 @@ use rustc_ast::expand::allocator::{ AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, default_fn_name, global_fn_name, }; use rustc_codegen_ssa::base::{allocator_kind_for_codegen, allocator_shim_contents}; -use rustc_session::config::OomStrategy; use rustc_symbol_mangling::mangle_internal_symbol; use crate::prelude::*; @@ -15,16 +14,11 @@ use crate::prelude::*; pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module) -> bool { let Some(kind) = allocator_kind_for_codegen(tcx) else { return false }; let methods = allocator_shim_contents(tcx, kind); - codegen_inner(tcx, module, &methods, tcx.sess.opts.unstable_opts.oom); + codegen_inner(tcx, module, &methods); true } -fn codegen_inner( - tcx: TyCtxt<'_>, - module: &mut dyn Module, - methods: &[AllocatorMethod], - oom_strategy: OomStrategy, -) { +fn codegen_inner(tcx: TyCtxt<'_>, module: &mut dyn Module, methods: &[AllocatorMethod]) { let usize_ty = module.target_config().pointer_type(); for method in methods { @@ -65,35 +59,6 @@ fn codegen_inner( ); } - { - let sig = Signature { - call_conv: module.target_config().default_call_conv, - params: vec![], - returns: vec![AbiParam::new(types::I8)], - }; - let func_id = module - .declare_function( - &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), - Linkage::Export, - &sig, - ) - .unwrap(); - let mut ctx = Context::new(); - ctx.func.signature = sig; - { - let mut func_ctx = FunctionBuilderContext::new(); - let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); - - let block = bcx.create_block(); - bcx.switch_to_block(block); - let value = bcx.ins().iconst(types::I8, oom_strategy.should_panic() as i64); - bcx.ins().return_(&[value]); - bcx.seal_all_blocks(); - bcx.finalize(); - } - module.define_function(func_id, &mut ctx).unwrap(); - } - { let sig = Signature { call_conv: module.target_config().default_call_conv, diff --git a/compiler/rustc_codegen_gcc/src/allocator.rs b/compiler/rustc_codegen_gcc/src/allocator.rs index 647569694b04d..1f464d7e1226f 100644 --- a/compiler/rustc_codegen_gcc/src/allocator.rs +++ b/compiler/rustc_codegen_gcc/src/allocator.rs @@ -1,12 +1,11 @@ #[cfg(feature = "master")] use gccjit::FnAttribute; -use gccjit::{Context, FunctionType, RValue, ToRValue, Type}; +use gccjit::{Context, FunctionType, ToRValue, Type}; use rustc_ast::expand::allocator::{ AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, default_fn_name, global_fn_name, }; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; -use rustc_session::config::OomStrategy; use rustc_symbol_mangling::mangle_internal_symbol; use crate::GccContext; @@ -59,14 +58,6 @@ pub(crate) unsafe fn codegen( create_wrapper_function(tcx, context, &from_name, Some(&to_name), &types, output); } - create_const_value_function( - tcx, - context, - &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), - i8, - context.new_rvalue_from_int(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as i32), - ); - create_wrapper_function( tcx, context, @@ -77,34 +68,6 @@ pub(crate) unsafe fn codegen( ); } -fn create_const_value_function( - tcx: TyCtxt<'_>, - context: &Context<'_>, - name: &str, - output: Type<'_>, - value: RValue<'_>, -) { - let func = context.new_function(None, FunctionType::Exported, output, &[], name, false); - - #[cfg(feature = "master")] - { - func.add_attribute(FnAttribute::Visibility(symbol_visibility_to_gcc( - tcx.sess.default_visibility(), - ))); - - // FIXME(antoyo): cg_llvm sets AlwaysInline, but AlwaysInline is different in GCC and using - // it here will causes linking errors when using LTO. - func.add_attribute(FnAttribute::Inline); - } - - if tcx.sess.must_emit_unwind_tables() { - // TODO(antoyo): emit unwind tables. - } - - let block = func.new_block("entry"); - block.end_with_return(None, value); -} - fn create_wrapper_function( tcx: TyCtxt<'_>, context: &Context<'_>, diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index de0b85ebb63b8..24b2bd37aa46c 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -7,13 +7,13 @@ use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _; use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::ty::TyCtxt; -use rustc_session::config::{DebugInfo, OomStrategy}; +use rustc_session::config::DebugInfo; use rustc_symbol_mangling::mangle_internal_symbol; use crate::attributes::llfn_attrs_from_instance; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; -use crate::llvm::{self, FALSE, FromGeneric, TRUE, Type, Value}; +use crate::llvm::{self, FromGeneric, TRUE, Type}; use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( @@ -28,7 +28,6 @@ pub(crate) unsafe fn codegen( 64 => cx.type_i64(), tws => bug!("Unsupported target word size for int: {}", tws), }; - let i8 = cx.type_i8(); let i8p = cx.type_ptr(); for method in methods { @@ -87,17 +86,6 @@ pub(crate) unsafe fn codegen( ); } - // __rust_alloc_error_handler_should_panic_v2 - create_const_value_function( - tcx, - &cx, - &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), - &i8, - unsafe { - llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, FALSE) - }, - ); - // __rust_no_alloc_shim_is_unstable_v2 create_wrapper_function( tcx, @@ -117,34 +105,6 @@ pub(crate) unsafe fn codegen( } } -fn create_const_value_function( - tcx: TyCtxt<'_>, - cx: &SimpleCx<'_>, - name: &str, - output: &Type, - value: &Value, -) { - let ty = cx.type_func(&[], output); - let llfn = declare_simple_fn( - &cx, - name, - llvm::CallConv::CCallConv, - llvm::UnnamedAddr::Global, - llvm::Visibility::from_generic(tcx.sess.default_visibility()), - ty, - ); - - attributes::apply_to_llfn( - llfn, - llvm::AttributePlace::Function, - &[llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx)], - ); - - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; - let mut bx = SBuilder::build(&cx, llbb); - bx.ret(value); -} - fn create_wrapper_function( tcx: TyCtxt<'_>, cx: &SimpleCx<'_>, diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 5bc18e2d7f8bb..aee64efa8fb57 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -15,7 +15,7 @@ use rustc_middle::middle::exported_symbols::{ use rustc_middle::query::LocalCrate; use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, Instance, SymbolName, Ty, TyCtxt}; use rustc_middle::util::Providers; -use rustc_session::config::{CrateType, OomStrategy}; +use rustc_session::config::CrateType; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::TlsModel; use tracing::debug; @@ -501,7 +501,6 @@ pub(crate) fn allocator_shim_symbols( .map(move |method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str())) .chain([ mangle_internal_symbol(tcx, global_fn_name(ALLOC_ERROR_HANDLER).as_str()), - mangle_internal_symbol(tcx, OomStrategy::SYMBOL), mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), ]) .map(move |symbol_name| { diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 7e5186db4eafe..9c4d2302f9665 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -14,10 +14,10 @@ use rustc_session::config::{ CoverageOptions, DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, FmtDebug, FunctionReturn, InliningThreshold, Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli, MirIncludeSpans, - NextSolverConfig, Offload, OomStrategy, Options, OutFileName, OutputType, OutputTypes, - PAuthKey, PacRet, Passes, PatchableFunctionEntry, Polonius, ProcMacroExecutionStrategy, Strip, - SwitchWithOptPath, SymbolManglingVersion, WasiExecModel, build_configuration, - build_session_options, rustc_optgroups, + NextSolverConfig, Offload, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet, + Passes, PatchableFunctionEntry, Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, + SymbolManglingVersion, WasiExecModel, build_configuration, build_session_options, + rustc_optgroups, }; use rustc_session::lint::Level; use rustc_session::search_paths::SearchPath; @@ -839,7 +839,6 @@ fn test_unstable_options_tracking_hash() { tracked!(no_unique_section_names, true); tracked!(offload, vec![Offload::Enable]); tracked!(on_broken_pipe, OnBrokenPipe::Kill); - tracked!(oom, OomStrategy::Panic); tracked!(osx_rpath_install_name, true); tracked!(packed_bundled_libs, true); tracked!(panic_abort_tests, true); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index d1426ff55fbd6..6ac48aba31ee8 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -3242,10 +3242,10 @@ pub(crate) mod dep_tracking { AutoDiff, BranchProtection, CFGuard, CFProtection, CollapseMacroDebuginfo, CoverageOptions, CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, - LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OomStrategy, OptLevel, OutFileName, - OutputType, OutputTypes, PatchableFunctionEntry, Polonius, RemapPathScopeComponents, - ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, - SymbolManglingVersion, WasiExecModel, + LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OptLevel, OutFileName, OutputType, + OutputTypes, PatchableFunctionEntry, Polonius, RemapPathScopeComponents, ResolveDocLinks, + SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, + WasiExecModel, }; use crate::lint; use crate::utils::NativeLib; @@ -3341,7 +3341,6 @@ pub(crate) mod dep_tracking { LocationDetail, FmtDebug, BranchProtection, - OomStrategy, LanguageIdentifier, NextSolverConfig, PatchableFunctionEntry, @@ -3454,27 +3453,6 @@ pub(crate) mod dep_tracking { } } -/// Default behavior to use in out-of-memory situations. -#[derive(Clone, Copy, PartialEq, Hash, Debug, Encodable, Decodable, HashStable_Generic)] -pub enum OomStrategy { - /// Generate a panic that can be caught by `catch_unwind`. - Panic, - - /// Abort the process immediately. - Abort, -} - -impl OomStrategy { - pub const SYMBOL: &'static str = "__rust_alloc_error_handler_should_panic_v2"; - - pub fn should_panic(self) -> u8 { - match self { - OomStrategy::Panic => 1, - OomStrategy::Abort => 0, - } - } -} - /// How to run proc-macro code when building this crate #[derive(Clone, Copy, PartialEq, Hash, Debug)] pub enum ProcMacroExecutionStrategy { diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 6dd90546de1b0..7b9f95d845996 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -806,7 +806,6 @@ mod desc { pub(crate) const parse_on_broken_pipe: &str = "either `kill`, `error`, or `inherit`"; pub(crate) const parse_patchable_function_entry: &str = "either two comma separated integers (total_nops,prefix_nops), with prefix_nops <= total_nops, or one integer (total_nops)"; pub(crate) const parse_opt_panic_strategy: &str = parse_panic_strategy; - pub(crate) const parse_oom_strategy: &str = "either `panic` or `abort`"; pub(crate) const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; pub(crate) const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `dataflow`, `hwaddress`, `kcfi`, `kernel-address`, `leak`, `memory`, `memtag`, `safestack`, `shadow-call-stack`, or `thread`"; pub(crate) const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2"; @@ -1218,15 +1217,6 @@ pub mod parse { false } - pub(crate) fn parse_oom_strategy(slot: &mut OomStrategy, v: Option<&str>) -> bool { - match v { - Some("panic") => *slot = OomStrategy::Panic, - Some("abort") => *slot = OomStrategy::Abort, - _ => return false, - } - true - } - pub(crate) fn parse_relro_level(slot: &mut Option, v: Option<&str>) -> bool { match v { Some(s) => match s.parse::() { @@ -2500,8 +2490,6 @@ options! { Currently the only option available"), on_broken_pipe: OnBrokenPipe = (OnBrokenPipe::Default, parse_on_broken_pipe, [TRACKED], "behavior of std::io::ErrorKind::BrokenPipe (SIGPIPE)"), - oom: OomStrategy = (OomStrategy::Abort, parse_oom_strategy, [TRACKED], - "panic strategy for out-of-memory handling"), osx_rpath_install_name: bool = (false, parse_bool, [TRACKED], "pass `-install_name @rpath/...` to the macOS linker (default: no)"), packed_bundled_libs: bool = (false, parse_bool, [TRACKED], diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index 39450f69ce30a..c5f16f3708d74 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -426,20 +426,9 @@ pub mod __alloc_error_handler { // `#[alloc_error_handler]`. #[rustc_std_internal_symbol] pub unsafe fn __rdl_alloc_error_handler(size: usize, _align: usize) -> ! { - unsafe extern "Rust" { - // This symbol is emitted by rustc next to __rust_alloc_error_handler. - // Its value depends on the -Zoom={panic,abort} compiler option. - #[rustc_std_internal_symbol] - fn __rust_alloc_error_handler_should_panic_v2() -> u8; - } - - if unsafe { __rust_alloc_error_handler_should_panic_v2() != 0 } { - panic!("memory allocation of {size} bytes failed") - } else { - core::panicking::panic_nounwind_fmt( - format_args!("memory allocation of {size} bytes failed"), - /* force_no_backtrace */ false, - ) - } + core::panicking::panic_nounwind_fmt( + format_args!("memory allocation of {size} bytes failed"), + /* force_no_backtrace */ false, + ) } } diff --git a/library/std/src/alloc.rs b/library/std/src/alloc.rs index daa25c5a50dd6..30c6d43f4a975 100644 --- a/library/std/src/alloc.rs +++ b/library/std/src/alloc.rs @@ -345,25 +345,14 @@ pub fn take_alloc_error_hook() -> fn(Layout) { } fn default_alloc_error_hook(layout: Layout) { - unsafe extern "Rust" { - // This symbol is emitted by rustc next to __rust_alloc_error_handler. - // Its value depends on the -Zoom={panic,abort} compiler option. - #[rustc_std_internal_symbol] - fn __rust_alloc_error_handler_should_panic_v2() -> u8; - } - - if unsafe { __rust_alloc_error_handler_should_panic_v2() != 0 } { - panic!("memory allocation of {} bytes failed", layout.size()); - } else { - // This is the default path taken on OOM, and the only path taken on stable with std. - // Crucially, it does *not* call any user-defined code, and therefore users do not have to - // worry about allocation failure causing reentrancy issues. That makes it different from - // the default `__rdl_alloc_error_handler` defined in alloc (i.e., the default alloc error - // handler that is called when there is no `#[alloc_error_handler]`), which triggers a - // regular panic and thus can invoke a user-defined panic hook, executing arbitrary - // user-defined code. - rtprintpanic!("memory allocation of {} bytes failed\n", layout.size()); - } + // This is the default path taken on OOM, and the only path taken on stable with std. + // Crucially, it does *not* call any user-defined code, and therefore users do not have to + // worry about allocation failure causing reentrancy issues. That makes it different from + // the default `__rdl_alloc_error_handler` defined in alloc (i.e., the default alloc error + // handler that is called when there is no `#[alloc_error_handler]`), which triggers a + // regular panic and thus can invoke a user-defined panic hook, executing arbitrary + // user-defined code. + rtprintpanic!("memory allocation of {} bytes failed\n", layout.size()); } #[cfg(not(test))] diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 74818cf0740bd..562c08e0785e1 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -12,7 +12,6 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::AllocInit; use rustc_middle::ty::{Instance, Ty}; use rustc_middle::{mir, ty}; -use rustc_session::config::OomStrategy; use rustc_span::Symbol; use rustc_target::callconv::FnAbi; @@ -305,18 +304,12 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { // Here we dispatch all the shims for foreign functions. If you have a platform specific // shim, add it to the corresponding submodule. match link_name.as_str() { - // Magic functions Rust emits (and not as part of the allocator shim). + // Magic function Rust emits (and not as part of the allocator shim). name if name == this.mangle_internal_symbol(NO_ALLOC_SHIM_IS_UNSTABLE) => { // This is a no-op shim that only exists to prevent making the allocator shims // instantly stable. let [] = this.check_shim_sig_lenient(abi, CanonAbi::Rust, link_name, args)?; } - name if name == this.mangle_internal_symbol(OomStrategy::SYMBOL) => { - // Gets the value of the `oom` option. - let [] = this.check_shim_sig_lenient(abi, CanonAbi::Rust, link_name, args)?; - let val = this.tcx.sess.opts.unstable_opts.oom.should_panic(); - this.write_int(val, dest)?; - } // Miri-specific extern functions "miri_alloc" => { diff --git a/src/tools/miri/tests/panic/alloc_error_handler_panic.rs b/src/tools/miri/tests/panic/alloc_error_handler_panic.rs deleted file mode 100644 index c434e8d3227a2..0000000000000 --- a/src/tools/miri/tests/panic/alloc_error_handler_panic.rs +++ /dev/null @@ -1,18 +0,0 @@ -//@compile-flags: -Zoom=panic -#![feature(allocator_api)] - -use std::alloc::*; - -struct Bomb; -impl Drop for Bomb { - fn drop(&mut self) { - eprintln!("yes we are unwinding!"); - } -} - -#[allow(unreachable_code, unused_variables)] -fn main() { - let bomb = Bomb; - handle_alloc_error(Layout::for_value(&0)); - std::mem::forget(bomb); // defuse unwinding bomb -} diff --git a/src/tools/miri/tests/panic/alloc_error_handler_panic.stderr b/src/tools/miri/tests/panic/alloc_error_handler_panic.stderr deleted file mode 100644 index 7c2d089f952ef..0000000000000 --- a/src/tools/miri/tests/panic/alloc_error_handler_panic.stderr +++ /dev/null @@ -1,6 +0,0 @@ - -thread 'main' ($TID) panicked at RUSTLIB/std/src/alloc.rs:LL:CC: -memory allocation of 4 bytes failed -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -note: in Miri, you may have to set `MIRIFLAGS=-Zmiri-env-forward=RUST_BACKTRACE` for the environment variable to have an effect -yes we are unwinding! diff --git a/tests/ui/panics/oom-panic-unwind.rs b/tests/ui/panics/alloc_error_hook-unwind.rs similarity index 84% rename from tests/ui/panics/oom-panic-unwind.rs rename to tests/ui/panics/alloc_error_hook-unwind.rs index 4f7939ce60b20..8a107bc390d4a 100644 --- a/tests/ui/panics/oom-panic-unwind.rs +++ b/tests/ui/panics/alloc_error_hook-unwind.rs @@ -1,17 +1,19 @@ -//! Test that out-of-memory conditions trigger catchable panics with `-Z oom=panic`. +//! Test that out-of-memory conditions trigger catchable panics with `set_alloc_error_hook`. -//@ compile-flags: -Z oom=panic //@ run-pass -//@ no-prefer-dynamic //@ needs-unwind //@ only-linux //@ ignore-backends: gcc +#![feature(alloc_error_hook)] + use std::hint::black_box; use std::mem::forget; use std::panic::catch_unwind; fn main() { + std::alloc::set_alloc_error_hook(|_| panic!()); + let panic = catch_unwind(|| { // This is guaranteed to exceed even the size of the address space for _ in 0..16 { From 6f46c85807c72b6bc14314964bc44aba07809484 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 15 Oct 2025 13:29:25 +0000 Subject: [PATCH 2/3] Show backtrace on allocation failures when possible --- library/std/src/alloc.rs | 73 +++++++++++++++++-- .../fail/alloc/alloc_error_handler.stderr | 8 +- tests/ui/alloc-error/alloc-error-backtrace.rs | 32 ++++++++ .../alloc-error/default-alloc-error-hook.rs | 11 ++- 4 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 tests/ui/alloc-error/alloc-error-backtrace.rs diff --git a/library/std/src/alloc.rs b/library/std/src/alloc.rs index 30c6d43f4a975..a2f3f4bc7bffa 100644 --- a/library/std/src/alloc.rs +++ b/library/std/src/alloc.rs @@ -57,7 +57,7 @@ #![stable(feature = "alloc_module", since = "1.28.0")] use core::ptr::NonNull; -use core::sync::atomic::{Atomic, AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use core::{hint, mem, ptr}; #[stable(feature = "alloc_module", since = "1.28.0")] @@ -287,7 +287,7 @@ unsafe impl Allocator for System { } } -static HOOK: Atomic<*mut ()> = AtomicPtr::new(ptr::null_mut()); +static HOOK: AtomicPtr<()> = AtomicPtr::new(ptr::null_mut()); /// Registers a custom allocation error hook, replacing any that was previously registered. /// @@ -344,7 +344,12 @@ pub fn take_alloc_error_hook() -> fn(Layout) { if hook.is_null() { default_alloc_error_hook } else { unsafe { mem::transmute(hook) } } } +#[optimize(size)] fn default_alloc_error_hook(layout: Layout) { + if cfg!(panic = "immediate-abort") { + return; + } + // This is the default path taken on OOM, and the only path taken on stable with std. // Crucially, it does *not* call any user-defined code, and therefore users do not have to // worry about allocation failure causing reentrancy issues. That makes it different from @@ -352,7 +357,57 @@ fn default_alloc_error_hook(layout: Layout) { // handler that is called when there is no `#[alloc_error_handler]`), which triggers a // regular panic and thus can invoke a user-defined panic hook, executing arbitrary // user-defined code. - rtprintpanic!("memory allocation of {} bytes failed\n", layout.size()); + + static PREV_ALLOC_FAILURE: AtomicBool = AtomicBool::new(false); + if PREV_ALLOC_FAILURE.swap(true, Ordering::Relaxed) { + // Don't try to print a backtrace if a previous alloc error happened. This likely means + // there is not enough memory to print a backtrace, although it could also mean that two + // threads concurrently run out of memory. + rtprintpanic!( + "memory allocation of {} bytes failed while handling another memory allocation error\n", + layout.size() + ); + return; + } else { + rtprintpanic!("memory allocation of {} bytes failed\n", layout.size()); + } + + let Some(mut out) = crate::sys::stdio::panic_output() else { + return; + }; + + // Use a lock to prevent mixed output in multithreading context. + // Some platforms also require it when printing a backtrace, like `SymFromAddr` on Windows. + // Make sure to not take this lock until after checking PREV_ALLOC_FAILURE to avoid deadlocks + // when there is too little memory to print a backtrace. + let mut lock = crate::sys::backtrace::lock(); + + match crate::panic::get_backtrace_style() { + // SAFETY: we took out a lock just a second ago. + Some(crate::panic::BacktraceStyle::Short) => { + drop(lock.print(&mut out, crate::backtrace_rs::PrintFmt::Short)) + } + Some(crate::panic::BacktraceStyle::Full) => { + drop(lock.print(&mut out, crate::backtrace_rs::PrintFmt::Full)) + } + Some(crate::panic::BacktraceStyle::Off) => { + use crate::io::Write; + let _ = writeln!( + out, + "note: run with `RUST_BACKTRACE=1` environment variable to display a \ + backtrace" + ); + if cfg!(miri) { + let _ = writeln!( + out, + "note: in Miri, you may have to set `MIRIFLAGS=-Zmiri-env-forward=RUST_BACKTRACE` \ + for the environment variable to have an effect" + ); + } + } + // If backtraces aren't supported or are forced-off, do nothing. + None => {} + } } #[cfg(not(test))] @@ -360,11 +415,13 @@ fn default_alloc_error_hook(layout: Layout) { #[alloc_error_handler] #[unstable(feature = "alloc_internals", issue = "none")] pub fn rust_oom(layout: Layout) -> ! { - let hook = HOOK.load(Ordering::Acquire); - let hook: fn(Layout) = - if hook.is_null() { default_alloc_error_hook } else { unsafe { mem::transmute(hook) } }; - hook(layout); - crate::process::abort() + crate::sys::backtrace::__rust_end_short_backtrace(|| { + let hook = HOOK.load(Ordering::Acquire); + let hook: fn(Layout) = + if hook.is_null() { default_alloc_error_hook } else { unsafe { mem::transmute(hook) } }; + hook(layout); + crate::process::abort() + }) } #[cfg(not(test))] diff --git a/src/tools/miri/tests/fail/alloc/alloc_error_handler.stderr b/src/tools/miri/tests/fail/alloc/alloc_error_handler.stderr index 7f7fc63ac0ddd..b85a23e3c7db1 100644 --- a/src/tools/miri/tests/fail/alloc/alloc_error_handler.stderr +++ b/src/tools/miri/tests/fail/alloc/alloc_error_handler.stderr @@ -1,11 +1,15 @@ memory allocation of 4 bytes failed +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +note: in Miri, you may have to set `MIRIFLAGS=-Zmiri-env-forward=RUST_BACKTRACE` for the environment variable to have an effect error: abnormal termination: the program aborted execution --> RUSTLIB/std/src/alloc.rs:LL:CC | -LL | crate::process::abort() - | ^^^^^^^^^^^^^^^^^^^^^^^ abnormal termination occurred here +LL | crate::process::abort() + | ^^^^^^^^^^^^^^^^^^^^^^^ abnormal termination occurred here | = note: BACKTRACE: + = note: inside closure at RUSTLIB/std/src/alloc.rs:LL:CC + = note: inside `std::sys::backtrace::__rust_end_short_backtrace::<{closure@std::alloc::rust_oom::{closure#0}}, !>` at RUSTLIB/std/src/sys/backtrace.rs:LL:CC = note: inside `std::alloc::rust_oom` at RUSTLIB/std/src/alloc.rs:LL:CC = note: inside `std::alloc::_::__rust_alloc_error_handler` at RUSTLIB/std/src/alloc.rs:LL:CC = note: inside `std::alloc::handle_alloc_error::rt_error` at RUSTLIB/alloc/src/alloc.rs:LL:CC diff --git a/tests/ui/alloc-error/alloc-error-backtrace.rs b/tests/ui/alloc-error/alloc-error-backtrace.rs new file mode 100644 index 0000000000000..4185586ba0448 --- /dev/null +++ b/tests/ui/alloc-error/alloc-error-backtrace.rs @@ -0,0 +1,32 @@ +//@ run-pass +//@ ignore-android FIXME #17520 +//@ needs-subprocess +//@ ignore-openbsd no support for libbacktrace without filename +//@ ignore-fuchsia Backtraces not symbolized +//@ compile-flags:-g +//@ compile-flags:-Cstrip=none + +use std::alloc::{Layout, handle_alloc_error}; +use std::process::Command; +use std::{env, str}; + +fn main() { + if env::args().len() > 1 { + handle_alloc_error(Layout::new::<[u8; 42]>()) + } + + let me = env::current_exe().unwrap(); + let output = Command::new(&me).env("RUST_BACKTRACE", "1").arg("next").output().unwrap(); + assert!(!output.status.success(), "{:?} is a success", output.status); + + let mut stderr = str::from_utf8(&output.stderr).unwrap(); + + // When running inside QEMU user-mode emulation, there will be an extra message printed by QEMU + // in the stderr whenever a core dump happens. Remove it before the check. + stderr = stderr + .strip_suffix("qemu: uncaught target signal 6 (Aborted) - core dumped\n") + .unwrap_or(stderr); + + assert!(stderr.contains("memory allocation of 42 bytes failed"), "{}", stderr); + assert!(stderr.contains("alloc_error_backtrace::main"), "{}", stderr); +} diff --git a/tests/ui/alloc-error/default-alloc-error-hook.rs b/tests/ui/alloc-error/default-alloc-error-hook.rs index 7fbc66ca5f472..3092f5044c080 100644 --- a/tests/ui/alloc-error/default-alloc-error-hook.rs +++ b/tests/ui/alloc-error/default-alloc-error-hook.rs @@ -2,9 +2,8 @@ //@ needs-subprocess use std::alloc::{Layout, handle_alloc_error}; -use std::env; use std::process::Command; -use std::str; +use std::{env, str}; fn main() { if env::args().len() > 1 { @@ -12,7 +11,7 @@ fn main() { } let me = env::current_exe().unwrap(); - let output = Command::new(&me).arg("next").output().unwrap(); + let output = Command::new(&me).env("RUST_BACKTRACE", "0").arg("next").output().unwrap(); assert!(!output.status.success(), "{:?} is a success", output.status); let mut stderr = str::from_utf8(&output.stderr).unwrap(); @@ -23,5 +22,9 @@ fn main() { .strip_suffix("qemu: uncaught target signal 6 (Aborted) - core dumped\n") .unwrap_or(stderr); - assert_eq!(stderr, "memory allocation of 42 bytes failed\n"); + assert_eq!( + stderr, + "memory allocation of 42 bytes failed\n\ + note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n" + ); } From 0c3cdacc60f3d9bce4300cf3401354b4e659cff5 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 15 Oct 2025 17:51:31 +0200 Subject: [PATCH 3/3] Update message for double allocation error --- library/std/src/alloc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/std/src/alloc.rs b/library/std/src/alloc.rs index a2f3f4bc7bffa..acf1383047371 100644 --- a/library/std/src/alloc.rs +++ b/library/std/src/alloc.rs @@ -364,7 +364,7 @@ fn default_alloc_error_hook(layout: Layout) { // there is not enough memory to print a backtrace, although it could also mean that two // threads concurrently run out of memory. rtprintpanic!( - "memory allocation of {} bytes failed while handling another memory allocation error\n", + "memory allocation of {} bytes failed\nskipping backtrace printing to avoid potential recursion\n", layout.size() ); return;