From b28efb11af6c3aea1117df81df9fc6ddf5b88077 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Fri, 21 Jun 2024 02:20:18 -0700 Subject: [PATCH] =?UTF-8?q?Save=202=20pointers=20in=20`TerminatorKind`=20(?= =?UTF-8?q?96=20=E2=86=92=2080=20bytes)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These things don't need to be `Vec`s; boxed slices are enough. The frequent one here is call arguments, but MIR building knows the number of arguments from the THIR, so the collect is always getting the allocation right in the first place, and thus this shouldn't ever add the shrink-in-place overhead. --- compiler/rustc_middle/src/mir/mod.rs | 4 +- compiler/rustc_middle/src/mir/syntax.rs | 8 +-- .../src/build/custom/parse/instruction.rs | 2 +- .../src/build/expr/as_rvalue.rs | 5 +- .../rustc_mir_build/src/build/expr/into.rs | 4 +- .../rustc_mir_build/src/build/matches/test.rs | 7 ++- .../rustc_mir_dataflow/src/elaborate_drops.rs | 6 +- .../rustc_mir_dataflow/src/framework/tests.rs | 4 +- compiler/rustc_mir_transform/src/coroutine.rs | 4 +- .../rustc_mir_transform/src/coverage/tests.rs | 2 +- compiler/rustc_mir_transform/src/inline.rs | 13 ++-- .../rustc_mir_transform/src/instsimplify.rs | 4 +- compiler/rustc_mir_transform/src/lib.rs | 10 ++- .../src/lower_intrinsics.rs | 61 +++++++------------ compiler/rustc_mir_transform/src/shim.rs | 4 +- compiler/rustc_type_ir/src/fold.rs | 6 ++ 16 files changed, 72 insertions(+), 72 deletions(-) diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 01cefc75194fd..ef88b253864bd 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1817,11 +1817,11 @@ mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; // tidy-alphabetical-start - static_assert_size!(BasicBlockData<'_>, 144); + static_assert_size!(BasicBlockData<'_>, 128); static_assert_size!(LocalDecl<'_>, 40); static_assert_size!(SourceScopeData<'_>, 64); static_assert_size!(Statement<'_>, 32); - static_assert_size!(Terminator<'_>, 112); + static_assert_size!(Terminator<'_>, 96); static_assert_size!(VarDebugInfo<'_>, 88); // tidy-alphabetical-end } diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 0fc84ea5c6212..2c2884f189743 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -730,7 +730,7 @@ pub enum TerminatorKind<'tcx> { /// reused across function calls without duplicating the contents. /// The span for each arg is also included /// (e.g. `a` and `b` in `x.foo(a, b)`). - args: Vec>>, + args: Box<[Spanned>]>, /// Where the returned value will be written destination: Place<'tcx>, /// Where to go after this call returns. If none, the call necessarily diverges. @@ -837,7 +837,7 @@ pub enum TerminatorKind<'tcx> { template: &'tcx [InlineAsmTemplatePiece], /// The operands for the inline assembly, as `Operand`s or `Place`s. - operands: Vec>, + operands: Box<[InlineAsmOperand<'tcx>]>, /// Miscellaneous options for the inline assembly. options: InlineAsmOptions, @@ -849,7 +849,7 @@ pub enum TerminatorKind<'tcx> { /// Valid targets for the inline assembly. /// The first element is the fallthrough destination, unless /// InlineAsmOptions::NORETURN is set. - targets: Vec, + targets: Box<[BasicBlock]>, /// Action to be taken if the inline assembly unwinds. This is present /// if and only if InlineAsmOptions::MAY_UNWIND is set. @@ -1561,6 +1561,6 @@ mod size_asserts { static_assert_size!(PlaceElem<'_>, 24); static_assert_size!(Rvalue<'_>, 40); static_assert_size!(StatementKind<'_>, 16); - static_assert_size!(TerminatorKind<'_>, 96); + static_assert_size!(TerminatorKind<'_>, 80); // tidy-alphabetical-end } diff --git a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs index 7549481c1b301..94ab2fb45818d 100644 --- a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs +++ b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs @@ -170,7 +170,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> { .map(|arg| Ok(Spanned { node: self.parse_operand(*arg)?, span: self.thir.exprs[*arg].span } ) ) - .collect::>>()?; + .collect::>>()?; Ok(TerminatorKind::Call { func: fun, args, diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs index 9f9b566bb8fa3..c5ee6db5999a5 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs @@ -152,10 +152,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { source_info, TerminatorKind::Call { func: exchange_malloc, - args: vec![ + args: [ Spanned { node: Operand::Move(size), span: DUMMY_SP }, Spanned { node: Operand::Move(align), span: DUMMY_SP }, - ], + ] + .into(), destination: storage, target: Some(success), unwind: UnwindAction::Continue, diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs index 6b4dd8b64a425..76bdc26a501a6 100644 --- a/compiler/rustc_mir_build/src/build/expr/into.rs +++ b/compiler/rustc_mir_build/src/build/expr/into.rs @@ -238,7 +238,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } ExprKind::Call { ty: _, fun, ref args, from_hir_call, fn_span } => { let fun = unpack!(block = this.as_local_operand(block, fun)); - let args: Vec<_> = args + let args: Box<[_]> = args .into_iter() .copied() .map(|arg| Spanned { @@ -485,7 +485,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { operands, options, line_spans, - targets, + targets: targets.into_boxed_slice(), unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) { UnwindAction::Continue } else { diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs index 11d3e2a8180f5..d29874a5ad4ba 100644 --- a/compiler/rustc_mir_build/src/build/matches/test.rs +++ b/compiler/rustc_mir_build/src/build/matches/test.rs @@ -324,7 +324,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { user_ty: None, const_: method, })), - args: vec![Spanned { node: Operand::Move(ref_src), span }], + args: [Spanned { node: Operand::Move(ref_src), span }].into(), destination: temp, target: Some(target_block), unwind: UnwindAction::Continue, @@ -486,10 +486,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { const_: method, })), - args: vec![ + args: [ Spanned { node: Operand::Copy(val), span: DUMMY_SP }, Spanned { node: expect, span: DUMMY_SP }, - ], + ] + .into(), destination: eq_result, target: Some(eq_block), unwind: UnwindAction::Continue, diff --git a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs index 654020164db8b..e0da9600ae37f 100644 --- a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs +++ b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs @@ -650,10 +650,8 @@ where [ty.into()], self.source_info.span, ), - args: vec![Spanned { - node: Operand::Move(Place::from(ref_place)), - span: DUMMY_SP, - }], + args: [Spanned { node: Operand::Move(Place::from(ref_place)), span: DUMMY_SP }] + .into(), destination: unit_temp, target: Some(succ), unwind: unwind.into_action(), diff --git a/compiler/rustc_mir_dataflow/src/framework/tests.rs b/compiler/rustc_mir_dataflow/src/framework/tests.rs index 6b338efd56974..52db931b68e1e 100644 --- a/compiler/rustc_mir_dataflow/src/framework/tests.rs +++ b/compiler/rustc_mir_dataflow/src/framework/tests.rs @@ -34,7 +34,7 @@ fn mock_body<'tcx>() -> mir::Body<'tcx> { 2, mir::TerminatorKind::Call { func: mir::Operand::Copy(dummy_place.clone()), - args: vec![], + args: [].into(), destination: dummy_place.clone(), target: Some(mir::START_BLOCK), unwind: mir::UnwindAction::Continue, @@ -48,7 +48,7 @@ fn mock_body<'tcx>() -> mir::Body<'tcx> { 4, mir::TerminatorKind::Call { func: mir::Operand::Copy(dummy_place.clone()), - args: vec![], + args: [].into(), destination: dummy_place.clone(), target: Some(mir::START_BLOCK), unwind: mir::UnwindAction::Continue, diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index bf79b4e133a91..05674792426f1 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -677,8 +677,8 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { fn eliminate_get_context_call<'tcx>(bb_data: &mut BasicBlockData<'tcx>) -> Local { let terminator = bb_data.terminator.take().unwrap(); - if let TerminatorKind::Call { mut args, destination, target, .. } = terminator.kind { - let arg = args.pop().unwrap(); + if let TerminatorKind::Call { args, destination, target, .. } = terminator.kind { + let [arg] = *Box::try_from(args).unwrap(); let local = arg.node.place().unwrap().local; let arg = Rvalue::Use(arg.node); diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 048547dc9f543..63a9f303b8521 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -134,7 +134,7 @@ impl<'tcx> MockBlocks<'tcx> { some_from_block, TerminatorKind::Call { func: Operand::Copy(self.dummy_place.clone()), - args: vec![], + args: [].into(), destination: self.dummy_place.clone(), target: Some(TEMP_BLOCK), unwind: UnwindAction::Continue, diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 32b5d8120252f..0a5fc697d0321 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -624,8 +624,7 @@ impl<'tcx> Inliner<'tcx> { }; // Copy the arguments if needed. - let args: Vec<_> = - self.make_call_args(args, &callsite, caller_body, &callee_body, return_block); + let args = self.make_call_args(args, &callsite, caller_body, &callee_body, return_block); let mut integrator = Integrator { args: &args, @@ -736,12 +735,12 @@ impl<'tcx> Inliner<'tcx> { fn make_call_args( &self, - args: Vec>>, + args: Box<[Spanned>]>, callsite: &CallSite<'tcx>, caller_body: &mut Body<'tcx>, callee_body: &Body<'tcx>, return_block: Option, - ) -> Vec { + ) -> Box<[Local]> { let tcx = self.tcx; // There is a bit of a mismatch between the *caller* of a closure and the *callee*. @@ -768,7 +767,8 @@ impl<'tcx> Inliner<'tcx> { // // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. if callsite.fn_sig.abi() == Abi::RustCall && callee_body.spread_arg.is_none() { - let mut args = args.into_iter(); + // FIXME(edition_2024): switch back to a normal method call. + let mut args = <_>::into_iter(args); let self_ = self.create_temp_if_necessary( args.next().unwrap().node, callsite, @@ -802,7 +802,8 @@ impl<'tcx> Inliner<'tcx> { closure_ref_arg.chain(tuple_tmp_args).collect() } else { - args.into_iter() + // FIXME(edition_2024): switch back to a normal method call. + <_>::into_iter(args) .map(|a| self.create_temp_if_necessary(a.node, callsite, caller_body, return_block)) .collect() } diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 6806c517c17de..8209e5e27111c 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -1,6 +1,7 @@ //! Performs various peephole optimizations. use crate::simplify::simplify_duplicate_switch_targets; +use crate::take_array; use rustc_ast::attr; use rustc_middle::bug; use rustc_middle::mir::*; @@ -285,7 +286,8 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> { return; } - let Some(arg_place) = args.pop().unwrap().node.place() else { return }; + let Ok([arg]) = take_array(args) else { return }; + let Some(arg_place) = arg.node.place() else { return }; statements.push(Statement { source_info: terminator.source_info, diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index fe195f0112f8d..f7056702cb4ed 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -160,8 +160,9 @@ fn remap_mir_for_const_eval_select<'tcx>( } if let ty::FnDef(def_id, _) = *const_.ty().kind() && tcx.is_intrinsic(def_id, sym::const_eval_select) => { - let [tupled_args, called_in_const, called_at_rt]: [_; 3] = - std::mem::take(args).try_into().unwrap(); + let Ok([tupled_args, called_in_const, called_at_rt]) = take_array(args) else { + unreachable!() + }; let ty = tupled_args.node.ty(&body.local_decls, tcx); let fields = ty.tuple_fields(); let num_args = fields.len(); @@ -211,6 +212,11 @@ fn remap_mir_for_const_eval_select<'tcx>( body } +fn take_array(b: &mut Box<[T]>) -> Result<[T; N], Box<[T]>> { + let b: Box<[T; N]> = std::mem::take(b).try_into()?; + Ok(*b) +} + fn is_mir_available(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { tcx.mir_keys(()).contains(&def_id) } diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 3ffc447217d7d..6aa9039435547 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -1,5 +1,6 @@ //! Lowers intrinsic calls +use crate::take_array; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, span_bug}; @@ -50,42 +51,34 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } sym::copy_nonoverlapping => { let target = target.unwrap(); - let mut args = args.drain(..); + let Ok([src, dst, count]) = take_array(args) else { + bug!("Wrong arguments for copy_non_overlapping intrinsic"); + }; block.statements.push(Statement { source_info: terminator.source_info, kind: StatementKind::Intrinsic(Box::new( NonDivergingIntrinsic::CopyNonOverlapping( rustc_middle::mir::CopyNonOverlapping { - src: args.next().unwrap().node, - dst: args.next().unwrap().node, - count: args.next().unwrap().node, + src: src.node, + dst: dst.node, + count: count.node, }, ), )), }); - assert_eq!( - args.next(), - None, - "Extra argument for copy_non_overlapping intrinsic" - ); - drop(args); terminator.kind = TerminatorKind::Goto { target }; } sym::assume => { let target = target.unwrap(); - let mut args = args.drain(..); + let Ok([arg]) = take_array(args) else { + bug!("Wrong arguments for assume intrinsic"); + }; block.statements.push(Statement { source_info: terminator.source_info, kind: StatementKind::Intrinsic(Box::new( - NonDivergingIntrinsic::Assume(args.next().unwrap().node), + NonDivergingIntrinsic::Assume(arg.node), )), }); - assert_eq!( - args.next(), - None, - "Extra argument for copy_non_overlapping intrinsic" - ); - drop(args); terminator.kind = TerminatorKind::Goto { target }; } sym::wrapping_add @@ -100,13 +93,9 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { | sym::unchecked_shl | sym::unchecked_shr => { let target = target.unwrap(); - let lhs; - let rhs; - { - let mut args = args.drain(..); - lhs = args.next().unwrap(); - rhs = args.next().unwrap(); - } + let Ok([lhs, rhs]) = take_array(args) else { + bug!("Wrong arguments for {} intrinsic", intrinsic.name); + }; let bin_op = match intrinsic.name { sym::wrapping_add => BinOp::Add, sym::wrapping_sub => BinOp::Sub, @@ -132,13 +121,9 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => { if let Some(target) = *target { - let lhs; - let rhs; - { - let mut args = args.drain(..); - lhs = args.next().unwrap(); - rhs = args.next().unwrap(); - } + let Ok([lhs, rhs]) = take_array(args) else { + bug!("Wrong arguments for {} intrinsic", intrinsic.name); + }; let bin_op = match intrinsic.name { sym::add_with_overflow => BinOp::AddWithOverflow, sym::sub_with_overflow => BinOp::SubWithOverflow, @@ -174,7 +159,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } } sym::read_via_copy => { - let [arg] = args.as_slice() else { + let Ok([arg]) = take_array(args) else { span_bug!(terminator.source_info.span, "Wrong number of arguments"); }; let derefed_place = if let Some(place) = arg.node.place() @@ -207,7 +192,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } sym::write_via_move => { let target = target.unwrap(); - let Ok([ptr, val]) = <[_; 2]>::try_from(std::mem::take(args)) else { + let Ok([ptr, val]) = take_array(args) else { span_bug!( terminator.source_info.span, "Wrong number of arguments for write_via_move intrinsic", @@ -247,7 +232,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } sym::offset => { let target = target.unwrap(); - let Ok([ptr, delta]) = <[_; 2]>::try_from(std::mem::take(args)) else { + let Ok([ptr, delta]) = take_array(args) else { span_bug!( terminator.source_info.span, "Wrong number of arguments for offset intrinsic", @@ -264,7 +249,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } sym::transmute | sym::transmute_unchecked => { let dst_ty = destination.ty(local_decls, tcx).ty; - let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else { + let Ok([arg]) = take_array(args) else { span_bug!( terminator.source_info.span, "Wrong number of arguments for transmute intrinsic", @@ -289,7 +274,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } } sym::aggregate_raw_ptr => { - let Ok([data, meta]) = <[_; 2]>::try_from(std::mem::take(args)) else { + let Ok([data, meta]) = take_array(args) else { span_bug!( terminator.source_info.span, "Wrong number of arguments for aggregate_raw_ptr intrinsic", @@ -317,7 +302,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { terminator.kind = TerminatorKind::Goto { target }; } sym::ptr_metadata => { - let Ok([ptr]) = <[_; 1]>::try_from(std::mem::take(args)) else { + let Ok([ptr]) = take_array(args) else { span_bug!( terminator.source_info.span, "Wrong number of arguments for ptr_metadata intrinsic", diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 825f8957187e1..25577e88e2831 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -562,7 +562,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { vec![statement], TerminatorKind::Call { func, - args: vec![Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }], + args: [Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }].into(), destination: dest, target: Some(next), unwind: UnwindAction::Cleanup(cleanup), @@ -843,7 +843,7 @@ fn build_call_shim<'tcx>( }; // BB #0 - let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect::>(); + let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect(); block( &mut blocks, statements, diff --git a/compiler/rustc_type_ir/src/fold.rs b/compiler/rustc_type_ir/src/fold.rs index 26e365f64faa3..64da068395c98 100644 --- a/compiler/rustc_type_ir/src/fold.rs +++ b/compiler/rustc_type_ir/src/fold.rs @@ -323,6 +323,12 @@ impl> TypeFoldable for Vec { } } +impl> TypeFoldable for Box<[T]> { + fn try_fold_with>(self, folder: &mut F) -> Result { + Vec::from(self).try_fold_with(folder).map(Vec::into_boxed_slice) + } +} + impl, Ix: Idx> TypeFoldable for IndexVec { fn try_fold_with>(self, folder: &mut F) -> Result { self.raw.try_fold_with(folder).map(IndexVec::from_raw)