diff --git a/compiler/rustc_borrowck/src/invalidation.rs b/compiler/rustc_borrowck/src/invalidation.rs index f5317a143aed7..f0c94f6848bc0 100644 --- a/compiler/rustc_borrowck/src/invalidation.rs +++ b/compiler/rustc_borrowck/src/invalidation.rs @@ -289,7 +289,7 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> { Rvalue::ThreadLocalRef(_) => {} Rvalue::Use(ref operand) - | Rvalue::Repeat(ref operand, _) + | Rvalue::Repeat(ref operand, _, _) | Rvalue::UnaryOp(_ /*un_op*/, ref operand) | Rvalue::Cast(_ /*cast_kind*/, ref operand, _ /*ty*/) | Rvalue::ShallowInitBox(ref operand, _ /*ty*/) => { diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 4a4887f19702f..722e2917943c8 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -1233,7 +1233,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { Rvalue::ThreadLocalRef(_) => {} Rvalue::Use(ref operand) - | Rvalue::Repeat(ref operand, _) + | Rvalue::Repeat(ref operand, _, _) | Rvalue::UnaryOp(_ /*un_op*/, ref operand) | Rvalue::Cast(_ /*cast_kind*/, ref operand, _ /*ty*/) | Rvalue::ShallowInitBox(ref operand, _ /*ty*/) => { diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 6ccc29b09c0a5..ca38c79900964 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1854,7 +1854,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { self.check_aggregate_rvalue(&body, rvalue, ak, ops, location) } - Rvalue::Repeat(operand, len) => { + Rvalue::Repeat(operand, len, _) => { self.check_operand(operand, location); // If the length cannot be evaluated we must assume that the length can be larger diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index ceebe4d417f7d..00c4f414bd7cd 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -7,6 +7,7 @@ #![feature(int_roundings)] #![feature(if_let_guard)] #![feature(never_type)] +#![feature(let_chains)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 4aab31fbfe7da..3f1f64c5c0396 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -73,7 +73,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx } - mir::Rvalue::Repeat(ref elem, count) => { + mir::Rvalue::Repeat(ref elem, count, enum_tag_only) => { let cg_elem = self.codegen_operand(&mut bx, elem); // Do not generate the loop for zero-sized elements or empty arrays. @@ -99,6 +99,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.memset(start, v, size, dest.align, MemFlags::empty()); return bx; } + } else if let OperandValue::Pair(tag, _) = cg_elem.val && enum_tag_only { + let zero = bx.const_usize(0); + let start = dest.project_index(&mut bx, zero).llval; + let size = bx.const_usize(dest.layout.size.bytes()); + let v = bx.from_immediate(tag); + if bx.cx().val_ty(v) == bx.cx().type_i8() { + bx.memset(start, v, size, dest.align, MemFlags::empty()); + return bx; + } } let count = diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 3c286fa61bec5..4c19db3d7cf59 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -207,7 +207,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } - Repeat(ref operand, _) => { + Repeat(ref operand, _, _) => { let src = self.eval_operand(operand, None)?; assert!(src.layout.is_sized()); let dest = self.force_allocation(&dest)?; diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs index d995d533ca3e4..fde6d903cfdf4 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs @@ -256,7 +256,7 @@ where Rvalue::CopyForDeref(place) => in_place::(cx, in_local, place.as_ref()), Rvalue::Use(operand) - | Rvalue::Repeat(operand, _) + | Rvalue::Repeat(operand, _, _) | Rvalue::UnaryOp(_, operand) | Rvalue::Cast(_, operand, _) | Rvalue::ShallowInitBox(operand, _) => in_operand::(cx, in_local, operand), diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs index f48bcd9080966..f7cb4047d411c 100644 --- a/compiler/rustc_const_eval/src/transform/promote_consts.rs +++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs @@ -487,7 +487,7 @@ impl<'tcx> Validator<'_, 'tcx> { fn validate_rvalue(&mut self, rvalue: &Rvalue<'tcx>) -> Result<(), Unpromotable> { match rvalue { - Rvalue::Use(operand) | Rvalue::Repeat(operand, _) => { + Rvalue::Use(operand) | Rvalue::Repeat(operand, _, _) => { self.validate_operand(operand)?; } Rvalue::CopyForDeref(place) => { diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index 81b82a21fa1a7..80de45e6ed6f1 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -591,7 +591,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } } - Rvalue::Repeat(_, _) + Rvalue::Repeat(_, _, _) | Rvalue::ThreadLocalRef(_) | Rvalue::AddressOf(_, _) | Rvalue::NullaryOp(_, _) diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 4781651071d38..0f638b912780c 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1920,7 +1920,7 @@ impl<'tcx> Rvalue<'tcx> { Rvalue::Use(_) | Rvalue::CopyForDeref(_) - | Rvalue::Repeat(_, _) + | Rvalue::Repeat(_, _, _) | Rvalue::Ref(_, _, _) | Rvalue::ThreadLocalRef(_) | Rvalue::AddressOf(_, _) @@ -1979,7 +1979,7 @@ impl<'tcx> Debug for Rvalue<'tcx> { match *self { Use(ref place) => write!(fmt, "{:?}", place), - Repeat(ref a, b) => { + Repeat(ref a, b, _) => { write!(fmt, "[{:?}; ", a)?; pretty_print_const(b, fmt, false)?; write!(fmt, "]") diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index fed943169dfb5..f2f471e7b5fb5 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -1004,10 +1004,13 @@ pub enum Rvalue<'tcx> { /// This is the cause of a bug in the case where the repetition count is zero because the value /// is not dropped, see [#74836]. /// + /// The third param is true if the value we're repeating is an enum variant + /// with no fields so we can emit a memset. + /// /// Corresponds to source code like `[x; 32]`. /// /// [#74836]: https://github.com/rust-lang/rust/issues/74836 - Repeat(Operand<'tcx>, ty::Const<'tcx>), + Repeat(Operand<'tcx>, ty::Const<'tcx>, bool), /// Creates a reference of the indicated kind to the place. /// diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs index fa3adafd4b85f..ead60c47eee83 100644 --- a/compiler/rustc_middle/src/mir/tcx.rs +++ b/compiler/rustc_middle/src/mir/tcx.rs @@ -162,7 +162,7 @@ impl<'tcx> Rvalue<'tcx> { { match *self { Rvalue::Use(ref operand) => operand.ty(local_decls, tcx), - Rvalue::Repeat(ref operand, count) => { + Rvalue::Repeat(ref operand, count, _) => { tcx.mk_ty(ty::Array(operand.ty(local_decls, tcx), count)) } Rvalue::ThreadLocalRef(did) => { diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index d87eb28970e41..21586dd1c70a9 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -627,7 +627,7 @@ macro_rules! make_mir_visitor { self.visit_operand(operand, location); } - Rvalue::Repeat(value, _) => { + Rvalue::Repeat(value, _, _) => { self.visit_operand(value, location); } diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs index 5c82fb1ddc0d5..6880b3bbd0f05 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs @@ -58,16 +58,21 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { if Some(0) == count.try_eval_usize(this.tcx, this.param_env) { this.build_zero_repeat(block, value, scope, source_info) } else { + let value = &this.thir[value]; + let enum_tag_only = if let ExprKind::Scope {value: val, ..} = value.kind + && let ExprKind::Adt(adt) = &this.thir[val].kind + && adt.adt_def.is_enum() { + // If this is an enum, and this variant has no fields, *and* other + // variants do have fields, store this information. This allows emitting + // a memset, instead of a series of single writes to the tag field + adt.fields.is_empty() && adt.adt_def.all_fields().count() > 0 + } else { + false + }; let value_operand = unpack!( - block = this.as_operand( - block, - scope, - &this.thir[value], - None, - NeedsTemporary::No - ) + block = this.as_operand(block, scope, value, None, NeedsTemporary::No) ); - block.and(Rvalue::Repeat(value_operand, count)) + block.and(Rvalue::Repeat(value_operand, count, enum_tag_only)) } } ExprKind::Binary { op, lhs, rhs } => { diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs index f46fd118bde5d..10358427edcc1 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs @@ -339,7 +339,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> { match *rvalue { Rvalue::ThreadLocalRef(_) => {} // not-a-move Rvalue::Use(ref operand) - | Rvalue::Repeat(ref operand, _) + | Rvalue::Repeat(ref operand, _, _) | Rvalue::Cast(_, ref operand, _) | Rvalue::ShallowInitBox(ref operand, _) | Rvalue::UnaryOp(_, ref operand) => self.gather_operand(operand), diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index 2f116aaa95849..c136955e05fda 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -208,7 +208,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData< | Rvalue::UnaryOp(_, Operand::Constant(_)) => return true, // These rvalues make things ambiguous - Rvalue::Repeat(_, _) + Rvalue::Repeat(_, _, _) | Rvalue::ThreadLocalRef(_) | Rvalue::Len(_) | Rvalue::BinaryOp(_, _) @@ -282,7 +282,7 @@ fn find_determining_place<'tcx>( | Rvalue::UnaryOp(_, Operand::Copy(new) | Operand::Move(new)) | Rvalue::CopyForDeref(new) | Rvalue::Cast(_, Operand::Move(new) | Operand::Copy(new), _) - | Rvalue::Repeat(Operand::Move(new) | Operand::Copy(new), _) + | Rvalue::Repeat(Operand::Move(new) | Operand::Copy(new), _, _) | Rvalue::Discriminant(new) => switch_place = new, @@ -297,7 +297,7 @@ fn find_determining_place<'tcx>( // The following rvalues definitely mean we cannot // or should not apply this optimization | Rvalue::Use(Operand::Constant(_)) - | Rvalue::Repeat(Operand::Constant(_), _) + | Rvalue::Repeat(Operand::Constant(_), _, _) | Rvalue::ThreadLocalRef(_) | Rvalue::AddressOf(_, _) | Rvalue::NullaryOp(_, _) diff --git a/src/test/codegen/enum-repeat.rs b/src/test/codegen/enum-repeat.rs new file mode 100644 index 0000000000000..6cba535d12672 --- /dev/null +++ b/src/test/codegen/enum-repeat.rs @@ -0,0 +1,19 @@ +// compile-flags: -O + +#![crate_type = "lib"] + +// CHECK-LABEL: @none_repeat +#[no_mangle] +pub fn none_repeat() -> [Option; 64] { + // CHECK: call void @llvm.memset + // CHECK-NEXT: ret void + [None; 64] +} + +// CHECK-LABEL: @some_repeat +#[no_mangle] +pub fn some_repeat() -> [Option; 64] { + // CHECK: call void @llvm.memset + // CHECK-NEXT: ret void + [Some(1); 64] +}