Support for a scalable simd representation

compiler/rustc_abi/src/layout.rs

@@ -383,6 +383,7 @@ where
                 hide_niches(b);
             }
             Abi::Vector { element, count: _ } => hide_niches(element),
+            Abi::ScalableVector { element, .. } => hide_niches(element),
             Abi::Aggregate { sized: _ } => {}
         }
         st.largest_niche = None;

compiler/rustc_abi/src/lib.rs

@@ -43,9 +43,11 @@ bitflags! {
         // If true, the type's layout can be randomized using
         // the seed stored in `ReprOptions.field_shuffle_seed`
         const RANDOMIZE_LAYOUT   = 1 << 4;
+        const IS_SCALABLE = 1 << 5;
         // Any of these flags being set prevent field reordering optimisation.
         const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
+                                 | ReprFlags::IS_SCALABLE.bits()
                                  | ReprFlags::IS_LINEAR.bits();
     }
 }
@@ -86,6 +88,7 @@ pub struct ReprOptions {
     pub align: Option<Align>,
     pub pack: Option<Align>,
     pub flags: ReprFlags,
+    pub scalable: Option<u32>,
     /// The seed to be used for randomizing a type's layout
     ///
     /// Note: This could technically be a `u128` which would
@@ -102,6 +105,11 @@ impl ReprOptions {
         self.flags.contains(ReprFlags::IS_SIMD)
     }
 
+    #[inline]
+    pub fn scalable(&self) -> bool {
+        self.flags.contains(ReprFlags::IS_SCALABLE)
+    }
+
     #[inline]
     pub fn c(&self) -> bool {
         self.flags.contains(ReprFlags::IS_C)
@@ -1266,6 +1274,10 @@ pub enum Abi {
     Uninhabited,
     Scalar(Scalar),
     ScalarPair(Scalar, Scalar),
+    ScalableVector {
+        element: Scalar,
+        elt: u64,
+    },
     Vector {
         element: Scalar,
         count: u64,
@@ -1283,6 +1295,7 @@ impl Abi {
         match *self {
             Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false,
             Abi::Aggregate { sized } => !sized,
+            Abi::ScalableVector { .. } => true,
         }
     }
 
@@ -1329,7 +1342,7 @@ impl Abi {
             Abi::Vector { element, count } => {
                 cx.data_layout().vector_align(element.size(cx) * count)
             }
-            Abi::Uninhabited | Abi::Aggregate { .. } => return None,
+            Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None,
         })
     }
 
@@ -1350,7 +1363,7 @@ impl Abi {
                 // to make the size a multiple of align (e.g. for vectors of size 3).
                 (element.size(cx) * count).align_to(self.inherent_align(cx)?.abi)
             }
-            Abi::Uninhabited | Abi::Aggregate { .. } => return None,
+            Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None,
         })
     }
 
@@ -1360,6 +1373,9 @@ impl Abi {
             Abi::Scalar(s) => Abi::Scalar(s.to_union()),
             Abi::ScalarPair(s1, s2) => Abi::ScalarPair(s1.to_union(), s2.to_union()),
             Abi::Vector { element, count } => Abi::Vector { element: element.to_union(), count },
+            Abi::ScalableVector { element, elt } => {
+                Abi::ScalableVector { element: element.to_union(), elt }
+            }
             Abi::Uninhabited | Abi::Aggregate { .. } => Abi::Aggregate { sized: true },
         }
     }
@@ -1646,6 +1662,11 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutS<FieldIdx, VariantIdx> {
         self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
     }
 
+    /// Returns true if the size of the type is only known at runtime.
+    pub fn is_runtime_sized(&self) -> bool {
+        matches!(self.abi, Abi::ScalableVector { .. })
+    }
+
     /// Returns `true` if the type is a ZST and not unsized.
     ///
     /// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -1655,6 +1676,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutS<FieldIdx, VariantIdx> {
             Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false,
             Abi::Uninhabited => self.size.bytes() == 0,
             Abi::Aggregate { sized } => sized && self.size.bytes() == 0,
+            Abi::ScalableVector { .. } => false,
         }
     }
 

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -251,6 +251,15 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                                 "SIMD types are experimental and possibly buggy"
                             );
                         }
+
+                        if item.has_name(sym::scalable) {
+                            gate!(
+                                &self,
+                                repr_scalable,
+                                attr.span,
+                                "Scalable SIMD types are experimental and possibly buggy"
+                            );
+                        }
                     }
                 }
             }

compiler/rustc_attr/messages.ftl

@@ -94,6 +94,9 @@ attr_rustc_allowed_unstable_pairing =
 attr_rustc_promotable_pairing =
     `rustc_promotable` attribute must be paired with either a `rustc_const_unstable` or a `rustc_const_stable` attribute
 
+attr_scalable_missing_n =
+    invalid `scalable(num)` attribute: `scalable` needs an argument
+    .suggestion = supply an argument here
 attr_soft_no_args =
     `soft` should not have any arguments
 

compiler/rustc_attr/src/builtin.rs

@@ -924,6 +924,7 @@ pub enum ReprAttr {
     ReprSimd,
     ReprTransparent,
     ReprAlign(Align),
+    ReprScalable(u32),
 }
 
 #[derive(Eq, PartialEq, Debug, Copy, Clone)]
@@ -979,6 +980,13 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         recognised = true;
                         None
                     }
+                    sym::scalable => {
+                        sess.dcx().emit_err(session_diagnostics::ScalableAttrMissingN {
+                            span: item.span(),
+                        });
+                        recognised = true;
+                        None
+                    }
                     name => int_type_of_word(name).map(ReprInt),
                 };
 
@@ -1007,6 +1015,12 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                             literal_error = Some(message)
                         }
                     };
+                } else if name == sym::scalable {
+                    recognised = true;
+                    match parse_scalable(&value.kind) {
+                        Ok(literal) => acc.push(ReprScalable(literal)),
+                        Err(message) => literal_error = Some(message),
+                    };
                 } else if matches!(name, sym::Rust | sym::C | sym::simd | sym::transparent)
                     || int_type_of_word(name).is_some()
                 {
@@ -1026,7 +1040,10 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
             } else if let Some(meta_item) = item.meta_item() {
                 match &meta_item.kind {
                     MetaItemKind::NameValue(value) => {
-                        if meta_item.has_name(sym::align) || meta_item.has_name(sym::packed) {
+                        if meta_item.has_name(sym::align)
+                            || meta_item.has_name(sym::packed)
+                            || meta_item.has_name(sym::scalable)
+                        {
                             let name = meta_item.name_or_empty().to_ident_string();
                             recognised = true;
                             sess.dcx().emit_err(session_diagnostics::IncorrectReprFormatGeneric {
@@ -1245,3 +1262,11 @@ pub fn parse_confusables(attr: &Attribute) -> Option<Vec<Symbol>> {
 
     return Some(candidates);
 }
+
+pub fn parse_scalable(node: &ast::LitKind) -> Result<u32, &'static str> {
+    if let ast::LitKind::Int(literal, ast::LitIntType::Unsuffixed) = node {
+        (literal.get()).try_into().map_err(|_| "integer too large")
+    } else {
+        Err("not an unsuffixed integer")
+    }
+}

compiler/rustc_attr/src/session_diagnostics.rs

@@ -400,3 +400,11 @@ pub(crate) struct UnknownVersionLiteral {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(attr_scalable_missing_n, code = E0798)]
+pub(crate) struct ScalableAttrMissingN {
+    #[primary_span]
+    #[suggestion(applicability = "has-placeholders", code = "scalable(...)")]
+    pub span: Span,
+}

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -531,7 +531,15 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
             place_ty = self.sanitize_projection(place_ty, elem, place, location, context);
         }
 
-        if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context {
+        // The Copy trait isn't implemented for scalable SIMD types.
+        // These types live somewhere between `Sized` and `Unsize`.
+        // The bounds on `Copy` disallow the trait from being
+        // implemented for them. As a result of this no bounds from
+        // `Copy` apply for the type, therefore, skipping this check
+        // should be perfectly legal.
+        if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context
+            && !place_ty.ty.is_scalable_simd()
+        {
             let tcx = self.tcx();
             let trait_ref =
                 ty::TraitRef::from_lang_item(tcx, LangItem::Copy, self.last_span, [place_ty.ty]);
@@ -1341,7 +1349,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 }
 
                 self.check_rvalue(body, rv, location);
-                if !self.unsized_feature_enabled() {
+                if !(self.unsized_feature_enabled() || place_ty.is_scalable_simd()) {
                     let trait_ref = ty::TraitRef::from_lang_item(
                         tcx,
                         LangItem::Sized,
@@ -1849,7 +1857,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         if !self.unsized_feature_enabled() {
             let span = local_decl.source_info.span;
             let ty = local_decl.ty;
-            self.ensure_place_sized(ty, span);
+            if !ty.is_scalable_simd() {
+                self.ensure_place_sized(ty, span);
+            }
         }
     }
 
@@ -1865,11 +1875,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             // expressions evaluate through `as_temp` or `into` a return
             // slot or local, so to find all unsized rvalues it is enough
             // to check all temps, return slots and locals.
-            if self.reported_errors.replace((ty, span)).is_none() {
-                // While this is located in `nll::typeck` this error is not
-                // an NLL error, it's a required check to prevent creation
-                // of unsized rvalues in a call expression.
-                self.tcx().dcx().emit_err(MoveUnsized { ty, span });
+            if !ty.is_scalable_simd() {
+                if self.reported_errors.replace((ty, span)).is_none() {
+                    // While this is located in `nll::typeck` this error is not
+                    // an NLL error, it's a required check to prevent creation
+                    // of unsized rvalues in a call expression.
+                    self.tcx().dcx().emit_err(MoveUnsized { ty, span });
+                }
             }
         }
     }

compiler/rustc_codegen_gcc/src/abi.rs

@@ -90,6 +90,7 @@ impl GccType for Reg {
                 _ => bug!("unsupported float: {:?}", self),
             },
             RegKind::Vector => unimplemented!(), //cx.type_vector(cx.type_i8(), self.size.bytes()),
+            RegKind::ScalableVector => unimplemented!(),
         }
     }
 }

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -296,7 +296,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 let layout = self.layout_of(tp_ty).layout;
                 let _use_integer_compare = match layout.abi() {
                     Scalar(_) | ScalarPair(_, _) => true,
-                    Uninhabited | Vector { .. } => false,
+                    Uninhabited | Vector { .. } | ScalableVector { .. } => false,
                     Aggregate { .. } => {
                         // For rusty ABIs, small aggregates are actually passed
                         // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -87,6 +87,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(
                 false,
             );
         }
+        Abi::ScalableVector { .. } => todo!(),
         Abi::Uninhabited | Abi::Aggregate { .. } => {}
     }
 
@@ -181,15 +182,15 @@ pub trait LayoutGccExt<'tcx> {
 impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
     fn is_gcc_immediate(&self) -> bool {
         match self.abi {
-            Abi::Scalar(_) | Abi::Vector { .. } => true,
+            Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } => true,
             Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false,
         }
     }
 
     fn is_gcc_scalar_pair(&self) -> bool {
         match self.abi {
             Abi::ScalarPair(..) => true,
-            Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false,
+            Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } | Abi::Aggregate { .. } => false,
         }
     }
 

compiler/rustc_codegen_llvm/src/abi.rs

@@ -123,6 +123,11 @@ impl LlvmType for Reg {
                 _ => bug!("unsupported float: {:?}", self),
             },
             RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()),
+            // Generate a LLVM type such as <vscale x 16 x i8>, like above for a non scalable
+            // vector. The use of 16 here is chosen as that will generate a valid type with both
+            // Arm SVE and RISC-V RVV. In the future with other architectures this might not be
+            // valid and might have to be configured by the target.
+            RegKind::ScalableVector => cx.type_scalable_vector(cx.type_i8(), 16),
         }
     }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -538,7 +538,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                 panic!("unsized locals must not be `extern` types");
             }
         }
-        assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
+
+        if !place.layout.is_runtime_sized() {
+            assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
+        }
 
         if place.layout.is_zst() {
             return OperandRef::zero_sized(place.layout);

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -1052,6 +1052,7 @@ fn build_struct_type_di_node<'ll, 'tcx>(
                         Cow::Borrowed(f.name.as_str())
                     };
                     let field_layout = struct_type_and_layout.field(cx, i);
+
                     build_field_di_node(
                         cx,
                         owner,

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -389,6 +389,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 let use_integer_compare = match layout.abi() {
                     Scalar(_) | ScalarPair(_, _) => true,
                     Uninhabited | Vector { .. } => false,
+                    ScalableVector { .. } => {
+                        tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
+                            span,
+                            name: sym::raw_eq,
+                            ty: tp_ty,
+                        });
+                        return Ok(());
+                    }
                     Aggregate { .. } => {
                         // For rusty ABIs, small aggregates are actually passed
                         // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
@@ -1084,6 +1092,18 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
     }
 
+    if name == sym::simd_reinterpret {
+        require_simd!(ret_ty, SimdReturn);
+
+        return Ok(match args[0].val {
+            OperandValue::Ref(val, _, _) | OperandValue::Immediate(val) => {
+                bx.bitcast(val, llret_ty)
+            }
+            OperandValue::ZeroSized => bx.const_undef(llret_ty),
+            OperandValue::Pair(_, _) => todo!(),
+        });
+    }
+
     // every intrinsic below takes a SIMD vector as its first argument
     let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput);
     let in_ty = arg_tys[0];
@@ -1296,12 +1316,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
         );
         match m_elem_ty.kind() {
-            ty::Int(_) => {}
+            // Arm SVE has a svbool type and we need to represent that as a bool in the type system.
+            ty::Int(_) | ty::Bool => {}
             _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }),
         }
         // truncate the mask to a vector of i1s
         let i1 = bx.type_i1();
-        let i1xn = bx.type_vector(i1, m_len as u64);
+        let i1xn = if arg_tys[1].is_scalable_simd() {
+            bx.type_scalable_vector(i1, m_len as u64)
+        } else {
+            bx.type_vector(i1, m_len as u64)
+        };
         let m_i1s = bx.trunc(args[0].immediate(), i1xn);
         return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
     }
@@ -2271,6 +2296,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             out_elem
         });
     }
+
     macro_rules! arith_binary {
         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
             $(if name == sym::$name {