I128

cranelift-codegen/meta/src/cdsl/ast.rs

@@ -8,6 +8,7 @@ use cranelift_entity::{entity_impl, PrimaryMap};
 
 use std::fmt;
 
+#[derive(Debug)]
 pub enum Expr {
     Var(VarIndex),
     Literal(Literal),
@@ -363,6 +364,7 @@ impl VarPool {
 ///
 /// An `Apply` AST expression is created by using function call syntax on instructions. This
 /// applies to both bound and unbound polymorphic instructions.
+#[derive(Debug)]
 pub struct Apply {
     pub inst: Instruction,
     pub args: Vec<Expr>,

cranelift-codegen/meta/src/cdsl/instructions.rs

@@ -14,7 +14,7 @@ use crate::cdsl::type_inference::Constraint;
 use crate::cdsl::types::{LaneType, ReferenceType, ValueType, VectorType};
 use crate::cdsl::typevar::TypeVar;
 
-#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
 pub struct OpcodeNumber(u32);
 entity_impl!(OpcodeNumber);
 
@@ -79,12 +79,14 @@ impl InstructionGroup {
     }
 }
 
+#[derive(Debug)]
 pub struct PolymorphicInfo {
     pub use_typevar_operand: bool,
     pub ctrl_typevar: TypeVar,
     pub other_typevars: Vec<TypeVar>,
 }
 
+#[derive(Debug)]
 pub struct InstructionContent {
     /// Instruction mnemonic, also becomes opcode name.
     pub name: String,
@@ -139,7 +141,7 @@ pub struct InstructionContent {
     pub writes_cpu_flags: bool,
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct Instruction {
     content: Rc<InstructionContent>,
 }
@@ -1125,6 +1127,11 @@ fn bind_vector(
     mut value_types: Vec<ValueTypeOrAny>,
 ) -> BoundInstruction {
     let num_lanes = vector_size_in_bits / lane_type.lane_bits();
+    assert!(
+        num_lanes >= 2,
+        "Minimum lane number for bind_vector is 2, found {}.",
+        num_lanes,
+    );
     let vector_type = ValueType::Vector(VectorType::new(lane_type, num_lanes));
     value_types.push(ValueTypeOrAny::ValueType(vector_type));
     verify_polymorphic_binding(&inst, &value_types);

cranelift-codegen/meta/src/cdsl/type_inference.rs

@@ -4,7 +4,7 @@ use crate::cdsl::typevar::{DerivedFunc, TypeSet, TypeVar};
 use std::collections::{HashMap, HashSet};
 use std::iter::FromIterator;
 
-#[derive(Hash, PartialEq, Eq)]
+#[derive(Debug, Hash, PartialEq, Eq)]
 pub enum Constraint {
     /// Constraint specifying that a type var tv1 must be wider than or equal to type var tv2 at
     /// runtime. This requires that:

cranelift-codegen/meta/src/cdsl/types.rs

@@ -215,12 +215,14 @@ impl LaneType {
                 LaneType::BoolType(shared_types::Bool::B16) => 2,
                 LaneType::BoolType(shared_types::Bool::B32) => 3,
                 LaneType::BoolType(shared_types::Bool::B64) => 4,
-                LaneType::IntType(shared_types::Int::I8) => 5,
-                LaneType::IntType(shared_types::Int::I16) => 6,
-                LaneType::IntType(shared_types::Int::I32) => 7,
-                LaneType::IntType(shared_types::Int::I64) => 8,
-                LaneType::FloatType(shared_types::Float::F32) => 9,
-                LaneType::FloatType(shared_types::Float::F64) => 10,
+                LaneType::BoolType(shared_types::Bool::B128) => 5,
+                LaneType::IntType(shared_types::Int::I8) => 6,
+                LaneType::IntType(shared_types::Int::I16) => 7,
+                LaneType::IntType(shared_types::Int::I32) => 8,
+                LaneType::IntType(shared_types::Int::I64) => 9,
+                LaneType::IntType(shared_types::Int::I128) => 10,
+                LaneType::FloatType(shared_types::Float::F32) => 11,
+                LaneType::FloatType(shared_types::Float::F64) => 12,
             }
     }
 
@@ -231,6 +233,7 @@ impl LaneType {
             16 => shared_types::Bool::B16,
             32 => shared_types::Bool::B32,
             64 => shared_types::Bool::B64,
+            128 => shared_types::Bool::B128,
             _ => unreachable!("unxpected num bits for bool"),
         })
     }
@@ -241,6 +244,7 @@ impl LaneType {
             16 => shared_types::Int::I16,
             32 => shared_types::Int::I32,
             64 => shared_types::Int::I64,
+            128 => shared_types::Int::I128,
             _ => unreachable!("unxpected num bits for int"),
         })
     }

cranelift-codegen/meta/src/cdsl/typevar.rs

@@ -9,7 +9,8 @@ use std::rc::Rc;
 use crate::cdsl::types::{BVType, LaneType, ReferenceType, SpecialType, ValueType};
 
 const MAX_LANES: u16 = 256;
-const MAX_BITS: u16 = 64;
+const MAX_BITS: u16 = 128;
+const MAX_FLOAT_BITS: u16 = 64;
 const MAX_BITVEC: u16 = MAX_BITS * MAX_LANES;
 
 /// Type variables can be used in place of concrete types when defining
@@ -177,7 +178,7 @@ impl TypeVar {
                     "can't double all integer types"
                 );
                 assert!(
-                    ts.floats.len() == 0 || *ts.floats.iter().max().unwrap() < MAX_BITS,
+                    ts.floats.len() == 0 || *ts.floats.iter().max().unwrap() < MAX_FLOAT_BITS,
                     "can't double all float types"
                 );
                 assert!(
@@ -503,7 +504,7 @@ impl TypeSet {
         copy.floats = NumSet::from_iter(
             self.floats
                 .iter()
-                .filter(|&&x| x < MAX_BITS)
+                .filter(|&&x| x < MAX_FLOAT_BITS)
                 .map(|&x| x * 2),
         );
         copy.bools = NumSet::from_iter(
@@ -621,7 +622,7 @@ impl TypeSet {
                 let mut copy = self.clone();
                 copy.bitvecs = NumSet::new();
                 if self.bools.contains(&1) {
-                    copy.ints = NumSet::from_iter(vec![8, 16, 32, 64]);
+                    copy.ints = NumSet::from_iter(vec![8, 16, 32, 64, 128]);
                     copy.floats = NumSet::from_iter(vec![32, 64]);
                 } else {
                     copy.ints = &self.bools - &NumSet::from_iter(vec![1]);
@@ -950,7 +951,7 @@ fn test_typevar_builder() {
     let type_set = TypeSetBuilder::new().ints(Interval::All).build();
     assert_eq!(type_set.lanes, num_set![1]);
     assert!(type_set.floats.is_empty());
-    assert_eq!(type_set.ints, num_set![8, 16, 32, 64]);
+    assert_eq!(type_set.ints, num_set![8, 16, 32, 64, 128]);
     assert!(type_set.bools.is_empty());
     assert!(type_set.bitvecs.is_empty());
     assert!(type_set.specials.is_empty());
@@ -959,7 +960,7 @@ fn test_typevar_builder() {
     assert_eq!(type_set.lanes, num_set![1]);
     assert!(type_set.floats.is_empty());
     assert!(type_set.ints.is_empty());
-    assert_eq!(type_set.bools, num_set![1, 8, 16, 32, 64]);
+    assert_eq!(type_set.bools, num_set![1, 8, 16, 32, 64, 128]);
     assert!(type_set.bitvecs.is_empty());
     assert!(type_set.specials.is_empty());
 
@@ -1101,7 +1102,7 @@ fn test_forward_images() {
     );
     assert_eq!(
         TypeSetBuilder::new().ints(32..64).build().double_width(),
-        TypeSetBuilder::new().ints(64..64).build()
+        TypeSetBuilder::new().ints(64..128).build()
     );
     assert_eq!(
         TypeSetBuilder::new().floats(32..32).build().double_width(),
@@ -1117,7 +1118,7 @@ fn test_forward_images() {
     );
     assert_eq!(
         TypeSetBuilder::new().bools(32..64).build().double_width(),
-        TypeSetBuilder::new().bools(64..64).build()
+        TypeSetBuilder::new().bools(64..128).build()
     );
 }
 
@@ -1145,7 +1146,7 @@ fn test_backward_images() {
     assert_eq!(
         TypeSetBuilder::new()
             .simd_lanes(1..4)
-            .bools(1..64)
+            .bools(1..128)
             .build()
             .preimage(DerivedFunc::AsBool),
         TypeSetBuilder::new()
@@ -1205,9 +1206,9 @@ fn test_backward_images() {
     // Half width.
     assert_eq!(
         TypeSetBuilder::new()
-            .ints(64..64)
+            .ints(128..128)
             .floats(64..64)
-            .bools(64..64)
+            .bools(128..128)
             .build()
             .preimage(DerivedFunc::HalfWidth)
             .size(),
@@ -1221,7 +1222,7 @@ fn test_backward_images() {
             .preimage(DerivedFunc::HalfWidth),
         TypeSetBuilder::new()
             .simd_lanes(64..256)
-            .bools(16..64)
+            .bools(16..128)
             .build(),
     );
 

cranelift-codegen/meta/src/cdsl/xform.rs

@@ -183,7 +183,14 @@ fn rewrite_expr(
     assert_eq!(
         apply_target.inst().operands_in.len(),
         dummy_args.len(),
-        "number of arguments in instruction is incorrect"
+        "number of arguments in instruction {} is incorrect\nexpected: {:?}",
+        apply_target.inst().name,
+        apply_target
+            .inst()
+            .operands_in
+            .iter()
+            .map(|operand| format!("{}: {}", operand.name, operand.kind.name))
+            .collect::<Vec<_>>(),
     );
 
     let mut args = Vec::new();

cranelift-codegen/meta/src/gen_legalizer.rs

@@ -61,10 +61,10 @@ fn unwrap_inst(
             fmtln!(fmt, "{},", field.member);
         }
 
-        if iform.num_value_operands == 1 {
-            fmt.line("arg,");
-        } else if iform.has_value_list || iform.num_value_operands > 1 {
+        if iform.has_value_list || iform.num_value_operands > 1 {
             fmt.line("ref args,");
+        } else if iform.num_value_operands == 1 {
+            fmt.line("arg,");
         }
 
         fmt.line("..");
@@ -87,6 +87,13 @@ fn unwrap_inst(
                 } else if op.is_value() {
                     let n = inst.value_opnums.iter().position(|&i| i == op_num).unwrap();
                     fmtln!(fmt, "func.dfg.resolve_aliases(args[{}]),", n);
+                } else if op.is_varargs() {
+                    let n = inst.imm_opnums.iter().chain(inst.value_opnums.iter()).max().map(|n| n + 1).unwrap_or(0);
+                    // We need to create a `Vec` here, as using a slice would result in a borrowck
+                    // error later on.
+                    fmtln!(fmt, "\
+                        args.iter().skip({}).map(|&arg| func.dfg.resolve_aliases(arg)).collect::<Vec<_>>(),\
+                    ", n);
                 }
             }
 
@@ -104,6 +111,19 @@ fn unwrap_inst(
     });
     fmtln!(fmt, "};");
 
+    assert_eq!(inst.operands_in.len(), apply.args.len());
+    for (i, op) in inst.operands_in.iter().enumerate() {
+        if op.is_varargs() {
+            let name = var_pool
+                .get(apply.args[i].maybe_var().expect("vararg without name"))
+                .name;
+
+            // Above name is set to an `Vec` representing the varargs. However it is expected to be
+            // `&[Value]` below, so we borrow it.
+            fmtln!(fmt, "let {} = &{};", name, name);
+        }
+    }
+
     for &op_num in &inst.value_opnums {
         let arg = &apply.args[op_num];
         if let Some(var_index) = arg.maybe_var() {
@@ -402,6 +422,13 @@ fn gen_transform<'a>(
             fmt.line("let removed = pos.remove_inst();");
             fmt.line("debug_assert_eq!(removed, inst);");
         }
+
+        if transform.def_pool.get(transform.src).apply.inst.is_branch {
+            // A branch might have been legalized into multiple branches, so we need to recompute
+            // the cfg.
+            fmt.line("cfg.recompute_ebb(pos.func, pos.current_ebb().unwrap());");
+        }
+
         fmt.line("return true;");
     });
     fmt.line("}");

cranelift-codegen/meta/src/isa/x86/encodings.rs

@@ -9,7 +9,7 @@ use crate::cdsl::instructions::{
 };
 use crate::cdsl::recipes::{EncodingRecipe, EncodingRecipeNumber, Recipes};
 use crate::cdsl::settings::{SettingGroup, SettingPredicateNumber};
-use crate::cdsl::types::ValueType;
+use crate::cdsl::types::{LaneType, ValueType};
 use crate::shared::types::Bool::{B1, B16, B32, B64, B8};
 use crate::shared::types::Float::{F32, F64};
 use crate::shared::types::Int::{I16, I32, I64, I8};
@@ -1735,6 +1735,8 @@ pub(crate) fn define(
     // legalize.rs for how this is done; once there, x86_pshuf* (below) is used for broadcasting the
     // value across the register
 
+    let allowed_simd_type = |t: &LaneType| t.lane_bits() >= 8 && t.lane_bits() < 128;
+
     // PSHUFB, 8-bit shuffle using two XMM registers
     for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 8) {
         let instruction = x86_pshufb.bind_vector_from_lane(ty, sse_vector_size);
@@ -1756,7 +1758,7 @@ pub(crate) fn define(
     // SIMD scalar_to_vector; this uses MOV to copy the scalar value to an XMM register; according
     // to the Intel manual: "When the destination operand is an XMM register, the source operand is
     // written to the low doubleword of the register and the regiser is zero-extended to 128 bits."
-    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8) {
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
         let instruction = scalar_to_vector.bind_vector_from_lane(ty, sse_vector_size);
         let template = rec_frurm.opcodes(vec![0x66, 0x0f, 0x6e]); // MOVD/MOVQ
         if ty.lane_bits() < 64 {
@@ -1774,7 +1776,7 @@ pub(crate) fn define(
     insertlane_mapping.insert(32, (vec![0x66, 0x0f, 0x3a, 0x22], Some(use_sse41_simd))); // PINSRD
     insertlane_mapping.insert(64, (vec![0x66, 0x0f, 0x3a, 0x22], Some(use_sse41_simd))); // PINSRQ, only x86_64
 
-    for ty in ValueType::all_lane_types() {
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
         if let Some((opcode, isap)) = insertlane_mapping.get(&ty.lane_bits()) {
             let instruction = insertlane.bind_vector_from_lane(ty, sse_vector_size);
             let template = rec_r_ib_unsigned_r.opcodes(opcode.clone());
@@ -1795,7 +1797,7 @@ pub(crate) fn define(
     extractlane_mapping.insert(32, (vec![0x66, 0x0f, 0x3a, 0x16], Some(use_sse41_simd))); // PEXTRD
     extractlane_mapping.insert(64, (vec![0x66, 0x0f, 0x3a, 0x16], Some(use_sse41_simd))); // PEXTRQ, only x86_64
 
-    for ty in ValueType::all_lane_types() {
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
         if let Some((opcode, isap)) = extractlane_mapping.get(&ty.lane_bits()) {
             let instruction = extractlane.bind_vector_from_lane(ty, sse_vector_size);
             let template = rec_r_ib_unsigned_gpr.opcodes(opcode.clone());
@@ -1816,8 +1818,9 @@ pub(crate) fn define(
     }
 
     // SIMD bitcast all 128-bit vectors to each other (for legalizing splat.x16x8)
-    for from_type in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8) {
-        for to_type in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8 && *t != from_type)
+    for from_type in ValueType::all_lane_types().filter(allowed_simd_type) {
+        for to_type in
+            ValueType::all_lane_types().filter(|t| allowed_simd_type(t) && *t != from_type)
         {
             let instruction = raw_bitcast
                 .bind_vector_from_lane(to_type, sse_vector_size)
@@ -1833,7 +1836,7 @@ pub(crate) fn define(
     // for that; alternately, constants could be loaded into XMM registers using a sequence like:
     // MOVQ + MOVHPD + MOVQ + MOVLPD (this allows the constants to be immediates instead of stored
     // in memory) but some performance measurements are needed.
-    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() >= 8) {
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
         let instruction = vconst.bind_vector_from_lane(ty, sse_vector_size);
         let template = rec_vconst.nonrex().opcodes(vec![0x0f, 0x10]);
         e.enc_32_64_maybe_isap(instruction, template, None); // from SSE

cranelift-codegen/meta/src/shared/instructions.rs

@@ -3143,7 +3143,7 @@ pub(crate) fn define(
         "WideInt",
         "An integer type with lanes from `i16` upwards",
         TypeSetBuilder::new()
-            .ints(16..64)
+            .ints(16..128)
             .simd_lanes(Interval::All)
             .build(),
     );
@@ -3171,9 +3171,9 @@ pub(crate) fn define(
 
     let NarrowInt = &TypeVar::new(
         "NarrowInt",
-        "An integer type with lanes type to `i32`",
+        "An integer type with lanes type to `i64`",
         TypeSetBuilder::new()
-            .ints(8..32)
+            .ints(8..64)
             .simd_lanes(Interval::All)
             .build(),
     );