Many multi-value returns

cranelift-codegen/meta/src/isa/x86/encodings.rs

@@ -1420,27 +1420,24 @@ pub(crate) fn define(
     // or 1.
     //
     // Encode movzbq as movzbl, because it's equivalent and shorter.
-    e.enc32(
-        bint.bind(I32).bind(B1),
-        rec_urm_noflags_abcd.opcodes(&MOVZX_BYTE),
-    );
-
-    e.enc64(
-        bint.bind(I64).bind(B1),
-        rec_urm_noflags.opcodes(&MOVZX_BYTE).rex(),
-    );
-    e.enc64(
-        bint.bind(I64).bind(B1),
-        rec_urm_noflags_abcd.opcodes(&MOVZX_BYTE),
-    );
-    e.enc64(
-        bint.bind(I32).bind(B1),
-        rec_urm_noflags.opcodes(&MOVZX_BYTE).rex(),
-    );
-    e.enc64(
-        bint.bind(I32).bind(B1),
-        rec_urm_noflags_abcd.opcodes(&MOVZX_BYTE),
-    );
+    for &to in &[I8, I16, I32, I64] {
+        for &from in &[B1, B8] {
+            e.enc64(
+                bint.bind(to).bind(from),
+                rec_urm_noflags.opcodes(&MOVZX_BYTE).rex(),
+            );
+            e.enc64(
+                bint.bind(to).bind(from),
+                rec_urm_noflags_abcd.opcodes(&MOVZX_BYTE),
+            );
+            if to != I64 {
+                e.enc32(
+                    bint.bind(to).bind(from),
+                    rec_urm_noflags_abcd.opcodes(&MOVZX_BYTE),
+                );
+            }
+        }
+    }
 
     // Numerical conversions.
 

cranelift-codegen/src/abi.rs

@@ -4,6 +4,7 @@
 //! `TargetIsa::legalize_signature()` method.
 
 use crate::ir::{AbiParam, ArgumentExtension, ArgumentLoc, Type};
+use alloc::borrow::Cow;
 use alloc::vec::Vec;
 use core::cmp::Ordering;
 
@@ -86,7 +87,9 @@ pub trait ArgAssigner {
 /// Legalize the arguments in `args` using the given argument assigner.
 ///
 /// This function can be used for both arguments and return values.
-pub fn legalize_args<AA: ArgAssigner>(args: &mut Vec<AbiParam>, aa: &mut AA) {
+pub fn legalize_args<AA: ArgAssigner>(args: &[AbiParam], aa: &mut AA) -> Option<Vec<AbiParam>> {
+    let mut args = Cow::Borrowed(args);
+
     // Iterate over the arguments.
     // We may need to mutate the vector in place, so don't use a normal iterator, and clone the
     // argument to avoid holding a reference.
@@ -102,20 +105,25 @@ pub fn legalize_args<AA: ArgAssigner>(args: &mut Vec<AbiParam>, aa: &mut AA) {
         match aa.assign(&arg) {
             // Assign argument to a location and move on to the next one.
             ArgAction::Assign(loc) => {
-                args[argno].location = loc;
+                args.to_mut()[argno].location = loc;
                 argno += 1;
             }
             // Split this argument into two smaller ones. Then revisit both.
             ArgAction::Convert(conv) => {
                 let value_type = conv.apply(arg.value_type);
                 let new_arg = AbiParam { value_type, ..arg };
-                args[argno].value_type = value_type;
+                args.to_mut()[argno].value_type = value_type;
                 if conv.is_split() {
-                    args.insert(argno + 1, new_arg);
+                    args.to_mut().insert(argno + 1, new_arg);
                 }
             }
         }
     }
+
+    match args {
+        Cow::Borrowed(_) => None,
+        Cow::Owned(a) => Some(a),
+    }
 }
 
 /// Determine the right action to take when passing a `have` value type to a call signature where

cranelift-codegen/src/ir/dfg.rs

@@ -62,6 +62,9 @@ pub struct DataFlowGraph {
     /// well as the external function references.
     pub signatures: PrimaryMap<SigRef, Signature>,
 
+    /// The pre-legalization signature for each entry in `signatures`, if any.
+    pub old_signatures: SecondaryMap<SigRef, Option<Signature>>,
+
     /// External function references. These are functions that can be called directly.
     pub ext_funcs: PrimaryMap<FuncRef, ExtFuncData>,
 
@@ -85,6 +88,7 @@ impl DataFlowGraph {
             value_lists: ValueListPool::new(),
             values: PrimaryMap::new(),
             signatures: PrimaryMap::new(),
+            old_signatures: SecondaryMap::new(),
             ext_funcs: PrimaryMap::new(),
             values_labels: None,
             constants: ConstantPool::new(),

cranelift-codegen/src/ir/extfunc.rs

@@ -55,6 +55,53 @@ impl Signature {
     pub fn special_param_index(&self, purpose: ArgumentPurpose) -> Option<usize> {
         self.params.iter().rposition(|arg| arg.purpose == purpose)
     }
+
+    /// Find the index of a presumed unique special-purpose parameter.
+    pub fn special_return_index(&self, purpose: ArgumentPurpose) -> Option<usize> {
+        self.returns.iter().rposition(|arg| arg.purpose == purpose)
+    }
+
+    /// Does this signature have a parameter whose `ArgumentPurpose` is
+    /// `purpose`?
+    pub fn uses_special_param(&self, purpose: ArgumentPurpose) -> bool {
+        self.special_param_index(purpose).is_some()
+    }
+
+    /// Does this signature have a return whose `ArgumentPurpose` is `purpose`?
+    pub fn uses_special_return(&self, purpose: ArgumentPurpose) -> bool {
+        self.special_return_index(purpose).is_some()
+    }
+
+    /// How many special parameters does this function have?
+    pub fn num_special_params(&self) -> usize {
+        self.params
+            .iter()
+            .filter(|p| p.purpose != ArgumentPurpose::Normal)
+            .count()
+    }
+
+    /// How many special returns does this function have?
+    pub fn num_special_returns(&self) -> usize {
+        self.returns
+            .iter()
+            .filter(|r| r.purpose != ArgumentPurpose::Normal)
+            .count()
+    }
+
+    /// Does this signature take an struct return pointer parameter?
+    pub fn uses_struct_return_param(&self) -> bool {
+        self.uses_special_param(ArgumentPurpose::StructReturn)
+    }
+
+    /// Does this return more than one normal value? (Pre-struct return
+    /// legalization)
+    pub fn is_multi_return(&self) -> bool {
+        self.returns
+            .iter()
+            .filter(|r| r.purpose == ArgumentPurpose::Normal)
+            .count()
+            > 1
+    }
 }
 
 /// Wrapper type capable of displaying a `Signature` with correct register names.

cranelift-codegen/src/ir/function.rs

@@ -34,6 +34,10 @@ pub struct Function {
     /// Signature of this function.
     pub signature: Signature,
 
+    /// The old signature of this function, before the most recent legalization,
+    /// if any.
+    pub old_signature: Option<Signature>,
+
     /// Stack slots allocated in this function.
     pub stack_slots: StackSlots,
 
@@ -96,6 +100,7 @@ impl Function {
         Self {
             name,
             signature: sig,
+            old_signature: None,
             stack_slots: StackSlots::new(),
             global_values: PrimaryMap::new(),
             heaps: PrimaryMap::new(),

cranelift-codegen/src/ir/stackslot.rs

@@ -64,6 +64,15 @@ pub enum StackSlotKind {
     /// stack slots are only valid while setting up a call.
     OutgoingArg,
 
+    /// Space allocated in the caller's frame for the callee's return values
+    /// that are passed out via return pointer.
+    ///
+    /// If there are more return values than registers available for the callee's calling
+    /// convention, or the return value is larger than the available registers' space, then we
+    /// allocate stack space in this frame and pass a pointer to the callee, which then writes its
+    /// return values into this space.
+    StructReturnSlot,
+
     /// An emergency spill slot.
     ///
     /// Emergency slots are allocated late when the register's constraint solver needs extra space
@@ -81,6 +90,7 @@ impl FromStr for StackSlotKind {
             "spill_slot" => Ok(SpillSlot),
             "incoming_arg" => Ok(IncomingArg),
             "outgoing_arg" => Ok(OutgoingArg),
+            "sret_slot" => Ok(StructReturnSlot),
             "emergency_slot" => Ok(EmergencySlot),
             _ => Err(()),
         }
@@ -95,6 +105,7 @@ impl fmt::Display for StackSlotKind {
             SpillSlot => "spill_slot",
             IncomingArg => "incoming_arg",
             OutgoingArg => "outgoing_arg",
+            StructReturnSlot => "sret_slot",
             EmergencySlot => "emergency_slot",
         })
     }

cranelift-codegen/src/isa/arm32/abi.rs

@@ -6,6 +6,7 @@ use crate::abi::{legalize_args, ArgAction, ArgAssigner, ValueConversion};
 use crate::ir::{self, AbiParam, ArgumentExtension, ArgumentLoc, Type};
 use crate::isa::RegClass;
 use crate::regalloc::RegisterSet;
+use alloc::borrow::Cow;
 use core::i32;
 use target_lexicon::Triple;
 
@@ -78,11 +79,13 @@ impl ArgAssigner for Args {
 }
 
 /// Legalize `sig`.
-pub fn legalize_signature(sig: &mut ir::Signature, triple: &Triple, _current: bool) {
+pub fn legalize_signature(sig: &mut Cow<ir::Signature>, triple: &Triple, _current: bool) {
     let bits = triple.pointer_width().unwrap().bits();
 
     let mut args = Args::new(bits);
-    legalize_args(&mut sig.params, &mut args);
+    if let Some(new_params) = legalize_args(&sig.params, &mut args) {
+        sig.to_mut().params = new_params;
+    }
 }
 
 /// Get register class for a type appearing in a legalized signature.

cranelift-codegen/src/isa/arm32/mod.rs

@@ -15,6 +15,7 @@ use crate::isa::enc_tables::{self as shared_enc_tables, lookup_enclist, Encoding
 use crate::isa::Builder as IsaBuilder;
 use crate::isa::{EncInfo, RegClass, RegInfo, TargetIsa};
 use crate::regalloc;
+use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use core::fmt;
 use target_lexicon::{Architecture, Triple};
@@ -100,7 +101,7 @@ impl TargetIsa for Isa {
         )
     }
 
-    fn legalize_signature(&self, sig: &mut ir::Signature, current: bool) {
+    fn legalize_signature(&self, sig: &mut Cow<ir::Signature>, current: bool) {
         abi::legalize_signature(sig, &self.triple, current)
     }
 

cranelift-codegen/src/isa/arm64/abi.rs

@@ -5,10 +5,11 @@ use crate::ir;
 use crate::isa::RegClass;
 use crate::regalloc::RegisterSet;
 use crate::settings as shared_settings;
+use alloc::borrow::Cow;
 
 /// Legalize `sig`.
 pub fn legalize_signature(
-    _sig: &mut ir::Signature,
+    _sig: &mut Cow<ir::Signature>,
     _flags: &shared_settings::Flags,
     _current: bool,
 ) {

cranelift-codegen/src/isa/arm64/mod.rs

@@ -15,6 +15,7 @@ use crate::isa::enc_tables::{lookup_enclist, Encodings};
 use crate::isa::Builder as IsaBuilder;
 use crate::isa::{EncInfo, RegClass, RegInfo, TargetIsa};
 use crate::regalloc;
+use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use core::fmt;
 use target_lexicon::Triple;
@@ -88,7 +89,7 @@ impl TargetIsa for Isa {
         )
     }
 
-    fn legalize_signature(&self, sig: &mut ir::Signature, current: bool) {
+    fn legalize_signature(&self, sig: &mut Cow<ir::Signature>, current: bool) {
         abi::legalize_signature(sig, &self.shared_flags, current)
     }
 

cranelift-codegen/src/isa/mod.rs

@@ -63,6 +63,7 @@ use crate::result::CodegenResult;
 use crate::settings;
 use crate::settings::SetResult;
 use crate::timing;
+use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use core::fmt;
@@ -315,7 +316,7 @@ pub trait TargetIsa: fmt::Display + Sync {
     /// Arguments and return values for the caller's frame pointer and other callee-saved registers
     /// should not be added by this function. These arguments are not added until after register
     /// allocation.
-    fn legalize_signature(&self, sig: &mut ir::Signature, current: bool);
+    fn legalize_signature(&self, sig: &mut Cow<ir::Signature>, current: bool);
 
     /// Get the register class that should be used to represent an ABI argument or return value of
     /// type `ty`. This should be the top-level register class that contains the argument

cranelift-codegen/src/isa/riscv/abi.rs

@@ -11,6 +11,7 @@ use crate::abi::{legalize_args, ArgAction, ArgAssigner, ValueConversion};
 use crate::ir::{self, AbiParam, ArgumentExtension, ArgumentLoc, ArgumentPurpose, Type};
 use crate::isa::RegClass;
 use crate::regalloc::RegisterSet;
+use alloc::borrow::Cow;
 use core::i32;
 use target_lexicon::Triple;
 
@@ -88,18 +89,22 @@ impl ArgAssigner for Args {
 
 /// Legalize `sig` for RISC-V.
 pub fn legalize_signature(
-    sig: &mut ir::Signature,
+    sig: &mut Cow<ir::Signature>,
     triple: &Triple,
     isa_flags: &settings::Flags,
     current: bool,
 ) {
     let bits = triple.pointer_width().unwrap().bits();
 
     let mut args = Args::new(bits, isa_flags.enable_e());
-    legalize_args(&mut sig.params, &mut args);
+    if let Some(new_params) = legalize_args(&sig.params, &mut args) {
+        sig.to_mut().params = new_params;
+    }
 
     let mut rets = Args::new(bits, isa_flags.enable_e());
-    legalize_args(&mut sig.returns, &mut rets);
+    if let Some(new_returns) = legalize_args(&sig.returns, &mut rets) {
+        sig.to_mut().returns = new_returns;
+    }
 
     if current {
         let ptr = Type::int(u16::from(bits)).unwrap();
@@ -110,8 +115,8 @@ pub fn legalize_signature(
         // in any register, but a micro-architecture with a return address predictor will only
         // recognize it as a return if the address is in `x1`.
         let link = AbiParam::special_reg(ptr, ArgumentPurpose::Link, GPR.unit(1));
-        sig.params.push(link);
-        sig.returns.push(link);
+        sig.to_mut().params.push(link);
+        sig.to_mut().returns.push(link);
     }
 }
 

cranelift-codegen/src/isa/riscv/mod.rs

@@ -15,6 +15,7 @@ use crate::isa::enc_tables::{self as shared_enc_tables, lookup_enclist, Encoding
 use crate::isa::Builder as IsaBuilder;
 use crate::isa::{EncInfo, RegClass, RegInfo, TargetIsa};
 use crate::regalloc;
+use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use core::fmt;
 use target_lexicon::{PointerWidth, Triple};
@@ -95,7 +96,7 @@ impl TargetIsa for Isa {
         )
     }
 
-    fn legalize_signature(&self, sig: &mut ir::Signature, current: bool) {
+    fn legalize_signature(&self, sig: &mut Cow<ir::Signature>, current: bool) {
         abi::legalize_signature(sig, &self.triple, &self.isa_flags, current)
     }