Skip to content

Port the remaining SIMD intrinsics to const-eval #147520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
+443 −217
Open

Port the remaining SIMD intrinsics to const-eval #147520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9666c78
Implement `simd_fma` and `simd_relaxed_fma` in const-eval
sayantn Oct 9, 2025
487d6d8
Refactor implementation of float minmax intrinsics
sayantn Oct 9, 2025
6c410ba
Refactor float rounding intrinsics a bit
sayantn Oct 10, 2025
7c594eb
Add some missing f16-f128 support in intrinsics
sayantn Oct 10, 2025
389bf9a
Add Miri tests for f16/f128 SIMD operations
sayantn Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 120 additions & 97 deletions compiler/rustc_const_eval/src/interpret/intrinsics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,31 @@ use super::{
};
use crate::fluent_generated as fluent;

#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum MulAddType {
/// Used with `fma` and `simd_fma`, always uses fused-multiply-add
Fused,
/// Used with `fmuladd` and `simd_relaxed_fma`, nondeterministically determines whether to use
/// fma or simple multiply-add
Nondeterministic,
}

#[derive(Copy, Clone)]
pub(crate) enum MinMax {
/// The IEEE `Minimum` operation - see `f32::minimum` etc
/// In particular, `-0.0` is considered smaller than `+0.0`.
Minimum,
/// The IEEE `MinNum` operation - see `f32::min` etc
/// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic.
MinNum,
/// The IEEE `Maximum` operation - see `f32::maximum` etc
/// In particular, `-0.0` is considered smaller than `+0.0`.
Maximum,
/// The IEEE `MaxNum` operation - see `f32::max` etc
/// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic.
MaxNum,
}

/// Directly returns an `Allocation` containing an absolute path representation of the given type.
pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
let path = crate::util::type_name(tcx, ty);
Expand Down Expand Up @@ -486,25 +511,33 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
}

sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
sym::minnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::MinNum, dest)?,
sym::minnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::MinNum, dest)?,
sym::minnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::MinNum, dest)?,
sym::minnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::MinNum, dest)?,

sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
sym::minimumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Minimum, dest)?,
sym::minimumf32 => {
self.float_minmax_intrinsic::<Single>(args, MinMax::Minimum, dest)?
}
sym::minimumf64 => {
self.float_minmax_intrinsic::<Double>(args, MinMax::Minimum, dest)?
}
sym::minimumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Minimum, dest)?,

sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
sym::maxnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::MaxNum, dest)?,
sym::maxnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::MaxNum, dest)?,
sym::maxnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::MaxNum, dest)?,
sym::maxnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::MaxNum, dest)?,

sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
sym::maximumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Maximum, dest)?,
sym::maximumf32 => {
self.float_minmax_intrinsic::<Single>(args, MinMax::Maximum, dest)?
}
sym::maximumf64 => {
self.float_minmax_intrinsic::<Double>(args, MinMax::Maximum, dest)?
}
sym::maximumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Maximum, dest)?,

sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
Expand Down Expand Up @@ -612,14 +645,22 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
dest,
rustc_apfloat::Round::NearestTiesToEven,
)?,
sym::fmaf16 => self.fma_intrinsic::<Half>(args, dest)?,
sym::fmaf32 => self.fma_intrinsic::<Single>(args, dest)?,
sym::fmaf64 => self.fma_intrinsic::<Double>(args, dest)?,
sym::fmaf128 => self.fma_intrinsic::<Quad>(args, dest)?,
sym::fmuladdf16 => self.float_muladd_intrinsic::<Half>(args, dest)?,
sym::fmuladdf32 => self.float_muladd_intrinsic::<Single>(args, dest)?,
sym::fmuladdf64 => self.float_muladd_intrinsic::<Double>(args, dest)?,
sym::fmuladdf128 => self.float_muladd_intrinsic::<Quad>(args, dest)?,
sym::fmaf16 => self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Fused)?,
sym::fmaf32 => self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Fused)?,
sym::fmaf64 => self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Fused)?,
sym::fmaf128 => self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Fused)?,
sym::fmuladdf16 => {
self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf32 => {
self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf64 => {
self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf128 => {
self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Nondeterministic)?
}

// Unsupported intrinsic: skip the return_to_block below.
_ => return interp_ok(false),
Expand Down Expand Up @@ -901,76 +942,45 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
}

fn float_min_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = if a == b {
// They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
// Let the machine decide which one to return.
M::equal_float_min_max(self, a, b)
} else {
self.adjust_nan(a.min(b), &[a, b])
};
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_max_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
fn float_minmax<F>(
&self,
a: Scalar<M::Provenance>,
b: Scalar<M::Provenance>,
op: MinMax,
) -> InterpResult<'tcx, Scalar<M::Provenance>>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = if a == b {
let a: F = a.to_float()?;
let b: F = b.to_float()?;
let res = if matches!(op, MinMax::MinNum | MinMax::MaxNum) && a == b {
// They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
// Let the machine decide which one to return.
M::equal_float_min_max(self, a, b)
} else {
self.adjust_nan(a.max(b), &[a, b])
let result = match op {
MinMax::Minimum => a.minimum(b),
MinMax::MinNum => a.min(b),
MinMax::Maximum => a.maximum(b),
MinMax::MaxNum => a.max(b),
};
self.adjust_nan(result, &[a, b])
};
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_minimum_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = a.minimum(b);
let res = self.adjust_nan(res, &[a, b]);
self.write_scalar(res, dest)?;
interp_ok(())
interp_ok(res.into())
}

fn float_maximum_intrinsic<F>(
fn float_minmax_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
op: MinMax,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = a.maximum(b);
let res = self.adjust_nan(res, &[a, b]);
let res =
self.float_minmax::<F>(self.read_scalar(&args[0])?, self.read_scalar(&args[1])?, op)?;
self.write_scalar(res, dest)?;
interp_ok(())
}
Expand Down Expand Up @@ -1004,56 +1014,69 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
interp_ok(())
}

fn float_round_intrinsic<F>(
fn float_round<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
x: Scalar<M::Provenance>,
mode: rustc_apfloat::Round,
) -> InterpResult<'tcx, ()>
) -> InterpResult<'tcx, Scalar<M::Provenance>>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let x: F = self.read_scalar(&args[0])?.to_float()?;
let x: F = x.to_float()?;
let res = x.round_to_integral(mode).value;
let res = self.adjust_nan(res, &[x]);
self.write_scalar(res, dest)?;
interp_ok(())
interp_ok(res.into())
}

fn fma_intrinsic<F>(
fn float_round_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
mode: rustc_apfloat::Round,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let c: F = self.read_scalar(&args[2])?.to_float()?;

let res = a.mul_add(b, c).value;
let res = self.adjust_nan(res, &[a, b, c]);
let res = self.float_round::<F>(self.read_scalar(&args[0])?, mode)?;
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_muladd<F>(
&self,
a: Scalar<M::Provenance>,
b: Scalar<M::Provenance>,
c: Scalar<M::Provenance>,
typ: MulAddType,
) -> InterpResult<'tcx, Scalar<M::Provenance>>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = a.to_float()?;
let b: F = b.to_float()?;
let c: F = c.to_float()?;

let fuse = typ == MulAddType::Fused || M::float_fuse_mul_add(self);

let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
let res = self.adjust_nan(res, &[a, b, c]);
interp_ok(res.into())
}

fn float_muladd_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
typ: MulAddType,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let c: F = self.read_scalar(&args[2])?.to_float()?;

let fuse = M::float_fuse_mul_add(self);
let a = self.read_scalar(&args[0])?;
let b = self.read_scalar(&args[1])?;
let c = self.read_scalar(&args[2])?;

let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
let res = self.adjust_nan(res, &[a, b, c]);
let res = self.float_muladd::<F>(a, b, c, typ)?;
self.write_scalar(res, dest)?;
interp_ok(())
}
Expand Down
Loading
Loading