Skip to content

Make X86 intrinsics const #1925

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 19 commits into
base: master
Choose a base branch
from
+6,855 −3,136
Draft

Make X86 intrinsics const #1925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
b9bd19c
Make some helpers `const`
sayantn Sep 14, 2025
3546cc9
Make `abm`, `bmi1`, `bmi2`, `bswap` and `tbm` functions const
sayantn Sep 23, 2025
72fdf42
Make `sse` functions const
sayantn Sep 23, 2025
52fb781
Make `sse2` functions const
sayantn Sep 23, 2025
09ae5b9
Make `sse3` functions const
sayantn Sep 23, 2025
0aba00b
Make `ssse3` functions const
sayantn Sep 23, 2025
4070bc1
Make `sse4.1` functions const
sayantn Sep 23, 2025
a5e94d8
Make `sse4.2` functions const
sayantn Sep 23, 2025
9a4f071
Make `avx` functions const
sayantn Sep 23, 2025
95fc2cc
Make `avx2` functions const
sayantn Sep 23, 2025
654546c
Make `fma` functions const
sayantn Oct 1, 2025
e525d0c
Make `avx512f` functions const
sayantn Sep 23, 2025
a5a6a6d
Make `avx512bw` functions const
sayantn Sep 23, 2025
6b05478
Make `avx512cd` functions const
sayantn Sep 23, 2025
e76d41f
Make `avx512dq` functions const
sayantn Sep 23, 2025
58df278
Make `avx512vbmi2` functions const
sayantn Sep 23, 2025
628a609
Make `avx512bitalg` functions const
sayantn Sep 23, 2025
6d5c57c
Make `avx512vpopcntdq` functions const
sayantn Sep 23, 2025
220fc5a
Make `avx512fp16` functions const
sayantn Sep 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion crates/core_arch/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
f16,
aarch64_unstable_target_feature,
bigint_helper_methods,
funnel_shifts
funnel_shifts,
const_trait_impl,
const_cmp,
const_convert
)]
#![cfg_attr(test, feature(test, abi_vectorcall, stdarch_internal))]
#![deny(clippy::missing_inline_in_public_items)]
Expand Down
12 changes: 7 additions & 5 deletions crates/core_arch/src/simd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ macro_rules! simd_ty {
}
// FIXME: Workaround rust@60637
#[inline(always)]
pub(crate) fn splat(value: $elem_type) -> Self {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub(crate) const fn splat(value: $elem_type) -> Self {
#[derive(Copy, Clone)]
#[repr(simd)]
struct JustOne([$elem_type; 1]);
Expand All @@ -38,12 +39,12 @@ macro_rules! simd_ty {
/// Use for testing only.
// FIXME: Workaround rust@60637
#[inline(always)]
pub(crate) fn extract(&self, index: usize) -> $elem_type {
pub(crate) const fn extract(&self, index: usize) -> $elem_type {
self.as_array()[index]
}

#[inline]
pub(crate) fn as_array(&self) -> &[$elem_type; $len] {
pub(crate) const fn as_array(&self) -> &[$elem_type; $len] {
let simd_ptr: *const Self = self;
let array_ptr: *const [$elem_type; $len] = simd_ptr.cast();
// SAFETY: We can always read the prefix of a simd type as an array.
Expand Down Expand Up @@ -89,7 +90,8 @@ macro_rules! simd_m_ty {

// FIXME: Workaround rust@60637
#[inline(always)]
pub(crate) fn splat(value: bool) -> Self {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub(crate) const fn splat(value: bool) -> Self {
#[derive(Copy, Clone)]
#[repr(simd)]
struct JustOne([$elem_type; 1]);
Expand All @@ -100,7 +102,7 @@ macro_rules! simd_m_ty {
}

#[inline]
pub(crate) fn as_array(&self) -> &[$elem_type; $len] {
pub(crate) const fn as_array(&self) -> &[$elem_type; $len] {
let simd_ptr: *const Self = self;
let array_ptr: *const [$elem_type; $len] = simd_ptr.cast();
// SAFETY: We can always read the prefix of a simd type as an array.
Expand Down
6 changes: 4 additions & 2 deletions crates/core_arch/src/x86/abm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ use stdarch_test::assert_instr;
#[target_feature(enable = "lzcnt")]
#[cfg_attr(test, assert_instr(lzcnt))]
#[stable(feature = "simd_x86", since = "1.27.0")]
pub fn _lzcnt_u32(x: u32) -> u32 {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _lzcnt_u32(x: u32) -> u32 {
x.leading_zeros()
}

Expand All @@ -40,7 +41,8 @@ pub fn _lzcnt_u32(x: u32) -> u32 {
#[target_feature(enable = "popcnt")]
#[cfg_attr(test, assert_instr(popcnt))]
#[stable(feature = "simd_x86", since = "1.27.0")]
pub fn _popcnt32(x: i32) -> i32 {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _popcnt32(x: i32) -> i32 {
x.count_ones() as i32
}

Expand Down
435 changes: 296 additions & 139 deletions crates/core_arch/src/x86/avx.rs
View file
Open in desktop

Large diffs are not rendered by default.

354 changes: 236 additions & 118 deletions crates/core_arch/src/x86/avx2.rs
View file
Open in desktop

Large diffs are not rendered by default.

54 changes: 36 additions & 18 deletions crates/core_arch/src/x86/avx512bitalg.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ unsafe extern "C" {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm512_popcnt_epi16(a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_popcnt_epi16(a: __m512i) -> __m512i {
unsafe { transmute(simd_ctpop(a.as_i16x32())) }
}

Expand All @@ -57,7 +58,8 @@ pub fn _mm512_popcnt_epi16(a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm512_maskz_popcnt_epi16(k: __mmask32, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_maskz_popcnt_epi16(k: __mmask32, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -77,7 +79,8 @@ pub fn _mm512_maskz_popcnt_epi16(k: __mmask32, a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm512_mask_popcnt_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_mask_popcnt_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -94,7 +97,8 @@ pub fn _mm512_mask_popcnt_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m51
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm256_popcnt_epi16(a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_popcnt_epi16(a: __m256i) -> __m256i {
unsafe { transmute(simd_ctpop(a.as_i16x16())) }
}

Expand All @@ -108,7 +112,8 @@ pub fn _mm256_popcnt_epi16(a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm256_maskz_popcnt_epi16(k: __mmask16, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_maskz_popcnt_epi16(k: __mmask16, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -128,7 +133,8 @@ pub fn _mm256_maskz_popcnt_epi16(k: __mmask16, a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm256_mask_popcnt_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_mask_popcnt_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -145,7 +151,8 @@ pub fn _mm256_mask_popcnt_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m25
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm_popcnt_epi16(a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_popcnt_epi16(a: __m128i) -> __m128i {
unsafe { transmute(simd_ctpop(a.as_i16x8())) }
}

Expand All @@ -159,7 +166,8 @@ pub fn _mm_popcnt_epi16(a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm_maskz_popcnt_epi16(k: __mmask8, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_maskz_popcnt_epi16(k: __mmask8, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -179,7 +187,8 @@ pub fn _mm_maskz_popcnt_epi16(k: __mmask8, a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm_mask_popcnt_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_mask_popcnt_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -196,7 +205,8 @@ pub fn _mm_mask_popcnt_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm512_popcnt_epi8(a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_popcnt_epi8(a: __m512i) -> __m512i {
unsafe { transmute(simd_ctpop(a.as_i8x64())) }
}

Expand All @@ -210,7 +220,8 @@ pub fn _mm512_popcnt_epi8(a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm512_maskz_popcnt_epi8(k: __mmask64, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_maskz_popcnt_epi8(k: __mmask64, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -230,7 +241,8 @@ pub fn _mm512_maskz_popcnt_epi8(k: __mmask64, a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm512_mask_popcnt_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_mask_popcnt_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -247,7 +259,8 @@ pub fn _mm512_mask_popcnt_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm256_popcnt_epi8(a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_popcnt_epi8(a: __m256i) -> __m256i {
unsafe { transmute(simd_ctpop(a.as_i8x32())) }
}

Expand All @@ -261,7 +274,8 @@ pub fn _mm256_popcnt_epi8(a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm256_maskz_popcnt_epi8(k: __mmask32, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_maskz_popcnt_epi8(k: __mmask32, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -281,7 +295,8 @@ pub fn _mm256_maskz_popcnt_epi8(k: __mmask32, a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm256_mask_popcnt_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_mask_popcnt_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -298,7 +313,8 @@ pub fn _mm256_mask_popcnt_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm_popcnt_epi8(a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_popcnt_epi8(a: __m128i) -> __m128i {
unsafe { transmute(simd_ctpop(a.as_i8x16())) }
}

Expand All @@ -312,7 +328,8 @@ pub fn _mm_popcnt_epi8(a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm_maskz_popcnt_epi8(k: __mmask16, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_maskz_popcnt_epi8(k: __mmask16, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -332,7 +349,8 @@ pub fn _mm_maskz_popcnt_epi8(k: __mmask16, a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm_mask_popcnt_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_mask_popcnt_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand Down
Loading
Loading