[X86] matchVPMADD52 - only use 512-bit MADD52 on AVX512IFMA targets #161011
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
If we have a AVX512 target capable of AVXIFMA but not AVX512IFMA then we must split 512-bit (or larger) types to 256-bits Fixes llvm#160928
|
@llvm/pr-subscribers-backend-x86 Author: Simon Pilgrim (RKSimon) ChangesIf we have a AVX512 target capable of AVXIFMA but not AVX512IFMA then we must split 512-bit (or larger) types to 256-bits Fixes #160928 Patch is 37.33 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/161011.diff 2 Files Affected:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3af673d951f65..efeddd7c9bd4b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -4452,11 +4452,12 @@ static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG,
template <typename F>
SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
- F Builder, bool CheckBWI = true) {
+ F Builder, bool CheckBWI = true,
+ bool AllowAVX512 = true) {
assert(Subtarget.hasSSE2() && "Target assumed to support at least SSE2");
unsigned NumSubs = 1;
if ((CheckBWI && Subtarget.useBWIRegs()) ||
- (!CheckBWI && Subtarget.useAVX512Regs())) {
+ (!CheckBWI && AllowAVX512 && Subtarget.useAVX512Regs())) {
if (VT.getSizeInBits() > 512) {
NumSubs = VT.getSizeInBits() / 512;
assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size");
@@ -58076,7 +58077,8 @@ static SDValue matchVPMADD52(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
};
return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Acc, X, Y}, VPMADD52Builder,
- /*CheckBWI*/ false);
+ /*CheckBWI*/ false,
+ /*AllowAVX512*/ Subtarget.hasIFMA());
}
static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
diff --git a/llvm/test/CodeGen/X86/ifma-combine-vpmadd52.ll b/llvm/test/CodeGen/X86/ifma-combine-vpmadd52.ll
index aebfc7d483d6f..3ece4beb9c22e 100644
--- a/llvm/test/CodeGen/X86/ifma-combine-vpmadd52.ll
+++ b/llvm/test/CodeGen/X86/ifma-combine-vpmadd52.ll
@@ -1,25 +1,26 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avxifma | FileCheck %s --check-prefixes=X64,AVX
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avxifma | FileCheck %s --check-prefixes=X64,AVX,AVXIFMA
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512ifma | FileCheck %s --check-prefixes=X64,AVX512,AVX512-NOVL
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512ifma,+avx512vl | FileCheck %s --check-prefixes=X64,AVX512,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avxifma,+avx512vl | FileCheck %s --check-prefixes=X64,AVX,AVX512-NOIFMA
; 67108863 == (1 << 26) - 1
; 4503599627370496 == (1 << 52)
; 4503599627370495 == (1 << 52) - 1
define <8 x i64> @test_512_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
-; AVX-LABEL: test_512_combine:
-; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastq {{.*#+}} ymm6 = [67108863,67108863,67108863,67108863]
-; AVX-NEXT: vpand %ymm6, %ymm2, %ymm2
-; AVX-NEXT: vpand %ymm6, %ymm0, %ymm0
-; AVX-NEXT: {vex} vpmadd52luq %ymm2, %ymm0, %ymm4
-; AVX-NEXT: vpand %ymm6, %ymm3, %ymm0
-; AVX-NEXT: vpand %ymm6, %ymm1, %ymm1
-; AVX-NEXT: {vex} vpmadd52luq %ymm0, %ymm1, %ymm5
-; AVX-NEXT: vmovdqa %ymm4, %ymm0
-; AVX-NEXT: vmovdqa %ymm5, %ymm1
-; AVX-NEXT: retq
+; AVXIFMA-LABEL: test_512_combine:
+; AVXIFMA: # %bb.0:
+; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [67108863,67108863,67108863,67108863]
+; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm2
+; AVXIFMA-NEXT: vpand %ymm6, %ymm0, %ymm0
+; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm0, %ymm4
+; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm0
+; AVXIFMA-NEXT: vpand %ymm6, %ymm1, %ymm1
+; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm1, %ymm5
+; AVXIFMA-NEXT: vmovdqa %ymm4, %ymm0
+; AVXIFMA-NEXT: vmovdqa %ymm5, %ymm1
+; AVXIFMA-NEXT: retq
;
; AVX512-LABEL: test_512_combine:
; AVX512: # %bb.0:
@@ -29,6 +30,19 @@ define <8 x i64> @test_512_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
; AVX512-NEXT: vpmadd52luq %zmm1, %zmm0, %zmm2
; AVX512-NEXT: vmovdqa64 %zmm2, %zmm0
; AVX512-NEXT: retq
+;
+; AVX512-NOIFMA-LABEL: test_512_combine:
+; AVX512-NOIFMA: # %bb.0:
+; AVX512-NOIFMA-NEXT: vpbroadcastq {{.*#+}} zmm3 = [67108863,67108863,67108863,67108863,67108863,67108863,67108863,67108863]
+; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm1, %zmm1
+; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm0, %ymm4
+; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm2, %ymm5
+; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm3, %ymm4, %ymm5
+; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm1, %ymm0, %ymm2
+; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm5, %zmm2, %zmm0
+; AVX512-NOIFMA-NEXT: retq
%x_masked = and <8 x i64> %x, splat (i64 67108863)
%y_masked = and <8 x i64> %y, splat (i64 67108863)
%mul = mul nuw nsw <8 x i64> %x_masked, %y_masked
@@ -37,19 +51,19 @@ define <8 x i64> @test_512_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
}
define <8 x i64> @test_512_combine_v2(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
-; AVX-LABEL: test_512_combine_v2:
-; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastq {{.*#+}} ymm6 = [3,3,3,3]
-; AVX-NEXT: vpand %ymm6, %ymm2, %ymm2
-; AVX-NEXT: vpbroadcastq {{.*#+}} ymm7 = [1125899906842623,1125899906842623,1125899906842623,1125899906842623]
-; AVX-NEXT: vpand %ymm7, %ymm0, %ymm0
-; AVX-NEXT: {vex} vpmadd52luq %ymm2, %ymm0, %ymm4
-; AVX-NEXT: vpand %ymm6, %ymm3, %ymm0
-; AVX-NEXT: vpand %ymm7, %ymm1, %ymm1
-; AVX-NEXT: {vex} vpmadd52luq %ymm0, %ymm1, %ymm5
-; AVX-NEXT: vmovdqa %ymm4, %ymm0
-; AVX-NEXT: vmovdqa %ymm5, %ymm1
-; AVX-NEXT: retq
+; AVXIFMA-LABEL: test_512_combine_v2:
+; AVXIFMA: # %bb.0:
+; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [3,3,3,3]
+; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm2
+; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm7 = [1125899906842623,1125899906842623,1125899906842623,1125899906842623]
+; AVXIFMA-NEXT: vpand %ymm7, %ymm0, %ymm0
+; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm0, %ymm4
+; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm0
+; AVXIFMA-NEXT: vpand %ymm7, %ymm1, %ymm1
+; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm1, %ymm5
+; AVXIFMA-NEXT: vmovdqa %ymm4, %ymm0
+; AVXIFMA-NEXT: vmovdqa %ymm5, %ymm1
+; AVXIFMA-NEXT: retq
;
; AVX512-LABEL: test_512_combine_v2:
; AVX512: # %bb.0:
@@ -58,6 +72,18 @@ define <8 x i64> @test_512_combine_v2(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z)
; AVX512-NEXT: vpmadd52luq %zmm1, %zmm0, %zmm2
; AVX512-NEXT: vmovdqa64 %zmm2, %zmm0
; AVX512-NEXT: retq
+;
+; AVX512-NOIFMA-LABEL: test_512_combine_v2:
+; AVX512-NOIFMA: # %bb.0:
+; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1
+; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm2, %ymm3
+; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm1, %ymm4
+; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm0, %ymm5
+; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm4, %ymm5, %ymm3
+; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm1, %ymm0, %ymm2
+; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm0
+; AVX512-NOIFMA-NEXT: retq
%x_masked = and <8 x i64> %x, splat (i64 1125899906842623) ; (1 << 50) - 1
%y_masked = and <8 x i64> %y, splat (i64 3)
%mul = mul nuw nsw <8 x i64> %x_masked, %y_masked
@@ -66,32 +92,32 @@ define <8 x i64> @test_512_combine_v2(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z)
}
define <8 x i64> @test_512_no_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
-; AVX-LABEL: test_512_no_combine:
-; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastq {{.*#+}} ymm6 = [4503599627370495,4503599627370495,4503599627370495,4503599627370495]
-; AVX-NEXT: vpand %ymm6, %ymm0, %ymm7
-; AVX-NEXT: vpand %ymm6, %ymm1, %ymm8
-; AVX-NEXT: vpand %ymm6, %ymm2, %ymm9
-; AVX-NEXT: vpand %ymm6, %ymm3, %ymm6
-; AVX-NEXT: vpsrlq $32, %ymm8, %ymm8
-; AVX-NEXT: vpmuludq %ymm3, %ymm8, %ymm8
-; AVX-NEXT: vpsrlq $32, %ymm6, %ymm6
-; AVX-NEXT: vpmuludq %ymm6, %ymm1, %ymm6
-; AVX-NEXT: vpaddq %ymm6, %ymm8, %ymm6
-; AVX-NEXT: vpsllq $32, %ymm6, %ymm6
-; AVX-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
-; AVX-NEXT: vpsrlq $32, %ymm7, %ymm3
-; AVX-NEXT: vpmuludq %ymm2, %ymm3, %ymm3
-; AVX-NEXT: vpsrlq $32, %ymm9, %ymm7
-; AVX-NEXT: vpmuludq %ymm7, %ymm0, %ymm7
-; AVX-NEXT: vpaddq %ymm3, %ymm7, %ymm3
-; AVX-NEXT: vpsllq $32, %ymm3, %ymm3
-; AVX-NEXT: vpmuludq %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm5, %ymm1, %ymm1
-; AVX-NEXT: vpaddq %ymm6, %ymm1, %ymm1
-; AVX-NEXT: retq
+; AVXIFMA-LABEL: test_512_no_combine:
+; AVXIFMA: # %bb.0:
+; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [4503599627370495,4503599627370495,4503599627370495,4503599627370495]
+; AVXIFMA-NEXT: vpand %ymm6, %ymm0, %ymm7
+; AVXIFMA-NEXT: vpand %ymm6, %ymm1, %ymm8
+; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm9
+; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm6
+; AVXIFMA-NEXT: vpsrlq $32, %ymm8, %ymm8
+; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm8, %ymm8
+; AVXIFMA-NEXT: vpsrlq $32, %ymm6, %ymm6
+; AVXIFMA-NEXT: vpmuludq %ymm6, %ymm1, %ymm6
+; AVXIFMA-NEXT: vpaddq %ymm6, %ymm8, %ymm6
+; AVXIFMA-NEXT: vpsllq $32, %ymm6, %ymm6
+; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
+; AVXIFMA-NEXT: vpsrlq $32, %ymm7, %ymm3
+; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm3, %ymm3
+; AVXIFMA-NEXT: vpsrlq $32, %ymm9, %ymm7
+; AVXIFMA-NEXT: vpmuludq %ymm7, %ymm0, %ymm7
+; AVXIFMA-NEXT: vpaddq %ymm3, %ymm7, %ymm3
+; AVXIFMA-NEXT: vpsllq $32, %ymm3, %ymm3
+; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm4, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm3, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm5, %ymm1, %ymm1
+; AVXIFMA-NEXT: vpaddq %ymm6, %ymm1, %ymm1
+; AVXIFMA-NEXT: retq
;
; AVX512-LABEL: test_512_no_combine:
; AVX512: # %bb.0:
@@ -108,6 +134,22 @@ define <8 x i64> @test_512_no_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z)
; AVX512-NEXT: vpaddq %zmm2, %zmm0, %zmm0
; AVX512-NEXT: vpaddq %zmm3, %zmm0, %zmm0
; AVX512-NEXT: retq
+;
+; AVX512-NOIFMA-LABEL: test_512_no_combine:
+; AVX512-NOIFMA: # %bb.0:
+; AVX512-NOIFMA-NEXT: vpbroadcastq {{.*#+}} zmm3 = [4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495]
+; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm0, %zmm4
+; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm1, %zmm3
+; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm4, %zmm4
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm4, %zmm4
+; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm3, %zmm3
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm3, %zmm0, %zmm3
+; AVX512-NOIFMA-NEXT: vpaddq %zmm4, %zmm3, %zmm3
+; AVX512-NOIFMA-NEXT: vpsllq $32, %zmm3, %zmm3
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpaddq %zmm2, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: retq
%x_masked = and <8 x i64> %x, splat (i64 4503599627370495)
%y_masked = and <8 x i64> %y, splat (i64 4503599627370495)
%mul = mul nuw nsw <8 x i64> %x_masked, %y_masked
@@ -116,27 +158,27 @@ define <8 x i64> @test_512_no_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z)
}
define <8 x i64> @test_512_no_combine_v2(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
-; AVX-LABEL: test_512_no_combine_v2:
-; AVX: # %bb.0:
-; AVX-NEXT: vpsrlq $32, %ymm1, %ymm6
-; AVX-NEXT: vpmuludq %ymm3, %ymm6, %ymm6
-; AVX-NEXT: vpsrlq $32, %ymm3, %ymm7
-; AVX-NEXT: vpmuludq %ymm7, %ymm1, %ymm7
-; AVX-NEXT: vpaddq %ymm6, %ymm7, %ymm6
-; AVX-NEXT: vpsllq $32, %ymm6, %ymm6
-; AVX-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
-; AVX-NEXT: vpsrlq $32, %ymm0, %ymm3
-; AVX-NEXT: vpmuludq %ymm2, %ymm3, %ymm3
-; AVX-NEXT: vpsrlq $32, %ymm2, %ymm7
-; AVX-NEXT: vpmuludq %ymm7, %ymm0, %ymm7
-; AVX-NEXT: vpaddq %ymm3, %ymm7, %ymm3
-; AVX-NEXT: vpsllq $32, %ymm3, %ymm3
-; AVX-NEXT: vpmuludq %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm5, %ymm1, %ymm1
-; AVX-NEXT: vpaddq %ymm6, %ymm1, %ymm1
-; AVX-NEXT: retq
+; AVXIFMA-LABEL: test_512_no_combine_v2:
+; AVXIFMA: # %bb.0:
+; AVXIFMA-NEXT: vpsrlq $32, %ymm1, %ymm6
+; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm6, %ymm6
+; AVXIFMA-NEXT: vpsrlq $32, %ymm3, %ymm7
+; AVXIFMA-NEXT: vpmuludq %ymm7, %ymm1, %ymm7
+; AVXIFMA-NEXT: vpaddq %ymm6, %ymm7, %ymm6
+; AVXIFMA-NEXT: vpsllq $32, %ymm6, %ymm6
+; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
+; AVXIFMA-NEXT: vpsrlq $32, %ymm0, %ymm3
+; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm3, %ymm3
+; AVXIFMA-NEXT: vpsrlq $32, %ymm2, %ymm7
+; AVXIFMA-NEXT: vpmuludq %ymm7, %ymm0, %ymm7
+; AVXIFMA-NEXT: vpaddq %ymm3, %ymm7, %ymm3
+; AVXIFMA-NEXT: vpsllq $32, %ymm3, %ymm3
+; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm4, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm3, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm5, %ymm1, %ymm1
+; AVXIFMA-NEXT: vpaddq %ymm6, %ymm1, %ymm1
+; AVXIFMA-NEXT: retq
;
; AVX512-LABEL: test_512_no_combine_v2:
; AVX512: # %bb.0:
@@ -150,6 +192,19 @@ define <8 x i64> @test_512_no_combine_v2(<8 x i64> %x, <8 x i64> %y, <8 x i64> %
; AVX512-NEXT: vpaddq %zmm2, %zmm0, %zmm0
; AVX512-NEXT: vpaddq %zmm3, %zmm0, %zmm0
; AVX512-NEXT: retq
+;
+; AVX512-NOIFMA-LABEL: test_512_no_combine_v2:
+; AVX512-NOIFMA: # %bb.0:
+; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm0, %zmm3
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm3, %zmm3
+; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm1, %zmm4
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm4, %zmm0, %zmm4
+; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm4, %zmm3
+; AVX512-NOIFMA-NEXT: vpsllq $32, %zmm3, %zmm3
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpaddq %zmm2, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: retq
%mul = mul <8 x i64> %x, %y
%res = add <8 x i64> %mul, %z
ret <8 x i64> %res
@@ -255,25 +310,25 @@ define <1 x i64> @test_scalar_no_ifma(<1 x i64> %x, <1 x i64> %y, <1 x i64> %z)
; 40-bit and 13-bit, too wide
define <8 x i64> @test_mixed_width_too_wide(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
-; AVX-LABEL: test_mixed_width_too_wide:
-; AVX: # %bb.0:
-; AVX-NEXT: vpbroadcastq {{.*#+}} ymm6 = [8191,8191,8191,8191]
-; AVX-NEXT: vpand %ymm6, %ymm2, %ymm2
-; AVX-NEXT: vpand %ymm6, %ymm3, %ymm3
-; AVX-NEXT: vpmovzxdq {{.*#+}} ymm6 = [2155905028,2155905036,2155905044,2155905052]
-; AVX-NEXT: vpshufb %ymm6, %ymm1, %ymm7
-; AVX-NEXT: vpmuludq %ymm3, %ymm7, %ymm7
-; AVX-NEXT: vpsllq $32, %ymm7, %ymm7
-; AVX-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
-; AVX-NEXT: vpshufb %ymm6, %ymm0, %ymm3
-; AVX-NEXT: vpmuludq %ymm2, %ymm3, %ymm3
-; AVX-NEXT: vpsllq $32, %ymm3, %ymm3
-; AVX-NEXT: vpmuludq %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm0, %ymm4, %ymm0
-; AVX-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; AVX-NEXT: vpaddq %ymm1, %ymm5, %ymm1
-; AVX-NEXT: vpaddq %ymm7, %ymm1, %ymm1
-; AVX-NEXT: retq
+; AVXIFMA-LABEL: test_mixed_width_too_wide:
+; AVXIFMA: # %bb.0:
+; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [8191,8191,8191,8191]
+; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm2
+; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm3
+; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm6 = [2155905028,2155905036,2155905044,2155905052]
+; AVXIFMA-NEXT: vpshufb %ymm6, %ymm1, %ymm7
+; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm7, %ymm7
+; AVXIFMA-NEXT: vpsllq $32, %ymm7, %ymm7
+; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
+; AVXIFMA-NEXT: vpshufb %ymm6, %ymm0, %ymm3
+; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm3, %ymm3
+; AVXIFMA-NEXT: vpsllq $32, %ymm3, %ymm3
+; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm0, %ymm4, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm3, %ymm0, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm1, %ymm5, %ymm1
+; AVXIFMA-NEXT: vpaddq %ymm7, %ymm1, %ymm1
+; AVXIFMA-NEXT: retq
;
; AVX512-LABEL: test_mixed_width_too_wide:
; AVX512: # %bb.0:
@@ -286,6 +341,18 @@ define <8 x i64> @test_mixed_width_too_wide(<8 x i64> %x, <8 x i64> %y, <8 x i64
; AVX512-NEXT: vpaddq %zmm3, %zmm2, %zmm1
; AVX512-NEXT: vpaddq %zmm0, %zmm1, %zmm0
; AVX512-NEXT: retq
+;
+; AVX512-NOIFMA-LABEL: test_mixed_width_too_wide:
+; AVX512-NOIFMA: # %bb.0:
+; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm3
+; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpsllq $32, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm2, %zmm1
+; AVX512-NOIFMA-NEXT: vpaddq %zmm0, %zmm1, %zmm0
+; AVX512-NOIFMA-NEXT: retq
%x40 = and <8 x i64> %x, splat (i64 1099511627775)
%y13 = and <8 x i64> %y, splat (i64 8191)
%mul = mul <8 x i64> %x40, %y13
@@ -294,19 +361,19 @@ define <8 x i64> @test_mixed_width_too_wide(<8 x i64> %x, <8 x i64> %y, <8 x i64
}
define <8 x i64> @test_zext32_inputs_not_safe(<8 x i32> %xi32, <8 x i32> %yi32, <8 x i64> %z) {
-; AVX-LABEL: test_zext32_inputs_not_safe:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX-NEXT: vpmovzxdq {{.*#+}} ymm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX-NEXT: vpmuludq %ymm5, %ymm4, %ymm4
-; AVX-NEXT: vextracti128 $1, %ymm1, %xmm1
-; AVX-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm1
-; AVX-NEXT: vpaddq %ymm4, %ymm2, %ymm0
-; AVX-NEXT: vpaddq %ymm1, %ymm3, %ymm1
-; AVX-NEXT: retq
+; AVXIFMA-LABEL: test_zext32_inputs_not_safe:
+; AVXIFMA: # %bb.0:
+; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVXIFMA-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVXIFMA-NEXT: vpmuludq %ymm5, %ymm4, %ymm4
+; AVXIFMA-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVXIFMA-NEXT: vpmuludq %ymm1, %ymm0, %ymm1
+; AVXIFMA-NEXT: vpaddq %ymm4, %ymm2, %ymm0
+; AVXIFMA-NEXT: vpaddq %ymm1, %ymm3, %ymm1
+; AVXIFMA-NEXT: retq
;
; AVX512-LABEL: test_zext32_inputs_not_safe:
; AVX512: # %bb.0:
@@ -315,6 +382,14 @@ define <8 x i64> @test_zext32_inputs_not_safe(<8 x i32> %xi32, <8 x i32> %yi32,
; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vpaddq %zmm0, %zmm2, %zmm0
; AVX512-NEXT: retq
+;
+; AVX512-NOIFMA-LABEL: test_zext32_inputs_not_safe:
+; AVX512-NOIFMA: # %bb.0:
+; AVX512-NOIFMA-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
+; AVX512-NOIFMA-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
+; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
+; AVX512-NOIFMA-NEXT: vpaddq %zmm0, %zmm2, %zmm0
+; AVX512-NOIFMA-NEXT: retq
%x = zext <8 x i32> %xi32 to <8 x i64>
%y = zext <8 x i32> %yi32 to <8 x i64>
%mul = mul <8 x i64> %x, %y
@@ -323,36 +398,36 @@ define <8 x i64> @test_zext32_inputs_not_safe(<8 x i32> %xi32, <8 x i32> %yi32,
}
define <16 x i64> @test_1024_combine_split(<16 x i64> %x, <16 x i64> %y, <16 x i64> %z) nounwind {
-; AVX-LABEL: test_1024_combine_split:
-; AVX: # %bb.0:
-; AVX-NEXT: pushq %rbp
-; AVX-NEXT: movq %rsp, %rbp
-; AVX-NEXT: andq $-32, %rsp
-; AVX-NEXT: subq $32, %rsp
-; AVX-NEXT: vmovdqa 112(%rbp), %ymm8
-; AVX-NEXT: vmovdqa 80(%rbp), %ymm9
-; AVX-NEXT: vmovdqa 48(%rbp), %ymm10
-; AVX-NEXT: vmovdqa 16(%rbp), %ymm11
-; AVX-NEXT: vpbroadcastq {{....
[truncated]
|
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/4/builds/9404 Here is the relevant piece of the build log for the reference |
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/141/builds/11848 Here is the relevant piece of the build log for the reference |
mahesh-attarde
pushed a commit
to mahesh-attarde/llvm-project
that referenced
this pull request
Oct 3, 2025
…lvm#161011) If we have a AVX512 target capable of AVXIFMA but not AVX512IFMA then we must split 512-bit (or larger) types to 256-bits Fixes llvm#160928
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
If we have a AVX512 target capable of AVXIFMA but not AVX512IFMA then we must split 512-bit (or larger) types to 256-bits
Fixes #160928