[AArch64] Expand vector ops when NEON and SVE are unavailable. #90833
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-llvm-selectiondag @llvm/pr-subscribers-backend-aarch64 Author: Sander de Smalen (sdesmalen-arm) ChangesUnlike Patch is 3.02 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/90833.diff 60 Files Affected:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index bfc3e08c1632de..3175d75eea0860 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3731,8 +3731,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
case ISD::SUB: {
EVT VT = Node->getValueType(0);
- assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
- TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
+ assert((VT.isFixedLengthVector() || // fixed length ADD can be expanded to
+ // scalar ADD
+ (TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
+ TLI.isOperationLegalOrCustom(ISD::XOR, VT))) &&
"Don't know how to expand this subtraction!");
Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2af679e0755b54..12a949a7733b28 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -357,7 +357,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
}
- if (Subtarget->hasNEON()) {
+ if (Subtarget->isNeonAvailable()) {
addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
// Someone set us up the NEON.
@@ -378,6 +378,28 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
addQRTypeForNEON(MVT::v2i64);
addQRTypeForNEON(MVT::v8f16);
addQRTypeForNEON(MVT::v8bf16);
+ } else if (Subtarget->hasNEON() ||
+ Subtarget->useSVEForFixedLengthVectors()) {
+ addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
+ addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
+
+ addRegisterClass(MVT::v2f32, &AArch64::FPR64RegClass);
+ addRegisterClass(MVT::v8i8, &AArch64::FPR64RegClass);
+ addRegisterClass(MVT::v4i16, &AArch64::FPR64RegClass);
+ addRegisterClass(MVT::v2i32, &AArch64::FPR64RegClass);
+ addRegisterClass(MVT::v1i64, &AArch64::FPR64RegClass);
+ addRegisterClass(MVT::v1f64, &AArch64::FPR64RegClass);
+ addRegisterClass(MVT::v4f16, &AArch64::FPR64RegClass);
+ addRegisterClass(MVT::v4bf16, &AArch64::FPR64RegClass);
+
+ addRegisterClass(MVT::v4f32, &AArch64::FPR128RegClass);
+ addRegisterClass(MVT::v2f64, &AArch64::FPR128RegClass);
+ addRegisterClass(MVT::v16i8, &AArch64::FPR128RegClass);
+ addRegisterClass(MVT::v8i16, &AArch64::FPR128RegClass);
+ addRegisterClass(MVT::v4i32, &AArch64::FPR128RegClass);
+ addRegisterClass(MVT::v2i64, &AArch64::FPR128RegClass);
+ addRegisterClass(MVT::v8f16, &AArch64::FPR128RegClass);
+ addRegisterClass(MVT::v8bf16, &AArch64::FPR128RegClass);
}
if (Subtarget->hasSVEorSME()) {
@@ -1125,7 +1147,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
- if (Subtarget->hasNEON()) {
+ if (Subtarget->isNeonAvailable()) {
// FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
// silliness like this:
for (auto Op :
@@ -1328,6 +1350,24 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// FADDP custom lowering
for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 })
setOperationAction(ISD::FADD, VT, Custom);
+ } else {
+ for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
+ for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
+ setOperationAction(Op, VT, Expand);
+
+ if (VT.is128BitVector() || VT.is64BitVector()) {
+ setOperationAction(ISD::LOAD, VT, Legal);
+ setOperationAction(ISD::STORE, VT, Legal);
+ setOperationAction(ISD::BITCAST, VT,
+ Subtarget->isLittleEndian() ? Legal : Expand);
+ }
+ for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
+ setTruncStoreAction(VT, InnerVT, Expand);
+ setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
+ setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
+ setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
+ }
+ }
}
if (Subtarget->hasSME()) {
@@ -9377,7 +9417,8 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
SelectionDAG &DAG) const {
- if (!Subtarget->hasNEON())
+ if (!Subtarget->isNeonAvailable() &&
+ !Subtarget->useSVEForFixedLengthVectors())
return SDValue();
EVT VT = Op.getValueType();
@@ -14110,6 +14151,13 @@ SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
return DAG.getNode(AArch64ISD::UZP1, dl, VT, ResultLo, ResultHi);
}
+bool AArch64TargetLowering::shouldExpandBuildVectorWithShuffles(
+ EVT VT, unsigned DefinedValues) const {
+ if (!Subtarget->isNeonAvailable())
+ return false;
+ return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
+}
+
bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
// Currently no fixed length shuffles that require SVE are legal.
if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
@@ -15979,7 +16027,8 @@ bool AArch64TargetLowering::isLegalInterleavedAccessType(
UseScalable = false;
- if (!VecTy->isScalableTy() && !Subtarget->hasNEON())
+ if (!VecTy->isScalableTy() && !Subtarget->isNeonAvailable() &&
+ !Subtarget->useSVEForFixedLengthVectors())
return false;
if (VecTy->isScalableTy() && !Subtarget->hasSVEorSME())
@@ -16003,18 +16052,20 @@ bool AArch64TargetLowering::isLegalInterleavedAccessType(
}
unsigned VecSize = DL.getTypeSizeInBits(VecTy);
- if (!Subtarget->isNeonAvailable() ||
- (Subtarget->useSVEForFixedLengthVectors() &&
- (VecSize % Subtarget->getMinSVEVectorSizeInBits() == 0 ||
- (VecSize < Subtarget->getMinSVEVectorSizeInBits() &&
- isPowerOf2_32(MinElts) && VecSize > 128)))) {
- UseScalable = true;
- return true;
+ if (Subtarget->useSVEForFixedLengthVectors()) {
+ unsigned MinSVEVectorSize =
+ std::max(Subtarget->getMinSVEVectorSizeInBits(), 128u);
+ if (VecSize % MinSVEVectorSize == 0 ||
+ (VecSize < MinSVEVectorSize && isPowerOf2_32(MinElts) &&
+ VecSize > 128)) {
+ UseScalable = true;
+ return true;
+ }
}
// Ensure the total vector size is 64 or a multiple of 128. Types larger than
// 128 will be split into multiple interleaved accesses.
- return VecSize == 64 || VecSize % 128 == 0;
+ return Subtarget->isNeonAvailable() && (VecSize == 64 || VecSize % 128 == 0);
}
static ScalableVectorType *getSVEContainerIRType(FixedVectorType *VTy) {
@@ -16105,8 +16156,7 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
// "legalize" wide vector types into multiple interleaved accesses as long as
// the vector types are divisible by 128.
bool UseScalable;
- if (!Subtarget->hasNEON() ||
- !isLegalInterleavedAccessType(VTy, DL, UseScalable))
+ if (!isLegalInterleavedAccessType(VTy, DL, UseScalable))
return false;
unsigned NumLoads = getNumInterleavedAccesses(VTy, DL, UseScalable);
@@ -16283,8 +16333,7 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
// Skip if we do not have NEON and skip illegal vector types. We can
// "legalize" wide vector types into multiple interleaved accesses as long as
// the vector types are divisible by 128.
- if (!Subtarget->hasNEON() ||
- !isLegalInterleavedAccessType(SubVecTy, DL, UseScalable))
+ if (!isLegalInterleavedAccessType(SubVecTy, DL, UseScalable))
return false;
unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL, UseScalable);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index fbdc4de5617fe9..5a402b8df099f0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1020,6 +1020,8 @@ class AArch64TargetLowering : public TargetLowering {
void addDRTypeForNEON(MVT VT);
void addQRTypeForNEON(MVT VT);
+ bool shouldExpandBuildVectorWithShuffles(EVT, unsigned) const override;
+
unsigned allocateLazySaveBuffer(SDValue &Chain, const SDLoc &DL,
SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll
index fd9259048df543..4c3188fd7b2381 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll
@@ -18,8 +18,15 @@ define <4 x i8> @vls_sve_and_4xi8(<4 x i8> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_4xi8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi d1, #0xff000000ff0000
-; NONEON-NOSVE-NEXT: and v0.8b, v0.8b, v1.8b
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #12]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #8]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%c = and <4 x i8> %b, <i8 0, i8 255, i8 0, i8 255>
ret <4 x i8> %c
@@ -37,8 +44,21 @@ define <8 x i8> @vls_sve_and_8xi8(<8 x i8> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_8xi8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi d1, #0xff00ff00ff00ff00
-; NONEON-NOSVE-NEXT: and v0.8b, v0.8b, v1.8b
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #14]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #12]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #15]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #10]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #13]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #8]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #11]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #9]
+; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%c = and <8 x i8> %b, <i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255>
ret <8 x i8> %c
@@ -56,8 +76,33 @@ define <16 x i8> @vls_sve_and_16xi8(<16 x i8> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_16xi8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi v1.2d, #0xff00ff00ff00ff00
-; NONEON-NOSVE-NEXT: and v0.16b, v0.16b, v1.16b
+; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #30]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #28]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #31]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #26]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #29]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #24]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #27]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #22]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #25]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #20]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #23]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #18]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #21]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #16]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #19]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #17]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%c = and <16 x i8> %b, <i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255>
ret <16 x i8> %c
@@ -78,9 +123,57 @@ define <32 x i8> @vls_sve_and_32xi8(<32 x i8> %ap) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_32xi8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi v2.2d, #0xff00ff00ff00ff00
-; NONEON-NOSVE-NEXT: and v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT: and v1.16b, v1.16b, v2.16b
+; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #-64]!
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #46]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #44]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #42]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #40]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #38]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #36]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #34]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #32]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #62]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #60]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #58]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #56]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #54]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #52]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #50]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; NONEON-NOSVE-NEXT: strb wzr, [sp, #48]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; NONEON-NOSVE-NEXT: add sp, sp, #64
; NONEON-NOSVE-NEXT: ret
%b = and <32 x i8> %ap, <i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255,
i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255>
@@ -102,9 +195,11 @@ define <2 x i16> @vls_sve_and_2xi16(<2 x i16> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_2xi16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT: mov v0.s[0], wzr
-; NONEON-NOSVE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; NONEON-NOSVE-NEXT: stp wzr, w8, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%c = and <2 x i16> %b, <i16 0, i16 65535>
ret <2 x i16> %c
@@ -122,8 +217,15 @@ define <4 x i16> @vls_sve_and_4xi16(<4 x i16> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_4xi16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi d1, #0xffff0000ffff0000
-; NONEON-NOSVE-NEXT: and v0.8b, v0.8b, v1.8b
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #12]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #8]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%c = and <4 x i16> %b, <i16 0, i16 65535, i16 0, i16 65535>
ret <4 x i16> %c
@@ -141,8 +243,21 @@ define <8 x i16> @vls_sve_and_8xi16(<8 x i16> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_8xi16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi v1.2d, #0xffff0000ffff0000
-; NONEON-NOSVE-NEXT: and v0.16b, v0.16b, v1.16b
+; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #28]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #24]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #30]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #20]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #26]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #16]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #22]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #18]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%c = and <8 x i16> %b, <i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535>
ret <8 x i16> %c
@@ -163,9 +278,33 @@ define <16 x i16> @vls_sve_and_16xi16(<16 x i16> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_16xi16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi v2.2d, #0xffff0000ffff0000
-; NONEON-NOSVE-NEXT: and v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT: and v1.16b, v1.16b, v2.16b
+; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #-64]!
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #44]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #40]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #36]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #32]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #60]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #56]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #52]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; NONEON-NOSVE-NEXT: strh wzr, [sp, #48]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; NONEON-NOSVE-NEXT: add sp, sp, #64
; NONEON-NOSVE-NEXT: ret
%c = and <16 x i16> %b, <i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535>
ret <16 x i16> %c
@@ -183,9 +322,11 @@ define <2 x i32> @vls_sve_and_2xi32(<2 x i32> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_2xi32:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT: mov v0.s[0], wzr
-; NONEON-NOSVE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; NONEON-NOSVE-NEXT: stp wzr, w8, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%c = and <2 x i32> %b, <i32 0, i32 4294967295>
ret <2 x i32> %c
@@ -203,8 +344,13 @@ define <4 x i32> @vls_sve_and_4xi32(<4 x i32> %b) nounwind {
;
; NONEON-NOSVE-LABEL: vls_sve_and_4xi32:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: movi v1.2d, #0xffffffff00000000
-; NONEON-NOSVE-NEXT: and v0.16b, v0.16b, v1.16b
+; NONEON-NOSVE-NEXT:...
[truncated]
|
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
sdesmalen-arm
force-pushed
the
decouple-codegen-sme-prep-4
branch
from
2270ec1
to
651e42f
Compare
kmclaughlin-arm
approved these changes
May 3, 2024
Comment on lines
382
to
385
| addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass); | ||
| addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass); | ||
|
|
||
| addRegisterClass(MVT::v2f32, &AArch64::FPR64RegClass); |
There was a problem hiding this comment.
I don't really like the duplication here. What about passing isNeonAvailable() into addDRTypeForNEON and addQRTypeForNEON to toggle the call to addTypeForNEON?
There was a problem hiding this comment.
I've moved this into (the renamed functions) addQRType and addDRType now.
| @@ -1328,6 +1349,24 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, | |||
| // FADDP custom lowering | |||
| for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 }) | |||
| setOperationAction(ISD::FADD, VT, Custom); | |||
| } else { | |||
There was a problem hiding this comment.
There's quite a gap here so perhaps worth adding /* !isNeonAvailable */?
There was a problem hiding this comment.
Alternatively should this be else if (!Subtarget->useSVEForFixedLengthVectors())? Just thinking there's several instances of "set everything to Expand" going on.
There was a problem hiding this comment.
Alternatively should this be
else if (!Subtarget->useSVEForFixedLengthVectors())? Just thinking there's several instances of "set everything to Expand" going on.
In practice this is not equivalent, because there are things missing from addTypeForFixedLengthSVE. For example, the loop that sets truncating-store actions:
MVT InnerVT = VT.changeVectorElementType(MVT::i8);
while (InnerVT != VT) {
setTruncStoreAction(VT, InnerVT, Default);
...
InnerVT = InnerVT.changeVectorElementType(
MVT::getIntegerVT(2 * InnerVT.getScalarSizeInBits()));
}
misses out on truncating store from v4i16 -> v4i1, which we'd want to Expand.
If we'd set it to Default (Custom) lowering, then it would try to lower it with SVE operations because the check useSVEForFixedLengthVectorVT is based on MVT::v4i16, not the MVT::v4i1, and the resulting truncating store would fail to select.
| @@ -3731,8 +3731,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { | |||
| } | |||
| case ISD::SUB: { | |||
| EVT VT = Node->getValueType(0); | |||
| assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && | |||
| TLI.isOperationLegalOrCustom(ISD::XOR, VT) && | |||
| assert((VT.isFixedLengthVector() || // fixed length ADD can be expanded to | |||
There was a problem hiding this comment.
This change looks weird. The assert is checking whether it is safe to create ADD and XOR nodes, which you're bypassing? What does the failing test case look like? Part of me thinks the original assert should be an if statement, but then I'm also wondering why VectorLegalizer::LegalizeOp hasn't already unrolled the operation.
There was a problem hiding this comment.
That's right. The way the code now works (if you'd ignore the assert) is that it expands a sub into a add + xor. If those are vector operations that also require expansion, they can be expanded further (in a subsequent step) into scalar operations. The assert is trying to match the way the code currently works, which allows the code to be successfully compiled. I figured updating the assert was the most sensible thing to do for now, given that it's not really a common use-case to optimise for.
There was a problem hiding this comment.
To me the assert exists to ensure optimal code, which is to say it doesn't really make sense to convert the sub to an inverted add unless both those operations are in someway legal. When not the better result would be to just expand the sub. Which I guess makes the assert a bear trap for the first person that needs the support.
paulwalker-arm
approved these changes
May 23, 2024
Unlike `+noneon` we must assume that vector types are available, i.e. it is valid to pass/return vector arguments to and from functions. However, the compiler must make sure to scalarize any vector operations.
Instead disable the 'performAddSubIntoVectorOp' which turned a scalar i64 sub back into a v1i64 sub. I tried disabling the combine before (operation) legalization, but that didn't work. So instead, I've just disabled it if the ISD::SUB needs Expand for v1i64.
sdesmalen-arm
force-pushed
the
decouple-codegen-sme-prep-4
branch
from
cf13b69
to
a52bc4f
Compare
vg0204
pushed a commit
to vg0204/llvm-project
that referenced
this pull request
May 29, 2024
…90833) Unlike `+noneon` we must assume that vector types are available, i.e. it is valid to pass/return vector arguments to and from functions. However, the compiler must make sure to scalarize any vector operations.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Unlike
+noneonwe must assume that vector types are available, i.e. it isvalid to pass/return vector arguments to and from functions. However, the
compiler must make sure to scalarize any vector operations.