diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 2ac6c5fbc471a..90883fd37d90d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -19932,8 +19932,8 @@ bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const { } bool AArch64TargetLowering::isLegalAddScalableImmediate(int64_t Imm) const { - // We will only emit addvl/inc* instructions if the subtarget allows it. - if (!Subtarget->useScalarIncVL()) + // We will only emit addvl/inc* instructions for SVE2 + if (!Subtarget->hasSVE2()) return false; // addvl's immediates are in terms of the number of bytes in a register. @@ -25266,6 +25266,18 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); + case Intrinsic::aarch64_sve_saddwb: + return DAG.getNode(AArch64ISD::SADDWB, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2)); + case Intrinsic::aarch64_sve_saddwt: + return DAG.getNode(AArch64ISD::SADDWT, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2)); + case Intrinsic::aarch64_sve_uaddwb: + return DAG.getNode(AArch64ISD::UADDWB, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2)); + case Intrinsic::aarch64_sve_uaddwt: + return DAG.getNode(AArch64ISD::UADDWT, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2)); case Intrinsic::aarch64_sve_eor_u: return DAG.getNode(ISD::XOR, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); @@ -31167,7 +31179,31 @@ static SDValue performPredicateLoadCombine(SDNode *N, DAG.makeEquivalentMemoryOrdering(Load, LoadPred); return LoadPred; } - +static SDValue performFPToIntToDivCombine(SDNode *N, SelectionDAG &DAG) { + unsigned Opc = N->getOpcode(); + bool IsSigned = (Opc == ISD::FP_TO_SINT); + SDValue FDiv = N->getOperand(0); + if (FDiv.getOpcode() != ISD::FDIV) + return SDValue(); + EVT IntVT = N->getValueType(0); + EVT FPVT = FDiv.getValueType(); + if (!IntVT.isVector() || !FPVT.isVector()) + return SDValue(); + if (IntVT.getVectorElementType() != MVT::i32 || + FPVT.getVectorElementType() != MVT::f64) + return SDValue(); + unsigned CastOpc = IsSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; + SDValue Op0 = FDiv.getOperand(0); + SDValue Op1 = FDiv.getOperand(1); + if (Op0.getOpcode() != CastOpc || Op1.getOpcode() != CastOpc) + return SDValue(); + if (Op0.getOperand(0).getValueType() != IntVT || + Op1.getOperand(0).getValueType() != IntVT) + return SDValue(); + unsigned DivOpc = IsSigned ? ISD::SDIV : ISD::UDIV; + return DAG.getNode(DivOpc, SDLoc(N), IntVT, Op0.getOperand(0), + Op1.getOperand(0)); +} SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; @@ -31235,6 +31271,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, return performIntToFpCombine(N, DAG, DCI, Subtarget); case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: + if (SDValue Res = performFPToIntToDivCombine(N, DAG)) + return Res; + [[fallthrough]]; case ISD::FP_TO_SINT_SAT: case ISD::FP_TO_UINT_SAT: return performFpToIntCombine(N, DAG, DCI, Subtarget); @@ -34719,8 +34758,8 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op, /// Lower a PARTIAL_REDUCE_MLA node. Three cases are handled: /// 1. (v2i32, v16i8): widen Acc to v4i32 and fold the high half with ADDP. -/// 2. (nx)v2i64/(nx)v16i8: accumulate in two steps via (nx)v4i32, using -/// (U|S)ADALP when available, otherwise add(add(Acc, ext(lo), ext(hi))). +/// 2. (nx)v2i64/(nx)v16i8: accumulate in two steps via v4i32, using +/// (U|S)ADDW(B|T) when available, otherwise add(add(Acc, ext(lo), ext(hi))). /// 3. SUMLA on (v4i32, v16i8) or (v2i32, v8i8) without +i8mm: rewrite as two /// UDOTs using the bias-128 identity sext(s) = zext(s ^ 128) - 128. SDValue @@ -34747,17 +34786,6 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op, return DAG.getExtractSubvector(DL, MVT::v2i32, Reduced, 0); } - // Handle (v2i64, v16i8) in two steps via v4i32 and Neon [SU]ADALP. - if (Subtarget->isNeonAvailable() && ResultVT == MVT::v2i64 && - OpVT == MVT::v16i8) { - SDValue Wide = DAG.getNode(Op.getOpcode(), DL, MVT::v4i32, - DAG.getConstant(0, DL, MVT::v4i32), LHS, RHS); - bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA; - unsigned Opc = IsUnsigned ? AArch64ISD::UADDLP : AArch64ISD::SADDLP; - return DAG.getNode(ISD::ADD, DL, ResultVT, Acc, - DAG.getNode(Opc, DL, ResultVT, Wide)); - } - // Lower PARTIAL_REDUCE_SUMLA on targets without +i8mm using udot via // sum(sext(LHS) * zext(RHS)) = // sum(zext(LHS ^ 128) * zext(RHS)) - sum(128 * zext(RHS)) @@ -34805,12 +34833,10 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op, bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA; if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) { - unsigned IID = IsUnsigned ? Intrinsic::aarch64_sve_uadalp - : Intrinsic::aarch64_sve_sadalp; - SDValue Pg = getPredicateForVector(DAG, DL, ResultVT); - SDValue Res = - DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResultVT, - DAG.getConstant(IID, DL, MVT::i64), Pg, Acc, DotNode); + unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB; + unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT; + SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode); + SDValue Res = DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode); return ConvertToScalable ? convertFromScalableVector(DAG, OrigResultVT, Res) : Res; } diff --git a/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll b/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll new file mode 100644 index 0000000000000..953187c7208d5 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-fdiv-int-fold.ll @@ -0,0 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s + +define @fptoui_fdiv_uitofp_nxv2i32( %a, %b) { +; CHECK-LABEL: fptoui_fdiv_uitofp_nxv2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.d, p0/m, z0.s +; CHECK-NEXT: ucvtf z1.d, p0/m, z1.s +; CHECK-NEXT: fdiv z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d +; CHECK-NEXT: ret + %fa = uitofp %a to + %fb = uitofp %b to + %fdiv = fdiv %fa, %fb + %res = fptoui %fdiv to + ret %res +} + +define @fptosi_fdiv_sitofp_nxv2i32( %a, %b) { +; CHECK-LABEL: fptosi_fdiv_sitofp_nxv2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: scvtf z0.d, p0/m, z0.s +; CHECK-NEXT: scvtf z1.d, p0/m, z1.s +; CHECK-NEXT: fdiv z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d +; CHECK-NEXT: ret + %fa = sitofp %a to + %fb = sitofp %b to + %fdiv = fdiv %fa, %fb + %res = fptosi %fdiv to + ret %res +}