171 changes: 96 additions & 75 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1929,8 +1929,8 @@ static SDValue lowerFROUND(SDValue Op, SelectionDAG &DAG,
DAG.getUNDEF(ContainerVT), SplatVal, VL);

// Add the adjustment.
SDValue Adjust =
DAG.getNode(RISCVISD::FADD_VL, DL, ContainerVT, Abs, Splat, Mask, VL);
SDValue Adjust = DAG.getNode(RISCVISD::FADD_VL, DL, ContainerVT, Abs, Splat,
DAG.getUNDEF(ContainerVT), Mask, VL);

// Truncate to integer and convert back to fp.
MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
Expand Down Expand Up @@ -2798,19 +2798,21 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
TrueMask = getAllOnesMask(HalfContainerVT, VL, DL, DAG);

// Widen V1 and V2 with 0s and add one copy of V2 to V1.
SDValue Add = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1,
V2, TrueMask, VL);
SDValue Add =
DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1, V2,
DAG.getUNDEF(WideIntContainerVT), TrueMask, VL);
// Create 2^eltbits - 1 copies of V2 by multiplying by the largest integer.
SDValue Multiplier = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntHalfVT,
DAG.getUNDEF(IntHalfVT),
DAG.getAllOnesConstant(DL, XLenVT));
SDValue WidenMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT,
V2, Multiplier, TrueMask, VL);
SDValue WidenMul =
DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT, V2, Multiplier,
DAG.getUNDEF(WideIntContainerVT), TrueMask, VL);
// Add the new copies to our previous addition giving us 2^eltbits copies of
// V2. This is equivalent to shifting V2 left by eltbits. This should
// combine with the vwmulu.vv above to form vwmaccu.vv.
Add = DAG.getNode(RISCVISD::ADD_VL, DL, WideIntContainerVT, Add, WidenMul,
TrueMask, VL);
DAG.getUNDEF(WideIntContainerVT), TrueMask, VL);
// Cast back to ContainerVT. We need to re-create a new ContainerVT in case
// WideIntContainerVT is a larger fractional LMUL than implied by the fixed
// vector VT.
Expand Down Expand Up @@ -3534,15 +3536,15 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::SETCC:
return lowerFixedLengthVectorSetccToRVV(Op, DAG);
case ISD::ADD:
return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL, /*HasMergeOp*/ true);
case ISD::SUB:
return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL, /*HasMergeOp*/ true);
case ISD::MUL:
return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL, /*HasMergeOp*/ true);
case ISD::MULHS:
return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL, /*HasMergeOp*/ true);
case ISD::MULHU:
return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL, /*HasMergeOp*/ true);
case ISD::AND:
return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
RISCVISD::AND_VL);
Expand All @@ -3553,13 +3555,13 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
RISCVISD::XOR_VL);
case ISD::SDIV:
return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL, /*HasMergeOp*/ true);
case ISD::SREM:
return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL, /*HasMergeOp*/ true);
case ISD::UDIV:
return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL, /*HasMergeOp*/ true);
case ISD::UREM:
return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL, /*HasMergeOp*/ true);
case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
Expand All @@ -3570,21 +3572,25 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
"Unexpected custom legalisation");
return SDValue();
case ISD::SADDSAT:
return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL,
/*HasMergeOp*/ true);
case ISD::UADDSAT:
return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL,
/*HasMergeOp*/ true);
case ISD::SSUBSAT:
return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL,
/*HasMergeOp*/ true);
case ISD::USUBSAT:
return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL,
/*HasMergeOp*/ true);
case ISD::FADD:
return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL, /*HasMergeOp*/ true);
case ISD::FSUB:
return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL, /*HasMergeOp*/ true);
case ISD::FMUL:
return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL, /*HasMergeOp*/ true);
case ISD::FDIV:
return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL, /*HasMergeOp*/ true);
case ISD::FNEG:
return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
case ISD::FABS:
Expand All @@ -3594,17 +3600,19 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::FMA:
return lowerToScalableOp(Op, DAG, RISCVISD::VFMADD_VL);
case ISD::SMIN:
return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL, /*HasMergeOp*/ true);
case ISD::SMAX:
return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL, /*HasMergeOp*/ true);
case ISD::UMIN:
return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL, /*HasMergeOp*/ true);
case ISD::UMAX:
return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL, /*HasMergeOp*/ true);
case ISD::FMINNUM:
return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL,
/*HasMergeOp*/ true);
case ISD::FMAXNUM:
return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL,
/*HasMergeOp*/ true);
case ISD::ABS:
return lowerABS(Op, DAG);
case ISD::CTLZ_ZERO_UNDEF:
Expand All @@ -3631,39 +3639,39 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::VP_MERGE:
return lowerVPOp(Op, DAG, RISCVISD::VP_MERGE_VL);
case ISD::VP_ADD:
return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
return lowerVPOp(Op, DAG, RISCVISD::ADD_VL, /*HasMergeOp*/ true);
case ISD::VP_SUB:
return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
return lowerVPOp(Op, DAG, RISCVISD::SUB_VL, /*HasMergeOp*/ true);
case ISD::VP_MUL:
return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
return lowerVPOp(Op, DAG, RISCVISD::MUL_VL, /*HasMergeOp*/ true);
case ISD::VP_SDIV:
return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL, /*HasMergeOp*/ true);
case ISD::VP_UDIV:
return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL, /*HasMergeOp*/ true);
case ISD::VP_SREM:
return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
return lowerVPOp(Op, DAG, RISCVISD::SREM_VL, /*HasMergeOp*/ true);
case ISD::VP_UREM:
return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
return lowerVPOp(Op, DAG, RISCVISD::UREM_VL, /*HasMergeOp*/ true);
case ISD::VP_AND:
return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
case ISD::VP_OR:
return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
case ISD::VP_XOR:
return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
case ISD::VP_ASHR:
return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
return lowerVPOp(Op, DAG, RISCVISD::SRA_VL, /*HasMergeOp*/ true);
case ISD::VP_LSHR:
return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
return lowerVPOp(Op, DAG, RISCVISD::SRL_VL, /*HasMergeOp*/ true);
case ISD::VP_SHL:
return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
return lowerVPOp(Op, DAG, RISCVISD::SHL_VL, /*HasMergeOp*/ true);
case ISD::VP_FADD:
return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
return lowerVPOp(Op, DAG, RISCVISD::FADD_VL, /*HasMergeOp*/ true);
case ISD::VP_FSUB:
return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL, /*HasMergeOp*/ true);
case ISD::VP_FMUL:
return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL, /*HasMergeOp*/ true);
case ISD::VP_FDIV:
return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL, /*HasMergeOp*/ true);
case ISD::VP_FNEG:
return lowerVPOp(Op, DAG, RISCVISD::FNEG_VL);
case ISD::VP_FMA:
Expand Down Expand Up @@ -4343,8 +4351,8 @@ SDValue RISCVTargetLowering::lowerVectorMaskTruncLike(SDValue Op,
DAG.getUNDEF(ContainerVT), SplatZero, VL);

MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
SDValue Trunc =
DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
DAG.getUNDEF(ContainerVT), Mask, VL);
Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
DAG.getCondCode(ISD::SETNE), Mask, VL);
if (MaskVT.isFixedLengthVector())
Expand Down Expand Up @@ -5807,8 +5815,8 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
VLMinus1, DAG.getRegister(RISCV::X0, XLenVT));

SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
SDValue Indices =
DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
DAG.getUNDEF(IntVT), Mask, VL);

return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices,
DAG.getUNDEF(VecVT), Mask, VL);
Expand Down Expand Up @@ -6071,9 +6079,10 @@ SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
MVT VT = Op.getSimpleValueType();

if (VT.getVectorElementType() == MVT::i1)
return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMergeOp*/ false,
/*HasMask*/ false);

return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
return lowerToScalableOp(Op, DAG, VecOpc, /*HasMergeOp*/ true);
}

SDValue
Expand All @@ -6087,7 +6096,7 @@ RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
}

return lowerToScalableOp(Op, DAG, Opc);
return lowerToScalableOp(Op, DAG, Opc, /*HasMergeOp*/ true);
}

// Lower vector ABS to smax(X, sub(0, X)).
Expand All @@ -6107,10 +6116,10 @@ SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
SDValue SplatZero = DAG.getNode(
RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
DAG.getConstant(0, DL, Subtarget.getXLenVT()));
SDValue NegX =
DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
SDValue Max =
DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
DAG.getUNDEF(ContainerVT), Mask, VL);
SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
DAG.getUNDEF(ContainerVT), Mask, VL);

return convertFromScalableVector(VT, Max, DAG, Subtarget);
}
Expand Down Expand Up @@ -6163,7 +6172,7 @@ SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
}

SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
unsigned NewOpc,
unsigned NewOpc, bool HasMergeOp,
bool HasMask) const {
MVT VT = Op.getSimpleValueType();
MVT ContainerVT = getContainerForFixedLengthVector(VT);
Expand All @@ -6188,6 +6197,8 @@ SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
SDLoc DL(Op);
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
if (HasMergeOp)
Ops.push_back(DAG.getUNDEF(ContainerVT));
if (HasMask)
Ops.push_back(Mask);
Ops.push_back(VL);
Expand All @@ -6202,14 +6213,22 @@ SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
// * Fixed-length vectors are converted to their scalable-vector container
// types.
SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
unsigned RISCVISDOpc) const {
unsigned RISCVISDOpc,
bool HasMergeOp) const {
SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
SmallVector<SDValue, 4> Ops;

MVT ContainerVT = VT;
if (VT.isFixedLengthVector())
ContainerVT = getContainerForFixedLengthVector(VT);

for (const auto &OpIdx : enumerate(Op->ops())) {
SDValue V = OpIdx.value();
assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
// Add dummy merge value before the mask.
if (HasMergeOp && *ISD::getVPMaskIdx(Op.getOpcode()) == OpIdx.index())
Ops.push_back(DAG.getUNDEF(ContainerVT));
// Pass through operands which aren't fixed-length vectors.
if (!V.getValueType().isFixedLengthVector()) {
Ops.push_back(V);
Expand All @@ -6226,8 +6245,6 @@ SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
if (!VT.isFixedLengthVector())
return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());

MVT ContainerVT = getContainerForFixedLengthVector(VT);

SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());

return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
Expand Down Expand Up @@ -6483,7 +6500,7 @@ SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
unsigned VecOpc) const {
MVT VT = Op.getSimpleValueType();
if (VT.getVectorElementType() != MVT::i1)
return lowerVPOp(Op, DAG, VecOpc);
return lowerVPOp(Op, DAG, VecOpc, true);

// It is safe to drop mask parameter as masked-off elements are undef.
SDValue Op1 = Op->getOperand(0);
Expand Down Expand Up @@ -7281,8 +7298,9 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
DAG.getUNDEF(ContainerVT),
DAG.getConstant(32, DL, XLenVT), VL);
SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
ThirtyTwoV, Mask, VL);
SDValue LShr32 =
DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
DAG.getUNDEF(ContainerVT), Mask, VL);

SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);

Expand Down Expand Up @@ -7389,8 +7407,8 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
SDValue ThirtyTwoV =
DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT),
DAG.getConstant(32, DL, XLenVT), VL);
SDValue LShr32 =
DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV,
DAG.getUNDEF(VecVT), Mask, VL);
SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);

Results.push_back(
Expand Down Expand Up @@ -8298,8 +8316,9 @@ static SDValue combineADDSUB_VLToVWADDSUB_VL(SDNode *N, SelectionDAG &DAG,
MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
VT.getVectorElementCount());

SDValue Mask = N->getOperand(2);
SDValue VL = N->getOperand(3);
SDValue Merge = N->getOperand(2);
SDValue Mask = N->getOperand(3);
SDValue VL = N->getOperand(4);

SDLoc DL(N);

Expand All @@ -8319,7 +8338,7 @@ static SDValue combineADDSUB_VLToVWADDSUB_VL(SDNode *N, SelectionDAG &DAG,
else
WOpc = IsAdd ? RISCVISD::VWADDU_W_VL : RISCVISD::VWSUBU_W_VL;

return DAG.getNode(WOpc, DL, VT, Op0, Op1, Mask, VL);
return DAG.getNode(WOpc, DL, VT, Op0, Op1, Merge, Mask, VL);
}

// FIXME: Is it useful to form a vwadd.wx or vwsub.wx if it removes a scalar
Expand All @@ -8333,8 +8352,9 @@ static SDValue combineADDSUB_VLToVWADDSUB_VL(SDNode *N, SelectionDAG &DAG,
static SDValue combineVWADD_W_VL_VWSUB_W_VL(SDNode *N, SelectionDAG &DAG) {
SDValue Op0 = N->getOperand(0);
SDValue Op1 = N->getOperand(1);
SDValue Mask = N->getOperand(2);
SDValue VL = N->getOperand(3);
SDValue Merge = N->getOperand(2);
SDValue Mask = N->getOperand(3);
SDValue VL = N->getOperand(4);

MVT VT = N->getSimpleValueType(0);
MVT NarrowVT = Op1.getSimpleValueType();
Expand Down Expand Up @@ -8364,7 +8384,7 @@ static SDValue combineVWADD_W_VL_VWSUB_W_VL(SDNode *N, SelectionDAG &DAG) {
// Re-introduce narrower extends if needed.
if (Op0.getValueType() != NarrowVT)
Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
return DAG.getNode(VOpc, DL, VT, Op0, Op1, Mask, VL);
return DAG.getNode(VOpc, DL, VT, Op0, Op1, Merge, Mask, VL);
}

bool IsAdd = N->getOpcode() == RISCVISD::VWADD_W_VL ||
Expand Down Expand Up @@ -8396,7 +8416,7 @@ static SDValue combineVWADD_W_VL_VWSUB_W_VL(SDNode *N, SelectionDAG &DAG) {

Op0 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT,
DAG.getUNDEF(NarrowVT), Op0, VL);
return DAG.getNode(VOpc, DL, VT, Op1, Op0, Mask, VL);
return DAG.getNode(VOpc, DL, VT, Op1, Op0, Merge, Mask, VL);
}

return SDValue();
Expand All @@ -8418,8 +8438,9 @@ static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
return SDValue();

SDValue Mask = N->getOperand(2);
SDValue VL = N->getOperand(3);
SDValue Merge = N->getOperand(2);
SDValue Mask = N->getOperand(3);
SDValue VL = N->getOperand(4);

// Make sure the mask and VL match.
if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
Expand Down Expand Up @@ -8497,7 +8518,7 @@ static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
unsigned WMulOpc = RISCVISD::VWMULSU_VL;
if (!IsVWMULSU)
WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Merge, Mask, VL);
}

static RISCVFPRndMode::RoundingMode matchRoundingOp(SDValue Op) {
Expand Down Expand Up @@ -9230,7 +9251,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
ShAmt.getOperand(1), VL);
return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
N->getOperand(2), N->getOperand(3));
N->getOperand(2), N->getOperand(3), N->getOperand(4));
}
break;
}
Expand Down
41 changes: 23 additions & 18 deletions llvm/lib/Target/RISCV/RISCVISelLowering.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,9 +213,8 @@ enum NodeType : unsigned {
VECREDUCE_FMIN_VL,
VECREDUCE_FMAX_VL,

// Vector binary and unary ops with a mask as a third operand, and VL as a
// fourth operand.
// FIXME: Can we replace these with ISD::VP_*?
// Vector binary ops with a merge as a third operand, a mask as a fourth
// operand, and VL as a fifth operand.
ADD_VL,
AND_VL,
MUL_VL,
Expand All @@ -229,40 +228,45 @@ enum NodeType : unsigned {
UDIV_VL,
UREM_VL,
XOR_VL,
SMIN_VL,
SMAX_VL,
UMIN_VL,
UMAX_VL,

SADDSAT_VL,
UADDSAT_VL,
SSUBSAT_VL,
USUBSAT_VL,

MULHS_VL,
MULHU_VL,
FADD_VL,
FSUB_VL,
FMUL_VL,
FDIV_VL,
FMINNUM_VL,
FMAXNUM_VL,

// Vector unary ops with a mask as a second operand and VL as a third operand.
FNEG_VL,
FABS_VL,
FSQRT_VL,
VFMADD_VL,
VFNMADD_VL,
VFMSUB_VL,
VFNMSUB_VL,
FCOPYSIGN_VL,
SMIN_VL,
SMAX_VL,
UMIN_VL,
UMAX_VL,
FMINNUM_VL,
FMAXNUM_VL,
MULHS_VL,
MULHU_VL,
FP_TO_SINT_VL,
FP_TO_UINT_VL,
SINT_TO_FP_VL,
UINT_TO_FP_VL,
FP_ROUND_VL,
FP_EXTEND_VL,

// Widening instructions
// Vector FMA ops with a mask as a fourth operand and VL as a fifth operand.
VFMADD_VL,
VFNMADD_VL,
VFMSUB_VL,
VFNMSUB_VL,

// Widening instructions with a merge value a third operand, a mask as a
// fourth operand, and VL as a fifth operand.
VWMUL_VL,
VWMULU_VL,
VWMULSU_VL,
Expand Down Expand Up @@ -679,8 +683,9 @@ class RISCVTargetLowering : public TargetLowering {
SDValue lowerFixedLengthVectorSelectToRVV(SDValue Op,
SelectionDAG &DAG) const;
SDValue lowerToScalableOp(SDValue Op, SelectionDAG &DAG, unsigned NewOpc,
bool HasMask = true) const;
SDValue lowerVPOp(SDValue Op, SelectionDAG &DAG, unsigned RISCVISDOpc) const;
bool HasMergeOp = false, bool HasMask = true) const;
SDValue lowerVPOp(SDValue Op, SelectionDAG &DAG, unsigned RISCVISDOpc,
bool HasMergeOp = false) const;
SDValue lowerLogicVPOp(SDValue Op, SelectionDAG &DAG, unsigned MaskOpc,
unsigned VecOpc) const;
SDValue lowerVPExtMaskOp(SDValue Op, SelectionDAG &DAG) const;
Expand Down
148 changes: 86 additions & 62 deletions llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,26 @@
// Helpers to define the VL patterns.
//===----------------------------------------------------------------------===//

def SDT_RISCVIntBinOp_VL : SDTypeProfile<1, 4, [SDTCisSameAs<0, 1>,
def SDT_RISCVIntBinOp_VL : SDTypeProfile<1, 5, [SDTCisSameAs<0, 1>,
SDTCisSameAs<0, 2>,
SDTCisVec<0>, SDTCisInt<0>,
SDTCVecEltisVT<3, i1>,
SDTCisSameNumEltsAs<0, 3>,
SDTCisVT<4, XLenVT>]>;
SDTCisSameAs<0, 3>,
SDTCVecEltisVT<4, i1>,
SDTCisSameNumEltsAs<0, 4>,
SDTCisVT<5, XLenVT>]>;

def SDT_RISCVFPUnOp_VL : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>,
SDTCisVec<0>, SDTCisFP<0>,
SDTCVecEltisVT<2, i1>,
SDTCisSameNumEltsAs<0, 2>,
SDTCisVT<3, XLenVT>]>;
def SDT_RISCVFPBinOp_VL : SDTypeProfile<1, 4, [SDTCisSameAs<0, 1>,
def SDT_RISCVFPBinOp_VL : SDTypeProfile<1, 5, [SDTCisSameAs<0, 1>,
SDTCisSameAs<0, 2>,
SDTCisVec<0>, SDTCisFP<0>,
SDTCVecEltisVT<3, i1>,
SDTCisSameNumEltsAs<0, 3>,
SDTCisVT<4, XLenVT>]>;
SDTCisSameAs<0, 3>,
SDTCVecEltisVT<4, i1>,
SDTCisSameNumEltsAs<0, 4>,
SDTCisVT<5, XLenVT>]>;

def SDT_RISCVCopySign_VL : SDTypeProfile<1, 5, [SDTCisSameAs<0, 1>,
SDTCisSameAs<0, 2>,
Expand Down Expand Up @@ -230,12 +232,13 @@ def riscv_trunc_vector_vl : SDNode<"RISCVISD::TRUNCATE_VECTOR_VL",
SDTCVecEltisVT<2, i1>,
SDTCisVT<3, XLenVT>]>>;

def SDT_RISCVVWBinOp_VL : SDTypeProfile<1, 4, [SDTCisVec<0>,
def SDT_RISCVVWBinOp_VL : SDTypeProfile<1, 5, [SDTCisVec<0>,
SDTCisSameNumEltsAs<0, 1>,
SDTCisSameAs<1, 2>,
SDTCisSameNumEltsAs<1, 3>,
SDTCVecEltisVT<3, i1>,
SDTCisVT<4, XLenVT>]>;
SDTCisSameAs<0, 3>,
SDTCisSameNumEltsAs<1, 4>,
SDTCVecEltisVT<4, i1>,
SDTCisVT<5, XLenVT>]>;
def riscv_vwmul_vl : SDNode<"RISCVISD::VWMUL_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
def riscv_vwmulu_vl : SDNode<"RISCVISD::VWMULU_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
def riscv_vwmulsu_vl : SDNode<"RISCVISD::VWMULSU_VL", SDT_RISCVVWBinOp_VL>;
Expand All @@ -244,13 +247,14 @@ def riscv_vwaddu_vl : SDNode<"RISCVISD::VWADDU_VL", SDT_RISCVVWBinOp_VL, [SDNPCo
def riscv_vwsub_vl : SDNode<"RISCVISD::VWSUB_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
def riscv_vwsubu_vl : SDNode<"RISCVISD::VWSUBU_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;

def SDT_RISCVVWBinOpW_VL : SDTypeProfile<1, 4, [SDTCisVec<0>,
def SDT_RISCVVWBinOpW_VL : SDTypeProfile<1, 5, [SDTCisVec<0>,
SDTCisSameAs<0, 1>,
SDTCisSameNumEltsAs<1, 2>,
SDTCisOpSmallerThanOp<2, 1>,
SDTCisSameNumEltsAs<1, 3>,
SDTCVecEltisVT<3, i1>,
SDTCisVT<4, XLenVT>]>;
SDTCisSameAs<0, 3>,
SDTCisSameNumEltsAs<1, 4>,
SDTCVecEltisVT<4, i1>,
SDTCisVT<5, XLenVT>]>;
def riscv_vwadd_w_vl : SDNode<"RISCVISD::VWADD_W_VL", SDT_RISCVVWBinOpW_VL>;
def riscv_vwaddu_w_vl : SDNode<"RISCVISD::VWADDU_W_VL", SDT_RISCVVWBinOpW_VL>;
def riscv_vwsub_w_vl : SDNode<"RISCVISD::VWSUB_W_VL", SDT_RISCVVWBinOpW_VL>;
Expand All @@ -261,27 +265,31 @@ def SDTRVVVecReduce : SDTypeProfile<1, 5, [
SDTCVecEltisVT<4, i1>, SDTCisSameNumEltsAs<2, 4>, SDTCisVT<5, XLenVT>
]>;

def riscv_mul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
def riscv_mul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
node:$E),
(riscv_mul_vl node:$A, node:$B, node:$C,
node:$D), [{
node:$D, node:$E), [{
return N->hasOneUse();
}]>;

def riscv_vwmul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
def riscv_vwmul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
node:$E),
(riscv_vwmul_vl node:$A, node:$B, node:$C,
node:$D), [{
node:$D, node:$E), [{
return N->hasOneUse();
}]>;

def riscv_vwmulu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
def riscv_vwmulu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
node:$E),
(riscv_vwmulu_vl node:$A, node:$B, node:$C,
node:$D), [{
node:$D, node:$E), [{
return N->hasOneUse();
}]>;

def riscv_vwmulsu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
def riscv_vwmulsu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
node:$E),
(riscv_vwmulsu_vl node:$A, node:$B, node:$C,
node:$D), [{
node:$D, node:$E), [{
return N->hasOneUse();
}]>;

Expand Down Expand Up @@ -331,15 +339,17 @@ multiclass VPatBinaryVL_V<SDNode vop,
ValueType mask_type,
int sew,
LMULInfo vlmul,
VReg result_reg_class,
VReg op1_reg_class,
VReg op2_reg_class> {
def : Pat<(result_type (vop
(op1_type op1_reg_class:$rs1),
(op2_type op2_reg_class:$rs2),
(result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_"#suffix#"_"# vlmul.MX#"_MASK")
(result_type (IMPLICIT_DEF)),
result_reg_class:$merge,
op1_reg_class:$rs1,
op2_reg_class:$rs2,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
Expand All @@ -354,16 +364,18 @@ multiclass VPatBinaryVL_XI<SDNode vop,
ValueType mask_type,
int sew,
LMULInfo vlmul,
VReg result_reg_class,
VReg vop_reg_class,
ComplexPattern SplatPatKind,
DAGOperand xop_kind> {
def : Pat<(result_type (vop
(vop1_type vop_reg_class:$rs1),
(vop2_type (SplatPatKind (XLenVT xop_kind:$rs2))),
(result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#_#suffix#_# vlmul.MX#"_MASK")
(result_type (IMPLICIT_DEF)),
result_reg_class:$merge,
vop_reg_class:$rs1,
xop_kind:$rs2,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
Expand All @@ -373,10 +385,12 @@ multiclass VPatBinaryVL_VV_VX<SDNode vop, string instruction_name> {
foreach vti = AllIntegerVectors in {
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
vti.RegClass>;
defm : VPatBinaryVL_XI<vop, instruction_name, "VX",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.RegClass, SplatPat, GPR>;
vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
SplatPat, GPR>;
}
}

Expand All @@ -386,7 +400,7 @@ multiclass VPatBinaryVL_VV_VX_VI<SDNode vop, string instruction_name,
foreach vti = AllIntegerVectors in {
defm : VPatBinaryVL_XI<vop, instruction_name, "VI",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.RegClass,
vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
!cast<ComplexPattern>(SplatPat#_#ImmType),
ImmType>;
}
Expand All @@ -398,10 +412,12 @@ multiclass VPatBinaryWVL_VV_VX<SDNode vop, string instruction_name> {
defvar wti = VtiToWti.Wti;
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
wti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass,
vti.RegClass>;
defm : VPatBinaryVL_XI<vop, instruction_name, "VX",
wti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.RegClass, SplatPat, GPR>;
vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass,
SplatPat, GPR>;
}
}
multiclass VPatBinaryWVL_VV_VX_WV_WX<SDNode vop, SDNode vop_w,
Expand All @@ -412,10 +428,12 @@ multiclass VPatBinaryWVL_VV_VX_WV_WX<SDNode vop, SDNode vop_w,
defvar wti = VtiToWti.Wti;
defm : VPatBinaryVL_V<vop_w, instruction_name, "WV",
wti.Vector, wti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass>;
vti.Log2SEW, vti.LMul, wti.RegClass, wti.RegClass,
vti.RegClass>;
defm : VPatBinaryVL_XI<vop_w, instruction_name, "WX",
wti.Vector, wti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, wti.RegClass, SplatPat, GPR>;
vti.Log2SEW, vti.LMul, wti.RegClass, wti.RegClass,
SplatPat, GPR>;
}
}

Expand All @@ -426,14 +444,16 @@ multiclass VPatBinaryVL_VF<SDNode vop,
ValueType mask_type,
int sew,
LMULInfo vlmul,
VReg result_reg_class,
VReg vop_reg_class,
RegisterClass scalar_reg_class> {
def : Pat<(result_type (vop (vop_type vop_reg_class:$rs1),
(vop_type (SplatFPOp scalar_reg_class:$rs2)),
(result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_"#vlmul.MX#"_MASK")
(result_type (IMPLICIT_DEF)),
result_reg_class:$merge,
vop_reg_class:$rs1,
scalar_reg_class:$rs2,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
Expand All @@ -443,21 +463,24 @@ multiclass VPatBinaryFPVL_VV_VF<SDNode vop, string instruction_name> {
foreach vti = AllFloatVectors in {
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
vti.RegClass>;
defm : VPatBinaryVL_VF<vop, instruction_name#"_V"#vti.ScalarSuffix,
vti.Vector, vti.Vector, vti.Mask, vti.Log2SEW,
vti.LMul, vti.RegClass, vti.ScalarRegClass>;
vti.LMul, vti.RegClass, vti.RegClass,
vti.ScalarRegClass>;
}
}

multiclass VPatBinaryFPVL_R_VF<SDNode vop, string instruction_name> {
foreach fvti = AllFloatVectors in {
def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2),
fvti.RegClass:$rs1,
(fvti.Vector fvti.RegClass:$merge),
(fvti.Mask V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_MASK")
(fvti.Vector (IMPLICIT_DEF)),
fvti.RegClass:$merge,
fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2,
(fvti.Mask V0), GPR:$vl, fvti.Log2SEW, TAIL_AGNOSTIC)>;
}
Expand Down Expand Up @@ -786,15 +809,15 @@ multiclass VPatWidenBinaryFPVL_VV_VF<SDNode op, PatFrags extop, string instructi
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Vector (extop (fvti.Vector fvti.RegClass:$rs1),
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Mask true_mask), VLOpFrag)),
srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VV_"#fvti.LMul.MX)
fvti.RegClass:$rs2, fvti.RegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
def : Pat<(fwti.Vector (op (fwti.Vector (extop (fvti.Vector fvti.RegClass:$rs2),
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Vector (extop (fvti.Vector (SplatFPOp fvti.ScalarRegClass:$rs1)),
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Mask true_mask), VLOpFrag)),
srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX)
fvti.RegClass:$rs2, fvti.ScalarRegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
Expand All @@ -808,14 +831,14 @@ multiclass VPatWidenBinaryFPVL_WV_WF<SDNode op, PatFrags extop, string instructi
def : Pat<(fwti.Vector (op (fwti.Vector fwti.RegClass:$rs2),
(fwti.Vector (extop (fvti.Vector fvti.RegClass:$rs1),
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Mask true_mask), VLOpFrag)),
srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WV_"#fvti.LMul.MX)
fwti.RegClass:$rs2, fvti.RegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
def : Pat<(fwti.Vector (op (fwti.Vector fwti.RegClass:$rs2),
(fwti.Vector (extop (fvti.Vector (SplatFPOp fvti.ScalarRegClass:$rs1)),
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Mask true_mask), VLOpFrag)),
srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_W"#fvti.ScalarSuffix#"_"#fvti.LMul.MX)
fwti.RegClass:$rs2, fvti.ScalarRegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
Expand All @@ -837,7 +860,7 @@ multiclass VPatNarrowShiftSplatExt_WX<SDNode op, PatFrags extop, string instruct
(op (wti.Vector wti.RegClass:$rs2),
(wti.Vector (extop (vti.Vector (SplatPat GPR:$rs1)),
(vti.Mask true_mask), VLOpFrag)),
(wti.Mask true_mask), VLOpFrag),
srcvalue, (wti.Mask true_mask), VLOpFrag),
(vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WX_"#vti.LMul.MX)
wti.RegClass:$rs2, GPR:$rs1, GPR:$vl, vti.Log2SEW)>;
Expand All @@ -853,8 +876,8 @@ multiclass VPatMultiplyAddVL_VV_VX<SDNode op, string instruction_name> {
(op vti.RegClass:$rs2,
(riscv_mul_vl_oneuse vti.RegClass:$rs1,
vti.RegClass:$rd,
(vti.Mask true_mask), VLOpFrag),
(vti.Mask true_mask), VLOpFrag)),
srcvalue, (vti.Mask true_mask), VLOpFrag),
srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VV_"# suffix)
vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
Expand All @@ -864,8 +887,8 @@ multiclass VPatMultiplyAddVL_VV_VX<SDNode op, string instruction_name> {
(op vti.RegClass:$rs2,
(riscv_mul_vl_oneuse (SplatPat XLenVT:$rs1),
vti.RegClass:$rd,
(vti.Mask true_mask), VLOpFrag),
(vti.Mask true_mask), VLOpFrag)),
srcvalue, (vti.Mask true_mask), VLOpFrag),
srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VX_" # suffix)
vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
Expand All @@ -880,17 +903,17 @@ multiclass VPatWidenMultiplyAddVL_VV_VX<PatFrag op1, string instruction_name> {
(riscv_add_vl wti.RegClass:$rd,
(op1 vti.RegClass:$rs1,
(vti.Vector vti.RegClass:$rs2),
(vti.Mask true_mask), VLOpFrag),
(vti.Mask true_mask), VLOpFrag)),
srcvalue, (vti.Mask true_mask), VLOpFrag),
srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VV_" # vti.LMul.MX)
wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
def : Pat<(wti.Vector
(riscv_add_vl wti.RegClass:$rd,
(op1 (SplatPat XLenVT:$rs1),
(vti.Vector vti.RegClass:$rs2),
(vti.Mask true_mask), VLOpFrag),
(vti.Mask true_mask), VLOpFrag)),
srcvalue, (vti.Mask true_mask), VLOpFrag),
srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VX_" # vti.LMul.MX)
wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
Expand All @@ -903,12 +926,12 @@ multiclass VPatNarrowShiftSplat_WX_WI<SDNode op, string instruction_name> {
defvar wti = vtiTowti.Wti;
def : Pat<(vti.Vector (riscv_trunc_vector_vl
(wti.Vector (op wti.RegClass:$rs1, (SplatPat XLenVT:$rs2),
true_mask, VLOpFrag)), true_mask, VLOpFrag)),
srcvalue, true_mask, VLOpFrag)), true_mask, VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WX_"#vti.LMul.MX)
wti.RegClass:$rs1, GPR:$rs2, GPR:$vl, vti.Log2SEW)>;
def : Pat<(vti.Vector (riscv_trunc_vector_vl
(wti.Vector (op wti.RegClass:$rs1, (SplatPat_uimm5 uimm5:$rs2),
true_mask, VLOpFrag)), true_mask, VLOpFrag)),
srcvalue, true_mask, VLOpFrag)), true_mask, VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WI_"#vti.LMul.MX)
wti.RegClass:$rs1, uimm5:$rs2, GPR:$vl, vti.Log2SEW)>;
}
Expand Down Expand Up @@ -991,16 +1014,16 @@ defm : VPatBinaryVL_VV_VX<riscv_sub_vl, "PseudoVSUB">;
// pattern operands
foreach vti = AllIntegerVectors in {
def : Pat<(riscv_sub_vl (vti.Vector (SplatPat (XLenVT GPR:$rs2))),
(vti.Vector vti.RegClass:$rs1), (vti.Mask V0),
VLOpFrag),
(vti.Vector vti.RegClass:$rs1),
vti.RegClass:$merge, (vti.Mask V0), VLOpFrag),
(!cast<Instruction>("PseudoVRSUB_VX_"# vti.LMul.MX#"_MASK")
(vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, GPR:$rs2,
vti.RegClass:$merge, vti.RegClass:$rs1, GPR:$rs2,
(vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
def : Pat<(riscv_sub_vl (vti.Vector (SplatPat_simm5 simm5:$rs2)),
(vti.Vector vti.RegClass:$rs1), (vti.Mask V0),
VLOpFrag),
(vti.Vector vti.RegClass:$rs1),
vti.RegClass:$merge, (vti.Mask V0), VLOpFrag),
(!cast<Instruction>("PseudoVRSUB_VI_"# vti.LMul.MX#"_MASK")
(vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, simm5:$rs2,
vti.RegClass:$merge, vti.RegClass:$rs1, simm5:$rs2,
(vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
}

Expand Down Expand Up @@ -1038,8 +1061,7 @@ foreach vti = AllIntegerVectors in {
// Emit shift by 1 as an add since it might be faster.
def : Pat<(riscv_shl_vl (vti.Vector vti.RegClass:$rs1),
(riscv_vmv_v_x_vl (vti.Vector undef), 1, (XLenVT srcvalue)),
(vti.Mask true_mask),
VLOpFrag),
srcvalue, (vti.Mask true_mask), VLOpFrag),
(!cast<Instruction>("PseudoVADD_VV_"# vti.LMul.MX)
vti.RegClass:$rs1, vti.RegClass:$rs1, GPR:$vl, vti.Log2SEW)>;
}
Expand Down Expand Up @@ -1140,8 +1162,10 @@ foreach vtiTowti = AllWidenableIntVectors in {
(riscv_add_vl wti.RegClass:$rd,
(riscv_vwmulsu_vl_oneuse (vti.Vector vti.RegClass:$rs1),
(SplatPat XLenVT:$rs2),
(vti.Mask true_mask), VLOpFrag),
(vti.Mask true_mask), VLOpFrag)),
srcvalue,
(vti.Mask true_mask),
VLOpFrag),
srcvalue, (vti.Mask true_mask),VLOpFrag)),
(!cast<Instruction>("PseudoVWMACCUS_VX_" # vti.LMul.MX)
wti.RegClass:$rd, vti.ScalarRegClass:$rs2, vti.RegClass:$rs1,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
Expand Down