Skip to content

[AArch64] Combine vector FNEG+FMA into FNML[A|S] #167900

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
+440 −4
Draft

[AArch64] Combine vector FNEG+FMA into FNML[A|S] #167900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9aa8dd5
Combine vector FNEG+FMA into `FNML[A|S]`
dheaton-arm Nov 13, 2025
d4e4360
Address review comments
dheaton-arm Nov 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1176,6 +1176,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setTargetDAGCombine(ISD::VECTOR_DEINTERLEAVE);
setTargetDAGCombine(ISD::CTPOP);

setTargetDAGCombine(ISD::FMA);

// In case of strict alignment, avoid an excessive number of byte wide stores.
MaxStoresPerMemsetOptSize = 8;
MaxStoresPerMemset =
Expand Down Expand Up @@ -20444,6 +20446,47 @@ static SDValue performFADDCombine(SDNode *N,
return SDValue();
}

static SDValue performFMACombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
SelectionDAG &DAG = DCI.DAG;
SDValue OpA = N->getOperand(0);
SDValue OpB = N->getOperand(1);
SDValue OpC = N->getOperand(2);
EVT VT = N->getValueType(0);
SDLoc DL(N);

// fma(a, b, neg(c)) -> fnmls(a, b, c)
// fma(neg(a), b, neg(c)) -> fnmla(a, b, c)
// fma(a, neg(b), neg(c)) -> fnmla(a, b, c)
if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
!Subtarget->isSVEorStreamingSVEAvailable() ||
OpC.getOpcode() != ISD::FNEG) {
return SDValue();
}
unsigned int Opcode;
if (OpA.getOpcode() == ISD::FNEG) {
OpA = OpA.getOperand(0);
Opcode = AArch64ISD::FNMLA_PRED;
} else if (OpB.getOpcode() == ISD::FNEG) {
OpB = OpB.getOperand(0);
Opcode = AArch64ISD::FNMLA_PRED;
} else {
Opcode = AArch64ISD::FNMLS_PRED;
Comment on lines +20468 to +20475
Copy link
Member

@MacDue MacDue Nov 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need these new ISD nodes? These ISEL patterns exist for scalable vectors (as shown by: https://godbolt.org/z/1Pn78GKox). So maybe you can promote the FNEG and FMA to scalable vectors, and rely on the existing patterns to lower them?

}
OpC = OpC.getOperand(0);
auto Pg = getPredicateForVector(DAG, DL, VT);
if (VT.isFixedLengthVector()) {
EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
OpA = convertToScalableVector(DAG, ContainerVT, OpA);
OpB = convertToScalableVector(DAG, ContainerVT, OpB);
OpC = convertToScalableVector(DAG, ContainerVT, OpC);
auto ScalableRes = DAG.getNode(Opcode, DL, ContainerVT, Pg, OpA, OpB, OpC);
return convertFromScalableVector(DAG, VT, ScalableRes);
}
return DAG.getNode(Opcode, DL, VT, Pg, OpA, OpB, OpC);
}

static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16) {
switch (Opcode) {
case ISD::STRICT_FADD:
Expand Down Expand Up @@ -27977,6 +28020,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performANDCombine(N, DCI);
case ISD::FADD:
return performFADDCombine(N, DCI);
case ISD::FMA:
return performFMACombine(N, DCI, Subtarget);
case ISD::INTRINSIC_WO_CHAIN:
return performIntrinsicCombine(N, DCI, Subtarget);
case ISD::ANY_EXTEND:
Expand Down
10 changes: 6 additions & 4 deletions llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,8 @@ def AArch64udiv_p : SDNode<"AArch64ISD::UDIV_PRED", SDT_AArch64Arith>;
def AArch64umax_p : SDNode<"AArch64ISD::UMAX_PRED", SDT_AArch64Arith>;
def AArch64umin_p : SDNode<"AArch64ISD::UMIN_PRED", SDT_AArch64Arith>;
def AArch64umulh_p : SDNode<"AArch64ISD::MULHU_PRED", SDT_AArch64Arith>;
def AArch64fnmla_p_node : SDNode<"AArch64ISD::FNMLA_PRED", SDT_AArch64FMA>;
def AArch64fnmls_p_node : SDNode<"AArch64ISD::FNMLS_PRED", SDT_AArch64FMA>;

def AArch64fadd_p_contract : PatFrag<(ops node:$op1, node:$op2, node:$op3),
(AArch64fadd_p node:$op1, node:$op2, node:$op3), [{
Expand Down Expand Up @@ -460,13 +462,13 @@ def AArch64fmlsidx : PatFrags<(ops node:$acc, node:$op1, node:$op2, node:$idx),


def AArch64fnmla_p : PatFrags<(ops node:$pg, node:$za, node:$zn, node:$zm),
[(int_aarch64_sve_fnmla_u node:$pg, node:$za, node:$zn, node:$zm),
(AArch64fma_p node:$pg, (AArch64fneg_mt node:$pg, node:$zn, (undef)), node:$zm, (AArch64fneg_mt node:$pg, node:$za, (undef))),
[(AArch64fnmla_p_node node:$pg, node:$zn, node:$zm, node:$za),
(int_aarch64_sve_fnmla_u node:$pg, node:$za, node:$zn, node:$zm),
(AArch64fneg_mt_nsz node:$pg, (AArch64fma_p node:$pg, node:$zn, node:$zm, node:$za), (undef))]>;

def AArch64fnmls_p : PatFrags<(ops node:$pg, node:$za, node:$zn, node:$zm),
[(int_aarch64_sve_fnmls_u node:$pg, node:$za, node:$zn, node:$zm),
(AArch64fma_p node:$pg, node:$zn, node:$zm, (AArch64fneg_mt node:$pg, node:$za, (undef)))]>;
[(AArch64fnmls_p_node node:$pg, node:$zn, node:$zm, node:$za),
(int_aarch64_sve_fnmls_u node:$pg, node:$za, node:$zn, node:$zm)]>;

def AArch64fsubr_p : PatFrag<(ops node:$pg, node:$op1, node:$op2),
(AArch64fsub_p node:$pg, node:$op2, node:$op1)>;
Expand Down
Loading
Loading