Skip to content

Commit

Permalink
[SelectionDAG][AArch64][SVE] Perform SETCC condition legalization in …
Browse files Browse the repository at this point in the history 
…LegalizeVectorOps

This is currently performed in SelectionDAGLegalize, here we make it also
happen in LegalizeVectorOps, allowing a target to lower the SETCC condition
codes first in LegalizeVectorOps and then lower to a custom node afterwards,
without having to duplicate all of the SETCC condition legalization in the
target specific lowering.

As a result of this, fixed length floating point SETCC nodes can now be
properly lowered for SVE.

Differential Revision: https://reviews.llvm.org/D98939
  • Loading branch information
@brads55
brads55 committed Mar 29, 2021
1 parent 0411b23 commit 9745dce
Show file tree
Hide file tree
Showing 10 changed files with 1,048 additions and 238 deletions.
23 changes: 23 additions & 0 deletions llvm/include/llvm/CodeGen/TargetLowering.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4513,6 +4513,29 @@ class TargetLowering : public TargetLoweringBase {
/// method accepts vectors as its arguments.
SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const;

/// Legalize a SETCC with given LHS and RHS and condition code CC on the
/// current target.
///
/// If the SETCC has been legalized using AND / OR, then the legalized node
/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
/// will be set to false.
///
/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
/// then the values of LHS and RHS will be swapped, CC will be set to the
/// new condition, and NeedInvert will be set to false.
///
/// If the SETCC has been legalized using the inverse condcode, then LHS and
/// RHS will be unchanged, CC will set to the inverted condcode, and
/// NeedInvert will be set to true. The caller must invert the result of the
/// SETCC with SelectionDAG::getLogicalNOT() or take equivalent action to swap
/// the effect of a true/false result.
///
/// \returns true if the SetCC has been legalized, false if it hasn't.
bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS,
SDValue &RHS, SDValue &CC, bool &NeedInvert,
const SDLoc &dl, SDValue &Chain,
bool IsSignaling = false) const;

//===--------------------------------------------------------------------===//
// Instruction Emitting Hooks
//
Expand Down
163 changes: 7 additions & 156 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,6 @@ class SelectionDAGLegalize {
SDValue N1, SDValue N2,
ArrayRef<int> Mask) const;

bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
bool &NeedInvert, const SDLoc &dl, SDValue &Chain,
bool IsSignaling = false);

SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);

void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
Expand Down Expand Up @@ -1685,152 +1681,6 @@ void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
Results.push_back(Tmp2);
}

/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
/// target.
///
/// If the SETCC has been legalized using AND / OR, then the legalized node
/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
/// will be set to false.
///
/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
/// then the values of LHS and RHS will be swapped, CC will be set to the
/// new condition, and NeedInvert will be set to false.
///
/// If the SETCC has been legalized using the inverse condcode, then LHS and
/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
/// will be set to true. The caller must invert the result of the SETCC with
/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
/// of a true/false result.
///
/// \returns true if the SetCC has been legalized, false if it hasn't.
bool SelectionDAGLegalize::LegalizeSetCCCondCode(
EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, bool &NeedInvert,
const SDLoc &dl, SDValue &Chain, bool IsSignaling) {
MVT OpVT = LHS.getSimpleValueType();
ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
NeedInvert = false;
switch (TLI.getCondCodeAction(CCCode, OpVT)) {
default: llvm_unreachable("Unknown condition code action!");
case TargetLowering::Legal:
// Nothing to do.
break;
case TargetLowering::Expand: {
ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
std::swap(LHS, RHS);
CC = DAG.getCondCode(InvCC);
return true;
}
// Swapping operands didn't work. Try inverting the condition.
bool NeedSwap = false;
InvCC = getSetCCInverse(CCCode, OpVT);
if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
// If inverting the condition is not enough, try swapping operands
// on top of it.
InvCC = ISD::getSetCCSwappedOperands(InvCC);
NeedSwap = true;
}
if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
CC = DAG.getCondCode(InvCC);
NeedInvert = true;
if (NeedSwap)
std::swap(LHS, RHS);
return true;
}

ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
unsigned Opc = 0;
switch (CCCode) {
default: llvm_unreachable("Don't know how to expand this condition!");
case ISD::SETUO:
if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;
break;
}
assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
"If SETUE is expanded, SETOEQ or SETUNE must be legal!");
NeedInvert = true;
LLVM_FALLTHROUGH;
case ISD::SETO:
assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)
&& "If SETO is expanded, SETOEQ must be legal!");
CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
case ISD::SETONE:
case ISD::SETUEQ:
// If the SETUO or SETO CC isn't legal, we might be able to use
// SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
// of SETOGT/SETOLT to be legal, the other can be emulated by swapping
// the operands.
CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
if (!TLI.isCondCodeLegal(CC2, OpVT) &&
(TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
CC1 = ISD::SETOGT;
CC2 = ISD::SETOLT;
Opc = ISD::OR;
NeedInvert = ((unsigned)CCCode & 0x8U);
break;
}
LLVM_FALLTHROUGH;
case ISD::SETOEQ:
case ISD::SETOGT:
case ISD::SETOGE:
case ISD::SETOLT:
case ISD::SETOLE:
case ISD::SETUNE:
case ISD::SETUGT:
case ISD::SETUGE:
case ISD::SETULT:
case ISD::SETULE:
// If we are floating point, assign and break, otherwise fall through.
if (!OpVT.isInteger()) {
// We can use the 4th bit to tell if we are the unordered
// or ordered version of the opcode.
CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
break;
}
// Fallthrough if we are unsigned integer.
LLVM_FALLTHROUGH;
case ISD::SETLE:
case ISD::SETGT:
case ISD::SETGE:
case ISD::SETLT:
case ISD::SETNE:
case ISD::SETEQ:
// If all combinations of inverting the condition and swapping operands
// didn't work then we have no means to expand the condition.
llvm_unreachable("Don't know how to expand this condition!");
}

SDValue SetCC1, SetCC2;
if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
// If we aren't the ordered or unorder operation,
// then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain,
IsSignaling);
SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain,
IsSignaling);
} else {
// Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain,
IsSignaling);
SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain,
IsSignaling);
}
if (Chain)
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
SetCC2.getValue(1));
LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
RHS = SDValue();
CC = SDValue();
return true;
}
}
return false;
}

/// Emit a store/load combination to the stack. This stores
/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
/// a load from the stack slot to DestVT, extending it if needed.
Expand Down Expand Up @@ -3729,8 +3579,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Tmp2 = Node->getOperand(1 + Offset);
Tmp3 = Node->getOperand(2 + Offset);
bool Legalized =
LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, Tmp3,
NeedInvert, dl, Chain, IsSignaling);
TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3,
NeedInvert, dl, Chain, IsSignaling);

if (Legalized) {
// If we expanded the SETCC by swapping LHS and RHS, or by inverting the
Expand Down Expand Up @@ -3825,8 +3675,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}

if (!Legalized) {
Legalized = LegalizeSetCCCondCode(getSetCCResultType(Tmp1.getValueType()),
Tmp1, Tmp2, CC, NeedInvert, dl, Chain);
Legalized = TLI.LegalizeSetCCCondCode(
DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
NeedInvert, dl, Chain);

assert(Legalized && "Can't legalize SELECT_CC with legal condition!");

Expand Down Expand Up @@ -3860,8 +3711,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Tmp4 = Node->getOperand(1); // CC

bool Legalized =
LegalizeSetCCCondCode(getSetCCResultType(Tmp2.getValueType()), Tmp2,
Tmp3, Tmp4, NeedInvert, dl, Chain);
TLI.LegalizeSetCCCondCode(DAG, getSetCCResultType(Tmp2.getValueType()),
Tmp2, Tmp3, Tmp4, NeedInvert, dl, Chain);
(void)Legalized;
assert(Legalized && "Can't legalize BR_CC with legal condition!");

Expand Down
56 changes: 54 additions & 2 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ class VectorLegalizer {
SDValue ExpandStore(SDNode *N);
SDValue ExpandFNEG(SDNode *Node);
void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
Expand Down Expand Up @@ -396,7 +397,6 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::SELECT:
case ISD::VSELECT:
case ISD::SELECT_CC:
case ISD::SETCC:
case ISD::ZERO_EXTEND:
case ISD::ANY_EXTEND:
case ISD::TRUNCATE:
Expand Down Expand Up @@ -495,6 +495,14 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(1).getValueType());
break;
case ISD::SETCC: {
MVT OpVT = Node->getOperand(0).getSimpleValueType();
ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
Action = TLI.getCondCodeAction(CCCode, OpVT);
if (Action == TargetLowering::Legal)
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
break;
}
}

LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
Expand Down Expand Up @@ -762,7 +770,7 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
ExpandFSUB(Node, Results);
return;
case ISD::SETCC:
Results.push_back(UnrollVSETCC(Node));
ExpandSETCC(Node, Results);
return;
case ISD::ABS:
if (TLI.expandABS(Node, Tmp, DAG)) {
Expand Down Expand Up @@ -1331,6 +1339,50 @@ void VectorLegalizer::ExpandFSUB(SDNode *Node,
Results.push_back(Tmp);
}

void VectorLegalizer::ExpandSETCC(SDNode *Node,
SmallVectorImpl<SDValue> &Results) {
bool NeedInvert = false;
SDLoc dl(Node);
MVT OpVT = Node->getOperand(0).getSimpleValueType();
ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();

if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
Results.push_back(UnrollVSETCC(Node));
return;
}

SDValue Chain;
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
SDValue CC = Node->getOperand(2);
bool Legalized = TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS,
RHS, CC, NeedInvert, dl, Chain);

if (Legalized) {
// If we expanded the SETCC by swapping LHS and RHS, or by inverting the
// condition code, create a new SETCC node.
if (CC.getNode())
LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
Node->getFlags());

// If we expanded the SETCC by inverting the condition code, then wrap
// the existing SETCC in a NOT to restore the intended condition.
if (NeedInvert)
LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
} else {
// Otherwise, SETCC for the given comparison type must be completely
// illegal; expand it into a SELECT_CC.
EVT VT = Node->getValueType(0);
LHS =
DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
LHS->setFlags(Node->getFlags());
}

Results.push_back(LHS);
}

void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
SmallVectorImpl<SDValue> &Results) {
SDValue Result, Overflow;
Expand Down
Loading

0 comments on commit 9745dce

Please sign in to comment.