[GlobalISel] convergent intrinsics

Introduced the convergent equivalent of the existing G_INTRINSIC opcodes:

- G_INTRINSIC_CONVERGENT
- G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS

Out of the targets that currently have some support for GlobalISel, the patch
assumes that the convergent intrinsics only relevant to SPIRV and AMDGPU.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D154766

llvm/docs/GlobalISel/GenericOpcode.rst

@@ -856,13 +856,25 @@ it during passes like legalization. This is needed because calls to exception
 throw routines do not return, so no code that must be on an executable path must
 be placed after throwing.
 
-G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+G_INTRINSIC, G_INTRINSIC_CONVERGENT
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Call an intrinsic
+Call an intrinsic that has no side-effects.
 
-The _W_SIDE_EFFECTS version is considered to have unknown side-effects and
-as such cannot be reordered across other side-effecting instructions.
+The _CONVERGENT variant corresponds to an LLVM IR intrinsic marked `convergent`.
+
+.. note::
+
+  Unlike SelectionDAG, there is no _VOID variant. Both of these are permitted
+  to have zero, one, or multiple results.
+
+G_INTRINSIC_W_SIDE_EFFECTS, G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Call an intrinsic that is considered to have unknown side-effects and as such
+cannot be reordered across other side-effecting instructions.
+
+The _CONVERGENT variant corresponds to an LLVM IR intrinsic marked `convergent`.
 
 .. note::
 

llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h

@@ -366,14 +366,33 @@ class GIntrinsic final : public GenericMachineInstr {
   }
 
   bool is(Intrinsic::ID ID) const { return getIntrinsicID() == ID; }
+
   bool hasSideEffects() const {
-    return getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
+    switch (getOpcode()) {
+    case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+    case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  bool isConvergent() const {
+    switch (getOpcode()) {
+    case TargetOpcode::G_INTRINSIC_CONVERGENT:
+    case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
+      return true;
+    default:
+      return false;
+    }
   }
 
   static bool classof(const MachineInstr *MI) {
     switch (MI->getOpcode()) {
     case TargetOpcode::G_INTRINSIC:
     case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+    case TargetOpcode::G_INTRINSIC_CONVERGENT:
+    case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
       return true;
     default:
       return false;

llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h

@@ -1108,20 +1108,25 @@ class MachineIRBuilder {
   MachineInstrBuilder buildInsert(const DstOp &Res, const SrcOp &Src,
                                   const SrcOp &Op, unsigned Index);
 
-  /// Build and insert either a G_INTRINSIC (if \p HasSideEffects is false) or
-  /// G_INTRINSIC_W_SIDE_EFFECTS instruction. Its first operand will be the
-  /// result register definition unless \p Reg is NoReg (== 0). The second
-  /// operand will be the intrinsic's ID.
+  /// Build and insert a G_INTRINSIC instruction.
   ///
-  /// Callers are expected to add the required definitions and uses afterwards.
+  /// There are four different opcodes based on combinations of whether the
+  /// intrinsic has side effects and whether it is convergent. These properties
+  /// can be specified as explicit parameters, or else they are retrieved from
+  /// the MCID for the intrinsic.
+  ///
+  /// The parameter \p Res provides the Registers or MOs that will be defined by
+  /// this instruction.
   ///
   /// \pre setBasicBlock or setMI must have been called.
   ///
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<Register> Res,
-                                     bool HasSideEffects);
+                                     bool HasSideEffects, bool isConvergent);
+  MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<Register> Res);
   MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<DstOp> Res,
-                                     bool HasSideEffects);
+                                     bool HasSideEffects, bool isConvergent);
+  MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<DstOp> Res);
 
   /// Build and insert \p Res = G_FPTRUNC \p Op
   ///

llvm/include/llvm/Support/TargetOpcodes.def

@@ -426,6 +426,12 @@ HANDLE_TARGET_OPCODE(G_INTRINSIC)
 /// Generic intrinsic use (with side effects).
 HANDLE_TARGET_OPCODE(G_INTRINSIC_W_SIDE_EFFECTS)
 
+/// Generic intrinsic use (without side effects).
+HANDLE_TARGET_OPCODE(G_INTRINSIC_CONVERGENT)
+
+/// Generic intrinsic use (with side effects).
+HANDLE_TARGET_OPCODE(G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS)
+
 /// Generic extension allowing rubbish in high bits.
 HANDLE_TARGET_OPCODE(G_ANYEXT)
 

llvm/include/llvm/Target/GenericOpcodes.td

@@ -1228,10 +1228,6 @@ def G_INTRINSIC : GenericInstruction {
   let OutOperandList = (outs);
   let InOperandList = (ins unknown:$intrin, variable_ops);
   let hasSideEffects = false;
-
-  // Conservatively assume this is convergent. If there turnes out to
-  // be a need, there should be separate convergent intrinsic opcodes.
-  let isConvergent = 1;
 }
 
 // Intrinsic with side effects.
@@ -1241,9 +1237,23 @@ def G_INTRINSIC_W_SIDE_EFFECTS : GenericInstruction {
   let hasSideEffects = true;
   let mayLoad = true;
   let mayStore = true;
+}
 
-  // Conservatively assume this is convergent. If there turnes out to
-  // be a need, there should be separate convergent intrinsic opcodes.
+// Convergent intrinsic without side effects.
+def G_INTRINSIC_CONVERGENT : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins unknown:$intrin, variable_ops);
+  let hasSideEffects = false;
+  let isConvergent = true;
+}
+
+// Convergent intrinsic with side effects.
+def G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins unknown:$intrin, variable_ops);
+  let hasSideEffects = true;
+  let mayLoad = true;
+  let mayStore = true;
   let isConvergent = true;
 }
 

llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td

@@ -39,6 +39,10 @@ class GINodeEquiv<Instruction i, SDNode node> {
   // SelectionDAG has one setcc for all compares. This differentiates
   // for G_ICMP and G_FCMP.
   Instruction IfFloatingPoint = ?;
+
+  // SelectionDAG does not differentiate between convergent and non-convergent
+  // intrinsics. This specifies an alternate opcode for a convergent intrinsic.
+  Instruction IfConvergent = ?;
 }
 
 // These are defined in the same order as the G_* instructions.
@@ -106,10 +110,17 @@ def : GINodeEquiv<G_FLOG2, flog2>;
 def : GINodeEquiv<G_FLDEXP, fldexp>;
 def : GINodeEquiv<G_FCANONICALIZE, fcanonicalize>;
 def : GINodeEquiv<G_IS_FPCLASS, is_fpclass>;
-def : GINodeEquiv<G_INTRINSIC, intrinsic_wo_chain>;
+
+def : GINodeEquiv<G_INTRINSIC, intrinsic_wo_chain> {
+  let IfConvergent = G_INTRINSIC_CONVERGENT;
+}
+
 // ISD::INTRINSIC_VOID can also be handled with G_INTRINSIC_W_SIDE_EFFECTS.
-def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_void>;
-def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_w_chain>;
+let IfConvergent = G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS in {
+  def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_void>;
+  def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_w_chain>;
+}
+
 def : GINodeEquiv<G_BR, br>;
 def : GINodeEquiv<G_BSWAP, bswap>;
 def : GINodeEquiv<G_BITREVERSE, bitreverse>;

llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp

@@ -48,6 +48,8 @@ Align GISelKnownBits::computeKnownAlignment(Register R, unsigned Depth) {
   }
   case TargetOpcode::G_INTRINSIC:
   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+  case TargetOpcode::G_INTRINSIC_CONVERGENT:
+  case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
   default:
     return TL.computeKnownAlignForTargetInstr(*this, R, MRI, Depth + 1);
   }
@@ -726,6 +728,8 @@ unsigned GISelKnownBits::computeNumSignBits(Register R,
   }
   case TargetOpcode::G_INTRINSIC:
   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+  case TargetOpcode::G_INTRINSIC_CONVERGENT:
+  case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
   default: {
     unsigned NumBits =
       TL.computeNumSignBitsForTargetInstr(*this, R, DemandedElts, MRI, Depth);

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

@@ -2533,8 +2533,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
 
   // Ignore the callsite attributes. Backend code is most likely not expecting
   // an intrinsic to sometimes have side effects and sometimes not.
-  MachineInstrBuilder MIB =
-      MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
+  MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
   if (isa<FPMathOperator>(CI))
     MIB->copyIRFlags(CI);
 
@@ -2885,7 +2884,7 @@ bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuil
     }
   }
 
-  MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>(), true);
+  MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
   return true;
 }
 

llvm/lib/CodeGen/GlobalISel/LegacyLegalizerInfo.cpp

@@ -76,6 +76,9 @@ LegacyLegalizerInfo::LegacyLegalizerInfo() {
 
   setScalarAction(TargetOpcode::G_INTRINSIC, 0, {{1, Legal}});
   setScalarAction(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS, 0, {{1, Legal}});
+  setScalarAction(TargetOpcode::G_INTRINSIC_CONVERGENT, 0, {{1, Legal}});
+  setScalarAction(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS, 0,
+                  {{1, Legal}});
 
   setLegalizeScalarToDifferentSizeStrategy(
       TargetOpcode::G_IMPLICIT_DEF, 0, narrowToSmallerAndUnsupportedIfTooSmall);

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

@@ -119,8 +119,7 @@ LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
 
   MIRBuilder.setInstrAndDebugLoc(MI);
 
-  if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
-      MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
+  if (isa<GIntrinsic>(MI))
     return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize;
   auto Step = LI.getAction(MI, MRI);
   switch (Step.Action) {

llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp

@@ -775,30 +775,55 @@ MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
   return buildInstr(TargetOpcode::G_INSERT, Res, {Src, Op, uint64_t(Index)});
 }
 
-MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
-                                                     ArrayRef<Register> ResultRegs,
-                                                     bool HasSideEffects) {
-  auto MIB =
-      buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
-                                : TargetOpcode::G_INTRINSIC);
+static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
+  if (HasSideEffects && IsConvergent)
+    return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
+  if (HasSideEffects)
+    return TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
+  if (IsConvergent)
+    return TargetOpcode::G_INTRINSIC_CONVERGENT;
+  return TargetOpcode::G_INTRINSIC;
+}
+
+MachineInstrBuilder
+MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
+                                 ArrayRef<Register> ResultRegs,
+                                 bool HasSideEffects, bool isConvergent) {
+  auto MIB = buildInstr(getIntrinsicOpcode(HasSideEffects, isConvergent));
   for (unsigned ResultReg : ResultRegs)
     MIB.addDef(ResultReg);
   MIB.addIntrinsicID(ID);
   return MIB;
 }
 
+MachineInstrBuilder
+MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
+                                 ArrayRef<Register> ResultRegs) {
+  auto Attrs = Intrinsic::getAttributes(getContext(), ID);
+  bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
+  bool isConvergent = Attrs.hasFnAttr(Attribute::Convergent);
+  return buildIntrinsic(ID, ResultRegs, HasSideEffects, isConvergent);
+}
+
 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
                                                      ArrayRef<DstOp> Results,
-                                                     bool HasSideEffects) {
-  auto MIB =
-      buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
-                                : TargetOpcode::G_INTRINSIC);
+                                                     bool HasSideEffects,
+                                                     bool isConvergent) {
+  auto MIB = buildInstr(getIntrinsicOpcode(HasSideEffects, isConvergent));
   for (DstOp Result : Results)
     Result.addDefToMIB(*getMRI(), MIB);
   MIB.addIntrinsicID(ID);
   return MIB;
 }
 
+MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
+                                                     ArrayRef<DstOp> Results) {
+  auto Attrs = Intrinsic::getAttributes(getContext(), ID);
+  bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
+  bool isConvergent = Attrs.hasFnAttr(Attribute::Convergent);
+  return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
+}
+
 MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
                                                  const SrcOp &Op) {
   return buildInstr(TargetOpcode::G_TRUNC, Res, Op);