[ISel] DAGCombine clmul -> clmul[hr]

Author: artagnon

llvm/include/llvm/CodeGen/SDPatternMatch.h

@@ -919,6 +919,11 @@ inline BinaryOpc_match<LHS, RHS> m_Rotr(const LHS &L, const RHS &R) {
   return BinaryOpc_match<LHS, RHS>(ISD::ROTR, L, R);
 }
 
+template <typename LHS, typename RHS>
+inline BinaryOpc_match<LHS, RHS> m_Clmul(const LHS &L, const RHS &R) {
+  return BinaryOpc_match<LHS, RHS>(ISD::CLMUL, L, R);
+}
+
 template <typename LHS, typename RHS>
 inline BinaryOpc_match<LHS, RHS, true> m_FAdd(const LHS &L, const RHS &R) {
   return BinaryOpc_match<LHS, RHS, true>(ISD::FADD, L, R);

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

@@ -10315,6 +10315,25 @@ SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
   if (SDValue R = combineShiftOfShiftedLogic(N, DAG))
     return R;
 
+  // Fold clmul(zext(x), zext(y)) >> (BW - 1 | BW) -> clmul(r|h)(x, y).
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+  SDValue X, Y;
+  if (sd_match(N, m_Srl(m_Clmul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y))),
+                        m_SpecificInt(VT.getScalarSizeInBits() / 2 - 1))))
+    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
+                       DAG.getNode(ISD::CLMULR, DL, X.getValueType(), X, Y));
+  if (sd_match(N, m_Srl(m_Clmul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y))),
+                        m_SpecificInt(VT.getScalarSizeInBits() / 2))))
+    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
+                       DAG.getNode(ISD::CLMULH, DL, X.getValueType(), X, Y));
+
+  // Fold bitreverse(clmul(bitreverse(x), bitreverse(y))) >> 1 -> clmulh(x, y).
+  if (sd_match(N, m_Srl(m_BitReverse(m_Clmul(m_BitReverse(m_Value(X)),
+                                             m_BitReverse(m_Value(Y)))),
+                        m_SpecificInt(1))))
+    return DAG.getNode(ISD::CLMULH, DL, VT, X, Y);
+
   // We want to pull some binops through shifts, so that we have (and (shift))
   // instead of (shift (and)), likewise for add, or, xor, etc.  This sort of
   // thing happens with address calculations, so it's important to canonicalize
@@ -10350,8 +10369,6 @@ SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
     return SDValue();
 
   // Attempt to fold the constants, shifting the binop RHS by the shift amount.
-  SDLoc DL(N);
-  EVT VT = N->getValueType(0);
   if (SDValue NewRHS = DAG.FoldConstantArithmetic(
           N->getOpcode(), DL, VT, {LHS.getOperand(1), N->getOperand(1)})) {
     SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0),
@@ -11771,6 +11788,11 @@ SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
       sd_match(N, m_BitReverse(m_Shl(m_BitReverse(m_Value(X)), m_Value(Y)))))
     return DAG.getNode(ISD::SRL, DL, VT, X, Y);
 
+  // fold bitreverse(clmul(bitreverse(x), bitreverse(y))) -> clmulr(x, y)
+  if (sd_match(N, m_BitReverse(m_Clmul(m_BitReverse(m_Value(X)),
+                                       m_BitReverse(m_Value(Y))))))
+    return DAG.getNode(ISD::CLMULR, DL, VT, X, Y);
+
   return SDValue();
 }
 

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

@@ -8307,13 +8307,14 @@ SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
   SDValue X = Node->getOperand(0);
   SDValue Y = Node->getOperand(1);
   unsigned BW = VT.getScalarSizeInBits();
+  unsigned Opcode = Node->getOpcode();
 
-  if (VT.isVector() && isOperationLegalOrCustomOrPromote(
-                           Node->getOpcode(), VT.getVectorElementType()))
+  if (VT.isVector() &&
+      isOperationLegalOrCustomOrPromote(Opcode, VT.getVectorElementType()))
     return DAG.UnrollVectorOp(Node);
 
   SDValue Res = DAG.getConstant(0, DL, VT);
-  switch (Node->getOpcode()) {
+  switch (Opcode) {
   case ISD::CLMUL: {
     for (unsigned I = 0; I < BW; ++I) {
       SDValue Mask = DAG.getConstant(APInt::getOneBitSet(BW, I), DL, VT);
@@ -8326,12 +8327,26 @@ SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
   case ISD::CLMULR:
   case ISD::CLMULH: {
     EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), 2 * BW);
+    // For example, ExtVT = i64 based operations aren't legal on rv32; use
+    // bitreverse-based lowering in this case.
+    if (!isOperationLegalOrCustom(ISD::ZERO_EXTEND, ExtVT) ||
+        !isOperationLegalOrCustom(ISD::SRL, ExtVT)) {
+      SDValue XRev = DAG.getNode(ISD::BITREVERSE, DL, VT, X);
+      SDValue YRev = DAG.getNode(ISD::BITREVERSE, DL, VT, Y);
+      SDValue ClMul = DAG.getNode(ISD::CLMUL, DL, VT, XRev, YRev);
+      Res = DAG.getNode(ISD::BITREVERSE, DL, VT, ClMul);
+      Res = Opcode == ISD::CLMULR
+                ? Res
+                : DAG.getNode(ISD::SRL, DL, VT, Res,
+                              DAG.getShiftAmountConstant(1, VT, DL));
+      break;
+    }
     SDValue XExt = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, X);
     SDValue YExt = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, Y);
     SDValue ClMul = DAG.getNode(ISD::CLMUL, DL, ExtVT, XExt, YExt);
-    unsigned ShtAmt = Node->getOpcode() == ISD::CLMULR ? BW - 1 : BW;
+    unsigned ShtAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
     SDValue HiBits = DAG.getNode(ISD::SRL, DL, ExtVT, ClMul,
-                                 DAG.getShiftAmountConstant(ShtAmt, VT, DL));
+                                 DAG.getShiftAmountConstant(ShtAmt, ExtVT, DL));
     Res = DAG.getNode(ISD::TRUNCATE, DL, VT, HiBits);
     break;
   }