Use cinv and cneg instead of csel when possible

src/coreclr/jit/codegen.h

@@ -895,7 +895,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     void genCodeForCompare(GenTreeOp* tree);
 #ifdef TARGET_ARM64
     void genCodeForCCMP(GenTreeCCMP* ccmp);
-    void genCodeForCinc(GenTreeOp* cinc);
+    void genCodeForCincOrCinv(GenTreeOp* cinc, const bool isCinc);
 #endif
     void genCodeForSelect(GenTreeOp* select);
     void genIntrinsic(GenTreeIntrinsic* treeNode);

src/coreclr/jit/codegenarm64.cpp

@@ -4681,47 +4681,47 @@ void CodeGen::genCodeForSelect(GenTreeOp* tree)
 }
 
 //------------------------------------------------------------------------
-// genCodeForCinc: Produce code for a GT_CINC/GT_CINCCC node.
+// genCodeForCincOrCinv: Produce code for a GT_CINC/GT_CINCCC/GT_CINV/GT_CINVCC node.
 //
 // Arguments:
-//    tree - the node
+//    tree   - The node.
+//    isCinc - If true, emit the 'cinc' instruction otherwise the 'cinv' instruction.
 //
-void CodeGen::genCodeForCinc(GenTreeOp* cinc)
+void CodeGen::genCodeForCincOrCinv(GenTreeOp* condOp, const bool isCinc)
 {
-    assert(cinc->OperIs(GT_CINC, GT_CINCCC));
+    assert(condOp->OperIs(GT_CINC, GT_CINCCC, GT_CINV, GT_CINVCC));
 
     GenTree* opcond = nullptr;
-    GenTree* op     = cinc->gtOp1;
-    if (cinc->OperIs(GT_CINC))
+    GenTree* op     = condOp->gtOp1;
+    if (condOp->OperIs(GT_CINC, GT_CINV))
     {
-        opcond = cinc->gtOp1;
-        op     = cinc->gtOp2;
+        opcond = condOp->gtOp1;
+        op     = condOp->gtOp2;
         genConsumeRegs(opcond);
     }
 
-    emitter*  emit   = GetEmitter();
     var_types opType = genActualType(op->TypeGet());
-    emitAttr  attr   = emitActualTypeSize(cinc->TypeGet());
+    emitAttr  attr   = emitActualTypeSize(condOp->TypeGet());
 
     assert(!op->isUsedFromMemory());
     genConsumeRegs(op);
 
     GenCondition cond;
 
-    if (cinc->OperIs(GT_CINC))
+    if (condOp->OperIs(GT_CINC, GT_CINV))
     {
         assert(!opcond->isContained());
         // Condition has been generated into a register - move it into flags.
-        emit->emitIns_R_I(INS_cmp, emitActualTypeSize(opcond), opcond->GetRegNum(), 0);
+        GetEmitter()->emitIns_R_I(INS_cmp, emitActualTypeSize(opcond), opcond->GetRegNum(), 0);
         cond = GenCondition::NE;
     }
     else
     {
-        assert(cinc->OperIs(GT_CINCCC));
-        cond = cinc->AsOpCC()->gtCondition;
+        assert(condOp->OperIs(GT_CINCCC, GT_CINVCC));
+        cond = condOp->AsOpCC()->gtCondition;
     }
     const GenConditionDesc& prevDesc  = GenConditionDesc::Get(cond);
-    regNumber               targetReg = cinc->GetRegNum();
+    regNumber               targetReg = condOp->GetRegNum();
     regNumber               srcReg;
 
     if (op->isContained())
@@ -4735,9 +4735,10 @@ void CodeGen::genCodeForCinc(GenTreeOp* cinc)
     }
 
     assert(prevDesc.oper != GT_OR && prevDesc.oper != GT_AND);
-    emit->emitIns_R_R_COND(INS_cinc, attr, targetReg, srcReg, JumpKindToInsCond(prevDesc.jumpKind1));
+    GetEmitter()->emitIns_R_R_COND(isCinc ? INS_cinc : INS_cinv, attr, targetReg, srcReg,
+                                   JumpKindToInsCond(prevDesc.jumpKind1));
     regSet.verifyRegUsed(targetReg);
-    genProduceReg(cinc);
+    genProduceReg(condOp);
 }
 
 //------------------------------------------------------------------------

src/coreclr/jit/codegenarmarch.cpp

@@ -361,7 +361,12 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
 
         case GT_CINC:
         case GT_CINCCC:
-            genCodeForCinc(treeNode->AsOp());
+            genCodeForCincOrCinv(treeNode->AsOp(), true);
+            break;
+
+        case GT_CINV:
+        case GT_CINVCC:
+            genCodeForCincOrCinv(treeNode->AsOp(), false);
             break;
 
         case GT_SELECTCC:

src/coreclr/jit/compiler.hpp

@@ -4146,6 +4146,7 @@ void GenTree::VisitOperands(TVisitor visitor)
 #ifdef TARGET_ARM64
         case GT_CNEG_LT:
         case GT_CINCCC:
+        case GT_CINVCC:
 #endif // TARGET_ARM64
         case GT_STORE_LCL_VAR:
         case GT_STORE_LCL_FLD:

src/coreclr/jit/gentree.cpp

@@ -6258,6 +6258,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse)
 #ifdef TARGET_ARM64
         case GT_CNEG_LT:
         case GT_CINCCC:
+        case GT_CINVCC:
 #endif // TARGET_ARM64
         case GT_STORE_LCL_VAR:
         case GT_STORE_LCL_FLD:
@@ -9516,6 +9517,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node)
 #ifdef TARGET_ARM64
         case GT_CNEG_LT:
         case GT_CINCCC:
+        case GT_CINVCC:
 #endif // TARGET_ARM64
         case GT_STORE_LCL_VAR:
         case GT_STORE_LCL_FLD:
@@ -12107,7 +12109,7 @@ void Compiler::gtDispTree(GenTree*     tree,
             printf(" cond=%s", tree->AsOpCC()->gtCondition.Name());
         }
 #ifdef TARGET_ARM64
-        else if (tree->OperIs(GT_CINCCC))
+        else if (tree->OperIs(GT_CINCCC, GT_CINVCC))
         {
             printf(" cond=%s", tree->AsOpCC()->gtCondition.Name());
         }

src/coreclr/jit/gentree.h

@@ -1691,7 +1691,7 @@ struct GenTree
         }
 #endif
 #if defined(TARGET_ARM64)
-        if (OperIs(GT_CCMP, GT_CINCCC))
+        if (OperIs(GT_CCMP, GT_CINCCC, GT_CINVCC))
         {
             return true;
         }
@@ -8735,7 +8735,11 @@ struct GenTreeOpCC : public GenTreeOp
     GenTreeOpCC(genTreeOps oper, var_types type, GenCondition condition, GenTree* op1 = nullptr, GenTree* op2 = nullptr)
         : GenTreeOp(oper, type, op1, op2 DEBUGARG(/*largeNode*/ FALSE)), gtCondition(condition)
     {
+#ifdef TARGET_ARM64
+        assert(OperIs(GT_SELECTCC, GT_CINCCC, GT_CINVCC));
+#else
         assert(OperIs(GT_SELECTCC));
+#endif
     }
 
 #if DEBUGGABLE_GENTREE

src/coreclr/jit/gtlist.h

@@ -250,6 +250,11 @@ GTNODE(CCMP             , GenTreeCCMP        ,0,GTK_BINOP|GTK_NOVALUE|DBK_NOTHIR
 GTNODE(CINC             , GenTreeOp          ,0,GTK_BINOP|DBK_NOTHIR)
 // Variant of CINC that reuses flags computed by a previous node with the specified condition.
 GTNODE(CINCCC           , GenTreeOpCC        ,0,GTK_UNOP|DBK_NOTHIR)
+// Maps to arm64 cinv instruction. It negates the operand when the condition is true.
+// Otherwise returns the unchanged operand. Optimises for patterns such as, result = condition ? ~op1 : op1
+GTNODE(CINV             , GenTreeOp          ,0,GTK_BINOP|DBK_NOTHIR)
+// Variant of CINV that reuses flags computed by a previous node with the specified condition.
+GTNODE(CINVCC           , GenTreeOpCC        ,0,GTK_UNOP|DBK_NOTHIR)
 #endif
 
 //-----------------------------------------------------------------------------

src/coreclr/jit/gtstructs.h

@@ -113,7 +113,7 @@ GTSTRUCT_1(ArrAddr     , GT_ARR_ADDR)
 GTSTRUCT_2(CC          , GT_JCC, GT_SETCC)
 #ifdef TARGET_ARM64
 GTSTRUCT_1(CCMP        , GT_CCMP)
-GTSTRUCT_2(OpCC        , GT_SELECTCC, GT_CINCCC)
+GTSTRUCT_3(OpCC        , GT_SELECTCC, GT_CINCCC, GT_CINVCC)
 #else
 GTSTRUCT_1(OpCC        , GT_SELECTCC)
 #endif

src/coreclr/jit/lower.cpp

@@ -3859,6 +3859,11 @@ GenTree* Lowering::LowerSelect(GenTreeConditional* select)
     {
         TryLowerCselToCinc(select, cond);
     }
+
+    if (((trueVal->gtOper == GT_NOT) || (falseVal->gtOper == GT_NOT)))
+    {
+        TryLowerCselToCinv(select, cond);
+    }
 #endif
     return newSelect != nullptr ? newSelect->gtNext : select->gtNext;
 }

src/coreclr/jit/lower.h

@@ -90,6 +90,8 @@ class Lowering final : public Phase
     void ContainCheckConditionalCompare(GenTreeCCMP* ccmp);
     void ContainCheckNeg(GenTreeOp* neg);
     void TryLowerCselToCinc(GenTreeOp* select, GenTree* cond);
+    void TryLowerCselToCinv(GenTreeOp* select, GenTree* cond);
+    void LowerToCincOrCinv(GenTreeOp* select, GenTree* cond, bool shouldReverseCondition, bool isCinc);
 #endif
     void ContainCheckSelect(GenTreeOp* select);
     void ContainCheckBitCast(GenTree* node);

src/coreclr/jit/lowerarmarch.cpp

@@ -2497,8 +2497,8 @@ void Lowering::ContainCheckNeg(GenTreeOp* neg)
 }
 
 //----------------------------------------------------------------------------------------------
-// Try converting SELECT/SELECTCC to CINC/CINCCC. Conversion is possible only if
-// both the trueVal and falseVal are integral constants and abs(trueVal - falseVal) = 1.
+// TryLowerCselToCinc: Try converting SELECT/SELECTCC to CINC/CINCCC. Conversion is possible only if both the trueVal
+// and falseVal are integral constants and abs(trueVal - falseVal) = 1.
 //
 // Arguments:
 //     select - The select node that is now SELECT or SELECTCC
@@ -2513,47 +2513,86 @@ void Lowering::TryLowerCselToCinc(GenTreeOp* select, GenTree* cond)
     size_t   op1Val   = (size_t)trueVal->AsIntCon()->IconValue();
     size_t   op2Val   = (size_t)falseVal->AsIntCon()->IconValue();
 
-    if (op1Val + 1 == op2Val || op2Val + 1 == op1Val)
+    if ((op1Val + 1 == op2Val) || (op2Val + 1 == op1Val))
     {
-        const bool shouldReverseCondition = op1Val + 1 == op2Val;
+        LowerToCincOrCinv(select, cond, (op1Val + 1 == op2Val), true);
+    }
+}
+
+//----------------------------------------------------------------------------------------------
+// TryLowerCselToCinv: Try converting SELECT/SELECTCC to CINV/CINVCC. Conversion is possible only if the
+// trueVal == ~(falseVal).
+//
+// Arguments:
+//     select - The select node that is now SELECT or SELECTCC
+//     cond   - The condition node that SELECT or SELECTCC uses
+//
+void Lowering::TryLowerCselToCinv(GenTreeOp* select, GenTree* cond)
+{
+    assert(select->OperIs(GT_SELECT, GT_SELECTCC));
+
+    GenTree* trueVal       = select->gtOp1;
+    GenTree* falseVal      = select->gtOp2;
+    GenTree* negatedVal    = (trueVal->gtOper == GT_NOT) ? trueVal : falseVal;
+    GenTree* nonNegatedVal = (trueVal->gtOper == GT_NOT) ? falseVal : trueVal;
 
-        // Create a cinc node, insert it and update the use.
-        if (select->OperIs(GT_SELECT))
+    if (GenTree::Compare(negatedVal->AsOp()->gtOp1, nonNegatedVal))
+    {
+        LowerToCincOrCinv(select, cond, (trueVal->gtOper == GT_NOT), false);
+    }
+}
+
+//----------------------------------------------------------------------------------------------
+// LowerToCincOrCinv: A helper method with common code that lowers node to CINC/CINCC/CINV/CINVCC.
+//
+// Arguments:
+//     select                   - The select node that is now SELECT or SELECTCC
+//     cond                     - The condition node that SELECT or SELECTCC uses
+//     shouldReverseCondition   - Should the condition be reversed or not
+//     isCinc                   - Lower the select node to CINC/CINCC if the value is true
+//
+void Lowering::LowerToCincOrCinv(GenTreeOp* select, GenTree* cond, bool shouldReverseCondition, bool isCinc)
+{
+    // Create a cinc/cinv node, insert it and update the use.
+    if (select->OperIs(GT_SELECT))
+    {
+        if (shouldReverseCondition)
         {
-            if (shouldReverseCondition)
+            // Reverse the condition so that op2 will be selected
+            if (!cond->OperIsCompare())
             {
-                // Reverse the condition so that op2 will be selected
-                if (!cond->OperIsCompare())
-                {
-                    // Non-compare nodes add additional GT_NOT node after reversing.
-                    // This would remove gains from this optimisation so don't proceed.
-                    return;
-                }
-                select->gtOp2    = select->gtOp1;
-                GenTree* revCond = comp->gtReverseCond(cond);
-                assert(cond == revCond); // Ensure `gtReverseCond` did not create a new node.
+                // Non-compare nodes add additional GT_NOT node after reversing.
+                // This would remove gains from this optimisation so don't proceed.
+                return;
             }
-            select->gtOp1 = cond->AsOp();
-            select->SetOper(GT_CINC);
-            DISPTREERANGE(BlockRange(), select);
+            select->gtOp2    = select->gtOp1;
+            GenTree* revCond = comp->gtReverseCond(cond);
+            assert(cond == revCond); // Ensure `gtReverseCond` did not create a new node.
+        }
+        select->gtOp1 = cond->AsOp();
+        select->SetOper(isCinc ? GT_CINC : GT_CINV);
+        JITDUMP("Converted to ", (isCinc ? "GT_CINC" : "GT_CINV"), " :\n");
+        DISPTREERANGE(BlockRange(), select);
+        JITDUMP("\n");
+    }
+    else
+    {
+        GenTreeOpCC* selectcc   = select->AsOpCC();
+        GenCondition selectCond = selectcc->gtCondition;
+        if (shouldReverseCondition)
+        {
+            // Reverse the condition so that op2 will be selected
+            selectcc->gtCondition = GenCondition::Reverse(selectCond);
         }
         else
         {
-            GenTreeOpCC* selectcc   = select->AsOpCC();
-            GenCondition selectCond = selectcc->gtCondition;
-            if (shouldReverseCondition)
-            {
-                // Reverse the condition so that op2 will be selected
-                selectcc->gtCondition = GenCondition::Reverse(selectCond);
-            }
-            else
-            {
-                std::swap(selectcc->gtOp1, selectcc->gtOp2);
-            }
-            BlockRange().Remove(selectcc->gtOp2);
-            selectcc->SetOper(GT_CINCCC);
-            DISPTREERANGE(BlockRange(), selectcc);
+            std::swap(selectcc->gtOp1, selectcc->gtOp2);
         }
+        BlockRange().Remove(selectcc->gtOp2, true);
+        selectcc->SetOper(isCinc ? GT_CINCCC : GT_CINVCC);
+        JITDUMP("Converted to ", (isCinc ? "GT_CINCCC" : "GT_CINVCC"), " :\n");
+        DISPTREERANGE(BlockRange(), selectcc);
+        JITDUMP("\n");
     }
 }
 #endif // TARGET_ARM64