JIT: Fix profile inconsistency asserts in flow graph optimization

src/coreclr/jit/fgopt.cpp

@@ -881,7 +881,9 @@ void Compiler::fgCompactBlock(BasicBlock* block)
     JITDUMP("\nCompacting " FMT_BB " into " FMT_BB ":\n", target->bbNum, block->bbNum);
     fgRemoveRefPred(block->GetTargetEdge());
 
-    if (target->countOfInEdges() > 0)
+    const bool targetHadOtherPreds = (target->countOfInEdges() > 0);
+
+    if (targetHadOtherPreds)
     {
         JITDUMP("Second block has %u other incoming edges\n", target->countOfInEdges());
         assert(block->isEmpty());
@@ -1028,6 +1030,28 @@ void Compiler::fgCompactBlock(BasicBlock* block)
         block->SetFlags(BBF_PROF_WEIGHT);
     }
 
+    // When the target had other incoming edges that were retargeted to block,
+    // the inherited weight may not precisely match block's actual incoming flow
+    // due to accumulated rounding from prior weight adjustments (e.g.,
+    // decreaseBBProfileWeight called during earlier optimizations in the same
+    // iteration of fgUpdateFlowGraph). Mark profile as potentially inconsistent.
+    if (targetHadOtherPreds && block->hasProfileWeight() && fgPgoConsistent)
+    {
+        weight_t incomingLikelyWeight = 0;
+        for (FlowEdge* const predEdge : block->PredEdges())
+        {
+            incomingLikelyWeight += predEdge->getLikelyWeight();
+        }
+
+        if (!fgProfileWeightsConsistentOrSmall(block->bbWeight, incomingLikelyWeight))
+        {
+            JITDUMP("fgCompactBlock: " FMT_BB " weight " FMT_WT " inconsistent with incoming " FMT_WT
+                    " after compaction. Data %s inconsistent.\n",
+                    block->bbNum, block->bbWeight, incomingLikelyWeight, fgPgoConsistent ? "is now" : "was already");
+            fgPgoConsistent = false;
+        }
+    }
+
     VarSetOps::AssignAllowUninitRhs(this, block->bbLiveOut, target->bbLiveOut);
 
     // Update the beginning and ending IL offsets (bbCodeOffs and bbCodeOffsEnd).
@@ -1331,12 +1355,28 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc
                     assert(!block->FalseTargetIs(bDest));
                     removedWeight = block->GetTrueEdge()->getLikelyWeight();
                     fgRedirectEdge(block->TrueEdgeRef(), bDest->GetTarget());
+
+                    // If the redirect caused an edge merge (both edges now point
+                    // to the same target), fix the likelihood. fgRedirectEdge
+                    // keeps the existing edge's likelihood and drops the redirected
+                    // edge's. Since this is a BBJ_COND whose outgoing likelihoods
+                    // must sum to 1.0, and both edges now go to the same block,
+                    // the merged edge must carry likelihood 1.0.
+                    if (block->GetTrueEdge() == block->GetFalseEdge())
+                    {
+                        block->GetTrueEdge()->setLikelihood(1.0);
+                    }
                 }
                 else
                 {
                     assert(block->FalseTargetIs(bDest));
                     removedWeight = block->GetFalseEdge()->getLikelyWeight();
                     fgRedirectEdge(block->FalseEdgeRef(), bDest->GetTarget());
+
+                    if (block->GetTrueEdge() == block->GetFalseEdge())
+                    {
+                        block->GetFalseEdge()->setLikelihood(1.0);
+                    }
                 }
                 break;
 
@@ -1350,6 +1390,11 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc
         //
         if (bDest->hasProfileWeight())
         {
+            if (fgPgoConsistent && (bDest->bbWeight < removedWeight))
+            {
+                JITDUMP("Clamping " FMT_BB " weight in fgOptimizeBranchToEmptyUnconditional\n", bDest->bbNum);
+                fgPgoConsistent = false;
+            }
             bDest->decreaseBBProfileWeight(removedWeight);
         }
 
@@ -1616,7 +1661,7 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block)
     if (modified)
     {
         JITDUMP(
-            "fgOptimizeSwitchBranches: Optimized switch flow. Profile needs to be re-propagated. Data %s consistent.\n",
+            "fgOptimizeSwitchBranches: Optimized switch flow. Profile needs to be re-propagated. Data %s inconsistent.\n",
             fgPgoConsistent ? "is now" : "was already");
         fgPgoConsistent = false;
     }
@@ -4780,7 +4825,7 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh
     // Mark the profile as inconsistent if we might have propagated the OSR entry weight.
     if (modified && opts.IsOSR())
     {
-        JITDUMP("fgUpdateFlowGraph: Inconsistent OSR entry weight may have been propagated. Data %s consistent.\n",
+        JITDUMP("fgUpdateFlowGraph: Inconsistent OSR entry weight may have been propagated. Data %s inconsistent.\n",
                 fgPgoConsistent ? "is now" : "was already");
         fgPgoConsistent = false;
     }