8299959: C2: CmpU::Value must filter overflow computation against loc…

…al sub computation

Backport-of: 5f66024e957e5e40ce8d5a65717ea7f82c9f0b8f

src/hotspot/share/opto/subnode.cpp

@@ -779,6 +779,9 @@ const Type* CmpUNode::Value(PhaseGVN* phase) const {
   if (t2 == TypeInt::INT) { // Compare to bottom?
     return bottom_type();
   }
+
+  const Type* t_sub = sub(t1, t2); // compare based on immediate inputs
+
   uint in1_op = in1->Opcode();
   if (in1_op == Op_AddI || in1_op == Op_SubI) {
     // The problem rise when result of AddI(SubI) may overflow
@@ -831,13 +834,15 @@ const Type* CmpUNode::Value(PhaseGVN* phase) const {
         const TypeInt* tr2 = TypeInt::make(lo_tr2, hi_tr2, w);
         const TypeInt* cmp1 = sub(tr1, t2)->is_int();
         const TypeInt* cmp2 = sub(tr2, t2)->is_int();
-        // compute union, so that cmp handles all possible results from the two cases
-        return cmp1->meet(cmp2);
+        // Compute union, so that cmp handles all possible results from the two cases
+        const Type* t_cmp = cmp1->meet(cmp2);
+        // Pick narrowest type, based on overflow computation and on immediate inputs
+        return t_sub->filter(t_cmp);
       }
     }
   }
 
-  return sub(t1, t2);            // Local flavor of type subtraction
+  return t_sub;
 }
 
 bool CmpUNode::is_index_range_check() const {

test/hotspot/jtreg/compiler/rangechecks/TestRangeCheckCmpUOverflowVsSub.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8299959
+ * @summary In CmpU::Value, the sub computation may be narrower than the overflow computation.
+ *
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+StressCCP -Xcomp -XX:-TieredCompilation
+ *                   -XX:CompileCommand=compileonly,compiler.rangechecks.TestRangeCheckCmpUOverflowVsSub::test
+ *                   -XX:RepeatCompilation=50
+ *                   compiler.rangechecks.TestRangeCheckCmpUOverflowVsSub
+*/
+
+package compiler.rangechecks;
+
+public class TestRangeCheckCmpUOverflowVsSub {
+    static int arr[] = new int[400];
+
+    public static void main(String[] strArr) {
+        for (int i = 0; i < 10; i++) {
+            test(); // repeat for multiple compilations
+        }
+    }
+
+    static void test() {
+        for(int i = 0; i < 50_000; i++) {} //empty loop - trigger OSR faster
+        int val;
+        int zero = arr[5];
+        int i = 1;
+        do {
+            for (int j = 1; j < 3; j++) {
+                for (int k = 2; k > i; k -= 3) {
+                    try {
+                        val = arr[i + 1] % k;
+                        val = arr[i - 1] % zero;
+                        val = arr[k - 1];
+                    } catch (ArithmeticException e) {} // catch div by zero
+                }
+            }
+        } while (++i < 3);
+    }
+}
+