8240195: some jaotc failures of fastdebug build with specific flags

Reviewed-by: kvn, thartmann

src/hotspot/share/opto/compile.cpp

@@ -4241,9 +4241,6 @@ int Compile::static_subtype_check(ciKlass* superk, ciKlass* subk) {
         // Add a dependency if there is a chance of a later subclass.
         dependencies()->assert_leaf_type(ik);
       }
-      if (ik->is_abstract()) {
-        return SSC_always_false;
-      }
       return SSC_easy_test;     // (3) caller can do a simple ptr comparison
     }
   } else {

src/hotspot/share/opto/subtypenode.cpp

@@ -36,6 +36,14 @@ const Type* SubTypeCheckNode::sub(const Type* sub_t, const Type* super_t) const
 
   bool xsubk = sub_t->isa_klassptr() ? sub_t->is_klassptr()->klass_is_exact() : sub_t->is_oopptr()->klass_is_exact();
 
+
+  // Oop can't be a subtype of abstract type that has no subclass.
+  if (sub_t->isa_oopptr() && superk->is_instance_klass() && superk->is_abstract() &&
+      !superk->as_instance_klass()->has_subklass()) {
+    Compile::current()->dependencies()->assert_leaf_type(superk);
+    return TypeInt::CC_GT;
+  }
+
   // Similar to logic in CmpPNode::sub()
 
   // Interfaces can't be trusted unless the subclass is an exact
@@ -93,9 +101,6 @@ const Type* SubTypeCheckNode::sub(const Type* sub_t, const Type* super_t) const
 }
 
 Node *SubTypeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) {
-  // Verify that optimizing the subtype check to a simple code pattern
-  // when possible would not constant fold better
-#ifdef ASSERT
   Node* obj_or_subklass = in(ObjOrSubKlass);
   Node* superklass = in(SuperKlass);
 
@@ -112,7 +117,28 @@ Node *SubTypeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     return NULL;
   }
 
+  Node* addr = NULL;
+  if (obj_or_subklass->is_DecodeNKlass()) {
+    if (obj_or_subklass->in(1) != NULL &&
+        obj_or_subklass->in(1)->Opcode() == Op_LoadNKlass) {
+      addr = obj_or_subklass->in(1)->in(MemNode::Address);
+    }
+  } else if (obj_or_subklass->Opcode() == Op_LoadKlass) {
+    addr = obj_or_subklass->in(MemNode::Address);
+  }
+
+  if (addr != NULL) {
+    intptr_t con = 0;
+    Node* obj = AddPNode::Ideal_base_and_offset(addr, phase, con);
+    if (con == oopDesc::klass_offset_in_bytes() && obj != NULL && phase->type(obj)->isa_oopptr()) {
+      set_req(ObjOrSubKlass, obj);
+      return this;
+    }
+  }
 
+  // Verify that optimizing the subtype check to a simple code pattern
+  // when possible would not constant fold better
+#ifdef ASSERT
   ciKlass* superk = super_t->is_klassptr()->klass();
   ciKlass* subk   = sub_t->isa_klassptr() ? sub_t->is_klassptr()->klass() : sub_t->is_oopptr()->klass();
 

test/hotspot/jtreg/compiler/types/TestSubTypeOfAsbtractClassWrongResult.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2020, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8240195
+ * @summary subtype check with abstract class that has no children classes can sometimes be constant folded
+ * @requires vm.compiler2.enabled
+ *
+ * @run main/othervm -XX:-BackgroundCompilation TestSubTypeOfAsbtractClassWrongResult
+ *
+ */
+
+
+public class TestSubTypeOfAsbtractClassWrongResult {
+    public static void main(String[] args) {
+        for (int i = 0; i < 20_000; i++) {
+            if (!test1(A.class)) {
+                throw new RuntimeException("Wrong result");
+            }
+            test2(new Object());
+            test3(new Exception());
+        }
+    }
+
+    private static boolean test1(Class c) {
+        return A.class.isAssignableFrom(c);
+    }
+
+    private static boolean test2(Object o) {
+        return o instanceof A;
+    }
+
+    private static void test3(Exception e) {
+        try {
+            throw e;
+        } catch (A ex1) {
+        } catch (Exception ex2) {
+        }
+    }
+
+    static abstract class A extends Exception {
+    }
+}