8308583: SIGSEGV in GraphKit::gen_checkcast

Reviewed-by: thartmann, kvn, never

src/hotspot/share/opto/graphKit.cpp

@@ -2888,6 +2888,7 @@ Node* GraphKit::type_check_receiver(Node* receiver, ciKlass* klass,
       // recv_xtype, since now we know what the type will be.
       Node* cast = new CheckCastPPNode(control(), receiver, recvx_type);
       (*casted_receiver) = _gvn.transform(cast);
+      assert(!(*casted_receiver)->is_top(), "that path should be unreachable");
       // (User must make the replace_in_map call.)
     }
   }
@@ -3519,19 +3520,19 @@ void GraphKit::shared_unlock(Node* box, Node* obj) {
 // This two-faced routine is useful because allocation sites
 // almost always feature constant types.
 Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) {
-  const TypeKlassPtr* inst_klass = _gvn.type(klass_node)->isa_klassptr();
-  if (!StressReflectiveCode && inst_klass != nullptr) {
-    bool    xklass = inst_klass->klass_is_exact();
-    if (xklass || inst_klass->isa_aryklassptr()) {
+  const TypeKlassPtr* klass_t = _gvn.type(klass_node)->isa_klassptr();
+  if (!StressReflectiveCode && klass_t != nullptr) {
+    bool xklass = klass_t->klass_is_exact();
+    if (xklass || (klass_t->isa_aryklassptr() && klass_t->is_aryklassptr()->elem() != Type::BOTTOM)) {
       jint lhelper;
-      if (inst_klass->isa_aryklassptr()) {
-        BasicType elem = inst_klass->as_instance_type()->isa_aryptr()->elem()->array_element_basic_type();
+      if (klass_t->isa_aryklassptr()) {
+        BasicType elem = klass_t->as_instance_type()->isa_aryptr()->elem()->array_element_basic_type();
         if (is_reference_type(elem, true)) {
           elem = T_OBJECT;
         }
         lhelper = Klass::array_layout_helper(elem);
       } else {
-        lhelper = inst_klass->is_instklassptr()->exact_klass()->layout_helper();
+        lhelper = klass_t->is_instklassptr()->exact_klass()->layout_helper();
       }
       if (lhelper != Klass::_lh_neutral_value) {
         constant_value = lhelper;

src/hotspot/share/opto/memnode.cpp

@@ -2360,7 +2360,7 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const {
 
   // Check for loading klass from an array
   const TypeAryPtr *tary = tp->isa_aryptr();
-  if (tary != nullptr && tary->elem() != Type::BOTTOM &&
+  if (tary != nullptr &&
       tary->offset() == oopDesc::klass_offset_in_bytes()) {
     return tary->as_klass_type(true);
   }

test/hotspot/jtreg/compiler/types/TestBottomArrayTypeCheck.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8308583
+ * @summary SIGSEGV in GraphKit::gen_checkcast
+ * @run main/othervm -Xbatch -XX:-TieredCompilation -XX:CompileCommand=compileonly,TestBottomArrayTypeCheck::test TestBottomArrayTypeCheck
+ */
+
+public class TestBottomArrayTypeCheck {
+
+    interface WordBase {
+    }
+
+    interface RelocatedPointer {
+    }
+
+    static final class Word implements WordBase {
+    }
+
+    static Object[] staticObjectFields;
+
+    static byte[] staticPrimitiveFields;
+
+
+    interface SnippetReflection {
+        Object forObject(Object o);
+    }
+
+    static class BaseSnippetReflection implements SnippetReflection {
+        public Object forObject(Object o) {
+            return null;
+        }
+
+    }
+    static class SubSnippetReflection extends BaseSnippetReflection {
+        public Object forObject(Object object) {
+            if (object instanceof WordBase word && !(object instanceof RelocatedPointer)) {
+                return word;
+            }
+            return super.forObject(object);
+        }
+    }
+
+    public static void main(String[] args) {
+        t1();
+        for (int i = 0; i < 10; i++) {
+            t2();
+        }
+    }
+
+    static void t1() {
+        Word w = new Word();
+        SnippetReflection base = new BaseSnippetReflection();
+        SnippetReflection sub = new SubSnippetReflection();
+        for (int i = 0; i < 10000; i++) {
+            invoke(base, w);
+            invoke(sub, w);
+        }
+    }
+
+    static void t2() {
+        SnippetReflection base = new BaseSnippetReflection();
+        SnippetReflection sub = new SubSnippetReflection();
+        for (int i = 0; i < 10000; i++) {
+            test(base, i % 2 == 0);
+            test(sub, i % 2 == 0);
+            test(sub, i % 2 == 0);
+        }
+    }
+
+    static Object test(SnippetReflection s, boolean b) {
+        return s.forObject(b ? staticObjectFields : staticPrimitiveFields);
+    }
+
+
+    static Object invoke(SnippetReflection s, Object o) {
+        return s.forObject(o);
+    }
+}