8266615: C2 incorrectly folds subtype checks involving an interface array

Backport-of: ce88b33

Author: rwestrel

src/hotspot/share/opto/compile.cpp

@@ -4316,16 +4316,22 @@ int Compile::static_subtype_check(ciKlass* superk, ciKlass* subk) {
   }
 
   ciType* superelem = superk;
-  if (superelem->is_array_klass())
+  ciType* subelem = subk;
+  if (superelem->is_array_klass()) {
     superelem = superelem->as_array_klass()->base_element_type();
+  }
+  if (subelem->is_array_klass()) {
+    subelem = subelem->as_array_klass()->base_element_type();
+  }
 
   if (!subk->is_interface()) {  // cannot trust static interface types yet
     if (subk->is_subtype_of(superk)) {
       return SSC_always_true;   // (1) false path dead; no dynamic test needed
     }
     if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
+        !(subelem->is_klass() && subelem->as_klass()->is_interface()) &&
         !superk->is_subtype_of(subk)) {
-      return SSC_always_false;
+      return SSC_always_false;  // (2) true path dead; no dynamic test needed
     }
   }
 

test/hotspot/jtreg/compiler/types/TestInterfaceArraySubtypeCheck.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @bug 8266615
+ * @summary C2 incorrectly folds subtype checks involving an interface array.
+ * @run main/othervm -Xbatch
+ *                   compiler.types.TestInterfaceArraySubtypeCheck
+ */
+
+package compiler.types;
+
+public class TestInterfaceArraySubtypeCheck {
+
+    static interface MyInterface { }
+
+    static class MyClassA { }
+
+    static class MyClassB extends MyClassA implements MyInterface { }
+
+    static MyInterface[] getMyInterfaceArray() {
+        return new MyClassB[0];
+    }
+
+    static MyInterface getMyInterface() {
+        return new MyClassB();
+    }
+
+    static MyClassA[] test1() {
+        return (MyClassA[])getMyInterfaceArray();
+    }
+
+    static void test2() {
+        if (!(getMyInterfaceArray() instanceof MyClassA[])) {
+            throw new RuntimeException("test2 failed");
+        }
+    }
+
+    static MyClassA test3() {
+        return (MyClassA)getMyInterface();
+    }
+
+    static void test4() {
+        if (!(getMyInterface() instanceof MyClassA)) {
+            throw new RuntimeException("test4 failed");
+        }
+    }
+
+    public static void main(String[] args) {
+        for (int i = 0; i < 50_000; ++i) {
+            test1();
+            test2();
+            test3();
+            test4();
+        }
+    }
+}