8236697: Stack overflow with cyclic hierarchy in class file

Implemented minimal necessary cycle detection to avoid stack overflow in c.s.t.j.code.Types::asSuper

Reviewed-by: vromero

src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java

@@ -2124,22 +2124,31 @@ public Type visitClassType(ClassType t, Symbol sym) {
                 if (t.tsym == sym)
                     return t;
 
-                Type st = supertype(t);
-                if (st.hasTag(CLASS) || st.hasTag(TYPEVAR)) {
-                    Type x = asSuper(st, sym);
-                    if (x != null)
-                        return x;
+                Symbol c = t.tsym;
+                if ((c.flags_field & LOCKED) != 0) {
+                    return null;
                 }
-                if ((sym.flags() & INTERFACE) != 0) {
-                    for (List<Type> l = interfaces(t); l.nonEmpty(); l = l.tail) {
-                        if (!l.head.hasTag(ERROR)) {
-                            Type x = asSuper(l.head, sym);
-                            if (x != null)
-                                return x;
+                try {
+                    c.flags_field |= LOCKED;
+                    Type st = supertype(t);
+                    if (st.hasTag(CLASS) || st.hasTag(TYPEVAR)) {
+                        Type x = asSuper(st, sym);
+                        if (x != null)
+                            return x;
+                    }
+                    if ((sym.flags() & INTERFACE) != 0) {
+                        for (List<Type> l = interfaces(t); l.nonEmpty(); l = l.tail) {
+                            if (!l.head.hasTag(ERROR)) {
+                                Type x = asSuper(l.head, sym);
+                                if (x != null)
+                                    return x;
+                            }
                         }
                     }
+                    return null;
+                } finally {
+                    c.flags_field &= ~LOCKED;
                 }
-                return null;
             }
 
             @Override

test/langtools/tools/javac/8236697/Cyclic.jcod

@@ -0,0 +1,29 @@
+class Cyclic {
+  0xCAFEBABE;
+  0; // minor version
+  57; // version
+  [] { // Constant Pool
+    ; // first element is empty
+    class #2; // #1
+    Utf8 "Cyclic"; // #2
+    class #4; // #3
+    Utf8 "java/lang/Object"; // #4
+  } // Constant Pool
+
+  0x0600; // access
+  #1;// this_cpx
+  #3;// super_cpx
+
+  [] { // Interfaces
+    #1;
+  } // Interfaces
+
+  [] { // fields
+  } // fields
+
+  [] { // methods
+  } // methods
+
+  [] { // Attributes
+  } // Attributes
+} // end class Cyclic

test/langtools/tools/javac/8236697/T8236697.java

@@ -0,0 +1,12 @@
+/*
+ * @test /nodynamiccopyright/
+ * @bug 8236697
+ * @summary Stack overflow with cyclic hierarchy in class file
+ * @build Cyclic
+ * @compile/fail/ref=T8236697.out -XDrawDiagnostics T8236697.java
+ */
+interface T8236697 extends Iterable {
+
+    public Cyclic iterator();
+
+}

test/langtools/tools/javac/8236697/T8236697.out

@@ -0,0 +1,2 @@
+T8236697.java:10:19: compiler.err.override.incompatible.ret: (compiler.misc.clashes.with: iterator(), T8236697, iterator(), java.lang.Iterable), Cyclic, java.util.Iterator
+1 error