8263452: Javac slow compilation due to algorithmic complexity

Reviewed-by: vromero, jfranck

Author: lahodaj

src/jdk.compiler/share/classes/com/sun/tools/javac/code/Flags.java

@@ -176,7 +176,12 @@ public static EnumSet<Flag> asFlagSet(long flags) {
 
     /** Flag for synthesized default constructors of anonymous classes.
      */
-    public static final int ANONCONSTR   = 1<<29;
+    public static final int ANONCONSTR   = 1<<29; //non-class members
+
+    /**
+     * Flag to indicate the super classes of this ClassSymbol has been attributed.
+     */
+    public static final int SUPER_OWNER_ATTRIBUTED = 1<<29; //ClassSymbols
 
     /** Flag for class symbols to indicate it has been checked and found
      *  acyclic.
@@ -380,6 +385,7 @@ public static EnumSet<Flag> asFlagSet(long flags) {
      */
     public static final long NON_SEALED = 1L<<63; // ClassSymbols
 
+
     /** Modifier masks.
      */
     public static final int

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Attr.java

@@ -5098,14 +5098,17 @@ void attribClass(ClassSymbol c) throws CompletionFailure {
         chk.checkNonCyclic(null, c.type);
 
         Type st = types.supertype(c.type);
-        if ((c.flags_field & Flags.COMPOUND) == 0) {
+        if ((c.flags_field & Flags.COMPOUND) == 0 &&
+            (c.flags_field & Flags.SUPER_OWNER_ATTRIBUTED) == 0) {
             // First, attribute superclass.
             if (st.hasTag(CLASS))
                 attribClass((ClassSymbol)st.tsym);
 
             // Next attribute owner, if it is a class.
             if (c.owner.kind == TYP && c.owner.type.hasTag(CLASS))
                 attribClass((ClassSymbol)c.owner);
+
+            c.flags_field |= Flags.SUPER_OWNER_ATTRIBUTED;
         }
 
         // The previous operations might have attributed the current class

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Check.java

@@ -2247,7 +2247,7 @@ void checkNonCyclicDecl(JCClassDecl tree) {
 
     class CycleChecker extends TreeScanner {
 
-        List<Symbol> seenClasses = List.nil();
+        Set<Symbol> seenClasses = new HashSet<>();
         boolean errorFound = false;
         boolean partialCheck = false;
 
@@ -2266,7 +2266,7 @@ private void checkSymbol(DiagnosticPosition pos, Symbol sym) {
                 } else if (sym.kind == TYP) {
                     checkClass(pos, sym, List.nil());
                 }
-            } else {
+            } else if (sym == null || sym.kind != PCK) {
                 //not completed yet
                 partialCheck = true;
             }
@@ -2315,7 +2315,7 @@ void checkClass(DiagnosticPosition pos, Symbol c, List<JCTree> supertypes) {
                 noteCyclic(pos, (ClassSymbol)c);
             } else if (!c.type.isErroneous()) {
                 try {
-                    seenClasses = seenClasses.prepend(c);
+                    seenClasses.add(c);
                     if (c.type.hasTag(CLASS)) {
                         if (supertypes.nonEmpty()) {
                             scan(supertypes);
@@ -2338,7 +2338,7 @@ void checkClass(DiagnosticPosition pos, Symbol c, List<JCTree> supertypes) {
                         }
                     }
                 } finally {
-                    seenClasses = seenClasses.tail;
+                    seenClasses.remove(c);
                 }
             }
         }

test/langtools/tools/javac/perf/SuperClassAndNesting.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8263452
+ * @summary Verify javac does not need a long time to process sources with deep class nesting
+ *          and deep inheritance hierarchies.
+ * @modules jdk.compiler
+ */
+
+import java.util.Arrays;
+import javax.tools.JavaCompiler;
+import javax.tools.ToolProvider;
+
+import com.sun.source.util.JavacTask;
+import java.io.IOException;
+import java.net.URI;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+
+public class SuperClassAndNesting {
+
+    private static final int SIZE = 100;
+
+    public static void main(String... args) throws IOException {
+        new SuperClassAndNesting().run();
+    }
+
+    void run() throws IOException {
+        compileTestClass(generateTestClass(SIZE));
+    }
+
+    String generateTestClass(int depth) {
+        StringBuilder clazz = new StringBuilder();
+        clazz.append("""
+                     class Test {
+                     class T0 extends java.util.ArrayList {
+                     """);
+        for (int i = 1; i < depth; i++) {
+            clazz.append("class T" + i + " extends T" + (i - 1) + " {\n");
+        }
+        for (int i = 0; i < depth; i++) {
+            clazz.append("}\n");
+        }
+        clazz.append("}\n");
+        return clazz.toString();
+    }
+
+    void compileTestClass(String code) throws IOException {
+        final JavaCompiler tool = ToolProvider.getSystemJavaCompiler();
+        assert tool != null;
+
+        JavacTask ct = (JavacTask) tool.getTask(null, null, null,
+            null, null, Arrays.asList(new MyFileObject(code)));
+        ct.analyze();
+    }
+
+    static class MyFileObject extends SimpleJavaFileObject {
+        private final String text;
+
+        public MyFileObject(String text) {
+            super(URI.create("myfo:/Test.java"), JavaFileObject.Kind.SOURCE);
+            this.text = text;
+        }
+
+        @Override
+        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+            return text;
+        }
+    }
+}