8210495: compiler crashes because of illegal signature in otherwise legal code

Disable strict verification of compiler signatures when they do not affect generated bytecode

Reviewed-by: clanger
Backport-of: c0d51dc

Author: Aditya Mandaleeka

src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java

@@ -5025,6 +5025,10 @@ protected SignatureGenerator(Types types) {
             this.types = types;
         }
 
+        protected void reportIllegalSignature(Type t) {
+            throw new InvalidSignatureException(t);
+        }
+
         /**
          * Assemble signature of given type in string buffer.
          */
@@ -5059,7 +5063,7 @@ public void assembleSig(Type type) {
                     break;
                 case CLASS:
                     if (type.isCompound()) {
-                        throw new InvalidSignatureException(type);
+                        reportIllegalSignature(type);
                     }
                     append('L');
                     assembleClassSig(type);
@@ -5104,7 +5108,7 @@ public void assembleSig(Type type) {
                 }
                 case TYPEVAR:
                     if (((TypeVar)type).isCaptured()) {
-                        throw new InvalidSignatureException(type);
+                        reportIllegalSignature(type);
                     }
                     append('T');
                     append(type.tsym.name);

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/LambdaToMethod.java

@@ -25,6 +25,9 @@
 
 package com.sun.tools.javac.comp;
 
+import com.sun.tools.javac.code.Types.SignatureGenerator.InvalidSignatureException;
+import com.sun.tools.javac.resources.CompilerProperties.Errors;
+import com.sun.tools.javac.resources.CompilerProperties.Fragments;
 import com.sun.tools.javac.tree.*;
 import com.sun.tools.javac.tree.JCTree.*;
 import com.sun.tools.javac.tree.JCTree.JCMemberReference.ReferenceKind;
@@ -2030,7 +2033,7 @@ private String serializedLambdaDisambiguation() {
                         owner.type != null ||
                         directlyEnclosingLambda() != null);
                 if (owner.type != null) {
-                    buf.append(typeSig(owner.type));
+                    buf.append(typeSig(owner.type, true));
                     buf.append(":");
                 }
 
@@ -2046,7 +2049,7 @@ private String serializedLambdaDisambiguation() {
                 //add captured locals info: type, name, order
                 for (Symbol fv : getSymbolMap(CAPTURED_VAR).keySet()) {
                     if (fv != self) {
-                        buf.append(typeSig(fv.type));
+                        buf.append(typeSig(fv.type, true));
                         buf.append(" ");
                         buf.append(fv.flatName());
                         buf.append(",");
@@ -2435,15 +2438,31 @@ boolean propagateAnnotations() {
      */
 
     private String typeSig(Type type) {
-        L2MSignatureGenerator sg = new L2MSignatureGenerator();
-        sg.assembleSig(type);
-        return sg.toString();
+        return typeSig(type, false);
+    }
+
+    private String typeSig(Type type, boolean allowIllegalSignature) {
+        try {
+            L2MSignatureGenerator sg = new L2MSignatureGenerator(allowIllegalSignature);
+            sg.assembleSig(type);
+            return sg.toString();
+        } catch (InvalidSignatureException ex) {
+            Symbol c = attrEnv.enclClass.sym;
+            log.error(Errors.CannotGenerateClass(c, Fragments.IllegalSignature(c, ex.type())));
+            return "<ERRONEOUS>";
+        }
     }
 
     private String classSig(Type type) {
-        L2MSignatureGenerator sg = new L2MSignatureGenerator();
-        sg.assembleClassSig(type);
-        return sg.toString();
+        try {
+            L2MSignatureGenerator sg = new L2MSignatureGenerator(false);
+            sg.assembleClassSig(type);
+            return sg.toString();
+        } catch (InvalidSignatureException ex) {
+            Symbol c = attrEnv.enclClass.sym;
+            log.error(Errors.CannotGenerateClass(c, Fragments.IllegalSignature(c, ex.type())));
+            return "<ERRONEOUS>";
+        }
     }
 
     /**
@@ -2456,8 +2475,22 @@ private class L2MSignatureGenerator extends Types.SignatureGenerator {
          */
         StringBuilder sb = new StringBuilder();
 
-        L2MSignatureGenerator() {
+        /**
+         * Are signatures incompatible with JVM spec allowed?
+         * Used by {@link LambdaTranslationContext#serializedLambdaDisambiguation()}.
+         */
+        boolean allowIllegalSignatures;
+
+        L2MSignatureGenerator(boolean allowIllegalSignatures) {
             super(types);
+            this.allowIllegalSignatures = allowIllegalSignatures;
+        }
+
+        @Override
+        protected void reportIllegalSignature(Type t) {
+            if (!allowIllegalSignatures) {
+                super.reportIllegalSignature(t);
+            }
         }
 
         @Override

test/langtools/tools/javac/lambda/8210495/T8210495.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8210495
+ * @summary compiler crashes because of illegal signature in otherwise legal code
+ * @compile T8210495.java
+ */
+
+import java.awt.*;
+import java.awt.event.ActionListener;
+import java.util.List;
+
+class T8210495 {
+    interface IFilter {
+        Component getComponent();
+    }
+
+    static class Filter implements IFilter {
+        @Override
+        public Component getComponent() {
+            return null;
+        }
+
+    }
+
+    public Component buildFilter(List<? extends Filter> l, Dialog dialog) {
+        Panel c = new Panel();
+        l.stream()
+                .map(f -> {
+                    Button btn = (Button)f.getComponent();
+                    btn.addActionListener((java.io.Serializable & ActionListener)evt -> {
+                        applyFilter(f);
+                        dialog.setVisible(false);
+                    });
+                    return btn;
+                })
+                .forEach(c::add);
+        return c;
+    }
+
+    private void applyFilter(IFilter f) { }
+}