8253944: Certain method references to VarHandle methods should fail

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Infer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -557,37 +557,24 @@ Type instantiatePolymorphicSignatureInstance(Env<AttrContext> env,
                                             List<Type> argtypes) {
         final Type restype;
 
-        if (spMethod == null || types.isSameType(spMethod.getReturnType(), syms.objectType)) {
-            // The return type of the polymorphic signature is polymorphic,
-            // and is computed from the enclosing tree E, as follows:
-            // if E is a cast, then use the target type of the cast expression
-            // as a return type; if E is an expression statement, the return
-            // type is 'void'; otherwise
-            // the return type is simply 'Object'. A correctness check ensures
-            // that env.next refers to the lexically enclosing environment in
-            // which the polymorphic signature call environment is nested.
-
-            switch (env.next.tree.getTag()) {
-                case TYPECAST:
-                    JCTypeCast castTree = (JCTypeCast)env.next.tree;
-                    restype = (TreeInfo.skipParens(castTree.expr) == env.tree) ?
-                              castTree.clazz.type :
-                              syms.objectType;
-                    break;
-                case EXEC:
-                    JCTree.JCExpressionStatement execTree =
-                            (JCTree.JCExpressionStatement)env.next.tree;
-                    restype = (TreeInfo.skipParens(execTree.expr) == env.tree) ?
-                              syms.voidType :
-                              syms.objectType;
-                    break;
-                default:
-                    restype = syms.objectType;
-            }
-        } else {
-            // The return type of the polymorphic signature is fixed
-            // (not polymorphic)
-            restype = spMethod.getReturnType();
+        Type spType = spMethod == null ? syms.objectType : spMethod.getReturnType();
+
+        switch (env.next.tree.getTag()) {
+            case TYPECAST:
+                JCTypeCast castTree = (JCTypeCast)env.next.tree;
+                restype = (TreeInfo.skipParens(castTree.expr) == env.tree) ?
+                          castTree.clazz.type :
+                          spType;
+                break;
+            case EXEC:
+                JCTree.JCExpressionStatement execTree =
+                        (JCTree.JCExpressionStatement)env.next.tree;
+                restype = (TreeInfo.skipParens(execTree.expr) == env.tree) ?
+                          syms.voidType :
+                          spType;
+                break;
+            default:
+                restype = spType;
         }
 
         List<Type> paramtypes = argtypes.map(new ImplicitArgType(spMethod, resolveContext.step));

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Resolve.java

@@ -2742,11 +2742,23 @@ Symbol findPolymorphicSignatureInstance(final Symbol spMethod,
             // Check that there is already a method symbol for the method
             // type and owner
             if (types.isSameType(mtype, sym.type) &&
-                spMethod.owner == sym.owner) {
+                    spMethod.owner == sym.owner) {
                 return sym;
             }
         }
 
+        Type spReturnType = spMethod.asType().getReturnType();
+        if (types.isSameType(spReturnType, syms.objectType)) {
+            // Polymorphic return, pass through mtype
+        } else if (!types.isSameType(spReturnType, mtype.getReturnType())) {
+            // Retain the sig poly method's return type, which differs from that of mtype
+            // Will result in an incompatible return type error
+            mtype = new MethodType(mtype.getParameterTypes(),
+                    spReturnType,
+                    mtype.getThrownTypes(),
+                    syms.methodClass);
+        }
+
         // Create the desired method
         // Retain static modifier is to support invocations to
         // MethodHandle.linkTo* methods

test/langtools/tools/javac/lambda/methodReferenceExecution/MethodReferenceTestVarHandle.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @summary test for VarHandle signature polymorphic methods
+ * @run testng MethodReferenceTestVarHandle
+ */
+
+import java.lang.invoke.*;
+import java.util.*;
+
+import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+
+@Test
+public class MethodReferenceTestVarHandle {
+
+  interface Setter {
+      void apply(int[] arr, int idx, int val);
+  }
+
+  interface Getter {
+      int apply(int[] arr, int idx);
+  }
+
+  public void testSet() throws Throwable {
+      VarHandle vh = MethodHandles.arrayElementVarHandle(int[].class);
+
+      Setter f = vh::set;
+
+      int[] data = {0};
+      f.apply(data, 0, 42);
+      assertEquals(42, data[0]);
+  }
+
+  public void testGet() throws Throwable {
+      VarHandle vh = MethodHandles.arrayElementVarHandle(int[].class);
+
+      Getter f = vh::get;
+
+      int[] data = {42};
+      int v = f.apply(data, 0);
+      assertEquals(42, v);
+  }
+}

test/langtools/tools/javac/lambda/methodReferenceExecution/MethodReferenceTestVarHandle_neg.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @summary test for VarHandle signature polymorphic methods with wrong return type
+ * @compile/fail/ref=MethodReferenceTestVarHandle_neg.out -XDrawDiagnostics MethodReferenceTestVarHandle_neg.java
+ */
+
+import java.lang.invoke.*;
+import java.util.*;
+
+public class MethodReferenceTestVarHandle_neg {
+
+  interface Setter {
+      int apply(int[] arr, int idx, int val);
+  }
+
+  public static void main(String[] args) {
+      VarHandle vh = MethodHandles.arrayElementVarHandle(int[].class);
+
+      // Return type of Setter::apply does not match return type of VarHandle::set
+      Setter f = vh::set;
+  }
+}

test/langtools/tools/javac/lambda/methodReferenceExecution/MethodReferenceTestVarHandle_neg.out

@@ -0,0 +1,2 @@
+MethodReferenceTestVarHandle_neg.java:43:18: compiler.err.prob.found.req: (compiler.misc.incompatible.ret.type.in.mref: (compiler.misc.inconvertible.types: void, int))
+1 error