8265369: [macos-aarch64] java/net/MulticastSocket/Promiscuous.java failed with "SocketException: Cannot allocate memory"

Reviewed-by: dfuchs, michaelm, chegar

Author: Mark Sheppard

src/java.base/unix/native/libnet/PlainDatagramSocketImpl.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1837,6 +1837,7 @@ static void mcast_join_leave(JNIEnv *env, jobject this,
     jint fd;
     jint family;
     jint ipv6_join_leave;
+    int res;
 
     if (IS_NULL(fdObj)) {
         JNU_ThrowByName(env, JNU_JAVANETPKG "SocketException",
@@ -1993,9 +1994,17 @@ static void mcast_join_leave(JNIEnv *env, jobject this,
         /*
          * Join the multicast group.
          */
-        if (setsockopt(fd, IPPROTO_IP, (join ? IP_ADD_MEMBERSHIP:IP_DROP_MEMBERSHIP),
-                       (char *) &mname, mname_len) < 0) {
+        res = setsockopt(fd, IPPROTO_IP, (join ? IP_ADD_MEMBERSHIP:IP_DROP_MEMBERSHIP),
+                       (char *) &mname, mname_len);
 
+#ifdef __APPLE__
+        if (res < 0 && errno == ENOMEM) {
+            res = setsockopt(fd, IPPROTO_IP, (join ? IP_ADD_MEMBERSHIP:IP_DROP_MEMBERSHIP),
+                       (char *) &mname, mname_len);
+        }
+#endif
+
+        if (res < 0) {
             /*
              * If IP_ADD_MEMBERSHIP returns ENOPROTOOPT on Linux and we've got
              * IPv6 enabled then it's possible that the kernel has been fixed
@@ -2096,9 +2105,16 @@ static void mcast_join_leave(JNIEnv *env, jobject this,
 #endif
 
         /* Join the multicast group */
-        if (setsockopt(fd, IPPROTO_IPV6, (join ? ADD_MEMBERSHIP : DRP_MEMBERSHIP),
-                       (char *) &mname6, sizeof (mname6)) < 0) {
+        res = setsockopt(fd, IPPROTO_IPV6, (join ? ADD_MEMBERSHIP : DRP_MEMBERSHIP),
+                       (char *) &mname6, sizeof(mname6));
 
+#ifdef __APPLE__
+        if (res < 0 && errno == ENOMEM) {
+            res = setsockopt(fd, IPPROTO_IPV6, (join ? ADD_MEMBERSHIP : DRP_MEMBERSHIP),
+                       (char *) &mname6, sizeof(mname6));
+        }
+#endif
+        if (res < 0) {
             if (join) {
                 NET_ThrowCurrent(env, "setsockopt " S_ADD_MEMBERSHIP " failed");
             } else {

test/jdk/ProblemList.txt

@@ -606,7 +606,6 @@ java/net/MulticastSocket/SetGetNetworkInterfaceTest.java        8219083 windows-
 
 java/net/ServerSocket/AcceptInheritHandle.java                  8211854 aix-ppc64
 
-java/net/MulticastSocket/Promiscuous.java                       8265369 macosx-aarch64
 
 ############################################################################