JRE-238 [736] java.awt.AWTError: access denied ("java.lang.RuntimePermission" "canInvokeInSystemThreadGroup")

Moved task execution on AppKit to the privileged block. Minor refactoring

(cherry picked from commit 5dbb88471115c9e4a536ae37d0e6794de9e5ac9c)

Author: avu

src/java.desktop/macosx/classes/sun/java2d/opengl/CGLGraphicsConfig.java

@@ -26,6 +26,7 @@
 package sun.java2d.opengl;
 
 import java.awt.AWTException;
+import java.awt.AWTError;
 import java.awt.BufferCapabilities;
 import java.awt.Component;
 import java.awt.Graphics;
@@ -60,6 +61,9 @@
 
 import sun.lwawt.LWComponentPeer;
 import sun.lwawt.macosx.CPlatformView;
+import sun.lwawt.macosx.CThreading;
+import java.security.PrivilegedAction;
+import java.util.concurrent.Callable;
 
 public final class CGLGraphicsConfig extends CGraphicsConfig
     implements OGLGraphicsConfig
@@ -134,40 +138,57 @@ public static CGLGraphicsConfig getConfig(CGraphicsDevice device,
             return null;
         }
 
-        long cfginfo = 0;
-        int textureSize = 0;
-        final String ids[] = new String[1];
-        OGLRenderQueue rq = OGLRenderQueue.getInstance();
-        rq.lock();
-        try {
-            // getCGLConfigInfo() creates and destroys temporary
-            // surfaces/contexts, so we should first invalidate the current
-            // Java-level context and flush the queue...
-            OGLContext.invalidateCurrentContext();
-
-            cfginfo = getCGLConfigInfo(device.getCGDisplayID(), pixfmt,
-                                       kOpenGLSwapInterval);
-            if (cfginfo != 0L) {
-                textureSize = nativeGetMaxTextureSize();
-                // 7160609: GL still fails to create a square texture of this
-                // size. Half should be safe enough.
-                // Explicitly not support a texture more than 2^14, see 8010999.
-                textureSize = textureSize <= 16384 ? textureSize / 2 : 8192;
-                OGLContext.setScratchSurface(cfginfo);
-                rq.flushAndInvokeNow(() -> {
-                    ids[0] = OGLContext.getOGLIdString();
-                });
+        // Move CGLGraphicsConfig creation code to AppKit thread in order to avoid the
+        // following deadlock:
+        // 1) CGLGraphicsConfig.getCGLConfigInfo (called from EDT) takes RenderQueue.lock
+        // 2) CGLLayer.drawInCGLContext is invoked on AppKit thread and
+        //    blocked on RenderQueue.lock
+        // 1) invokes native block on AppKit and wait
+
+        Callable<CGLGraphicsConfig> command = () -> {
+            long cfginfo;
+            int textureSize = 0;
+            final String ids[] = new String[1];
+            OGLRenderQueue rq = OGLRenderQueue.getInstance();
+            rq.lock();
+            try {
+                // getCGLConfigInfo() creates and destroys temporary
+                // surfaces/contexts, so we should first invalidate the current
+                // Java-level context and flush the queue...
+                OGLContext.invalidateCurrentContext();
+
+                cfginfo = getCGLConfigInfo(device.getCGDisplayID(), pixfmt,
+                        kOpenGLSwapInterval);
+                if (cfginfo != 0L) {
+                    textureSize = nativeGetMaxTextureSize();
+                    // 7160609: GL still fails to create a square texture of this
+                    // size. Half should be safe enough.
+                    // Explicitly not support a texture more than 2^14, see 8010999.
+                    textureSize = textureSize <= 16384 ? textureSize / 2 : 8192;
+                    OGLContext.setScratchSurface(cfginfo);
+                    rq.flushAndInvokeNow(() -> ids[0] = OGLContext.getOGLIdString());
+                }
+            } finally {
+                rq.unlock();
+            }
+            if (cfginfo == 0) {
+                return null;
             }
-        } finally {
-            rq.unlock();
-        }
-        if (cfginfo == 0) {
-            return null;
-        }
 
-        int oglCaps = getOGLCapabilities(cfginfo);
-        ContextCapabilities caps = new OGLContextCaps(oglCaps, ids[0]);
-        return new CGLGraphicsConfig(device, pixfmt, cfginfo, textureSize, caps);
+            int oglCaps = getOGLCapabilities(cfginfo);
+            ContextCapabilities caps = new OGLContextCaps(oglCaps, ids[0]);
+            return new CGLGraphicsConfig(
+                    device, pixfmt, cfginfo, textureSize, caps);
+        };
+
+        return java.security.AccessController.doPrivileged(
+                (PrivilegedAction<CGLGraphicsConfig>) () -> {
+                    try {
+                        return CThreading.executeOnAppKit(command);
+                    } catch (Throwable throwable) {
+                        throw new AWTError(throwable.getMessage());
+                    }
+                });
     }
 
     static void refPConfigInfo(long pConfigInfo) {