8256465: [macos] Java frame and dialog presented full screen freeze application

src/java.desktop/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,6 +45,8 @@
 import java.beans.PropertyChangeEvent;
 import java.beans.PropertyChangeListener;
 import java.lang.reflect.InvocationTargetException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -85,6 +87,7 @@ private static native void nativeSetNSWindowStandardFrame(long nsWindowPtr,
     private static native void nativeRevalidateNSWindowShadow(long nsWindowPtr);
     private static native void nativeSetNSWindowMinimizedIcon(long nsWindowPtr, long nsImage);
     private static native void nativeSetNSWindowRepresentedFilename(long nsWindowPtr, String representedFilename);
+    private static native void nativeSetAllowAutomaticTabbingProperty(boolean allowAutomaticWindowTabbing);
     private static native void nativeSetEnabled(long nsWindowPtr, boolean isEnabled);
     private static native void nativeSynthesizeMouseEnteredExitedEvents();
     private static native void nativeSynthesizeMouseEnteredExitedEvents(long nsWindowPtr, int eventType);
@@ -124,6 +127,10 @@ private static native void nativeSetNSWindowStandardFrame(long nsWindowPtr,
     public static final String WINDOW_TRANSPARENT_TITLE_BAR = "apple.awt.transparentTitleBar";
     public static final String WINDOW_TITLE_VISIBLE = "apple.awt.windowTitleVisible";
 
+    // This system property is used in javafx as well
+    public static final String MAC_OS_TABBED_WINDOW = System.getProperty("jdk.allowMacOSTabbedWindows");
+    public static boolean allowMacOSAutomaticWindowTabbing;
+
     // Yeah, I know. But it's easier to deal with ints from JNI
     static final int MODELESS = 0;
     static final int DOCUMENT_MODAL = 1;
@@ -187,6 +194,16 @@ static boolean IS(final int bits, final int mask) {
         return (bits & mask) != 0;
     }
 
+    static {
+        AccessController.doPrivileged(
+                (PrivilegedAction<Object>) () -> {
+                    allowMacOSAutomaticWindowTabbing = Boolean.parseBoolean(
+                            MAC_OS_TABBED_WINDOW);
+                    return null;
+                });
+        nativeSetAllowAutomaticTabbingProperty(allowMacOSAutomaticWindowTabbing);
+    }
+
     @SuppressWarnings({"unchecked", "rawtypes"})
     static ClientPropertyApplicator<JRootPane, CPlatformWindow> CLIENT_PROPERTY_APPLICATOR = new ClientPropertyApplicator<JRootPane, CPlatformWindow>(new Property[] {
         new Property<CPlatformWindow>(WINDOW_DOCUMENT_MODIFIED) { public void applyProperty(final CPlatformWindow c, final Object value) {

src/java.desktop/macosx/native/libawt_lwawt/awt/AWTWindow.m

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1094,6 +1094,24 @@ + (AWTWindow *) lastKeyWindow {
 
 @end // AWTWindow
 
+/*
+ * Class:     sun_lwawt_macosx_CPlatformWindow
+ * Method:    nativeSetAllAllowAutomaticTabbingProperty
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_sun_lwawt_macosx_CPlatformWindow_nativeSetAllowAutomaticTabbingProperty
+        (JNIEnv *env, jclass clazz, jboolean allowAutomaticTabbing)
+{
+    JNI_COCOA_ENTER(env);
+    [ThreadUtilities performOnMainThreadWaiting:NO block:^(){
+        if (allowAutomaticTabbing) {
+            [NSWindow setAllowsAutomaticWindowTabbing:YES];
+        } else {
+            [NSWindow setAllowsAutomaticWindowTabbing:NO];
+        }
+    }];
+    JNI_COCOA_EXIT(env);
+}
 
 /*
  * Class:     sun_lwawt_macosx_CPlatformWindow

test/jdk/java/awt/Window/TestAppFreeze.java

@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import javax.swing.AbstractAction;
+import javax.swing.JButton;
+import javax.swing.JDialog;
+import javax.swing.JFrame;
+import javax.swing.JPanel;
+import javax.swing.JTextArea;
+import javax.swing.SwingUtilities;
+import javax.swing.WindowConstants;
+import java.awt.BorderLayout;
+import java.awt.FlowLayout;
+import java.awt.event.ActionEvent;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *  @test
+ *  @bug 8256465
+ *  @requires (os.family == "Mac")
+ *  @summary Application should not freeze when opening dialog
+ *  @key headful
+ *  @run main/manual TestAppFreeze
+ */
+
+public class TestAppFreeze {
+    private static volatile CountDownLatch countDownLatch;
+    private static JFrame instructionFrame;
+    private static JFrame testFrame;
+    private static volatile boolean testPassed = false;
+
+    private static boolean validatePlatform() {
+        String osName = System.getProperty("os.name");
+        if (osName == null) {
+            throw new RuntimeException("Name of the current OS could not be" +
+                    " retrieved.");
+        }
+        return osName.startsWith("Mac");
+    }
+
+    private static void createInstructionUI() {
+        SwingUtilities.invokeLater(() -> {
+            String instruction = "1. This test is only for Mac OS Version 11 " +
+                    "or later, on other Mac OS just press PASS\n" +
+                    "2. Go to System Preference -> General\n"+
+                    "3. Set prefer tab while opening document to Always.\n" +
+                    "4. Then click on the click button of the test frame\n" +
+                    "5. The dialog should open in new window and Application " +
+                    "should not freeze\n" +
+                    "6. IF the conditions are met then press PASS else " +
+                    "press FAIL";
+            instructionFrame = new JFrame("Instruction Frame");
+            JTextArea  textArea = new JTextArea(instruction);
+            textArea.setEditable(false);
+            final JButton passButton = new JButton("PASS");
+            passButton.addActionListener((e -> {
+                testPassed = true;
+                instructionFrame.dispose();
+                testFrame.dispose();
+                countDownLatch.countDown();
+            }));
+            final JButton failButton = new JButton("FAIL");
+            failButton.addActionListener((e) -> {
+                instructionFrame.dispose();
+                testFrame.dispose();
+                countDownLatch.countDown();
+            });
+
+            JPanel mainPanel = new JPanel(new BorderLayout());
+            mainPanel.add(textArea, BorderLayout.CENTER);
+
+            JPanel buttonPanel =  new JPanel(new FlowLayout());
+            buttonPanel.add(passButton);
+            buttonPanel.add(failButton);
+            mainPanel.add(buttonPanel, BorderLayout.SOUTH);
+            instructionFrame.setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE);
+            instructionFrame.setBounds(0, 0, 500, 500);
+            instructionFrame.add(mainPanel);
+            instructionFrame.pack();
+            instructionFrame.setVisible(true);
+        });
+    }
+
+    private static void testApp() {
+        testFrame = new JFrame("TestFrame");
+        testFrame.setBounds(600,0,1000,200);
+        testFrame.getContentPane().add(new JButton(new AbstractAction("Click") {
+            @Override
+            public void actionPerformed(ActionEvent e) {
+                JDialog dlg = new JDialog(testFrame, false);
+                dlg.setSize(500, 500);
+                dlg.getContentPane().add(new JTextArea());
+                dlg.setVisible(true);
+            }
+        }));
+        testFrame.setVisible(true);
+    }
+
+    public static void main(String[] args) throws Exception{
+        if (!validatePlatform()) {
+            System.out.println("This test is only for Mac OS");
+            return;
+        }
+        countDownLatch = new CountDownLatch(1);
+        TestAppFreeze testAppFreeze = new TestAppFreeze();
+        testAppFreeze.createInstructionUI();
+        testAppFreeze.testApp();
+        countDownLatch.await(15, TimeUnit.MINUTES);
+
+        if(!testPassed) {
+            throw new RuntimeException("Test failed!");
+        }
+    }
+}