8280993: [XWayland] Popup is not closed on click outside of area cont…

…rolled by XWayland

Reviewed-by: lucy
Backport-of: 3d550f748521443dc2e148aa4f786060419f1113

src/java.desktop/share/classes/javax/swing/JPopupMenu.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,7 @@
 import java.awt.Point;
 import java.awt.Rectangle;
 import java.awt.Toolkit;
+import java.awt.Window;
 import java.awt.event.FocusEvent;
 import java.awt.event.KeyEvent;
 import java.awt.event.MouseEvent;
@@ -757,6 +758,16 @@ public void pack() {
         }
     }
 
+    private Window getMenuInvoker() {
+        if (invoker instanceof Window menuInvoker) {
+            return menuInvoker;
+        } else {
+            return invoker == null
+                    ? null
+                    : SwingUtilities.getWindowAncestor(invoker);
+        }
+    }
+
     /**
      * Sets the visibility of the popup menu.
      *
@@ -798,14 +809,24 @@ public void setVisible(boolean b) {
             }
         }
 
-        if(b) {
+        if (b) {
             firePopupMenuWillBecomeVisible();
+
+            if (Toolkit.getDefaultToolkit() instanceof SunToolkit sunToolkit) {
+                sunToolkit.dismissPopupOnFocusLostIfNeeded(getMenuInvoker());
+            }
+
             showPopup();
             firePropertyChange("visible", Boolean.FALSE, Boolean.TRUE);
 
 
-        } else if(popup != null) {
+        } else if (popup != null) {
             firePopupMenuWillBecomeInvisible();
+
+            if (Toolkit.getDefaultToolkit() instanceof SunToolkit sunToolkit) {
+                sunToolkit.dismissPopupOnFocusLostIfNeededCleanUp(getMenuInvoker());
+            }
+
             popup.hide();
             popup = null;
             firePropertyChange("visible", Boolean.TRUE, Boolean.FALSE);

src/java.desktop/share/classes/sun/awt/SunToolkit.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1883,6 +1883,15 @@ public boolean isNativeGTKAvailable() {
         return false;
     }
 
+    public boolean isRunningOnWayland() {
+        return false;
+    }
+
+    public void dismissPopupOnFocusLostIfNeeded(Window invoker) {}
+
+    public void dismissPopupOnFocusLostIfNeededCleanUp(Window invoker) {}
+
+
     private static final Object DEACTIVATION_TIMES_MAP_KEY = new Object();
 
     public synchronized void setWindowDeactivationTime(Window w, long time) {

src/java.desktop/unix/classes/sun/awt/UNIXToolkit.java

@@ -25,15 +25,37 @@
 package sun.awt;
 
 import java.awt.RenderingHints;
-import static java.awt.RenderingHints.*;
+
+import static java.awt.RenderingHints.KEY_TEXT_ANTIALIASING;
+import static java.awt.RenderingHints.VALUE_TEXT_ANTIALIAS_DEFAULT;
+import static java.awt.RenderingHints.VALUE_TEXT_ANTIALIAS_LCD_HBGR;
+import static java.awt.RenderingHints.VALUE_TEXT_ANTIALIAS_LCD_HRGB;
+import static java.awt.RenderingHints.VALUE_TEXT_ANTIALIAS_LCD_VBGR;
+import static java.awt.RenderingHints.VALUE_TEXT_ANTIALIAS_LCD_VRGB;
+import static java.awt.RenderingHints.VALUE_TEXT_ANTIALIAS_ON;
+
 import static java.util.concurrent.TimeUnit.SECONDS;
 import java.awt.color.ColorSpace;
-import java.awt.image.*;
+
+import java.awt.Window;
+import java.awt.event.WindowAdapter;
+import java.awt.event.WindowEvent;
+import java.awt.event.WindowFocusListener;
+import java.awt.image.BufferedImage;
+import java.awt.image.ColorModel;
+import java.awt.image.ComponentColorModel;
+import java.awt.image.DataBuffer;
+import java.awt.image.DataBufferByte;
+import java.awt.image.Raster;
+import java.awt.image.WritableRaster;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
+import java.util.Arrays;
+
+import sun.awt.X11.XBaseWindow;
 
 import sun.security.action.GetIntegerAction;
 import com.sun.java.swing.plaf.gtk.GTKConstants.TextDirection;
@@ -491,4 +513,91 @@ public static boolean isGtkVerbose() {
         return AccessController.doPrivileged((PrivilegedAction<Boolean>)()
                 -> Boolean.getBoolean("jdk.gtk.verbose"));
     }
+
+    private static volatile Boolean isOnWayland = null;
+
+    @SuppressWarnings("removal")
+    public static boolean isOnWayland() {
+        Boolean result = isOnWayland;
+        if (result == null) {
+            synchronized (GTK_LOCK) {
+                result = isOnWayland;
+                if (result == null) {
+                    isOnWayland
+                            = result
+                            = AccessController.doPrivileged(
+                            (PrivilegedAction<Boolean>) () -> {
+                                final String display =
+                                        System.getenv("WAYLAND_DISPLAY");
+
+                                return display != null
+                                        && !display.trim().isEmpty();
+                            }
+                    );
+                }
+            }
+        }
+        return result;
+    }
+
+    @Override
+    public boolean isRunningOnWayland() {
+        return isOnWayland();
+    }
+
+    // We rely on the X11 input grab mechanism, but for the Wayland session
+    // it only works inside the XWayland server, so mouse clicks outside of it
+    // will not be detected.
+    // (window decorations, pure Wayland applications, desktop, etc.)
+    //
+    // As a workaround, we can dismiss menus when the window loses focus.
+    //
+    // However, there are "blind spots" though, which, when clicked, don't
+    // transfer the focus away and don't dismiss the menu
+    // (e.g. the window's own title or the area in the side dock without
+    // application icons).
+    private static final WindowFocusListener waylandWindowFocusListener;
+
+    static {
+        if (isOnWayland()) {
+            waylandWindowFocusListener = new WindowAdapter() {
+                @Override
+                public void windowLostFocus(WindowEvent e) {
+                    Window window = e.getWindow();
+                    window.removeWindowFocusListener(this);
+
+                    // AWT
+                    XBaseWindow.ungrabInput();
+
+                    // Swing
+                    window.dispatchEvent(new UngrabEvent(window));
+                }
+            };
+        } else {
+            waylandWindowFocusListener = null;
+        }
+    }
+
+    @Override
+    public void dismissPopupOnFocusLostIfNeeded(Window invoker) {
+        if (!isOnWayland()
+                || invoker == null
+                || Arrays
+                    .asList(invoker.getWindowFocusListeners())
+                    .contains(waylandWindowFocusListener)
+        ) {
+            return;
+        }
+
+        invoker.addWindowFocusListener(waylandWindowFocusListener);
+    }
+
+    @Override
+    public void dismissPopupOnFocusLostIfNeededCleanUp(Window invoker) {
+        if (!isOnWayland() || invoker == null) {
+            return;
+        }
+
+        invoker.removeWindowFocusListener(waylandWindowFocusListener);
+    }
 }

src/java.desktop/unix/classes/sun/awt/X11/XBaseWindow.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,9 +25,13 @@
 
 package sun.awt.X11;
 
-import java.awt.*;
-import sun.awt.*;
-import java.util.*;
+import java.awt.Dimension;
+import java.awt.Point;
+import java.awt.Rectangle;
+import java.util.HashSet;
+import java.util.Set;
+
+import sun.awt.SunToolkit;
 import sun.util.logging.PlatformLogger;
 
 public class XBaseWindow {
@@ -912,7 +916,7 @@ public boolean grabInput() {
         }
     }
 
-    static void ungrabInput() {
+    public static void ungrabInput() {
         XToolkit.awtLock();
         try {
             XBaseWindow grabWindow = XAwtState.getGrabWindow();

src/java.desktop/unix/classes/sun/awt/X11/XMenuWindow.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,16 +24,21 @@
  */
 package sun.awt.X11;
 
-import java.awt.*;
-import java.awt.peer.*;
-import java.awt.event.*;
-
-import java.awt.image.BufferedImage;
-import java.awt.geom.Point2D;
+import java.awt.Dimension;
+import java.awt.Graphics;
+import java.awt.MenuItem;
+import java.awt.Point;
+import java.awt.Rectangle;
+import java.awt.Toolkit;
+import java.awt.Window;
 
 import java.util.Vector;
+
+import sun.awt.SunToolkit;
 import sun.util.logging.PlatformLogger;
 
+import javax.swing.SwingUtilities;
+
 public class XMenuWindow extends XBaseMenuWindow {
 
     /************************************************
@@ -389,6 +394,16 @@ boolean ensureCreated() {
         return true;
     }
 
+    protected Window getMenuTarget() {
+        if (target instanceof Window targetWindow) {
+            return targetWindow;
+        } else {
+            return target == null
+                    ? null
+                    : SwingUtilities.getWindowAncestor(target);
+        }
+    }
+
     /**
      * Init window if it's not inited yet
      * and show it at specified coordinates
@@ -405,6 +420,9 @@ void show(Rectangle bounds) {
         XToolkit.awtLock();
         try {
             reshape(bounds.x, bounds.y, bounds.width, bounds.height);
+            if (Toolkit.getDefaultToolkit() instanceof SunToolkit sunToolkit) {
+                sunToolkit.dismissPopupOnFocusLostIfNeeded(getMenuTarget());
+            }
             xSetVisible(true);
             //Fixed 6267182: PIT: Menu is not visible after
             //showing and disposing a file dialog, XToolkit
@@ -421,6 +439,9 @@ void show(Rectangle bounds) {
     void hide() {
         selectItem(null, false);
         xSetVisible(false);
+        if (Toolkit.getDefaultToolkit() instanceof SunToolkit sunToolkit) {
+            sunToolkit.dismissPopupOnFocusLostIfNeededCleanUp(getMenuTarget());
+        }
     }
 
     /************************************************

src/java.desktop/unix/classes/sun/awt/X11/XPopupMenuPeer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,12 +24,25 @@
  */
 package sun.awt.X11;
 
-import java.awt.*;
-import java.awt.peer.*;
-import java.awt.event.*;
+import java.awt.AWTEvent;
+import java.awt.Component;
+import java.awt.Dimension;
+import java.awt.Event;
+import java.awt.Font;
+import java.awt.FontMetrics;
+import java.awt.Graphics;
+import java.awt.MenuItem;
+import java.awt.Point;
+import java.awt.PopupMenu;
+import java.awt.Rectangle;
+import java.awt.Toolkit;
+import java.awt.event.KeyEvent;
+import java.awt.event.MouseEvent;
 
+import java.awt.peer.PopupMenuPeer;
 import java.util.Vector;
 import sun.awt.AWTAccessor;
+import sun.awt.SunToolkit;
 import sun.util.logging.PlatformLogger;
 
 public class XPopupMenuPeer extends XMenuWindow implements PopupMenuPeer {
@@ -125,6 +138,9 @@ public void show(Event e) {
             //near the periphery of the screen, XToolkit
             Rectangle bounds = getWindowBounds(pt, dim);
             reshape(bounds);
+            if (Toolkit.getDefaultToolkit() instanceof SunToolkit sunToolkit) {
+                sunToolkit.dismissPopupOnFocusLostIfNeeded(getMenuTarget());
+            }
             xSetVisible(true);
             toFront();
             selectItem(null, false);