8359601: Fix window button states of an extended stage

modules/javafx.graphics/src/main/java/com/sun/glass/ui/HeaderButtonOverlay.java

@@ -26,7 +26,6 @@
 package com.sun.glass.ui;
 
 import com.sun.glass.events.MouseEvent;
-import com.sun.javafx.binding.ObjectConstant;
 import com.sun.javafx.util.Utils;
 import javafx.beans.property.DoubleProperty;
 import javafx.beans.property.ObjectProperty;
@@ -45,6 +44,7 @@
 import javafx.css.StyleableIntegerProperty;
 import javafx.css.StyleableObjectProperty;
 import javafx.css.StyleableProperty;
+import javafx.event.Event;
 import javafx.geometry.Dimension2D;
 import javafx.geometry.HPos;
 import javafx.geometry.Insets;
@@ -57,6 +57,7 @@
 import javafx.scene.paint.Paint;
 import javafx.stage.Stage;
 import javafx.stage.StageStyle;
+import javafx.stage.WindowEvent;
 import javafx.util.Subscription;
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -264,12 +265,15 @@ protected void invalidated() {
     private final ButtonRegion maximizeButton = new ButtonRegion(HeaderButtonType.MAXIMIZE, "-FX-INTERNAL-maximize-button", 1);
     private final ButtonRegion closeButton = new ButtonRegion(HeaderButtonType.CLOSE, "-FX-INTERNAL-close-button", 2);
     private final Subscription subscriptions;
+    private final boolean modalOrOwned;
     private final boolean utility;
     private final boolean rightToLeft;
 
     private Node buttonAtMouseDown;
 
-    public HeaderButtonOverlay(ObservableValue<String> stylesheet, boolean utility, boolean rightToLeft) {
+    public HeaderButtonOverlay(ObservableValue<String> stylesheet, boolean modalOrOwned,
+                               boolean utility, boolean rightToLeft) {
+        this.modalOrOwned = modalOrOwned;
         this.utility = utility;
         this.rightToLeft = rightToLeft;
 
@@ -310,10 +314,7 @@ public HeaderButtonOverlay(ObservableValue<String> stylesheet, boolean utility,
         getStyleClass().setAll("-FX-INTERNAL-header-button-container");
 
         if (utility) {
-            iconifyButton.managedProperty().bind(ObjectConstant.valueOf(false));
-            maximizeButton.managedProperty().bind(ObjectConstant.valueOf(false));
             getChildren().add(closeButton);
-            getStyleClass().add(UTILITY_STYLE_CLASS);
         } else {
             getChildren().addAll(iconifyButton, maximizeButton, closeButton);
         }
@@ -438,7 +439,7 @@ private void handleMouseUp(Node node, HeaderButtonType buttonType) {
             switch (buttonType) {
                 case ICONIFY -> stage.setIconified(true);
                 case MAXIMIZE -> stage.setMaximized(!stage.isMaximized());
-                case CLOSE -> stage.close();
+                case CLOSE -> Event.fireEvent(stage, new WindowEvent(stage, WindowEvent.WINDOW_CLOSE_REQUEST));
             }
         }
     }
@@ -450,6 +451,9 @@ private void onFocusedChanged(boolean focused) {
     }
 
     private void onResizableChanged(boolean resizable) {
+        boolean utilityStyle = utility || (modalOrOwned && !resizable);
+        toggleStyleClass(this, UTILITY_STYLE_CLASS, utilityStyle);
+        iconifyButton.setDisable(utility || modalOrOwned);
         maximizeButton.setDisable(!resizable);
     }
 

modules/javafx.graphics/src/main/java/com/sun/glass/ui/Window.java

@@ -280,6 +280,7 @@ protected Window(Window owner, Screen screen, int styleMask) {
         this.styleMask = styleMask;
         this.isDecorated = (this.styleMask & Window.TITLED) != 0;
         this.isPopup = (this.styleMask & Window.POPUP) != 0;
+        this.isModal = (this.styleMask & Window.MODAL) != 0;
 
         this.screen = screen != null ? screen : Screen.getMainScreen();
         if (PrismSettings.allowHiDPIScaling) {
@@ -1157,43 +1158,6 @@ public void toBack() {
         _toBack(this.ptr);
     }
 
-    // *****************************************************
-    // modality (prototype using native platform feature)
-    // *****************************************************
-    protected abstract void _enterModal(long ptr);
-    /**
-     * Enter modal state blocking everything except our window.
-     */
-    public void enterModal() {
-        checkNotClosed();
-        if (this.isModal == false) {
-            this.isModal = true;
-            _enterModal(this.ptr);
-        }
-    }
-
-    protected abstract void _enterModalWithWindow(long dialog, long window);
-    /**
-     * Enter modal state only blocking the given window.
-     * On Mac OS X this is done using a dialog sheet.
-     */
-    public void enterModal(final Window window) {
-        checkNotClosed();
-        if (this.isModal == false) {
-            this.isModal = true;
-            _enterModalWithWindow(this.ptr, window.getNativeHandle());
-        }
-    }
-
-    protected abstract void _exitModal(long ptr);
-    public void exitModal() {
-        checkNotClosed();
-        if (this.isModal == true) {
-            _exitModal(this.ptr);
-            this.isModal = false;
-        }
-    }
-
     public boolean isModal() {
         return this.isModal;
     }

modules/javafx.graphics/src/main/java/com/sun/glass/ui/gtk/GtkWindow.java

@@ -115,15 +115,6 @@ protected boolean _setMenubar(long ptr, long menubarPtr) {
     @Override
     protected native void _toBack(long ptr);
 
-    @Override
-    protected native void _enterModal(long ptr);
-
-    @Override
-    protected native void _enterModalWithWindow(long dialog, long window);
-
-    @Override
-    protected native void _exitModal(long ptr);
-
     protected native long _getNativeWindowImpl(long ptr);
 
     private native void _showSystemMenu(long ptr, int x, int y);
@@ -255,7 +246,7 @@ private void onPrefHeaderButtonHeightChanged(Number height) {
     private HeaderButtonOverlay createHeaderButtonOverlay() {
         var overlay = new HeaderButtonOverlay(
             PlatformThemeObserver.getInstance().stylesheetProperty(),
-            isUtilityWindow(),
+            isModal() || getOwner() != null, isUtilityWindow(),
             (getStyleMask() & RIGHT_TO_LEFT) != 0);
 
         // Set the system-defined absolute minimum size to the size of the window buttons area,

modules/javafx.graphics/src/main/java/com/sun/glass/ui/ios/IosWindow.java

@@ -61,9 +61,6 @@ protected IosWindow(Window owner, Screen screen, int styleMask) {
     @Override native protected void _setIcon(long ptr, Pixels pixels);
     @Override native protected void _toFront(long ptr);
     @Override native protected void _toBack(long ptr);
-    @Override native protected void _enterModal(long ptr);
-    @Override native protected void _enterModalWithWindow(long dialog, long window);
-    @Override native protected void _exitModal(long ptr);
     @Override native protected boolean _grabFocus(long ptr);
     @Override native protected void _ungrabFocus(long ptr);
 

modules/javafx.graphics/src/main/java/com/sun/glass/ui/mac/MacWindow.java

@@ -115,9 +115,6 @@ protected MacWindow(Window owner, Screen screen, int styleMask) {
 
     @Override native protected void _toFront(long ptr);
     @Override native protected void _toBack(long ptr);
-    @Override native protected void _enterModal(long ptr);
-    @Override native protected void _enterModalWithWindow(long dialog, long window);
-    @Override native protected void _exitModal(long ptr);
 
     @Override native protected boolean _grabFocus(long ptr);
     @Override native protected void _ungrabFocus(long ptr);

modules/javafx.graphics/src/main/java/com/sun/glass/ui/monocle/MonocleWindow.java

@@ -392,28 +392,6 @@ protected void _ungrabFocus(long ptr) {
         MonocleWindowManager.getInstance().ungrabFocus(this);
     }
 
-    /**
-     * The functions below are used when the platform support modality natively.
-     * Currently only GTK is using it. This functionality is disabled by
-     * default. In order to enable it this class need to override Window::
-     * supportsPlatformModality() to return true.
-     *
-     */
-    @Override
-    protected void _enterModal(long ptr) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected void _enterModalWithWindow(long dialog, long window) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected void _exitModal(long ptr) {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     protected void notifyClose() {
         super.notifyClose();

modules/javafx.graphics/src/main/java/com/sun/glass/ui/win/WinHeaderButtonOverlay.java

@@ -56,8 +56,8 @@ public class WinHeaderButtonOverlay extends HeaderButtonOverlay {
         { 2.5, 0.95 },
     };
 
-    public WinHeaderButtonOverlay(boolean utility, boolean rightToLeft) {
-        super(getStylesheet(), utility, rightToLeft);
+    public WinHeaderButtonOverlay(boolean modalOrOwned, boolean utility, boolean rightToLeft) {
+        super(getStylesheet(), modalOrOwned, utility, rightToLeft);
 
         var windowProperty = sceneProperty().flatMap(Scene::windowProperty);
 

modules/javafx.graphics/src/main/java/com/sun/glass/ui/win/WinWindow.java

@@ -289,9 +289,6 @@ protected boolean _setBackground(long ptr, float r, float g, float b) {
     @Override native protected void _setIcon(long ptr, Pixels pixels);
     @Override native protected void _toFront(long ptr);
     @Override native protected void _toBack(long ptr);
-    @Override native protected void _enterModal(long ptr);
-    @Override native protected void _enterModalWithWindow(long dialog, long window);
-    @Override native protected void _exitModal(long ptr);
     @Override native protected boolean _grabFocus(long ptr);
     @Override native protected void _ungrabFocus(long ptr);
     @Override native protected void _setCursor(long ptr, Cursor cursor);
@@ -381,7 +378,7 @@ private void onPrefHeaderButtonHeightChanged(Number height) {
      */
     private HeaderButtonOverlay createHeaderButtonOverlay() {
         var overlay = new WinHeaderButtonOverlay(
-            isUtilityWindow(),
+            isModal() || getOwner() != null, isUtilityWindow(),
             (getStyleMask() & RIGHT_TO_LEFT) != 0);
 
         overlay.prefButtonHeightProperty().bind(prefHeaderButtonHeightProperty());

modules/javafx.graphics/src/main/java/com/sun/javafx/scene/layout/HeaderButtonBehavior.java

@@ -28,17 +28,21 @@
 import com.sun.javafx.PlatformUtil;
 import javafx.beans.value.ObservableValue;
 import javafx.css.PseudoClass;
+import javafx.event.Event;
 import javafx.event.EventHandler;
 import javafx.scene.Node;
 import javafx.scene.Scene;
+import javafx.scene.input.MouseButton;
 import javafx.scene.input.MouseEvent;
 import javafx.scene.layout.HeaderButtonType;
 import javafx.stage.Modality;
 import javafx.stage.Stage;
+import javafx.stage.StageStyle;
+import javafx.stage.WindowEvent;
 import javafx.util.Subscription;
 import java.util.Objects;
 
-public final class HeaderButtonBehavior implements EventHandler<MouseEvent> {
+public final class HeaderButtonBehavior implements EventHandler<Event> {
 
     private static final PseudoClass MAXIMIZED_PSEUDO_CLASS = PseudoClass.getPseudoClass("maximized");
 
@@ -54,20 +58,24 @@ public HeaderButtonBehavior(Node node, HeaderButtonType type) {
             .flatMap(Scene::windowProperty)
             .map(w -> w instanceof Stage s ? s : null);
 
-        if (type == HeaderButtonType.MAXIMIZE) {
-            subscription = Subscription.combine(
+        Subscription subscription = Subscription.combine(
+            stage.flatMap(Stage::fullScreenProperty).subscribe(this::onFullScreenChanged),
+            () -> node.removeEventHandler(MouseEvent.MOUSE_RELEASED, this));
+
+        if (type != HeaderButtonType.CLOSE) {
+            subscription = Subscription.combine(subscription,
+                stage.subscribe(this::onStageChanged),
                 stage.flatMap(Stage::resizableProperty).subscribe(this::onResizableChanged),
-                stage.flatMap(Stage::fullScreenProperty).subscribe(this::onFullScreenChanged),
-                stage.flatMap(Stage::maximizedProperty).subscribe(this::onMaximizedChanged),
-                () -> node.removeEventHandler(MouseEvent.MOUSE_RELEASED, this)
-            );
-        } else {
-            subscription = Subscription.combine(
-                stage.flatMap(Stage::fullScreenProperty).subscribe(this::onFullScreenChanged),
-                () -> node.removeEventHandler(MouseEvent.MOUSE_RELEASED, this)
-            );
+                () -> { if (getStage() instanceof Stage s) s.removeEventFilter(WindowEvent.WINDOW_SHOWING, this); });
+        }
+
+        if (type == HeaderButtonType.MAXIMIZE) {
+            subscription = Subscription.combine(subscription,
+                stage.flatMap(Stage::maximizedProperty).subscribe(this::onMaximizedChanged));
         }
 
+        this.subscription = subscription;
+
         node.addEventHandler(MouseEvent.MOUSE_RELEASED, this);
 
         if (!node.focusTraversableProperty().isBound()) {
@@ -80,8 +88,16 @@ public void dispose() {
     }
 
     @Override
-    public void handle(MouseEvent event) {
-        if (!node.getLayoutBounds().contains(event.getX(), event.getY())) {
+    public void handle(Event event) {
+        if (event instanceof MouseEvent mouseEvent) {
+            handleMouseEvent(mouseEvent);
+        } else if (event instanceof WindowEvent windowEvent) {
+            handleWindowEvent(windowEvent);
+        }
+    }
+
+    private void handleMouseEvent(MouseEvent event) {
+        if (!node.getLayoutBounds().contains(event.getX(), event.getY()) || event.getButton() != MouseButton.PRIMARY) {
             return;
         }
 
@@ -91,7 +107,7 @@ public void handle(MouseEvent event) {
         }
 
         switch (type) {
-            case CLOSE -> stage.close();
+            case CLOSE -> Event.fireEvent(stage, new WindowEvent(stage, WindowEvent.WINDOW_CLOSE_REQUEST));
             case ICONIFY -> stage.setIconified(true);
             case MAXIMIZE -> {
                 // On macOS, a non-modal window is put into full-screen mode when the maximize button is clicked,
@@ -105,6 +121,12 @@ public void handle(MouseEvent event) {
         }
     }
 
+    private void handleWindowEvent(WindowEvent event) {
+        if (event.getEventType() == WindowEvent.WINDOW_SHOWING && getStage() instanceof Stage stage) {
+            updateButtonDisableState(stage);
+        }
+    }
+
     private Stage getStage() {
         Scene scene = node.getScene();
         if (scene == null) {
@@ -114,9 +136,32 @@ private Stage getStage() {
         return scene.getWindow() instanceof Stage stage ? stage : null;
     }
 
-    private void onResizableChanged(Boolean resizable) {
+    private void onStageChanged(Stage oldStage, Stage newStage) {
+        if (oldStage != null) {
+            oldStage.removeEventFilter(WindowEvent.WINDOW_SHOWING, this);
+        }
+
+        if (newStage != null) {
+            newStage.addEventFilter(WindowEvent.WINDOW_SHOWING, this);
+        }
+    }
+
+    private void onResizableChanged(Boolean unused) {
+        if (getStage() instanceof Stage stage) {
+            updateButtonDisableState(stage);
+        }
+    }
+
+    private void updateButtonDisableState(Stage stage) {
         if (!node.disableProperty().isBound()) {
-            node.setDisable(resizable == Boolean.FALSE);
+            boolean utility = stage.getStyle() == StageStyle.UTILITY;
+            boolean modalOrOwned = stage.getOwner() != null || stage.getModality() != Modality.NONE;
+            boolean resizable = stage.isResizable();
+
+            switch (type) {
+                case ICONIFY -> node.setDisable(utility || modalOrOwned);
+                case MAXIMIZE -> node.setDisable(!resizable);
+            }
         }
     }