fix(windows): fix undecorated resizing (#878)

* fix(windows): fix undecorated resizing

also disable resizing for fullscreen windows on Linux and Window

closes #877

* remove redundant unsafe

* check the atomic bool only if needed

* one less clone

* remove unneeded checks

* Update window.rs

* Update window.rs

.changes/hit-test.md

@@ -0,0 +1,5 @@
+---
+"tao": "minor"
+---
+
+**Breaking change**: Removed `window::hit_test` function.

.changes/undecorated-resizing-fullscreen.md

@@ -0,0 +1,5 @@
+---
+"tao": "patch"
+---
+
+On Windows and Linux, disable resizing undecorated windows when in fullscreen.

.changes/win-undecorated-resizing.md

@@ -0,0 +1,5 @@
+---
+"tao": "patch"
+---
+
+On Windows, fix undecorated window resizing.

src/platform_impl/linux/event_loop.rs

@@ -17,7 +17,11 @@ use crossbeam_channel::SendError;
 use gdk::{Cursor, CursorType, EventKey, EventMask, ScrollDirection, WindowEdge, WindowState};
 use gio::Cancellable;
 use glib::{source::Priority, MainContext};
-use gtk::{cairo, gdk, gio, glib, prelude::*};
+use gtk::{
+  cairo, gdk, gio,
+  glib::{self},
+  prelude::*,
+};
 
 use crate::{
   dpi::{LogicalPosition, LogicalSize, PhysicalPosition},
@@ -388,6 +392,7 @@ impl<T: 'static> EventLoop<T> {
           WindowRequest::ProgressBarState(_) => unreachable!(),
           WindowRequest::WireUpEvents {
             transparent,
+            fullscreen,
             cursor_moved,
           } => {
             window.add_events(
@@ -400,41 +405,64 @@ impl<T: 'static> EventLoop<T> {
                 | EventMask::SCROLL_MASK,
             );
 
-            // Allow resizing unmaximized borderless window
-            window.connect_motion_notify_event(|window, event| {
+            let fullscreen = Rc::new(AtomicBool::new(fullscreen));
+            let fullscreen_ = fullscreen.clone();
+            window.connect_window_state_event(move |_window, event| {
+              let state = event.changed_mask();
+              if state.contains(WindowState::FULLSCREEN) {
+                fullscreen_.store(
+                  event.new_window_state().contains(WindowState::FULLSCREEN),
+                  Ordering::Relaxed,
+                );
+              }
+              glib::Propagation::Proceed
+            });
+
+            // Allow resizing unmaximized non-fullscreen undecorated window
+            let fullscreen_ = fullscreen.clone();
+            window.connect_motion_notify_event(move |window, event| {
               if !window.is_decorated() && window.is_resizable() && !window.is_maximized() {
                 if let Some(window) = window.window() {
                   let (cx, cy) = event.root();
                   let (left, top) = window.position();
                   let (w, h) = (window.width(), window.height());
                   let (right, bottom) = (left + w, top + h);
+                  let border = window.scale_factor() * 5;
                   let edge = crate::window::hit_test(
                     (left, top, right, bottom),
                     cx as _,
                     cy as _,
-                    window.scale_factor() as _,
+                    border,
+                    border,
                   );
+
                   let edge = match &edge {
-                    Some(e) => e.to_cursor_str(),
-                    None => "default",
+                    Some(e) if !fullscreen_.load(Ordering::Relaxed) => e.to_cursor_str(),
+                    _ => "default",
                   };
                   window.set_cursor(Cursor::from_name(&window.display(), edge).as_ref());
                 }
               }
               glib::Propagation::Proceed
             });
-            window.connect_button_press_event(|window, event| {
-              if !window.is_decorated() && window.is_resizable() && event.button() == 1 {
+            window.connect_button_press_event(move |window, event| {
+              if !window.is_decorated()
+                && window.is_resizable()
+                && !window.is_maximized()
+                && event.button() == 1
+              {
                 if let Some(window) = window.window() {
                   let (cx, cy) = event.root();
                   let (left, top) = window.position();
                   let (w, h) = (window.width(), window.height());
                   let (right, bottom) = (left + w, top + h);
+                  let border = window.scale_factor() * 5;
                   let edge = crate::window::hit_test(
                     (left, top, right, bottom),
                     cx as _,
                     cy as _,
-                    window.scale_factor() as _,
+                    border,
+                    border,
                   )
                   .map(|d| d.to_gtk_edge())
                   // we return `WindowEdge::__Unknown` to be ignored later.
@@ -453,19 +481,21 @@ impl<T: 'static> EventLoop<T> {
 
               glib::Propagation::Proceed
             });
-            window.connect_touch_event(|window, event| {
-              if !window.is_decorated() && window.is_resizable() {
+            window.connect_touch_event(move |window, event| {
+              if !window.is_decorated() && window.is_resizable() && !window.is_maximized() {
                 if let Some(window) = window.window() {
                   if let Some((cx, cy)) = event.root_coords() {
                     if let Some(device) = event.device() {
                       let (left, top) = window.position();
                       let (w, h) = (window.width(), window.height());
                       let (right, bottom) = (left + w, top + h);
+                      let border = window.scale_factor() * 5;
                       let edge = crate::window::hit_test(
                         (left, top, right, bottom),
                         cx as _,
                         cy as _,
-                        window.scale_factor() as _,
+                        border,
+                        border,
                       )
                       .map(|d| d.to_gtk_edge())
                       // we return `WindowEdge::__Unknown` to be ignored later.

src/platform_impl/linux/window.rs

@@ -282,6 +282,7 @@ impl Window {
       window_id,
       WindowRequest::WireUpEvents {
         transparent,
+        fullscreen: attributes.fullscreen.is_some(),
         cursor_moved,
       },
     )) {
@@ -905,6 +906,7 @@ pub enum WindowRequest {
   CursorIgnoreEvents(bool),
   WireUpEvents {
     transparent: bool,
+    fullscreen: bool,
     cursor_moved: bool,
   },
   SetVisibleOnAllWorkspaces(bool),

src/platform_impl/windows/event_loop.rs

@@ -2132,31 +2132,39 @@ unsafe fn public_window_callback_inner<T: 'static>(
 
     win32wm::WM_NCHITTEST => {
       let window_state = subclass_input.window_state.lock();
-      // Allow resizing unmaximized borderless window
-      if !util::is_maximized(window).unwrap_or(false)
-        && !window_state
-          .window_flags()
-          .contains(WindowFlags::MARKER_DECORATIONS)
+      let window_flags = window_state.window_flags();
+
+      // Allow resizing unmaximized non-fullscreen undecorated window
+      if !window_flags.contains(WindowFlags::MARKER_DECORATIONS)
+        && window_flags.contains(WindowFlags::RESIZABLE)
+        && window_state.fullscreen.is_none()
+        && !util::is_maximized(window).unwrap_or(false)
       {
         // cursor location
         let (cx, cy) = (
-          i32::from(util::GET_X_LPARAM(lparam)),
-          i32::from(util::GET_Y_LPARAM(lparam)),
+          util::GET_X_LPARAM(lparam) as i32,
+          util::GET_Y_LPARAM(lparam) as i32,
         );
 
         let mut rect = RECT::default();
-        let _ = GetClientRect(window, &mut rect);
+        let _ = GetWindowRect(window, &mut rect);
+
+        let padded_border = GetSystemMetrics(SM_CXPADDEDBORDER);
+        let border_x = GetSystemMetrics(SM_CXFRAME) + padded_border;
+        let border_y = GetSystemMetrics(SM_CYFRAME) + padded_border;
 
         let hit_result = crate::window::hit_test(
           (rect.left, rect.top, rect.right, rect.bottom),
           cx,
           cy,
-          window_state.scale_factor,
+          border_x,
+          border_y,
         )
-        .map(|d| d.to_win32())
-        .unwrap_or(HTCLIENT);
+        .map(|d| d.to_win32());
 
-        result = ProcResult::Value(LRESULT(hit_result as _));
+        result = hit_result
+          .map(|r| ProcResult::Value(LRESULT(r as _)))
+          .unwrap_or(ProcResult::DefSubclassProc);
       } else {
         result = ProcResult::DefSubclassProc;
       }

src/window.rs

@@ -1560,11 +1560,12 @@ pub enum ResizeDirection {
   West,
 }
 
-pub fn hit_test(
+pub(crate) fn hit_test(
   (left, top, right, bottom): (i32, i32, i32, i32),
   cx: i32,
   cy: i32,
-  scale_factor: f64,
+  border_x: i32,
+  border_y: i32,
 ) -> Option<ResizeDirection> {
   const LEFT: isize = 0b0001;
   const RIGHT: isize = 0b0010;
@@ -1575,14 +1576,11 @@ pub fn hit_test(
   const BOTTOMLEFT: isize = BOTTOM | LEFT;
   const BOTTOMRIGHT: isize = BOTTOM | RIGHT;
 
-  let inset = (5 as f64 * scale_factor) as i32;
-
   #[rustfmt::skip]
-      let result =
-          (LEFT * (if cx < (left + inset) { 1 } else { 0 }))
-        | (RIGHT * (if cx >= (right - inset) { 1 } else { 0 }))
-        | (TOP * (if cy < (top + inset) { 1 } else { 0 }))
-        | (BOTTOM * (if cy >= (bottom - inset) { 1 } else { 0 }));
+  let result = (LEFT * (cx < left + border_x) as isize)
+             | (RIGHT * (cx >= right - border_x) as isize)
+             | (TOP * (cy < top + border_y) as isize)
+             | (BOTTOM * (cy >= bottom - border_y) as isize);
 
   match result {
     LEFT => Some(ResizeDirection::West),