[x11 layering] Fix bug with multiple transients (#4417)

Google Earth identified a couple of issues with the layering code:
- The `transient_for` property for some windows returned the ID of a
  window that was not being managed by qtile. As a result, the parent
  window would not have been stacked and placing the new window above
  this would therefore put the new window in the wrong stack position.
- Transient windows were excluded from the list of windows in different
  layers and so, if an app has multiple transient windows, a new window
  would not be stacked after the last transient.

CHANGELOG

@@ -7,6 +7,7 @@ Qtile x.xx.x, released XXXX-XX-XX:
       - Updated `Bluetooth` widget which allows users to manage multiple devices in a single widget
       - Add `align` option to `Columns` layout so new windows can be added to left or right column.
     * bugfixes
+      - Fix two bugs in stacking transient windows in X11
 
 Qtile 0.23.0, released 2023-09-24:
     !!! Dependency Changes !!!

libqtile/backend/x11/window.py

@@ -1044,15 +1044,32 @@ def change_layer(self, up=True, top_bottom=False):
         windows.sort(key=lambda w: stack.index(w[0].wid))
 
         # Get lists of windows on lower, higher or same "layer" as window
-        lower = [w[0].wid for w in windows if w[1] is None and w[2] > layering]
-        higher = [w[0].wid for w in windows if w[1] is None and w[2] < layering]
-        same = [w[0].wid for w in windows if w[1] is None and w[2] == layering]
+        lower = [w[0].wid for w in windows if w[2] > layering]
+        higher = [w[0].wid for w in windows if w[2] < layering]
+        same = [w[0].wid for w in windows if w[2] == layering]
 
         # We now need to identify the new position in the stack
 
         # If the window has a parent, the window should just be put above it
-        if parent:
-            sibling = parent
+        # If the parent isn't being managed by qtile then it may not be stacked correctly
+        if parent and parent in self.qtile.windows_map:
+            # If the window is modal then it should be placed above every other window that is in that window group
+            # e.g. the parent of the dialog and any other window that is also transient for the same parent.
+            if "_NET_WM_STATE_MODAL" in self.window.get_net_wm_state():
+                window_group = [parent]
+                window_group.extend(
+                    k
+                    for k, v in self.qtile.windows_map.items()
+                    if v.window.get_wm_transient_for() == parent
+                )
+                window_group.sort(key=stack.index)
+
+                # Make sure we're above the last window in that group
+                sibling = window_group[-1]
+
+            else:
+                sibling = parent
+
             above = True
 
         # Now we just check whether the window has changed layer.
@@ -1166,9 +1183,31 @@ def change_layer(self, up=True, top_bottom=False):
             stackmode=xcffib.xproto.StackMode.Above if above else xcffib.xproto.StackMode.Below,
             sibling=sibling,
         )
-        # TODO: also move our children if we were moved upwards
+
+        # Move window's children if we were moved upwards
+        if above:
+            self.raise_children(stack=stack)
+
         self.qtile.core.update_client_lists()
 
+    def raise_children(self, stack=None):
+        """Ensure any transient windows are moved up with the parent."""
+        children = [
+            k
+            for k, v in self.qtile.windows_map.items()
+            if v.window.get_wm_transient_for() == self.window.wid
+        ]
+        if children:
+            if stack is None:
+                stack = list(self.qtile.core._root.query_tree())
+            parent = self.window.wid
+            children.sort(key=stack.index)
+            for child in children:
+                self.qtile.windows_map[child].window.configure(
+                    stackmode=xcffib.xproto.StackMode.Above, sibling=parent
+                )
+                parent = child
+
     def paint_borders(self, color, width):
         self.borderwidth = width
         self.bordercolor = color
@@ -1622,6 +1661,8 @@ def handle_PropertyNotify(self, e):  # noqa: N802
     def bring_to_front(self):
         if self.get_wm_type() != "desktop":
             self.window.configure(stackmode=xcffib.xproto.StackMode.Above)
+            self.raise_children()
+            self.qtile.core.update_client_lists()
 
 
 class Window(_Window, base.Window):
@@ -2243,6 +2284,8 @@ def disable_fullscreen(self):
     def bring_to_front(self):
         if self.get_wm_type() != "desktop":
             self.window.configure(stackmode=xcffib.xproto.StackMode.Above)
+            self.raise_children()
+            self.qtile.core.update_client_lists()
 
     def _is_in_window(self, x, y, window):
         return window.edges[0] <= x <= window.edges[2] and window.edges[1] <= y <= window.edges[3]