Drop the X.Org server dependency

Start GNOME Kiosk as a Wayland compositor and run Anaconda as a native
Wayland client.

This commit is a follow up on the work done by Neal Gompa [1], Martin
Kolman and Ray Strode [2]. Credit goes to them for the code I copied
and pasted.

[1] rhinstaller#5401
[2] rhinstaller#5309

Co-authored-by: Neal Gompa <neal@gompa.dev>
Co-authored-by: Martin Kolman <mkolman@redhat.com>
Co-authored-by: Ray Strode <rstrode@redhat.com>

Resolves: RHEL-38399

anaconda.py

@@ -144,6 +144,12 @@ def setup_environment():
     if "LD_PRELOAD" in os.environ:
         del os.environ["LD_PRELOAD"]
 
+    # Go ahead and set $WAYLAND_DISPLAY whether we're going to use Wayland or not
+    if "WAYLAND_DISPLAY" in os.environ:
+        flags.preexisting_wayland = True
+    else:
+        os.environ["WAYLAND_DISPLAY"] = constants.WAYLAND_SOCKET_NAME
+
     # Go ahead and set $DISPLAY whether we're going to use X or not
     if "DISPLAY" in os.environ:
         flags.preexisting_x11 = True
@@ -307,10 +313,11 @@ def setup_environment():
     except pid.PidFileError as e:
         log.error("Unable to create %s, exiting", pidfile.filename)
 
-        # If we had a $DISPLAY at start and zenity is available, we may be
-        # running in a live environment and we can display an error dialog.
+        # If we had a Wayland/X11 display at start and zenity is available, we may
+        # be running in a live environment and we can display an error dialog.
         # Otherwise just print an error.
-        if flags.preexisting_x11 and os.access("/usr/bin/zenity", os.X_OK):
+        preexisting_graphics = flags.preexisting_wayland or flags.preexisting_x11
+        if preexisting_graphics and os.access("/usr/bin/zenity", os.X_OK):
             # The module-level _() calls are ok here because the language may
             # be set from the live environment in this case, and anaconda's
             # language setup hasn't happened yet.

anaconda.spec.in

@@ -249,14 +249,13 @@ Requires: zram-generator
 # needed for proper driver disk support - if RPMs must be installed, a repo is needed
 Requires: createrepo_c
 # Display stuff moved from lorax templates
-Requires: xorg-x11-drivers
-Requires: xorg-x11-server-Xorg
 Requires: dbus-x11
 Requires: gsettings-desktop-schemas
 Requires: nm-connection-editor
 Requires: librsvg2
 Requires: gnome-kiosk
 Requires: brltty
+Requires: python3-pam
 # dependencies for rpm-ostree payload module
 Requires: rpm-ostree >= %{rpmostreever}
 Requires: ostree
@@ -399,6 +398,7 @@ rm -rf \
 %{_sbindir}/anaconda
 %{_sbindir}/handle-sshpw
 %{_datadir}/anaconda
+%{_sysconfdir}/pam.d/anaconda
 %{_prefix}/libexec/anaconda
 %exclude %{_datadir}/anaconda/gnome
 %exclude %{_datadir}/anaconda/pixmaps

configure.ac

@@ -112,6 +112,7 @@ AC_CONFIG_FILES([Makefile
                  data/systemd/Makefile
                  data/dbus/Makefile
                  data/gtk-4.0/Makefile
+                 data/pam/Makefile
                  data/window-manager/Makefile
                  data/window-manager/config/Makefile
                  po/Makefile

data/Makefile.am

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-SUBDIRS = command-stubs gtk-4.0 liveinst systemd pixmaps window-manager dbus conf.d profile.d
+SUBDIRS = command-stubs gtk-4.0 liveinst systemd pam pixmaps window-manager dbus conf.d profile.d
 
 CLEANFILES = *~
 

data/pam/Makefile.am

@@ -0,0 +1,21 @@
+# Copyright (C) 2024  Neal Gompa.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation; either version 2.1 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+CLEANFILES = *~
+
+pamdir           = $(sysconfdir)/pam.d
+dist_pam_DATA    = anaconda
+
+MAINTAINERCLEANFILES = Makefile.in

data/pam/anaconda

@@ -0,0 +1,8 @@
+#%PAM-1.0
+auth       sufficient  pam_permit.so
+account    sufficient  pam_permit.so
+password   sufficient  pam_permit.so
+session    required    pam_loginuid.so
+-session   optional    pam_keyinit.so revoke
+-session   optional    pam_limits.so
+session    required    pam_systemd.so

data/systemd/anaconda.service

@@ -5,6 +5,6 @@ Wants=anaconda-noshell.service
 
 [Service]
 Type=forking
-Environment=HOME=/root MALLOC_CHECK_=2 MALLOC_PERTURB_=204 PATH=/usr/bin:/bin:/sbin:/usr/sbin:/mnt/sysimage/bin:/mnt/sysimage/usr/bin:/mnt/sysimage/usr/sbin:/mnt/sysimage/sbin LANG=en_US.UTF-8 GDK_BACKEND=x11 XDG_RUNTIME_DIR=/tmp GIO_USE_VFS=local
+Environment=HOME=/root MALLOC_CHECK_=2 MALLOC_PERTURB_=204 PATH=/usr/bin:/bin:/sbin:/usr/sbin:/mnt/sysimage/bin:/mnt/sysimage/usr/bin:/mnt/sysimage/usr/sbin:/mnt/sysimage/sbin LANG=en_US.UTF-8 GDK_BACKEND=wayland XDG_RUNTIME_DIR=/run/user/0 GIO_USE_VFS=local
 WorkingDirectory=/root
 ExecStart=/usr/bin/tmux -u -f /usr/share/anaconda/tmux.conf start

pyanaconda/core/constants.py

@@ -269,6 +269,9 @@ class SecretStatus(Enum):
 IPMI_ABORTED = 0x9          # installation finished unsuccessfully, due to some non-exn error
 IPMI_FAILED = 0xA           # installation hit an exception
 
+# Wayland socket name to use
+WAYLAND_SOCKET_NAME = "wl-sysinstall-0"
+
 # X display number to use
 X_DISPLAY_NUMBER = 1
 

pyanaconda/core/util.py

@@ -255,7 +255,7 @@ def sigusr1_preexec():
             log.debug("Exception handler test suspended to prevent accidental activation by "
                       "delayed Xorg start. Next SIGUSR1 will be handled as delayed Xorg start.")
             # Raise an exception to notify the caller that things went wrong. This affects
-            # particularly pyanaconda.display.do_startup_x11_actions(), where the window manager
+            # particularly pyanaconda.display.do_startup_wl_actions(), where the window manager
             # is started immediately after this. The WM would just wait forever.
             raise TimeoutError("Timeout trying to start %s" % argv[0])
 

pyanaconda/display.py

@@ -20,7 +20,6 @@
 # Author(s):  Martin Kolman <mkolman@redhat.com>
 #
 import os
-import subprocess
 import time
 import textwrap
 import pkgutil
@@ -52,9 +51,9 @@
 log = get_module_logger(__name__)
 stdout_log = get_stdout_logger()
 
-X_TIMEOUT_ADVICE = \
+WAYLAND_TIMEOUT_ADVICE = \
     "Do not load the stage2 image over a slow network link.\n" \
-    "Wait longer for the X server startup with the inst.xtimeout=<SECONDS> boot option." \
+    "Wait longer for Wayland startup with the inst.xtimeout=<SECONDS> boot option." \
     "The default is 60 seconds.\n" \
     "Load the stage2 image into memory with the rd.live.ram boot option to decrease access " \
     "time.\n" \
@@ -172,22 +171,7 @@ def check_vnc_can_be_started(anaconda):
     return vnc_startup_possible, error_messages
 
 
-# X11
-
-def start_x11(xtimeout):
-    """Start the X server for the Anaconda GUI."""
-
-    # Start Xorg and wait for it become ready
-    util.startX(["Xorg", "-br", "-logfile", "/tmp/X.log",
-                 ":%s" % constants.X_DISPLAY_NUMBER, "vt6", "-s", "1440", "-ac",
-                 "-nolisten", "tcp", "-dpi", "96",
-                 "-noreset"],
-                output_redirect=subprocess.DEVNULL, timeout=xtimeout)
-
-
-# function to handle X startup special issues for anaconda
-
-def do_startup_x11_actions():
+def do_startup_wl_actions(timeout):
     """Start the window manager.
 
     When window manager actually connects to the X server is unknowable, but
@@ -209,14 +193,35 @@ def do_startup_x11_actions():
         xdg_config_dirs = datadir + ':' + os.environ['XDG_CONFIG_DIRS']
     os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs
 
-    def x11_preexec():
+    os.environ["XDG_SESSION_TYPE"] = "wayland"
+
+    def wl_preexec():
         # to set GUI subprocess SIGINT handler
         signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-    childproc = util.startProgram(["gnome-kiosk", "--display", ":1", "--sm-disable", "--x11"],
-                                  env_add={'XDG_DATA_DIRS': xdg_data_dirs},
-                                  preexec_fn=x11_preexec)
-    WatchProcesses.watch_process(childproc, "gnome-kiosk")
+    argv = ["/usr/libexec/anaconda/run-in-new-session",
+            "--user", "root",
+            "--service", "anaconda",
+            "--vt", "6",
+            "--session-type", "wayland",
+            "--session-class", "user",
+            "gnome-kiosk", "--sm-disable", "--wayland", "--no-x11",
+            "--wayland-display", constants.WAYLAND_SOCKET_NAME]
+
+    childproc = util.startProgram(argv, env_add={'XDG_DATA_DIRS': xdg_data_dirs},
+                                  preexec_fn=wl_preexec)
+    WatchProcesses.watch_process(childproc, argv[0])
+
+    for _i in range(0, int(timeout / 0.1)):
+        wl_socket_path = os.path.join(os.getenv("XDG_RUNTIME_DIR"), constants.WAYLAND_SOCKET_NAME)
+        if os.path.exists(wl_socket_path):
+            return
+
+        time.sleep(0.1)
+
+    WatchProcesses.unwatch_process(childproc)
+    childproc.terminate()
+    raise TimeoutError("Timeout trying to start gnome-kiosk")
 
 
 def set_resolution(runres):
@@ -324,17 +329,6 @@ def setup_display(anaconda, options):
         for error_message in vnc_error_messages:
             stdout_log.warning(error_message)
 
-    # Should we try to start Xorg?
-    want_x = anaconda.gui_mode and not (flags.preexisting_x11 or flags.usevnc)
-
-    # Is Xorg is actually available?
-    if want_x and not os.access("/usr/bin/Xorg", os.X_OK):
-        stdout_log.warning(_("Graphical installation is not available. "
-                             "Starting text mode."))
-        time.sleep(2)
-        anaconda.display_mode = constants.DisplayModes.TUI
-        want_x = False
-
     if anaconda.tui_mode and flags.vncquestion:
         # we prefer vnc over text mode, so ask about that
         message = _("Text mode provides a limited set of installation "
@@ -350,27 +344,26 @@ def setup_display(anaconda, options):
     startup_utils.check_memory(anaconda, options)
 
     # check_memory may have changed the display mode
-    want_x = want_x and (anaconda.gui_mode)
-    if want_x:
+    want_gui = anaconda.gui_mode and not (flags.preexisting_wayland or flags.usevnc)
+    if want_gui:
         try:
-            start_x11(xtimeout)
-            do_startup_x11_actions()
+            do_startup_wl_actions(xtimeout)
         except TimeoutError as e:
-            log.warning("X startup failed: %s", e)
-            print("\nX did not start in the expected time, falling back to text mode. There are "
-                  "multiple ways to avoid this issue:")
+            log.warning("Wayland startup failed: %s", e)
+            print("\nWayland did not start in the expected time, falling back to text mode. "
+                  "There are multiple ways to avoid this issue:")
             wrapper = textwrap.TextWrapper(initial_indent=" * ", subsequent_indent="   ",
                                            width=os.get_terminal_size().columns - 3)
-            for line in X_TIMEOUT_ADVICE.split("\n"):
+            for line in WAYLAND_TIMEOUT_ADVICE.split("\n"):
                 print(wrapper.fill(line))
             util.vtActivate(1)
             anaconda.display_mode = constants.DisplayModes.TUI
             anaconda.gui_startup_failed = True
             time.sleep(2)
 
         except (OSError, RuntimeError) as e:
-            log.warning("X or window manager startup failed: %s", e)
-            print("\nX or window manager startup failed, falling back to text mode.")
+            log.warning("Wayland startup failed: %s", e)
+            print("\nWayland startup failed, falling back to text mode.")
             util.vtActivate(1)
             anaconda.display_mode = constants.DisplayModes.TUI
             anaconda.gui_startup_failed = True
@@ -394,7 +387,7 @@ def on_mutter_ready(observer):
     # if they want us to use VNC do that now
     if anaconda.gui_mode and flags.usevnc:
         vnc_server.startServer()
-        do_startup_x11_actions()
+        do_startup_wl_actions(xtimeout)
 
     # with X running we can initialize the UI interface
     anaconda.initialize_interface()

pyanaconda/flags.py

@@ -35,6 +35,7 @@ def __init__(self):
         self.__dict__['_in_init'] = True
         self.usevnc = False
         self.vncquestion = True
+        self.preexisting_wayland = False
         self.preexisting_x11 = False
         self.automatedInstall = False
         self.eject = True

scripts/Makefile.am

@@ -16,7 +16,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 scriptsdir = $(libexecdir)/$(PACKAGE_NAME)
-dist_scripts_SCRIPTS = run-anaconda anaconda-pre-log-gen log-capture start-module apply-updates
+dist_scripts_SCRIPTS = run-anaconda anaconda-pre-log-gen log-capture start-module apply-updates \
+                       run-in-new-session
 
 dist_noinst_SCRIPTS  = makeupdates makebumpver
 

scripts/makeupdates

@@ -35,7 +35,8 @@ RPM_RELEASE_DIR_TEMPLATE = "for_%s"
 SITE_PACKAGES_PATH = "./usr/lib64/python3.12/site-packages/"
 
 # Anaconda scripts that should be installed into the libexec folder
-LIBEXEC_SCRIPTS = ["log-capture", "start-module", "apply-updates", "anaconda-pre-log-gen"]
+LIBEXEC_SCRIPTS = ["log-capture", "start-module", "apply-updates", "anaconda-pre-log-gen",
+                   "run-in-new-session"]
 
 # Anaconda scripts that should be installed into /usr/bin
 USR_BIN_SCRIPTS = ["anaconda-disable-nm-ibft-plugin", "anaconda-nm-disable-autocons"]