Introducing ostree-grub-generator

ostree-grub-generator can be used to customize
the generated grub.cfg file. Compile time
decision ostree-grub-generator vs grub2-mkconfig
can be overwritten with the OSTREE_GRUB2_EXEC
envvar - useful for auto tests and OS installers.

Why this alternative approach:

1) The current approach is less flexible than using a
   custom 'ostree-grub-generator' script. Each system can
   adjust this script for its needs, instead of using the
   hardcoded values from ostree-bootloader-grub2.c.

2) Too much overhead on embedded to generate grub.cfg
   via /etc/grub.d/ configuration files. It is still
   possible to do so, even with this patch applied.
   No need to install grub2 package on a target device.

3) The grub2-mkconfig code path has other issues:
   https://bugzilla.gnome.org/show_bug.cgi?id=761180

Task: https://bugzilla.gnome.org/show_bug.cgi?id=762220

Closes: #228
Approved by: cgwalters

Makefile-boot.am

@@ -1,4 +1,4 @@
-# Makefile for dracut module
+# Makefile for boot module
 #
 # Copyright (C) 2013 Colin Walters <walters@verbum.org>
 #
@@ -39,19 +39,24 @@ systemdsystemunit_DATA = src/boot/ostree-prepare-root.service \
 	src/boot/ostree-remount.service
 endif
 
-dist_pkglibexec_SCRIPTS = src/boot/grub2-15_ostree
-
-if BUILDOPT_GRUB2
+if !BUILDOPT_BUILTIN_GRUB2_MKCONFIG
+# We're using the system grub2-mkconfig generator
+libexec_SCRIPTS = src/boot/grub2/grub2-15_ostree
 install-grub2-config-hook:
 	mkdir -p $(DESTDIR)$(grub2configdir)
-	ln -sf $(pkglibexecdir)/grub2-15_ostree $(DESTDIR)$(grub2configdir)/15_ostree
+	ln -sf $(libexecdir)/grub2-15_ostree $(DESTDIR)$(grub2configdir)/15_ostree
 grub2configdir = $(sysconfdir)/grub.d
 INSTALL_DATA_HOOKS += install-grub2-config-hook
+else
+# We're using our internal generator
+libexec_SCRIPTS = src/boot/grub2/ostree-grub-generator
 endif
 
 EXTRA_DIST += src/boot/dracut/module-setup.sh \
 	src/boot/dracut/ostree.conf \
 	src/boot/mkinitcpio/ostree \
 	src/boot/ostree-prepare-root.service \
 	src/boot/ostree-remount.service \
+	src/boot/grub2/grub2-15_ostree \
+	src/boot/grub2/ostree-grub-generator \
 	$(NULL)

Makefile-tests.am

@@ -59,6 +59,7 @@ test_scripts = \
 	tests/test-admin-deploy-switch.sh \
 	tests/test-admin-deploy-etcmerge-cornercases.sh \
 	tests/test-admin-deploy-uboot.sh \
+	tests/test-admin-deploy-grub2.sh \
 	tests/test-admin-instutil-set-kargs.sh \
 	tests/test-admin-upgrade-not-backwards.sh \
 	tests/test-admin-pull-deploy-commit.sh \
@@ -97,7 +98,8 @@ installed_test_data = tests/archive-test.sh \
 	tests/pre-endian-deltas-repo-little.tar.xz \
 	$(NULL)
 
-test_extra_scripts = tests/syslinux-entries-crosscheck.py
+test_extra_scripts = tests/bootloader-entries-crosscheck.py \
+     tests/ostree-grub-generator
 
 # We can't use nobase_ as we need to strip off the tests/, can't
 # use plain installed_ as we do need the gpghome/ prefix.

configure.ac

@@ -251,11 +251,20 @@ AS_IF([test "x$with_dracut" = "xyes" || test "x$with_mkinitcpio" = "xyes"], [
 ])
 AM_CONDITIONAL(BUILDOPT_SYSTEMD, test x$with_systemd = xyes)
 
-AC_ARG_WITH(grub2,
-            AS_HELP_STRING([--with-grub2],
-                           [Install grub2 hook (default: yes)]),,
-              [with_grub2=yes])
-AM_CONDITIONAL(BUILDOPT_GRUB2, test x$with_grub2 = xyes)
+AC_ARG_WITH(builtin-grub2-mkconfig,
+            AS_HELP_STRING([--with-builtin-grub-mkconfig],
+                           [Use a builtin minimal grub2-mkconfig to generate a GRUB2 configuration file (default: no)]),,
+              [with_builtin_grub2_mkconfig=no])
+AM_CONDITIONAL(BUILDOPT_BUILTIN_GRUB2_MKCONFIG, test x$with_builtin_grub2_mkconfig = xyes)
+AM_COND_IF(BUILDOPT_BUILTIN_GRUB2_MKCONFIG,
+           AC_DEFINE([USE_BUILTIN_GRUB2_MKCONFIG], 1, [Define if using internal ostree-grub-generator]),
+[
+  # Otherwise, look for the path to the system generator.  On some
+  # distributions GRUB2 *-mkconfig executable has 'grub2' prefix and
+  # on some 'grub'.
+  AC_CHECK_PROG(GRUB2_MKCONFIG, grub2-mkconfig, grub2-mkconfig, grub-mkconfig)
+  AC_DEFINE_UNQUOTED([GRUB2_MKCONFIG_PATH], ["$GRUB2_MKCONFIG"], [The system grub2-mkconfig executible name])
+])
 
 dnl for tests
 AS_IF([test "x$found_introspection" = xyes], [
@@ -292,7 +301,8 @@ echo "
     api docs (gtk-doc):                           $enable_gtk_doc
     gjs-based tests:                              $have_gjs
     dracut:                                       $with_dracut
-    mkinitcpio:                                   $with_mkinitcpio"
+    mkinitcpio:                                   $with_mkinitcpio
+    builtin grub2-mkconfig (instead of system):   $with_builtin_grub2_mkconfig"
 AS_IF([test "x$with_systemd" = "xyes"], [
     echo "    systemd unit dir:                             $with_systemdsystemunitdir"
 ])

src/boot/grub2/ostree-grub-generator

@@ -0,0 +1,101 @@
+#!/bin/sh
+
+# To use a custrom script for generating grub.cfg, set the --with-grub2-mkconfig=no
+# configure switch when configuring and building OSTree.
+#
+# This script is called by ostree/src/libostree/ostree-bootloader-grub2.c whenever
+# boot loader configuration file needs to be updated. It can be used as a template
+# for a custom grub.cfg generator. What to consider when writing a custom grub.cfg
+# generator:
+#
+#   - The populate_menu() function converts boot loader entries as defined by
+#   https://www.freedesktop.org/wiki/Specifications/BootLoaderSpec/ into GRUB2
+#   menuentry sections. This is the core logic that is required by OSTree
+#   based system.
+#
+#   - Embedded systems: Be aware that this script is executed not only on a host machine by OS
+#   installer, but also on a target device, thus think about shell portability. A target device
+#   for example might be using busybox with a limited shell.
+#
+# Feel free to edit this script to fit your requirements.
+
+set -e
+
+script=$(basename ${0})
+# Atomically safe location where to generete grub.cfg when executing system upgrade.
+new_grub2_cfg=${2}
+entries_path=$(dirname $new_grub2_cfg)/entries
+
+read_config()
+{
+    config_file=${entries_path}/${1}
+    title=""
+    initrd=""
+    options=""
+    linux=""
+
+    while read -r line
+    do
+        record=$(echo ${line} | cut -f 1 -d ' ')
+        value=$(echo ${line} | cut -s -f2- -d ' ')
+        case "${record}" in
+            "title")
+                title=${value}
+                ;;
+            "initrd")
+                initrd=${value}
+                ;;
+            "linux")
+                linux=${value}
+                ;;
+            "options")
+                options=${value}
+                ;;
+        esac
+    done < ${config_file}
+
+    if [ -z "${title}" ]; then
+        title="(Untitled)"
+    fi
+}
+
+populate_menu()
+{
+    boot_prefix="${OSTREE_BOOT_PARTITION}"
+    for config in $(ls ${entries_path}); do
+        read_config ${config}
+        menu="${menu}menuentry '${title}' {\n"
+        menu="${menu}\t linux ${boot_prefix}${linux} ${options}\n"
+        menu="${menu}\t initrd ${boot_prefix}${initrd}\n"
+        menu="${menu}}\n\n"
+    done
+    # The printf command seems to be more reliable across shells for special character (\n, \t) evaluation
+    printf "$menu" >> ${new_grub2_cfg}
+}
+
+populate_warning()
+{
+cat >> ${new_grub2_cfg} <<EOF
+# This file was generated by ${script}. Do not modify the generated file - all changes will
+# be lost the next time file is regenerated. For more details refer to the ${script} script.
+EOF
+}
+
+populate_header()
+{
+cat >> ${new_grub2_cfg} <<EOF
+serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
+default=boot
+timeout=10
+
+EOF
+}
+
+generate_grub2_cfg()
+{
+    populate_warning
+    populate_header
+    populate_menu
+}
+
+generate_grub2_cfg

src/libostree/ostree-bootloader-grub2.c

@@ -299,8 +299,25 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader      *bootloader,
   g_autofree char *bootversion_str = g_strdup_printf ("%u", (guint)bootversion);
   g_autoptr(GFile) config_path_efi_dir = NULL;
   g_autofree char *grub2_mkconfig_chroot = NULL;
+  gboolean use_system_grub2_mkconfig = TRUE;
+  const gchar *grub_exec = NULL;
+
+#ifdef USE_BUILTIN_GRUB2_MKCONFIG
+  use_system_grub2_mkconfig = FALSE;
+#endif
+  /* Autotests can set this envvar to select which code path to test, useful for OS installers as well */
+  grub_exec = g_getenv ("OSTREE_GRUB2_EXEC");
+  if (grub_exec)
+    {
+      if (g_str_has_suffix (grub_exec, GRUB2_MKCONFIG_PATH))
+        use_system_grub2_mkconfig = TRUE;
+      else
+        use_system_grub2_mkconfig = FALSE;
+    }
+  else
+    grub_exec = use_system_grub2_mkconfig ? GRUB2_MKCONFIG_PATH : LIBEXECDIR "/ostree-grub-generator";
 
-  if (ostree_sysroot_get_booted_deployment (self->sysroot) == NULL
+  if (use_system_grub2_mkconfig && ostree_sysroot_get_booted_deployment (self->sysroot) == NULL
       && g_file_has_parent (self->sysroot->path, NULL))
     {
       g_autoptr(GPtrArray) deployments = NULL;
@@ -318,6 +335,9 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader      *bootloader,
        *
        * In the case of an installer, use the first deployment root (which
        * will most likely be the only one.
+       *
+       * This all only applies if we're not using the builtin
+       * generator, which handles being run outside of the root.
        */
       tool_deployment_root = ostree_sysroot_get_deployment_directory (self->sysroot, tool_deployment);
       grub2_mkconfig_chroot = g_file_get_path (tool_deployment_root);
@@ -361,7 +381,7 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader      *bootloader,
      Upstream is fixed though.
   */
   proc = g_subprocess_launcher_spawn (launcher, error,
-                                      "grub2-mkconfig", "-o",
+                                      grub_exec, "-o",
                                       gs_file_get_path_cached (new_config_path),
                                       NULL);
 

tests/admin-test.sh

@@ -21,10 +21,17 @@ set -euo pipefail
 echo "1..16"
 
 function validate_bootloader() {
-    (cd ${test_tmpdir};
-     if test -f sysroot/boot/syslinux/syslinux.cfg; then
-	$(dirname $0)/syslinux-entries-crosscheck.py sysroot
-     fi)
+    cd ${test_tmpdir};
+    bootloader=""
+    if test -f sysroot/boot/syslinux/syslinux.cfg; then
+	    bootloader="syslinux"
+    elif test -f sysroot/boot/grub2/grub.cfg; then
+	    bootloader="grub2"
+    fi
+    if test -n "${bootloader}"; then
+        $(dirname $0)/bootloader-entries-crosscheck.py sysroot ${bootloader}
+    fi
+    cd -
 }
 
 orig_mtime=$(stat -c '%.Y' sysroot/ostree/deploy)

tests/syslinux-entries-crosscheck.py → tests/bootloader-entries-crosscheck.py

@@ -25,9 +25,14 @@
 else:
     sysroot = sys.argv[1]
 
+bootloader = sys.argv[2]
 loaderpath = sysroot + '/boot/loader/entries'
 syslinuxpath = sysroot + '/boot/syslinux/syslinux.cfg'
 
+if bootloader == "grub2":
+    sys.stdout.write('GRUB2 configuration validation not implemented.\n')
+    sys.exit(0)
+
 def fatal(msg):
     sys.stderr.write(msg)
     sys.stderr.write('\n')

tests/libtest.sh

@@ -229,6 +229,20 @@ setup_os_boot_uboot() {
     ln -s loader/uEnv.txt sysroot/boot/uEnv.txt
 }
 
+setup_os_boot_grub2() {
+    grub2_options=$1
+    mkdir -p sysroot/boot/grub2/
+    ln -s ../loader/grub.cfg sysroot/boot/grub2/grub.cfg
+    export OSTREE_BOOT_PARTITION="/boot"
+    case "$grub2_options" in
+        *ostree-grub-generator*)
+            cp ${test_srcdir}/ostree-grub-generator ${test_tmpdir}
+            chmod +x ${test_tmpdir}/ostree-grub-generator
+            export OSTREE_GRUB2_EXEC=${test_tmpdir}/ostree-grub-generator
+            ;;
+    esac
+}
+
 setup_os_repository () {
     mode=$1
     bootmode=$2
@@ -301,6 +315,9 @@ EOF
         "uboot")
 	    setup_os_boot_uboot
             ;;
+        *grub2*)
+        setup_os_boot_grub2 "${bootmode}"
+            ;;
     esac
 
     cd ${test_tmpdir}

tests/ostree-grub-generator

@@ -0,0 +1 @@
+../src/boot/grub2/ostree-grub-generator

tests/test-admin-deploy-grub2.sh

@@ -21,11 +21,7 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
-echo "1..1"
-
 # Exports OSTREE_SYSROOT so --sysroot not needed.
-setup_os_repository "archive-z2" "grub2"
-
-echo "ok setup"
+setup_os_repository "archive-z2" "grub2 ostree-grub-generator"
 
 . $(dirname $0)/admin-test.sh