compose: Add ostree-layers and ostree-overrride-layers

The use case for `ostree-layers` is to support injecting non-RPM
content in a more flexible way than can be done with `add-files`,
and also without dropping all the way to split composes.

This starts with support on the `compose tree` side but down the
line I'd like to make it more convenient to do *client* side too.

For `ostree-override-layers` this is mainly a development thing
for tools like coreos-assembler.  Rather than building an RPM
we just `make install DESTDIR` then commit and add to
`ostree-override-layers`.

docs/manual/treefile.md

@@ -61,6 +61,14 @@ It supports the following parameters:
  * `packages-$basearch`: Array of strings, optional: Set of installed packages, used
     only if $basearch matches the target architecture name.
 
+ * `ostree-layers`: Array of strings, optional: After all packages are unpacked,
+    check out these OSTree refs, which must already be in the destination repository.
+    Any conflicts with packages will be an error.
+
+ * `ostree-override-layers`: Array of strings, optional: Like above, but any
+    files present in packages and prior layers will be silently overriden.
+    This is useful for development builds to replace parts of the base tree.
+
  * `bootstrap_packages`: Array of strings, optional: Deprecated; you should
     now just include this set in the main `packages` array.
 

rust/src/treefile.rs

@@ -338,6 +338,8 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
     merge_vecs!(
         repos,
         packages,
+        ostree_layers,
+        ostree_override_layers,
         install_langs,
         initramfs_args,
         units,
@@ -593,6 +595,12 @@ struct TreeComposeConfig {
     // Deprecated option
     #[serde(skip_serializing_if = "Option::is_none")]
     bootstrap_packages: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "ostree-layers")]
+    ostree_layers: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "ostree-override-layers")]
+    ostree_override_layers: Option<Vec<String>>,
 
     // Content installation opts
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -994,6 +1002,7 @@ packages:
 mod ffi {
     use super::*;
     use glib_sys;
+    use glib::translate::*;
     use libc;
     use std::io::Seek;
     use std::os::unix::io::{AsRawFd, RawFd};
@@ -1077,6 +1086,40 @@ mod ffi {
         ref_from_raw_ptr(tf).serialized.as_ptr()
     }
 
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char  {
+        let tf = ref_from_raw_ptr(tf);
+        if let Some(ref layers) = tf.parsed.ostree_layers {
+            layers.to_glib_full()
+        } else {
+            ptr::null_mut()
+        }
+    }
+
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_ostree_override_layers(tf: *mut Treefile) -> *mut *mut libc::c_char  {
+        let tf = ref_from_raw_ptr(tf);
+        if let Some(ref layers) = tf.parsed.ostree_override_layers {
+            layers.to_glib_full()
+        } else {
+            ptr::null_mut()
+        }
+    }
+
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_all_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char  {
+        let tf = ref_from_raw_ptr(tf);
+        let mut ret : Vec<String> = Vec::new();
+        if let Some(ref layers) = tf.parsed.ostree_layers {
+            ret.extend(layers.iter().cloned())
+        }
+        if let Some(ref layers) = tf.parsed.ostree_override_layers {
+            ret.extend(layers.iter().cloned())
+        }
+        ret.to_glib_full()
+    }
+
+
     #[no_mangle]
     pub extern "C" fn ror_treefile_get_rojig_spec_path(tf: *mut Treefile) -> *const libc::c_char {
         let tf = ref_from_raw_ptr(tf);

src/app/rpmostree-compose-builtin-rojig.c

@@ -166,6 +166,7 @@ install_packages (RpmOstreeRojigCompose  *self,
 
   g_autofree char *ret_new_inputhash = NULL;
   if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx),
+                                       self->repo,
                                        self->treefile_rs, self->treefile,
                                        &ret_new_inputhash, error))
     return FALSE;

src/app/rpmostree-compose-builtin-tree.c

@@ -51,6 +51,13 @@
 
 #include "libglnx.h"
 
+static gboolean
+pull_local_into_target_repo (OstreeRepo   *src_repo,
+                             OstreeRepo   *dest_repo,
+                             const char   *checksum,
+                             GCancellable *cancellable,
+                             GError      **error);
+
 static char *opt_workdir;
 static gboolean opt_workdir_tmpfs;
 static char *opt_cachedir;
@@ -336,7 +343,7 @@ install_packages (RpmOstreeTreeComposeContext  *self,
 
   /* FIXME - just do a depsolve here before we compute download requirements */
   g_autofree char *ret_new_inputhash = NULL;
-  if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx),
+  if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx), self->repo,
                                        self->treefile_rs, self->treefile,
                                        &ret_new_inputhash, error))
     return FALSE;
@@ -689,6 +696,18 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
                                error))
     return FALSE;
 
+  if (self->build_repo != self->repo)
+    {
+      g_auto(GStrv) layers = ror_treefile_get_all_ostree_layers (self->treefile_rs);
+      for (char **iter = layers; iter && *iter; iter++)
+        {
+          const char *layer = *iter;
+          if (!pull_local_into_target_repo (self->repo, self->build_repo, layer,
+                                            cancellable, error))
+            return FALSE;
+        }
+    }
+
   self->treefile_rootval = json_parser_get_root (self->treefile_parser);
   if (!JSON_NODE_HOLDS_OBJECT (self->treefile_rootval))
     return glnx_throw (error, "Treefile root is not an object");

src/app/rpmostree-composeutil.c

@@ -50,6 +50,7 @@
 
 gboolean
 rpmostree_composeutil_checksum (HyGoal             goal,
+                                OstreeRepo        *repo,
                                 RORTreefile       *tf,
                                 JsonObject        *treefile,
                                 char             **out_checksum,
@@ -92,6 +93,24 @@ rpmostree_composeutil_checksum (HyGoal             goal,
   if (!rpmostree_dnf_add_checksum_goal (checksum, goal, NULL, error))
     return FALSE;
 
+  if (tf)
+    {
+      GLNX_AUTO_PREFIX_ERROR ("Resolving ostree-layers", error);
+      g_auto(GStrv) layers = ror_treefile_get_all_ostree_layers (tf);
+      for (char **iter = layers; iter && *iter; iter++)
+        {
+          const char *layer = *iter;
+          g_autofree char *rev = NULL;
+          if (!ostree_repo_resolve_rev (repo, layer, FALSE, &rev, error))
+            return FALSE;
+          g_autoptr(GVariant) commit = NULL;
+          if (!ostree_repo_load_commit (repo, rev, &commit, NULL, error))
+            return FALSE;
+          const char *content_checksum = ostree_commit_get_content_checksum (commit);
+          g_checksum_update (checksum, (const guint8*) content_checksum, strlen (content_checksum));
+        }
+    }
+
   *out_checksum = g_strdup (g_checksum_get_string (checksum));
   return TRUE;
 }

src/app/rpmostree-composeutil.h

@@ -29,6 +29,7 @@ G_BEGIN_DECLS
 
 gboolean
 rpmostree_composeutil_checksum (HyGoal             goal,
+                                OstreeRepo        *repo,
                                 RORTreefile       *tf,
                                 JsonObject        *treefile,
                                 char             **out_checksum,

src/libpriv/rpmostree-core.c

@@ -3683,6 +3683,81 @@ rpmostree_context_get_kernel_changed (RpmOstreeContext *self)
   return self->kernel_changed;
 }
 
+static gboolean
+process_one_ostree_layer (RpmOstreeContext *self,
+                          int               rootfs_dfd,
+                          const char       *ref,
+                          OstreeRepoCheckoutOverwriteMode ovw_mode,
+                          GCancellable     *cancellable,
+                          GError          **error)
+{
+  OstreeRepo *repo = self->ostreerepo;
+  OstreeRepoCheckoutAtOptions opts = { OSTREE_REPO_CHECKOUT_MODE_USER,
+                                       ovw_mode };
+
+  /* We want the checkout to match the repo type so that we get hardlinks. */
+  if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE)
+    opts.mode = OSTREE_REPO_CHECKOUT_MODE_NONE;
+
+  /* Explicitly don't provide a devino cache here.  We can't do it reliably because
+   * of SELinux.  The ostree layering infrastructure doesn't have the relabeling
+   * that we do for the pkgcache repo because we can't assume that we can mutate
+   * the input commits right now.
+   * opts.devino_to_csum_cache = self->devino_cache;
+   */
+  /* Always want hardlinks */
+  opts.no_copy_fallback = TRUE;
+
+  g_autofree char *rev = NULL;
+  if (!ostree_repo_resolve_rev (repo, ref, FALSE, &rev, error))
+    return FALSE;
+
+  return ostree_repo_checkout_at (repo, &opts, rootfs_dfd, ".",
+                                  rev, cancellable, error);
+}
+
+static gboolean
+process_ostree_layers (RpmOstreeContext *self,
+                       int               rootfs_dfd,
+                       GCancellable     *cancellable,
+                       GError          **error)
+{
+  if (!self->treefile_rs)
+    return TRUE;
+
+  g_auto(GStrv) layers = ror_treefile_get_ostree_layers (self->treefile_rs);
+  g_auto(GStrv) override_layers = ror_treefile_get_ostree_override_layers (self->treefile_rs);
+  const size_t n = (layers ? g_strv_length (layers) : 0) + (override_layers ? g_strv_length (override_layers) : 0);
+  if (n == 0)
+    return TRUE;
+
+  g_auto(RpmOstreeProgress) checkout_progress = { 0, };
+  rpmostree_output_progress_nitems_begin (&checkout_progress, n, "Checking out ostree layers");
+  size_t i = 0;
+  for (char **iter = layers; iter && *iter; iter++)
+    {
+      const char *ref = *iter;
+      if (!process_one_ostree_layer (self, rootfs_dfd, ref,
+                                     OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL,
+                                     cancellable, error))
+        return FALSE;
+      i++;
+      rpmostree_output_progress_n_items (i);
+    }
+  for (char **iter = override_layers; iter && *iter; iter++)
+    {
+      const char *ref = *iter;
+      if (!process_one_ostree_layer (self, rootfs_dfd, ref,
+                                     OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES,
+                                     cancellable, error))
+        return FALSE;
+      i++;
+      rpmostree_output_progress_n_items (i);
+    }
+  rpmostree_output_progress_end (&checkout_progress);
+  return TRUE;
+}
+
 gboolean
 rpmostree_context_assemble (RpmOstreeContext      *self,
                             GCancellable          *cancellable,
@@ -4134,6 +4209,10 @@ rpmostree_context_assemble (RpmOstreeContext      *self,
         }
       }
 
+      /* Any ostree refs to overlay */
+      if (!process_ostree_layers (self, tmprootfs_dfd, cancellable, error))
+        return FALSE;
+
       {
       g_auto(RpmOstreeProgress) task = { 0, };
       rpmostree_output_task_begin (&task, "Running posttrans scripts");

tests/compose-tests/test-misc-tweaks.sh

@@ -56,10 +56,35 @@ postprocess:
    set -xeuo pipefail
    test -f /usr/share/included-postprocess-test
 EOF
+
+for x in $(seq 3); do
+  rm tmp/usr -rf
+  mkdir -p tmp/usr/{bin,share}
+  mkdir tmp/usr/share/testsubdir-${x}
+  echo sometest${x} > tmp/usr/bin/sometestbinary-${x}
+  chmod a+x tmp/usr/bin/sometestbinary-${x}
+  echo sometestdata${x} > tmp/usr/share/sometestdata-${x}
+  echo sometestdata-subdir-${x} > tmp/usr/share/testsubdir-${x}/test
+  ostree --repo="${repobuild}" commit --consume --no-xattrs --owner-uid=0 --owner-gid=0 -b testlayer-${x} --tree=dir=tmp
+done
+rm tmp/usr -rf
+mkdir -p tmp/usr/share/info
+echo some info | gzip > tmp/usr/share/info/bash.info.gz
+ostree --repo="${repobuild}" commit --consume --no-xattrs --owner-uid=0 --owner-gid=0 -b testoverride-1 --tree=dir=tmp
+cat >> ${new_treefile} <<EOF
+ostree-layers:
+  - testlayer-1
+  - testlayer-2
+  - testlayer-3
+ostree-override-layers:
+  - testoverride-1
+EOF
+
 export treefile=${new_treefile}
 
 
 # Do the compose
+compose_base_argv="${compose_base_argv} --unified-core"
 runcompose
 echo "ok compose"
 
@@ -113,6 +138,17 @@ assert_file_has_content_literal pkglist.txt 'systemd-'
 assert_not_file_has_content pkglist.txt 'systemd-bootchart'
 echo "ok recommends"
 
+# Test overlays/overrides
+for x in $(seq 3); do
+  ostree --repo=${repobuild} cat ${treeref} /usr/bin/sometestbinary-${x} > t
+  assert_file_has_content t "sometest${x}"
+  ostree --repo=${repobuild} cat ${treeref} /usr/share/testsubdir-${x}/test > t
+  assert_file_has_content t sometestdata-subdir-${x}
+done
+ostree --repo=${repobuild} cat ${treeref} /usr/share/info/bash.info.gz | gunzip > bash.info
+assert_file_has_content bash.info 'some info'
+echo "ok layers"
+
 # Check that add-files with bad paths are rejected
 prepare_compose_test "add-files-failure"
 pysetjsonmember "add-files" '[["foo.txt", "/var/lib/foo.txt"]]'