ModuleIndex: add compressed file loading support

Fixes: fedora-modularity#208

Signed-off-by: Stephen Gallagher <sgallagh@redhat.com>

.travis/archlinux/Dockerfile.deps.tmpl

@@ -6,6 +6,7 @@ ARG TARBALL
 
 RUN pacman -Syu --needed --noconfirm \
 	base-devel \
+	file \
 	glib2 \
 	glib2-docs \
 	gobject-introspection \

.travis/archlinux/travis-tasks.sh

@@ -10,14 +10,15 @@ JOB_NAME=${TRAVIS_JOB_NAME:-Arch Linux}
 os_name='Arch Linux'
 release=base
 
-COMMON_MESON_ARGS="-Dtest_dirty_git=false -Ddeveloper_build=false -Dpython_name=python3"
-
 
 pushd /builddir/
 
 # Build the code under GCC and run standard tests
 meson --buildtype=debug \
-      $COMMON_MESON_ARGS \
+      -Dtest_dirty_git=false \
+      -Ddeveloper_build=false \
+      -Dpython_name=python3 \
+      -Drpmio=disabled \
       travis
 
 ninja -C travis test

modulemd/modulemd-compression.c

@@ -153,6 +153,8 @@ modulemd_detect_compression (const gchar *filename, int fd, GError **error)
                    magic_error (magic));
       return MODULEMD_COMPRESSION_TYPE_DETECTION_FAILED;
     }
+  /* Reset the file descriptor to the start of the file, if it has moved */
+  lseek (fd, 0, SEEK_SET);
 #endif /* HAVE_LIBMAGIC */
 
   return type;

modulemd/modulemd-module-index.c

@@ -14,13 +14,20 @@
 #include <errno.h>
 #include <glib.h>
 #include <inttypes.h>
+#include <stdio.h>
 #include <yaml.h>
 
+#ifdef HAVE_RPMIO
+#include <rpm/rpmio.h>
+#endif
+
 #include "modulemd-errors.h"
+#include "modulemd-compression.h"
 #include "modulemd-module-index.h"
 #include "modulemd-subdocument-info.h"
 #include "private/glib-extensions.h"
 #include "private/modulemd-module-private.h"
+#include "private/modulemd-compression-private.h"
 #include "private/modulemd-defaults-private.h"
 #include "private/modulemd-defaults-v1-private.h"
 #include "private/modulemd-subdocument-info-private.h"
@@ -472,6 +479,10 @@ modulemd_module_index_update_from_file (ModulemdModuleIndex *self,
 
   int saved_errno;
   g_autoptr (FILE) yaml_stream = NULL;
+  g_autoptr (GError) nested_error = NULL;
+  int fd;
+  ModulemdCompressionTypeEnum comtype;
+  g_autofree gchar *fmode = NULL;
 
   yaml_stream = g_fopen (yaml_file, "rb");
   saved_errno = errno;
@@ -486,8 +497,93 @@ modulemd_module_index_update_from_file (ModulemdModuleIndex *self,
       return FALSE;
     }
 
-  return modulemd_module_index_update_from_stream (
-    self, yaml_stream, strict, failures, error);
+  /* To avoid TOCTOU race conditions, do everything from the same opened
+   * file
+   */
+  fd = fileno (yaml_stream);
+
+  /* Determine if the file is compressed */
+  comtype = modulemd_detect_compression (yaml_file, fd, &nested_error);
+  if (comtype == MODULEMD_COMPRESSION_TYPE_DETECTION_FAILED)
+    {
+      g_propagate_error (error, g_steal_pointer (&nested_error));
+      return FALSE;
+    }
+  else if (comtype == MODULEMD_COMPRESSION_TYPE_NO_COMPRESSION ||
+           comtype == MODULEMD_COMPRESSION_TYPE_UNKNOWN_COMPRESSION)
+    {
+      /* If it's not compressed (or we can't figure out what compression is in
+       * use), just use the libyaml function. It's fast and will fail quickly
+       * if the file is unreadable.
+       */
+      return modulemd_module_index_update_from_stream (
+        self, yaml_stream, strict, failures, error);
+    }
+
+#ifdef HAVE_RPMIO
+  /* We're handling a compressed input file, so we'll use librpm's "rpmio"
+   * suite of tools to deal with it. We need to construct a special "mode"
+   * argument to pass to Fdopen().
+   */
+  FD_t rpmio_fd = NULL;
+  g_auto (FD_t) fd_dup = NULL;
+
+  fmode = modulemd_get_rpmio_fmode ("r", comtype);
+  if (!fmode)
+    {
+      g_set_error (error,
+                   MODULEMD_ERROR,
+                   MODULEMD_ERROR_FILE_ACCESS,
+                   "Unable to construct rpmio fmode from comtype [%d]",
+                   comtype);
+      return FALSE;
+    }
+
+  /* Create an rpmio file descriptor object from the current file descriptor,
+   * setting the appropriate "mode" so that librpm will read it with the
+   * correct handlers.
+   */
+  fd_dup = fdDup (fd);
+  saved_errno = errno;
+  if (!fd_dup)
+    {
+      g_set_error (
+        error,
+        MODULEMD_ERROR,
+        MODULEMD_ERROR_NOT_IMPLEMENTED,
+        "Cannot open compressed file. Error in rpmio::fdDup(%d): %s",
+        fd,
+        strerror (saved_errno));
+      return FALSE;
+    }
+
+  g_debug ("Calling rpmio::Fdopen (%p, %s)", fd_dup, fmode);
+  rpmio_fd = Fdopen (fd_dup, fmode);
+
+  if (!rpmio_fd)
+    {
+      g_set_error_literal (
+        error,
+        MODULEMD_ERROR,
+        MODULEMD_ERROR_NOT_IMPLEMENTED,
+        "Cannot open compressed file. Error in rpmio::Fdopen().");
+      return FALSE;
+    }
+
+  g_debug ("rpmio::Fdopen (%p, %s) succeeded", fd_dup, fmode);
+
+  return modulemd_module_index_update_from_custom (
+    self, compressed_stream_read_fn, rpmio_fd, strict, failures, error);
+
+#else /* HAVE_RPMIO */
+  g_set_error_literal (
+    error,
+    MODULEMD_ERROR,
+    MODULEMD_ERROR_NOT_IMPLEMENTED,
+    "Cannot open compressed file. libmodulemd was not compiled "
+    "with rpmio support.");
+  return FALSE;
+#endif /* HAVE_RPMIO */
 }
 
 

modulemd/tests/test-modulemd-moduleindex.c

@@ -17,6 +17,7 @@
 #include <signal.h>
 #include <yaml.h>
 
+#include "config.h"
 #include "modulemd-defaults.h"
 #include "modulemd-module.h"
 #include "modulemd-module-index.h"
@@ -1018,6 +1019,173 @@ module_index_test_dump_empty_index (void)
 }
 
 
+struct expected_compressed_read_t
+{
+  const gchar *filename;
+
+  /* Whether this should succeed at reading */
+  gboolean succeeds;
+
+  /* If it fails, the expected error */
+  GQuark error_domain;
+  int error_code;
+};
+
+static void
+test_module_index_read_compressed (void)
+{
+  gboolean bret;
+  g_autoptr (ModulemdModuleIndex) baseline_idx = NULL;
+  g_autoptr (ModulemdModuleIndex) compressed_idx = NULL;
+  g_autofree gchar *file_path = NULL;
+  g_autoptr (GError) error = NULL;
+  g_autoptr (GPtrArray) failures = NULL;
+  g_autofree gchar *baseline_text = NULL;
+  g_autofree gchar *compressed_text = NULL;
+
+#ifdef HAVE_RPMIO
+#ifdef HAVE_LIBMAGIC
+  struct expected_compressed_read_t expected[] = {
+    {
+      .filename = "bzipped",
+      .succeeds = TRUE,
+    },
+    {
+      .filename = "bzipped.yaml.bz2",
+      .succeeds = TRUE,
+    },
+    { .filename = "gzipped", .succeeds = TRUE },
+    { .filename = "gzipped.yaml.gz", .succeeds = TRUE },
+    { .filename = "xzipped", .succeeds = TRUE },
+    { .filename = "xzipped.yaml.xz", .succeeds = TRUE },
+    { .filename = NULL }
+  };
+#else /* HAVE_LIBMAGIC */
+  struct expected_compressed_read_t expected[] = {
+    { .filename = "bzipped",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_YAML_ERROR,
+      .error_code = MODULEMD_YAML_ERROR_UNPARSEABLE },
+    {
+      .filename = "bzipped.yaml.bz2",
+      .succeeds = TRUE,
+    },
+    { .filename = "gzipped",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_YAML_ERROR,
+      .error_code = MODULEMD_YAML_ERROR_UNPARSEABLE },
+    { .filename = "gzipped.yaml.gz", .succeeds = TRUE },
+    { .filename = "xzipped",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_YAML_ERROR,
+      .error_code = MODULEMD_YAML_ERROR_UNPARSEABLE },
+    { .filename = "xzipped.yaml.xz", .succeeds = TRUE },
+    { .filename = NULL }
+  };
+#endif /* HAVE_LIBMAGIC */
+
+#else /* HAVE_RPMIO */
+  struct expected_compressed_read_t expected[] = {
+    { .filename = "bzipped",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_ERROR,
+      .error_code = MODULEMD_ERROR_NOT_IMPLEMENTED },
+    { .filename = "bzipped.yaml.bz2",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_ERROR,
+      .error_code = MODULEMD_ERROR_NOT_IMPLEMENTED },
+    { .filename = "gzipped",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_ERROR,
+      .error_code = MODULEMD_ERROR_NOT_IMPLEMENTED },
+    { .filename = "gzipped.yaml.gz",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_ERROR,
+      .error_code = MODULEMD_ERROR_NOT_IMPLEMENTED },
+    { .filename = "xzipped",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_ERROR,
+      .error_code = MODULEMD_ERROR_NOT_IMPLEMENTED },
+    { .filename = "xzipped.yaml.xz",
+      .succeeds = FALSE,
+      .error_domain = MODULEMD_ERROR,
+      .error_code = MODULEMD_ERROR_NOT_IMPLEMENTED },
+    { .filename = NULL }
+  };
+#endif /* HAVE_RPMIO */
+
+  baseline_idx = modulemd_module_index_new ();
+  g_assert_nonnull (baseline_idx);
+
+  file_path = g_strdup_printf ("%s/compression/uncompressed.yaml",
+                               g_getenv ("TEST_DATA_PATH"));
+  g_assert_nonnull (file_path);
+
+  bret = modulemd_module_index_update_from_file (
+    baseline_idx, file_path, TRUE, &failures, &error);
+  g_assert_true (bret);
+  g_assert_no_error (error);
+  g_assert_cmpint (failures->len, ==, 0);
+
+  baseline_text = modulemd_module_index_dump_to_string (baseline_idx, &error);
+  g_assert_nonnull (baseline_text);
+  g_assert_no_error (error);
+
+  g_clear_pointer (&file_path, g_free);
+
+  for (size_t i = 0; expected[i].filename; i++)
+    {
+      compressed_idx = modulemd_module_index_new ();
+      g_assert_nonnull (compressed_idx);
+
+      file_path = g_strdup_printf ("%s/compression/%s",
+                                   g_getenv ("TEST_DATA_PATH"),
+                                   expected[i].filename);
+      g_assert_nonnull (file_path);
+
+      g_debug ("Processing %s, expecting %s",
+               file_path,
+               expected[i].succeeds ? "success" : "failure");
+      bret = modulemd_module_index_update_from_file (
+        compressed_idx, file_path, TRUE, &failures, &error);
+
+      if (error)
+        {
+          g_debug ("Error: %s", error->message);
+        }
+
+      if (expected[i].succeeds)
+        {
+          g_assert_true (bret);
+          g_assert_no_error (error);
+        }
+      else
+        {
+          g_assert_false (bret);
+          g_assert_error (
+            error, expected[i].error_domain, expected[i].error_code);
+
+          g_clear_error (&error);
+          g_clear_pointer (&file_path, g_free);
+          g_clear_object (&compressed_idx);
+          continue;
+        }
+      g_assert_cmpint (failures->len, ==, 0);
+
+      compressed_text =
+        modulemd_module_index_dump_to_string (compressed_idx, &error);
+      g_assert_nonnull (compressed_text);
+      g_assert_no_error (error);
+
+      g_assert_cmpstr (baseline_text, ==, compressed_text);
+
+      g_clear_pointer (&compressed_text, g_free);
+      g_clear_pointer (&file_path, g_free);
+      g_clear_object (&compressed_idx);
+    }
+}
+
+
 int
 main (int argc, char *argv[])
 {
@@ -1097,5 +1265,8 @@ main (int argc, char *argv[])
   g_test_add_func ("/modulemd/v2/module/index/empty",
                    module_index_test_dump_empty_index);
 
+  g_test_add_func ("/modulemd/v2/module/index/compressed",
+                   test_module_index_read_compressed);
+
   return g_test_run ();
 }