Add helpers for reading compressed files

Currently supports gzip, bzip2 and xz compression formats.

Signed-off-by: Stephen Gallagher <sgallagh@redhat.com>

.travis/centos/Dockerfile.deps.tmpl

@@ -9,6 +9,7 @@ RUN yum -y install epel-release \
         clang \
 	createrepo_c \
 	elinks \
+	file-devel \
 	gcc \
 	gcc-c++ \
 	git-core \
@@ -28,6 +29,7 @@ RUN yum -y install epel-release \
 	python3-rpm-macros \
 	redhat-rpm-config \
 	rpm-build \
+	rpm-devel \
 	rpmdevtools \
 	sudo \
     && yum -y clean all

.travis/fedora/Dockerfile.deps.tmpl

@@ -11,6 +11,7 @@ RUN dnf -y --setopt=install_weak_deps=False --setopt=tsflags='' install \
 	"libmodulemd >= 2.3" \
 	curl \
 	elinks \
+	file-devel \
 	gcc \
 	gcc-c++ \
 	git-core \
@@ -35,6 +36,7 @@ RUN dnf -y --setopt=install_weak_deps=False --setopt=tsflags='' install \
 	python3-rpm-macros \
 	redhat-rpm-config \
 	rpm-build \
+	rpm-devel \
 	rpmdevtools \
 	ruby \
 	"rubygem(json)" \

libmodulemd.spec.in

@@ -33,6 +33,8 @@ BuildRequires:  python3-devel
 BuildRequires:  python3-gobject-base
 BuildRequires:  valgrind
 BuildRequires:  glib2-doc
+BuildRequires:  rpm-devel
+BuildRequires:  file-devel
 
 # Patches
 

meson.build

@@ -37,6 +37,7 @@ test_cflags = [
   '-Werror=return-type',
   '-Werror=array-bounds',
   '-Werror=write-strings',
+  '-D_POSIX_SOURCE',
   '-DG_LOG_USE_STRUCTURED',
   '-DG_LOG_DOMAIN="libmodulemd"',
 ]
@@ -53,6 +54,12 @@ gobject = dependency('gobject-2.0')
 yaml = dependency('yaml-0.1')
 gtkdoc = dependency('gtk-doc')
 
+with_rpmio = get_option('rpmio')
+with_libmagic = get_option('libmagic')
+
+rpm = dependency('rpm', required : with_rpmio)
+magic = cc.find_library('magic', required : with_libmagic)
+
 glib_prefix = dependency('glib-2.0').get_pkgconfig_variable('prefix')
 glib_docpath = join_paths(glib_prefix, 'share', 'gtk-doc', 'html')
 

meson_options.txt

@@ -16,3 +16,5 @@ option('skip_introspection', type : 'boolean', value : false)
 option('python_name', type : 'string')
 option('with_py2_overrides', type : 'boolean', value : true)
 option('with_py3_overrides', type : 'boolean', value : true)
+option('rpmio', type : 'feature', value : 'enabled')
+option('libmagic', type : 'feature', value : 'enabled')

modulemd/include/modulemd-2.0/modulemd-compression.h

@@ -0,0 +1,69 @@
+/*
+ * This file is part of libmodulemd
+ * Copyright (C) 2019 Red Hat, Inc.
+ *
+ * Fedora-License-Identifier: MIT
+ * SPDX-2.0-License-Identifier: MIT
+ * SPDX-3.0-License-Identifier: MIT
+ *
+ * This program is free software.
+ * For more information on the license, see COPYING.
+ * For more information on free software, see <https://www.gnu.org/philosophy/free-sw.en.html>.
+ */
+
+#pragma once
+
+#include <glib.h>
+
+G_BEGIN_DECLS
+
+/**
+ * SECTION: modulemd-compression
+ * @title: Modulemd Compression Helpers
+ * @stability: stable
+ * @short_description: Utility functions for working with compressed files.
+ */
+
+
+/**
+ * ModulemdCompressionTypeEnum
+ * @MODULEMD_COMPRESSION_TYPE_AUTO_DETECT_COMPRESSION: Autodetection
+ * @MODULEMD_COMPRESSION_TYPE_UNKNOWN_COMPRESSION: Unknown compression
+ * @MODULEMD_COMPRESSION_TYPE_NO_COMPRESSION: No compression
+ * @MODULEMD_COMPRESSION_TYPE_GZ_COMPRESSION: gzip compression
+ * @MODULEMD_COMPRESSION_TYPE_BZ2_COMPRESSION: bzip2 compression
+ * @MODULEMD_COMPRESSION_TYPE_XZ_COMPRESSION: LZMA compression
+ * @MODULEMD_COMPRESSION_TYPE_ZCK_COMPRESSION: zchunk compression
+ *
+ * Since: 2.8
+ */
+typedef enum
+{
+  MODULEMD_COMPRESSION_TYPE_DETECTION_FAILED = -2,
+  MODULEMD_COMPRESSION_TYPE_UNKNOWN_COMPRESSION,
+  MODULEMD_COMPRESSION_TYPE_NO_COMPRESSION,
+  MODULEMD_COMPRESSION_TYPE_GZ_COMPRESSION,
+  MODULEMD_COMPRESSION_TYPE_BZ2_COMPRESSION,
+  MODULEMD_COMPRESSION_TYPE_XZ_COMPRESSION,
+  MODULEMD_COMPRESSION_TYPE_ZCK_COMPRESSION,
+  MODULEMD_COMPRESSION_TYPE_SENTINEL,
+} ModulemdCompressionTypeEnum;
+
+
+/**
+ * modulemd_compression_type:
+ * @name: (in): The name of the compression type. Valid options are:
+ * "gz", "gzip", "bz2", "bzip2", "xz" and "zck".
+ *
+ * Returns: The #ModulemdCompressionTypeEnum value corresponding to the
+ * provided string if available or
+ * #MODULEMD_COMPRESSION_TYPE_UNKNOWN_COMPRESSION if the string does not match
+ * a known type.
+ *
+ * Since: 2.8
+ */
+ModulemdCompressionTypeEnum
+modulemd_compression_type (const gchar *name);
+
+
+G_END_DECLS

modulemd/include/modulemd-2.0/modulemd-errors.h

@@ -38,6 +38,8 @@ G_BEGIN_DECLS
  * @MODULEMD_ERROR_TOO_MANY_MATCHES: Represents an error indicating that
  * multiple streams matched when searching for a specific module
  * stream. Since: 2.2
+ * @MODULEMD_ERROR_MAGIC: Could not detect the mime type of a file for
+ * automatic detection of compression format. Since: 2.8
  *
  * Since: 2.0
  */
@@ -47,7 +49,9 @@ typedef enum
   MODULEMD_ERROR_VALIDATE,
   MODULEMD_ERROR_FILE_ACCESS,
   MODULEMD_ERROR_NO_MATCHES,
-  MODULEMD_ERROR_TOO_MANY_MATCHES
+  MODULEMD_ERROR_TOO_MANY_MATCHES,
+  MODULEMD_ERROR_MAGIC,
+  MODULEMD_ERROR_NOT_IMPLEMENTED
 } ModulemdErrorEnum;
 
 /**

modulemd/include/private/modulemd-compression-private.h

@@ -0,0 +1,95 @@
+/*
+ * This file is part of libmodulemd
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * Fedora-License-Identifier: MIT
+ * SPDX-2.0-License-Identifier: MIT
+ * SPDX-3.0-License-Identifier: MIT
+ *
+ * This program is free software.
+ * For more information on the license, see COPYING.
+ * For more information on free software, see <https://www.gnu.org/philosophy/free-sw.en.html>.
+ */
+
+#pragma once
+
+#include <glib.h>
+
+#include "config.h"
+#include "modulemd-compression.h"
+
+#ifdef HAVE_RPMIO
+#include <rpm/rpmio.h>
+void
+mmd_Fclose (FD_t fd);
+G_DEFINE_AUTO_CLEANUP_FREE_FUNC (FD_t, mmd_Fclose, NULL);
+#endif
+
+/**
+ * SECTION: modulemd-compression-private
+ * @title: Modulemd Compression Helpers (Private)
+ * @stability: Private
+ * @short_description: Internal utility functions for working with compressed
+ * files.
+ */
+
+
+/**
+ * modulemd_detect_compression:
+ * @filename: (in): The original filename that matches @fd.
+ * @fd: (in): An open file descriptor pointing at a real file.
+ * @error: (out): A #GError containing the reason this function failed.
+ *
+ * Returns: #The ModulemdCompressionTypeEnum detected from this file
+ * descriptor. In the event of an error, returns
+ * #MODULEMD_COMPRESSION_TYPE_DETECTION_FAILED and sets @error
+ * appropriately. Returns #MODULEMD_COMPRESSION_TYPE_UNKNOWN_COMPRESSION if
+ * all detection methods complete but the type is still indeterminate.
+ *
+ * Since: 2.8
+ */
+ModulemdCompressionTypeEnum
+modulemd_detect_compression (const gchar *filename, int fd, GError **error);
+
+
+/**
+ * modulemd_compression_suffix:
+ * @comtype: (in): A #ModulemdCompressionTypeEnum
+ *
+ * Returns: (transfer none): A static string representing the filename suffix
+ * that a file of this compression type should have.
+ *
+ * Since: 2.8
+ */
+const gchar *
+modulemd_compression_suffix (ModulemdCompressionTypeEnum comtype);
+
+
+/**
+ * modulemd_get_rpmio_fmode:
+ * @mode: (in): A mode argument that will be passed to `fopen(3)`.
+ * @comtype: (in): A #ModulemdCompressionTypeEnum
+ *
+ * Returns: (transfer full): A string suitable for passing to rpmio's
+ * `Fopen()` function. NULL if @mode is NULL or the @comtype is invalid.
+ *
+ * Since: 2.8
+ */
+gchar *
+modulemd_get_rpmio_fmode (const gchar *mode,
+                          ModulemdCompressionTypeEnum comtype);
+
+
+/**
+ * compressed_stream_read_fn:
+ *
+ * A #ModulemdReadHandler that uses rpmio's `Fread()` function to handle
+ * compressed files.
+ *
+ * Since: 2.8
+ */
+gint
+compressed_stream_read_fn (void *data,
+                           unsigned char *buffer,
+                           size_t size,
+                           size_t *size_read);

modulemd/meson.build

@@ -57,6 +57,7 @@ modulemd_srcs = files(
     'modulemd-component.c',
     'modulemd-component-module.c',
     'modulemd-component-rpm.c',
+    'modulemd-compression.c',
     'modulemd-defaults.c',
     'modulemd-defaults-v1.c',
     'modulemd-dependencies.c',
@@ -86,6 +87,7 @@ modulemd_hdrs = files(
     'include/modulemd-2.0/modulemd-component.h',
     'include/modulemd-2.0/modulemd-component-module.h',
     'include/modulemd-2.0/modulemd-component-rpm.h',
+    'include/modulemd-2.0/modulemd-compression.h',
     'include/modulemd-2.0/modulemd-defaults.h',
     'include/modulemd-2.0/modulemd-defaults-v1.h',
     'include/modulemd-2.0/modulemd-dependencies.h',
@@ -111,6 +113,7 @@ modulemd_priv_hdrs = files(
     'include/private/modulemd-component-private.h',
     'include/private/modulemd-component-module-private.h',
     'include/private/modulemd-component-rpm-private.h',
+    'include/private/modulemd-compression-private.h',
     'include/private/modulemd-dependencies-private.h',
     'include/private/modulemd-profile-private.h',
     'include/private/modulemd-defaults-private.h',
@@ -133,6 +136,7 @@ test_srcs = files(
     'tests/test-modulemd-buildopts.c',
     'tests/test-modulemd-component-module.c',
     'tests/test-modulemd-component-rpm.c',
+    'tests/test-modulemd-compression.c',
     'tests/test-modulemd-defaults.c',
     'tests/test-modulemd-defaults-v1.c',
     'tests/test-modulemd-dependencies.c',
@@ -180,6 +184,8 @@ enums = gnome.mkenums_simple ('modulemd-enums', sources : modulemd_hdrs)
 
 cdata = configuration_data()
 cdata.set_quoted('LIBMODULEMD_VERSION', libmodulemd_version)
+cdata.set('HAVE_RPMIO', rpm.found())
+cdata.set('HAVE_LIBMAGIC', magic.found())
 configure_file(
   output : 'config.h',
   configuration : cdata
@@ -191,6 +197,8 @@ modulemd_lib = library(
     include_directories : include_dirs,
     dependencies : [
         gobject,
+        magic,
+        rpm,
         yaml,
         build_lib,
     ],
@@ -298,6 +306,7 @@ c_tests = {
 'buildopts'           : [ 'tests/test-modulemd-buildopts.c' ],
 'component_module'    : [ 'tests/test-modulemd-component-module.c' ],
 'component_rpm'       : [ 'tests/test-modulemd-component-rpm.c' ],
+'compression'         : [ 'tests/test-modulemd-compression.c' ],
 'defaults'            : [ 'tests/test-modulemd-defaults.c' ],
 'defaultsv1'          : [ 'tests/test-modulemd-defaults-v1.c' ],
 'dependencies'        : [ 'tests/test-modulemd-dependencies.c' ],