Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use rpmio (add support for zstd) #251

Closed
wants to merge 2 commits into from
Closed

Use rpmio (add support for zstd) #251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e933ffb
Add zstd compression support through rpmio
kontura Aug 17, 2021
e2dd20e
Add unittests for zstd compression
kontura Aug 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions doc/createrepo_c.8
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
.TH CREATEREPO_C 8 "2020-07-02" "" ""
.TH CREATEREPO_C 8 "2021-08-18" "" ""
.SH NAME
createrepo_c \- Create rpm-md format (xml-rpm-metadata) repository
.
Expand Down Expand Up @@ -161,7 +161,7 @@ Number of workers to spawn to read rpms.
Use xz for repodata compression.
.SS \-\-compress\-type COMPRESSION_TYPE
.sp
Which compression type to use.
Which compression type to use. Supported compressions are: bzip2, gzip, zck, zstd, xz.
.SS \-\-general\-compress\-type COMPRESSION_TYPE
.sp
Which compression type to use (even for primary, filelists and other xml).
Expand Down
4 changes: 3 additions & 1 deletion src/cmd_parser.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ static GOptionEntry cmd_entries[] =
{ "xz", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.xz_compression),
"Use xz for repodata compression.", NULL },
{ "compress-type", 0, 0, G_OPTION_ARG_STRING, &(_cmd_options.compress_type),
"Which compression type to use.", "COMPRESSION_TYPE" },
"Which compression type to use. Supported compressions are: bzip2, gzip, zck, zstd, xz.", "COMPRESSION_TYPE" },
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be great if "none" were also an option

{ "general-compress-type", 0, 0, G_OPTION_ARG_STRING, &(_cmd_options.general_compress_type),
"Which compression type to use (even for primary, filelists and other xml).",
"COMPRESSION_TYPE" },
Expand Down Expand Up @@ -279,6 +279,8 @@ check_and_set_compression_type(const char *type_str,
*type = CR_CW_BZ2_COMPRESSION;
} else if (!strcmp(compress_str->str, "xz")) {
*type = CR_CW_XZ_COMPRESSION;
} else if (!strcmp(compress_str->str, "zstd") || !strcmp(compress_str->str, "zst")) {
*type = CR_CW_ZSTD_COMPRESSION;
} else {
g_set_error(err, ERR_DOMAIN, CRE_BADARG,
"Unknown/Unsupported compression type \"%s\"", type_str);
Expand Down
60 changes: 60 additions & 0 deletions src/compression_wrapper.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#endif // WITH_ZCHUNK
#include "error.h"
#include "compression_wrapper.h"
#include <rpm/rpmio.h>


#define ERR_DOMAIN CREATEREPO_C_ERROR
Expand Down Expand Up @@ -156,6 +157,9 @@ cr_detect_compression(const char *filename, GError **err)
} else if (g_str_has_suffix(filename, ".zck"))
{
return CR_CW_ZCK_COMPRESSION;
} else if (g_str_has_suffix(filename, ".zst"))
{
return CR_CW_ZSTD_COMPRESSION;
} else if (g_str_has_suffix(filename, ".xml") ||
g_str_has_suffix(filename, ".tar") ||
g_str_has_suffix(filename, ".yaml") ||
Expand Down Expand Up @@ -197,6 +201,11 @@ cr_detect_compression(const char *filename, GError **err)
type = CR_CW_GZ_COMPRESSION;
}

else if (g_str_has_prefix(mime_type, "application/zstd"))
{
type = CR_CW_ZSTD_COMPRESSION;
}

else if (g_str_has_prefix(mime_type, "application/x-bzip2") ||
g_str_has_prefix(mime_type, "application/x-bz2") ||
g_str_has_prefix(mime_type, "application/bzip2") ||
Expand Down Expand Up @@ -260,6 +269,8 @@ cr_compression_type(const char *name)
type = CR_CW_XZ_COMPRESSION;
if (!g_strcmp0(name_lower, "zck"))
type = CR_CW_ZCK_COMPRESSION;
if (!g_strcmp0(name_lower, "zst") || !g_strcmp0(name_lower, "zstd"))
type = CR_CW_ZSTD_COMPRESSION;
g_free(name_lower);

return type;
Expand All @@ -277,6 +288,8 @@ cr_compression_suffix(cr_CompressionType comtype)
return ".xz";
case CR_CW_ZCK_COMPRESSION:
return ".zck";
case CR_CW_ZSTD_COMPRESSION:
return ".zst";
default:
return NULL;
}
Expand Down Expand Up @@ -416,6 +429,16 @@ cr_sopen(const char *filename,
}
break;

case (CR_CW_ZSTD_COMPRESSION): { // ------------------------------------
mode_str = (mode == CR_CW_MODE_WRITE) ? "wb.zstdio" : "rb.zstdio";
file->FILE = (void *) Fopen(filename, mode_str);
if (!file->FILE || Ferror((FD_t) file->FILE))
g_set_error(err, ERR_DOMAIN, CRE_ZSTD,
"Fopen() from rpmio for zstd failed: %s",
Fstrerror((FD_t) file->FILE));
break;
}

case (CR_CW_BZ2_COMPRESSION): { // ------------------------------------
FILE *f = fopen(filename, mode_str);
file->INNERFILE = f;
Expand Down Expand Up @@ -772,6 +795,17 @@ cr_close(CR_FILE *cr_file, GError **err)
}
break;

case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------
if (Fclose((FD_t) cr_file->FILE) == 0) {
ret = CRE_OK;
} else {
ret = CRE_ZSTD;
g_set_error(err, ERR_DOMAIN, CRE_ZSTD,
"Fclose() from rpmio for zstd failed: %s",
Fstrerror((FD_t) cr_file->FILE));
}
break;

case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
if (cr_file->mode == CR_CW_MODE_READ)
BZ2_bzReadClose(&rc, (BZFILE *) cr_file->FILE);
Expand Down Expand Up @@ -983,6 +1017,15 @@ cr_read(CR_FILE *cr_file, void *buffer, unsigned int len, GError **err)
}
break;

case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
ret = Fread(buffer, sizeof(*buffer), len, (FD_t) cr_file->FILE);
if (ret < 0) {
ret = CR_CW_ERR;
g_set_error(err, ERR_DOMAIN, CRE_ZSTD,
"Fread() zstd rpmio: %d", Ferror((FD_t) cr_file->FILE));
}
break;

case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
ret = BZ2_bzRead(&bzerror, (BZFILE *) cr_file->FILE, buffer, len);
if (!ret && bzerror == BZ_SEQUENCE_ERROR)
Expand Down Expand Up @@ -1217,6 +1260,19 @@ cr_write(CR_FILE *cr_file, const void *buffer, unsigned int len, GError **err)
}
break;

case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
if (len == 0) {
ret = 0;
break;
}
if ((ret = Fwrite(buffer, 1, len, (FD_t) cr_file->FILE)) != (int) len) {
ret = CR_CW_ERR;
g_set_error(err, ERR_DOMAIN, CRE_ZSTD,
"Fwrite() from rpmio for zstd failed: %s",
Fstrerror((FD_t) cr_file->FILE));
}
break;

case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
BZ2_bzWrite(&bzerror, (BZFILE *) cr_file->FILE, (void *) buffer, len);
if (bzerror == BZ_OK) {
Expand Down Expand Up @@ -1364,6 +1420,7 @@ cr_puts(CR_FILE *cr_file, const char *str, GError **err)
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
case (CR_CW_ZCK_COMPRESSION): // --------------------------------------
case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------
len = strlen(str);
ret = cr_write(cr_file, str, len, err);
if (ret != (int) len)
Expand Down Expand Up @@ -1401,6 +1458,7 @@ cr_end_chunk(CR_FILE *cr_file, GError **err)
case (CR_CW_GZ_COMPRESSION): // ---------------------------------------
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
break;
case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------
#ifdef WITH_ZCHUNK
Expand Down Expand Up @@ -1453,6 +1511,7 @@ cr_set_autochunk(CR_FILE *cr_file, gboolean auto_chunk, GError **err)
case (CR_CW_GZ_COMPRESSION): // ---------------------------------------
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
case (CR_CW_ZSTD_COMPRESSION): // ---------------------------------------
break;
case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------
#ifdef WITH_ZCHUNK
Expand Down Expand Up @@ -1527,6 +1586,7 @@ cr_printf(GError **err, CR_FILE *cr_file, const char *format, ...)
case (CR_CW_BZ2_COMPRESSION): // --------------------------------------
case (CR_CW_XZ_COMPRESSION): // ---------------------------------------
case (CR_CW_ZCK_COMPRESSION): // --------------------------------------
case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------
tmp_ret = cr_write(cr_file, buf, ret, err);
if (tmp_ret != (int) ret)
ret = CR_CW_ERR;
Expand Down
1 change: 1 addition & 0 deletions src/compression_wrapper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ typedef enum {
CR_CW_BZ2_COMPRESSION, /*!< BZip2 compression */
CR_CW_XZ_COMPRESSION, /*!< XZ compression */
CR_CW_ZCK_COMPRESSION, /*!< ZCK compression */
CR_CW_ZSTD_COMPRESSION, /*!< ZSTD compression */
CR_CW_COMPRESSION_SENTINEL, /*!< Sentinel of the list */
} cr_CompressionType;

Expand Down
2 changes: 2 additions & 0 deletions src/error.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ typedef enum {
(34) ZCK library related error */
CRE_MODULEMD, /*!<
(35) modulemd related error */
CRE_ZSTD, /*!<
(36) Zstd from rpmio library related error */
CRE_SENTINEL, /*!<
(XX) Sentinel */
} cr_Error;
Expand Down
3 changes: 3 additions & 0 deletions src/python/createrepo_c/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@
#: Zchunk compression alias
ZCK = _createrepo_c.ZCK_COMPRESSION

#: Zstd compression alias
ZSTD = _createrepo_c.ZSTD_COMPRESSION

HT_KEY_DEFAULT = _createrepo_c.HT_KEY_DEFAULT #: Default key (hash)
HT_KEY_HASH = _createrepo_c.HT_KEY_HASH #: Package hash as a key
HT_KEY_NAME = _createrepo_c.HT_KEY_NAME #: Package name as a key
Expand Down
1 change: 1 addition & 0 deletions src/python/createrepo_cmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ PyInit__createrepo_c(void)
PyModule_AddIntConstant(m, "BZ2_COMPRESSION", CR_CW_BZ2_COMPRESSION);
PyModule_AddIntConstant(m, "XZ_COMPRESSION", CR_CW_XZ_COMPRESSION);
PyModule_AddIntConstant(m, "ZCK_COMPRESSION", CR_CW_ZCK_COMPRESSION);
PyModule_AddIntConstant(m, "ZSTD_COMPRESSION", CR_CW_ZSTD_COMPRESSION);

/* Zchunk support */
#ifdef WITH_ZCHUNK
Expand Down
8 changes: 8 additions & 0 deletions tests/python/tests/test_compression_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def test_compression_suffix(self):
self.assertEqual(cr.compression_suffix(cr.BZ2), ".bz2")
self.assertEqual(cr.compression_suffix(cr.XZ), ".xz")
self.assertEqual(cr.compression_suffix(cr.ZCK), ".zck")
self.assertEqual(cr.compression_suffix(cr.ZSTD), ".zst")

def test_detect_compression(self):

Expand Down Expand Up @@ -69,6 +70,11 @@ def test_detect_compression(self):
#comtype = cr.detect_compression(path)
#self.assertEqual(comtype, cr.ZCK)

# Bad suffix - zstd compression
path = os.path.join(COMPRESSED_FILES_PATH, "01_plain.foo5")
comtype = cr.detect_compression(path)
self.assertEqual(comtype, cr.ZSTD)

def test_compression_type(self):
self.assertEqual(cr.compression_type(None), cr.UNKNOWN_COMPRESSION)
self.assertEqual(cr.compression_type(""), cr.UNKNOWN_COMPRESSION)
Expand All @@ -77,4 +83,6 @@ def test_compression_type(self):
self.assertEqual(cr.compression_type("xz"), cr.XZ)
self.assertEqual(cr.compression_type("XZ"), cr.XZ)
self.assertEqual(cr.compression_type("zck"), cr.ZCK)
self.assertEqual(cr.compression_type("zstd"), cr.ZSTD)
self.assertEqual(cr.compression_type("zst"), cr.ZSTD)

Loading