From 1b99aa3e962708b809ebeb7933664eee2e2a03fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Tue, 17 Aug 2021 14:27:47 +0200 Subject: [PATCH 1/2] Add zstd compression support --- CMakeLists.txt | 9 ++ README.md | 1 + createrepo_c.spec | 9 +- doc/createrepo_c.8 | 2 +- src/CMakeLists.txt | 1 + src/cmd_parser.c | 6 +- src/compression_wrapper.c | 192 ++++++++++++++++++++++++++++ src/compression_wrapper.h | 1 + src/error.h | 2 + src/python/createrepo_c/__init__.py | 3 + src/python/createrepo_cmodule.c | 1 + 11 files changed, 223 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fff38c6e..071a29df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,8 @@ if(NOT BUILD_LIBCREATEREPO_C_SHARED) set(CMAKE_POSITION_INDEPENDENT_CODE 1) endif() +option(WITH_ZSTD "Build with zstd support" ON) + option(CREATEREPO_C_INSTALL_DEVELOPMENT "Install createrepo_c development files." ON) option(CREATEREPO_C_INSTALL_MANPAGES "Install createrepo_c man-pages." ON) @@ -102,6 +104,13 @@ IF (WITH_LIBMODULEMD) SET (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DWITH_LIBMODULEMD") ENDIF (WITH_LIBMODULEMD) +if (WITH_ZSTD) + pkg_check_modules(ZSTD REQUIRED libzstd) + include_directories(${ZSTD_INCLUDE_DIRS}) + SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWITH_ZSTD") + SET (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DWITH_ZSTD") +endif() + # Threaded XZ Compression # Note: This option is disabled by default, because Createrepo_c # parallelize a lot of tasks (including compression) by default, this diff --git a/README.md b/README.md index de2cd017..95b03ceb 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Package build requires - Pkg name in Fedora/Ubuntu: * xz (http://tukaani.org/xz/) - xz-devel/liblzma-dev * zchunk (https://github.com/zchunk/zchunk) - zchunk-devel/ * zlib (http://www.zlib.net/) - zlib-devel/zlib1g-dev +* libzstd (http://facebook.github.io/zstd/) - libzstd-devel/libzstd-dev * *Documentation:* doxygen (http://doxygen.org/) - doxygen/doxygen * *Documentation:* sphinx (http://sphinx-doc.org/) - python3-sphinx/python3-sphinx * **Test requires:** check (http://check.sourceforge.net/) - check-devel/check diff --git a/createrepo_c.spec b/createrepo_c.spec index 92ab0c6f..c5acea69 100644 --- a/createrepo_c.spec +++ b/createrepo_c.spec @@ -18,8 +18,11 @@ %if 0%{?rhel} && 0%{?rhel} < 7 %bcond_with libmodulemd +# dnf supports zstd since 8.4: https://bugzilla.redhat.com/show_bug.cgi?id=1914876 +%bcond_with zstd %else %bcond_without libmodulemd +%bcond_without zstd %endif %if 0%{?rhel} && 0%{?rhel} <= 8 @@ -70,6 +73,9 @@ Requires: rpm >= 4.9.0 %if %{with drpm} BuildRequires: drpm-devel >= 0.4.0 %endif +%if %{with zstd} +BuildRequires: pkgconfig(libzstd) +%endif %if 0%{?fedora} || 0%{?rhel} > 7 Obsoletes: createrepo < 0.11.0 @@ -120,7 +126,8 @@ pushd build-py3 -DWITH_ZCHUNK=%{?with_zchunk:ON}%{!?with_zchunk:OFF} \ -DWITH_LIBMODULEMD=%{?with_libmodulemd:ON}%{!?with_libmodulemd:OFF} \ -DWITH_LEGACY_HASHES=%{?with_legacy_hashes:ON}%{!?with_legacy_hashes:OFF} \ - -DENABLE_DRPM=%{?with_drpm:ON}%{!?with_drpm:OFF} + -DENABLE_DRPM=%{?with_drpm:ON}%{!?with_drpm:OFF} \ + -DWITH_ZSTD=%{?with_zstd:ON}%{!?with_zstd:OFF} make %{?_smp_mflags} RPM_OPT_FLAGS="%{optflags}" # Build C documentation make doc-c diff --git a/doc/createrepo_c.8 b/doc/createrepo_c.8 index 86d0dc7e..bf9862b6 100644 --- a/doc/createrepo_c.8 +++ b/doc/createrepo_c.8 @@ -161,7 +161,7 @@ Number of workers to spawn to read rpms. Use xz for repodata compression. .SS \-\-compress\-type COMPRESSION_TYPE .sp -Which compression type to use. +Which compression type to use. Supported compressions are: bzip2, gzip, zck, zstd, xz. .SS \-\-general\-compress\-type COMPRESSION_TYPE .sp Which compression type to use (even for primary, filelists and other xml). diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 41e3772b..3217e95c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -81,6 +81,7 @@ TARGET_LINK_LIBRARIES(libcreaterepo_c ${SQLITE3_LIBRARIES}) TARGET_LINK_LIBRARIES(libcreaterepo_c ${ZLIB_LIBRARY}) TARGET_LINK_LIBRARIES(libcreaterepo_c ${ZCK_LIBRARIES}) TARGET_LINK_LIBRARIES(libcreaterepo_c ${DRPM_LIBRARIES}) +TARGET_LINK_LIBRARIES(libcreaterepo_c ${ZSTD_LIBRARIES}) SET_TARGET_PROPERTIES(libcreaterepo_c PROPERTIES OUTPUT_NAME "createrepo_c" diff --git a/src/cmd_parser.c b/src/cmd_parser.c index 97c9ea7d..39ff99d5 100644 --- a/src/cmd_parser.c +++ b/src/cmd_parser.c @@ -180,9 +180,9 @@ static GOptionEntry cmd_entries[] = { "xz", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.xz_compression), "Use xz for repodata compression.", NULL }, { "compress-type", 0, 0, G_OPTION_ARG_STRING, &(_cmd_options.compress_type), - "Which compression type to use.", "COMPRESSION_TYPE" }, + "Which compression type to use for additional metadata files (comps, updateinfo, etc). Supported compressions are: bzip2, gzip, zck, zstd, xz.", "COMPRESSION_TYPE" }, { "general-compress-type", 0, 0, G_OPTION_ARG_STRING, &(_cmd_options.general_compress_type), - "Which compression type to use (even for primary, filelists and other xml).", + "Which compression type to use (even for primary, filelists and other xml). Supported compressions are: bzip2, gzip, zck, zstd, xz.", "COMPRESSION_TYPE" }, #ifdef WITH_ZCHUNK { "zck", 0, 0, G_OPTION_ARG_NONE, &(_cmd_options.zck_compression), @@ -315,6 +315,8 @@ check_and_set_compression_type(const char *type_str, *type = CR_CW_BZ2_COMPRESSION; } else if (!strcmp(compress_str->str, "xz")) { *type = CR_CW_XZ_COMPRESSION; + } else if (!strcmp(compress_str->str, "zstd")) { + *type = CR_CW_ZSTD_COMPRESSION; } else { g_set_error(err, ERR_DOMAIN, CRE_BADARG, "Unknown/Unsupported compression type \"%s\"", type_str); diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c index b23c345f..91002226 100644 --- a/src/compression_wrapper.c +++ b/src/compression_wrapper.c @@ -35,6 +35,9 @@ #endif // WITH_ZCHUNK #include "error.h" #include "compression_wrapper.h" +#ifdef WITH_ZSTD +#include +#endif #define ERR_DOMAIN CREATEREPO_C_ERROR @@ -118,6 +121,17 @@ typedef struct { unsigned char buffer[XZ_BUFFER_SIZE]; } XzFile; +#ifdef WITH_ZSTD +#define CR_CW_ZSTD_COMPRESSION_LEVEL 9 +typedef struct { + void *buffer; + size_t buffer_size; + ZSTD_inBuffer zib; + ZSTD_outBuffer zob; + void * context; //ZSTD_{C,D}Ctx +} ZstdFile; +#endif + cr_CompressionType cr_detect_compression(const char *filename, GError **err) { @@ -151,6 +165,9 @@ cr_detect_compression(const char *filename, GError **err) } else if (g_str_has_suffix(filename, ".zck")) { return CR_CW_ZCK_COMPRESSION; + } else if (g_str_has_suffix(filename, ".zst")) + { + return CR_CW_ZSTD_COMPRESSION; } else if (g_str_has_suffix(filename, ".xml") || g_str_has_suffix(filename, ".tar") || g_str_has_suffix(filename, ".yaml") || @@ -192,6 +209,11 @@ cr_detect_compression(const char *filename, GError **err) type = CR_CW_GZ_COMPRESSION; } + else if (g_str_has_prefix(mime_type, "application/zstd")) + { + type = CR_CW_ZSTD_COMPRESSION; + } + else if (g_str_has_prefix(mime_type, "application/x-bzip2") || g_str_has_prefix(mime_type, "application/x-bz2") || g_str_has_prefix(mime_type, "application/bzip2") || @@ -255,6 +277,8 @@ cr_compression_type(const char *name) type = CR_CW_XZ_COMPRESSION; if (!g_strcmp0(name_lower, "zck")) type = CR_CW_ZCK_COMPRESSION; + if (!g_strcmp0(name_lower, "zstd")) + type = CR_CW_ZSTD_COMPRESSION; g_free(name_lower); return type; @@ -272,6 +296,8 @@ cr_compression_suffix(cr_CompressionType comtype) return ".xz"; case CR_CW_ZCK_COMPRESSION: return ".zck"; + case CR_CW_ZSTD_COMPRESSION: + return ".zst"; default: return NULL; } @@ -413,6 +439,56 @@ cr_sopen(const char *filename, } break; + case (CR_CW_ZSTD_COMPRESSION): { // ------------------------------------ +#ifdef WITH_ZSTD + FILE *f = fopen(filename, mode_str); + + if (!f) { + g_set_error(err, ERR_DOMAIN, CRE_IO, "fopen(): %s", g_strerror(errno)); + break; + } + + file->INNERFILE = f; + + ZstdFile *zstd_file = g_malloc0(sizeof(ZstdFile)); + + if (mode == CR_CW_MODE_WRITE) { + if ((zstd_file->context = (void *) ZSTD_createCCtx()) == NULL) { + g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", + "Failed to create ZSTD context."); + g_free(zstd_file); + fclose(f); + break; + } + size_t ret = ZSTD_CCtx_setParameter(zstd_file->context, ZSTD_c_compressionLevel, CR_CW_ZSTD_COMPRESSION_LEVEL); + if (ZSTD_isError(ret)) { + g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", + ZSTD_getErrorName(ret)); + g_free(zstd_file); + fclose(f); + break; + } + zstd_file->buffer_size = ZSTD_CStreamOutSize(); + } else { + if ((zstd_file->context = (void *) ZSTD_createDCtx()) == NULL) { + g_free(zstd_file); + fclose(f); + g_set_error(err, ERR_DOMAIN, CRE_IO, "%s", + "Failed to create ZSTD context."); + break; + } + zstd_file->buffer_size = ZSTD_DStreamInSize(); + } + zstd_file->buffer = g_malloc(zstd_file->buffer_size); + file->FILE = (void *) zstd_file; + + break; +#else + g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support"); + break; +#endif // WITH_ZSTD + } + case (CR_CW_BZ2_COMPRESSION): { // ------------------------------------ FILE *f = fopen(filename, mode_str); file->INNERFILE = f; @@ -769,6 +845,43 @@ cr_close(CR_FILE *cr_file, GError **err) } break; + case (CR_CW_ZSTD_COMPRESSION): // -------------------------------------- +#ifdef WITH_ZSTD + ZstdFile * zstd = (ZstdFile *) cr_file->FILE; + if (cr_file->mode == CR_CW_MODE_READ) { + ZSTD_freeDCtx(zstd->context); + } else { + size_t remaining; + // No more new input just finish flushing compression data + ZSTD_inBuffer zip = { NULL, 0, 0 }; + do { + zstd->zob.dst = zstd->buffer; + zstd->zob.size = zstd->buffer_size; + zstd->zob.pos = 0; + + remaining = ZSTD_compressStream2(zstd->context, &zstd->zob , &zip, ZSTD_e_end); + if (ZSTD_isError(remaining)) { + g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", ZSTD_getErrorName(remaining)); + break; + } else if (zstd->zob.pos != fwrite(zstd->buffer, 1, zstd->zob.pos, cr_file->INNERFILE)) { + g_set_error(err, ERR_DOMAIN, CRE_IO, "cr_close ZSTD fwrite failed"); + break; + } + } while(remaining != 0); + ZSTD_freeCCtx(zstd->context); + } + + fclose(cr_file->INNERFILE); + g_free(zstd->buffer); + g_free(cr_file->FILE); + + ret = CRE_OK; + break; +#else + g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support"); + break; +#endif // WITH_ZSTD + case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- if (cr_file->mode == CR_CW_MODE_READ) BZ2_bzReadClose(&rc, (BZFILE *) cr_file->FILE); @@ -980,6 +1093,43 @@ cr_read(CR_FILE *cr_file, void *buffer, unsigned int len, GError **err) } break; + case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------- +#ifdef WITH_ZSTD + ZstdFile * zstd = (ZstdFile *) cr_file->FILE; + + ZSTD_outBuffer zob = {buffer, len, 0}; + + while (zob.pos < zob.size) { + // Re-fill compressed data buffer + if (zstd->zib.pos >= zstd->zib.size) { + zstd->zib.size = fread(zstd->buffer, 1, zstd->buffer_size, cr_file->INNERFILE); + if (zstd->zib.size == 0) { + break; //EOF + } + zstd->zib.src = zstd->buffer; + zstd->zib.pos = 0; + } + + // Decompress chunk + int decomp_ret = ZSTD_decompressStream(zstd->context, &zob, &zstd->zib); + if (ZSTD_isError(decomp_ret)) { + ret = CR_CW_ERR; + g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", ZSTD_getErrorName(decomp_ret)); + break; + } + + } + + if (!(err && *err)) { + ret = zob.pos; + } + + break; +#else + g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support"); + break; +#endif // WITH_ZSTD + case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- ret = BZ2_bzRead(&bzerror, (BZFILE *) cr_file->FILE, buffer, len); if (!ret && bzerror == BZ_SEQUENCE_ERROR) @@ -1214,6 +1364,44 @@ cr_write(CR_FILE *cr_file, const void *buffer, unsigned int len, GError **err) } break; + case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------- +#ifdef WITH_ZSTD + ZstdFile * zstd = (ZstdFile *) cr_file->FILE; + ZSTD_inBuffer zib = {buffer, len, 0}; + + while (zib.pos < zib.size) { + zstd->zob.dst = zstd->buffer; + zstd->zob.size = zstd->buffer_size; + zstd->zob.pos = 0; + + // Compress chunk into buffer + size_t remaining = ZSTD_compressStream2(zstd->context, &zstd->zob , &zib, ZSTD_e_continue); + if (ZSTD_isError(remaining)) { + g_set_error(err, ERR_DOMAIN, CRE_ZSTD, "%s", ZSTD_getErrorName(remaining)); + break; + } + + // Write compressed buffer + if (zstd->zob.pos > 0) { + size_t nw = fwrite(zstd->buffer, 1, zstd->zob.pos, cr_file->INNERFILE); + if (nw != zstd->zob.pos) { + g_set_error(err, ERR_DOMAIN, CRE_IO, "cr_write zstd write failed"); + break; + } + } + + } + + if (!(err && *err)) { + ret = zib.pos; + } + + break; +#else + g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled with zstd support"); + break; +#endif // WITH_ZSTD + case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- BZ2_bzWrite(&bzerror, (BZFILE *) cr_file->FILE, (void *) buffer, len); if (bzerror == BZ_OK) { @@ -1361,6 +1549,7 @@ cr_puts(CR_FILE *cr_file, const char *str, GError **err) case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- case (CR_CW_XZ_COMPRESSION): // --------------------------------------- case (CR_CW_ZCK_COMPRESSION): // -------------------------------------- + case (CR_CW_ZSTD_COMPRESSION): // -------------------------------------- len = strlen(str); ret = cr_write(cr_file, str, len, err); if (ret != (int) len) @@ -1398,6 +1587,7 @@ cr_end_chunk(CR_FILE *cr_file, GError **err) case (CR_CW_GZ_COMPRESSION): // --------------------------------------- case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- case (CR_CW_XZ_COMPRESSION): // --------------------------------------- + case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------- break; case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------ #ifdef WITH_ZCHUNK @@ -1450,6 +1640,7 @@ cr_set_autochunk(CR_FILE *cr_file, gboolean auto_chunk, GError **err) case (CR_CW_GZ_COMPRESSION): // --------------------------------------- case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- case (CR_CW_XZ_COMPRESSION): // --------------------------------------- + case (CR_CW_ZSTD_COMPRESSION): // --------------------------------------- break; case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------ #ifdef WITH_ZCHUNK @@ -1524,6 +1715,7 @@ cr_printf(GError **err, CR_FILE *cr_file, const char *format, ...) case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- case (CR_CW_XZ_COMPRESSION): // --------------------------------------- case (CR_CW_ZCK_COMPRESSION): // -------------------------------------- + case (CR_CW_ZSTD_COMPRESSION): // -------------------------------------- tmp_ret = cr_write(cr_file, buf, ret, err); if (tmp_ret != (int) ret) ret = CR_CW_ERR; diff --git a/src/compression_wrapper.h b/src/compression_wrapper.h index 72e0078f..32936d0b 100644 --- a/src/compression_wrapper.h +++ b/src/compression_wrapper.h @@ -42,6 +42,7 @@ typedef enum { CR_CW_BZ2_COMPRESSION, /*!< BZip2 compression */ CR_CW_XZ_COMPRESSION, /*!< XZ compression */ CR_CW_ZCK_COMPRESSION, /*!< ZCK compression */ + CR_CW_ZSTD_COMPRESSION, /*!< ZSTD compression */ CR_CW_COMPRESSION_SENTINEL, /*!< Sentinel of the list */ } cr_CompressionType; diff --git a/src/error.h b/src/error.h index b925bc7a..87380327 100644 --- a/src/error.h +++ b/src/error.h @@ -98,6 +98,8 @@ typedef enum { (34) ZCK library related error */ CRE_MODULEMD, /*!< (35) modulemd related error */ + CRE_ZSTD, /*!< + (36) Zstd library related error */ CRE_SENTINEL, /*!< (XX) Sentinel */ } cr_Error; diff --git a/src/python/createrepo_c/__init__.py b/src/python/createrepo_c/__init__.py index 8d7abf75..dc6a7810 100644 --- a/src/python/createrepo_c/__init__.py +++ b/src/python/createrepo_c/__init__.py @@ -59,6 +59,9 @@ #: Zchunk compression alias ZCK = _createrepo_c.ZCK_COMPRESSION +#: Zstd compression alias +ZSTD = _createrepo_c.ZSTD_COMPRESSION + HT_KEY_DEFAULT = _createrepo_c.HT_KEY_DEFAULT #: Default key (hash) HT_KEY_HASH = _createrepo_c.HT_KEY_HASH #: Package hash as a key HT_KEY_NAME = _createrepo_c.HT_KEY_NAME #: Package name as a key diff --git a/src/python/createrepo_cmodule.c b/src/python/createrepo_cmodule.c index a8e41027..0caaa46f 100644 --- a/src/python/createrepo_cmodule.c +++ b/src/python/createrepo_cmodule.c @@ -266,6 +266,7 @@ PyInit__createrepo_c(void) PyModule_AddIntConstant(m, "BZ2_COMPRESSION", CR_CW_BZ2_COMPRESSION); PyModule_AddIntConstant(m, "XZ_COMPRESSION", CR_CW_XZ_COMPRESSION); PyModule_AddIntConstant(m, "ZCK_COMPRESSION", CR_CW_ZCK_COMPRESSION); + PyModule_AddIntConstant(m, "ZSTD_COMPRESSION", CR_CW_ZSTD_COMPRESSION); /* Zchunk support */ #ifdef WITH_ZCHUNK From 6ced4263ab4d99fa4d5fdacd38a17f85cc43f443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Tue, 17 Aug 2021 14:28:05 +0200 Subject: [PATCH 2/2] Add unittests for zstd compression --- .../python/tests/test_compression_wrapper.py | 8 +- tests/test_compression_wrapper.c | 161 ++++++++++++++++++ tests/testdata/compressed_files/00_plain.foo5 | Bin 0 -> 13 bytes .../compressed_files/00_plain.txt.zst | Bin 0 -> 13 bytes tests/testdata/compressed_files/01_plain.foo5 | Bin 0 -> 51 bytes .../compressed_files/01_plain.txt.zst | Bin 0 -> 51 bytes 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 tests/testdata/compressed_files/00_plain.foo5 create mode 100644 tests/testdata/compressed_files/00_plain.txt.zst create mode 100644 tests/testdata/compressed_files/01_plain.foo5 create mode 100644 tests/testdata/compressed_files/01_plain.txt.zst diff --git a/tests/python/tests/test_compression_wrapper.py b/tests/python/tests/test_compression_wrapper.py index c8957a29..7719fdd6 100644 --- a/tests/python/tests/test_compression_wrapper.py +++ b/tests/python/tests/test_compression_wrapper.py @@ -15,6 +15,7 @@ def test_compression_suffix(self): self.assertEqual(cr.compression_suffix(cr.BZ2), ".bz2") self.assertEqual(cr.compression_suffix(cr.XZ), ".xz") self.assertEqual(cr.compression_suffix(cr.ZCK), ".zck") + self.assertEqual(cr.compression_suffix(cr.ZSTD), ".zst") def test_detect_compression(self): @@ -69,6 +70,11 @@ def test_detect_compression(self): #comtype = cr.detect_compression(path) #self.assertEqual(comtype, cr.ZCK) + # Bad suffix - zstd compression + path = os.path.join(COMPRESSED_FILES_PATH, "01_plain.foo5") + comtype = cr.detect_compression(path) + self.assertEqual(comtype, cr.ZSTD) + def test_compression_type(self): self.assertEqual(cr.compression_type(None), cr.UNKNOWN_COMPRESSION) self.assertEqual(cr.compression_type(""), cr.UNKNOWN_COMPRESSION) @@ -77,4 +83,4 @@ def test_compression_type(self): self.assertEqual(cr.compression_type("xz"), cr.XZ) self.assertEqual(cr.compression_type("XZ"), cr.XZ) self.assertEqual(cr.compression_type("zck"), cr.ZCK) - + self.assertEqual(cr.compression_type("zstd"), cr.ZSTD) diff --git a/tests/test_compression_wrapper.c b/tests/test_compression_wrapper.c index edd4fe5a..63261563 100644 --- a/tests/test_compression_wrapper.c +++ b/tests/test_compression_wrapper.c @@ -37,10 +37,12 @@ #define FILE_COMPRESSED_0_GZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.gz" #define FILE_COMPRESSED_0_BZ2 TEST_COMPRESSED_FILES_PATH"/00_plain.txt.bz2" #define FILE_COMPRESSED_0_XZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.xz" +#define FILE_COMPRESSED_0_ZSTD TEST_COMPRESSED_FILES_PATH"/00_plain.txt.zst" #define FILE_COMPRESSED_0_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo0" #define FILE_COMPRESSED_0_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo1" #define FILE_COMPRESSED_0_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo2" #define FILE_COMPRESSED_0_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo3" +#define FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo5" #define FILE_COMPRESSED_1_CONTENT "foobar foobar foobar foobar test test\nfolkjsaflkjsadokf\n" #define FILE_COMPRESSED_1_CONTENT_LEN 56 @@ -48,11 +50,13 @@ #define FILE_COMPRESSED_1_GZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.gz" #define FILE_COMPRESSED_1_BZ2 TEST_COMPRESSED_FILES_PATH"/01_plain.txt.bz2" #define FILE_COMPRESSED_1_XZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.xz" +#define FILE_COMPRESSED_1_ZSTD TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zst" #define FILE_COMPRESSED_1_ZCK TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zck" #define FILE_COMPRESSED_1_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo0" #define FILE_COMPRESSED_1_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo1" #define FILE_COMPRESSED_1_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo2" #define FILE_COMPRESSED_1_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo3" +#define FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo5" static void @@ -98,6 +102,11 @@ test_cr_compression_suffix(void) suffix = cr_compression_suffix(CR_CW_XZ_COMPRESSION); g_assert_cmpstr(suffix, ==, ".xz"); + +#ifdef WITH_ZSTD + suffix = cr_compression_suffix(CR_CW_ZSTD_COMPRESSION); + g_assert_cmpstr(suffix, ==, ".zst"); +#endif // WITH_ZSTD } static void @@ -134,6 +143,11 @@ test_cr_compression_type(void) type = cr_compression_type("xz"); g_assert_cmpint(type, ==, CR_CW_XZ_COMPRESSION); + +#ifdef WITH_ZSTD + type = cr_compression_type("zstd"); + g_assert_cmpint(type, ==, CR_CW_ZSTD_COMPRESSION); +#endif // WITH_ZSTD } static void @@ -177,6 +191,17 @@ test_cr_detect_compression(void) ret = cr_detect_compression(FILE_COMPRESSED_1_XZ, &tmp_err); g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // Zstd + + ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); + ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); +#endif // WITH_ZSTD } @@ -221,6 +246,17 @@ test_cr_detect_compression_bad_suffix(void) ret = cr_detect_compression(FILE_COMPRESSED_1_XZ_BAD_SUFFIX, &tmp_err); g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // Zstd + + ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); + ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX, &tmp_err); + g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION); + g_assert(!tmp_err); +#endif // WITH_ZSTD } @@ -235,6 +271,16 @@ test_helper_cw_input(const char *filename, char buffer[COMPRESSED_BUFFER_LEN+1]; GError *tmp_err = NULL; + if (ctype != CR_CW_AUTO_DETECT_COMPRESSION) { + cr_CompressionType detected_type = cr_detect_compression(filename, &tmp_err); + g_assert(!tmp_err); + if (ctype != detected_type) { + printf("detected_type: %i does not match passed type: %i for filename: %s\n", + detected_type, ctype, filename); + g_assert(0); + } + } + file = cr_open(filename, CR_CW_MODE_READ, ctype, &tmp_err); g_assert(file); g_assert(!tmp_err); @@ -281,6 +327,15 @@ test_cr_read_with_autodetection(void) FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN); test_helper_cw_input(FILE_COMPRESSED_1_XZ, CR_CW_AUTO_DETECT_COMPRESSION, FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN); + +#ifdef WITH_ZSTD + // Zstd + + test_helper_cw_input(FILE_COMPRESSED_0_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION, + FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_input(FILE_COMPRESSED_1_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION, + FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN); +#endif // WITH_ZSTD } @@ -447,6 +502,29 @@ outputtest_cw_output(Outputtest *outputtest, test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename, CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN); + +#ifdef WITH_ZSTD + // Zstd + + test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT, + FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT, + FILE_COMPRESSED_1_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT, + FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT, + FILE_COMPRESSED_1_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT, + FILE_COMPRESSED_0_CONTENT_LEN); + test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename, + CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT, + FILE_COMPRESSED_1_CONTENT_LEN); +#endif // WITH_ZSTD } @@ -508,6 +586,15 @@ test_cr_error_handling(void) g_error_free(tmp_err); tmp_err = NULL; +#ifdef WITH_ZSTD + f = cr_open("/", CR_CW_MODE_WRITE, CR_CW_ZSTD_COMPRESSION, &tmp_err); + g_assert(!f); + g_assert(tmp_err); + g_assert_cmpint(tmp_err->code, ==, CRE_IO); + g_error_free(tmp_err); + tmp_err = NULL; +#endif // WITH_ZSTD + // Opening plain text file as compressed char buf[256]; @@ -550,6 +637,21 @@ test_cr_error_handling(void) ret = cr_close(f, &tmp_err); g_assert_cmpint(ret, ==, CRE_OK); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + f = cr_open(FILE_COMPRESSED_1_PLAIN, CR_CW_MODE_READ, + CR_CW_ZSTD_COMPRESSION, &tmp_err); + g_assert(f); + ret = cr_read(f, buf, 256, &tmp_err); + g_assert_cmpint(ret, ==, -1); + g_assert(tmp_err); + g_assert_cmpint(tmp_err->code, ==, CRE_ZSTD); + g_error_free(tmp_err); + tmp_err = NULL; + ret = cr_close(f, &tmp_err); + g_assert_cmpint(ret, ==, CRE_OK); + g_assert(!tmp_err); +#endif // WITH_ZSTD } @@ -670,6 +772,33 @@ test_contentstating_singlewrite(Outputtest *outputtest, g_assert_cmpstr(stat->checksum, ==, content_sha256); cr_contentstat_free(stat, &tmp_err); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // zstd compression + stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err); + g_assert(stat); + g_assert(!tmp_err); + + f = cr_sopen(outputtest->tmp_filename, + CR_CW_MODE_WRITE, + CR_CW_ZSTD_COMPRESSION, + stat, + &tmp_err); + g_assert(f); + g_assert(!tmp_err); + + ret = cr_write(f, content, content_len, &tmp_err); + g_assert_cmpint(ret, ==, content_len); + g_assert(!tmp_err); + + cr_close(f, &tmp_err); + g_assert(!tmp_err); + + g_assert_cmpint(stat->size, ==, content_len); + g_assert_cmpstr(stat->checksum, ==, content_sha256); + cr_contentstat_free(stat, &tmp_err); + g_assert(!tmp_err); +#endif // WITH_ZSTD } static void @@ -715,6 +844,38 @@ test_contentstating_multiwrite(Outputtest *outputtest, g_assert_cmpstr(stat->checksum, ==, content_sha256); cr_contentstat_free(stat, &tmp_err); g_assert(!tmp_err); + +#ifdef WITH_ZSTD + // Zstd compression + + stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err); + g_assert(stat); + g_assert(!tmp_err); + + f = cr_sopen(outputtest->tmp_filename, + CR_CW_MODE_WRITE, + CR_CW_ZSTD_COMPRESSION, + stat, + &tmp_err); + g_assert(f); + g_assert(!tmp_err); + + ret = cr_write(f, content, 10, &tmp_err); + g_assert_cmpint(ret, ==, 10); + g_assert(!tmp_err); + + ret = cr_write(f, content+10, 29, &tmp_err); + g_assert_cmpint(ret, ==, 29); + g_assert(!tmp_err); + + cr_close(f, &tmp_err); + g_assert(!tmp_err); + + g_assert_cmpint(stat->size, ==, content_len); + g_assert_cmpstr(stat->checksum, ==, content_sha256); + cr_contentstat_free(stat, &tmp_err); + g_assert(!tmp_err); +#endif // WITH_ZSTD } static void diff --git a/tests/testdata/compressed_files/00_plain.foo5 b/tests/testdata/compressed_files/00_plain.foo5 new file mode 100644 index 0000000000000000000000000000000000000000..e58c09d56ab1845ad8c524844d4acd0515de2df5 GIT binary patch literal 13 UcmdPcs{dDofsuh>=F1y_03PE6+5i9m literal 0 HcmV?d00001 diff --git a/tests/testdata/compressed_files/00_plain.txt.zst b/tests/testdata/compressed_files/00_plain.txt.zst new file mode 100644 index 0000000000000000000000000000000000000000..e58c09d56ab1845ad8c524844d4acd0515de2df5 GIT binary patch literal 13 UcmdPcs{dDofsuh>=F1y_03PE6+5i9m literal 0 HcmV?d00001 diff --git a/tests/testdata/compressed_files/01_plain.foo5 b/tests/testdata/compressed_files/01_plain.foo5 new file mode 100644 index 0000000000000000000000000000000000000000..9d9c470fc36aae29007d667145a3dedf23a825c2 GIT binary patch literal 51 zcmdPcs{dET!jzHWLt1`*Qeu%pNosKkS6Y5fc2;p>8kkJU&raiFV$fi^7~aE^zvB)7 D=VTJ` literal 0 HcmV?d00001 diff --git a/tests/testdata/compressed_files/01_plain.txt.zst b/tests/testdata/compressed_files/01_plain.txt.zst new file mode 100644 index 0000000000000000000000000000000000000000..9d9c470fc36aae29007d667145a3dedf23a825c2 GIT binary patch literal 51 zcmdPcs{dET!jzHWLt1`*Qeu%pNosKkS6Y5fc2;p>8kkJU&raiFV$fi^7~aE^zvB)7 D=VTJ` literal 0 HcmV?d00001