Skip to content

Commit

Permalink
Add unittests for zstd compression
Browse files Browse the repository at this point in the history
  • Loading branch information
@kontura
kontura committed Feb 2, 2023
1 parent 8c83b37 commit e75ce31
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 0 deletions.
8 changes: 8 additions & 0 deletions tests/python/tests/test_compression_wrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def test_compression_suffix(self):
self.assertEqual(cr.compression_suffix(cr.BZ2), ".bz2")
self.assertEqual(cr.compression_suffix(cr.XZ), ".xz")
self.assertEqual(cr.compression_suffix(cr.ZCK), ".zck")
self.assertEqual(cr.compression_suffix(cr.ZSTD), ".zst")

def test_detect_compression(self):

Expand Down Expand Up @@ -69,6 +70,11 @@ def test_detect_compression(self):
#comtype = cr.detect_compression(path)
#self.assertEqual(comtype, cr.ZCK)

# Bad suffix - zstd compression
path = os.path.join(COMPRESSED_FILES_PATH, "01_plain.foo5")
comtype = cr.detect_compression(path)
self.assertEqual(comtype, cr.ZSTD)

def test_compression_type(self):
self.assertEqual(cr.compression_type(None), cr.UNKNOWN_COMPRESSION)
self.assertEqual(cr.compression_type(""), cr.UNKNOWN_COMPRESSION)
Expand All @@ -77,4 +83,6 @@ def test_compression_type(self):
self.assertEqual(cr.compression_type("xz"), cr.XZ)
self.assertEqual(cr.compression_type("XZ"), cr.XZ)
self.assertEqual(cr.compression_type("zck"), cr.ZCK)
self.assertEqual(cr.compression_type("zstd"), cr.ZSTD)
self.assertEqual(cr.compression_type("zst"), cr.ZSTD)

164 changes: 164 additions & 0 deletions tests/test_compression_wrapper.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,26 @@
#define FILE_COMPRESSED_0_GZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.gz"
#define FILE_COMPRESSED_0_BZ2 TEST_COMPRESSED_FILES_PATH"/00_plain.txt.bz2"
#define FILE_COMPRESSED_0_XZ TEST_COMPRESSED_FILES_PATH"/00_plain.txt.xz"
#define FILE_COMPRESSED_0_ZSTD TEST_COMPRESSED_FILES_PATH"/00_plain.txt.zst"
#define FILE_COMPRESSED_0_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo0"
#define FILE_COMPRESSED_0_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo1"
#define FILE_COMPRESSED_0_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo2"
#define FILE_COMPRESSED_0_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo3"
#define FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/00_plain.foo5"

#define FILE_COMPRESSED_1_CONTENT "foobar foobar foobar foobar test test\nfolkjsaflkjsadokf\n"
#define FILE_COMPRESSED_1_CONTENT_LEN 56
#define FILE_COMPRESSED_1_PLAIN TEST_COMPRESSED_FILES_PATH"/01_plain.txt"
#define FILE_COMPRESSED_1_GZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.gz"
#define FILE_COMPRESSED_1_BZ2 TEST_COMPRESSED_FILES_PATH"/01_plain.txt.bz2"
#define FILE_COMPRESSED_1_XZ TEST_COMPRESSED_FILES_PATH"/01_plain.txt.xz"
#define FILE_COMPRESSED_1_ZSTD TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zst"
#define FILE_COMPRESSED_1_ZCK TEST_COMPRESSED_FILES_PATH"/01_plain.txt.zck"
#define FILE_COMPRESSED_1_PLAIN_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo0"
#define FILE_COMPRESSED_1_GZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo1"
#define FILE_COMPRESSED_1_BZ2_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo2"
#define FILE_COMPRESSED_1_XZ_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo3"
#define FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX TEST_COMPRESSED_FILES_PATH"/01_plain.foo5"


static void
Expand Down Expand Up @@ -98,6 +102,11 @@ test_cr_compression_suffix(void)

suffix = cr_compression_suffix(CR_CW_XZ_COMPRESSION);
g_assert_cmpstr(suffix, ==, ".xz");

#ifdef WITH_ZSTD
suffix = cr_compression_suffix(CR_CW_ZSTD_COMPRESSION);
g_assert_cmpstr(suffix, ==, ".zst");
#endif // WITH_ZSTD
}

static void
Expand Down Expand Up @@ -134,6 +143,14 @@ test_cr_compression_type(void)

type = cr_compression_type("xz");
g_assert_cmpint(type, ==, CR_CW_XZ_COMPRESSION);

#ifdef WITH_ZSTD
type = cr_compression_type("zst");
g_assert_cmpint(type, ==, CR_CW_ZSTD_COMPRESSION);

type = cr_compression_type("zstd");
g_assert_cmpint(type, ==, CR_CW_ZSTD_COMPRESSION);
#endif // WITH_ZSTD
}

static void
Expand Down Expand Up @@ -177,6 +194,17 @@ test_cr_detect_compression(void)
ret = cr_detect_compression(FILE_COMPRESSED_1_XZ, &tmp_err);
g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION);
g_assert(!tmp_err);

#ifdef WITH_ZSTD
// Zstd

ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD, &tmp_err);
g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
g_assert(!tmp_err);
ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD, &tmp_err);
g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
g_assert(!tmp_err);
#endif // WITH_ZSTD
}


Expand Down Expand Up @@ -221,6 +249,17 @@ test_cr_detect_compression_bad_suffix(void)
ret = cr_detect_compression(FILE_COMPRESSED_1_XZ_BAD_SUFFIX, &tmp_err);
g_assert_cmpint(ret, ==, CR_CW_XZ_COMPRESSION);
g_assert(!tmp_err);

#ifdef WITH_ZSTD
// Zstd

ret = cr_detect_compression(FILE_COMPRESSED_0_ZSTD_BAD_SUFFIX, &tmp_err);
g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
g_assert(!tmp_err);
ret = cr_detect_compression(FILE_COMPRESSED_1_ZSTD_BAD_SUFFIX, &tmp_err);
g_assert_cmpint(ret, ==, CR_CW_ZSTD_COMPRESSION);
g_assert(!tmp_err);
#endif // WITH_ZSTD
}


Expand All @@ -235,6 +274,16 @@ test_helper_cw_input(const char *filename,
char buffer[COMPRESSED_BUFFER_LEN+1];
GError *tmp_err = NULL;

if (ctype != CR_CW_AUTO_DETECT_COMPRESSION) {
cr_CompressionType detected_type = cr_detect_compression(filename, &tmp_err);
g_assert(!tmp_err);
if (ctype != detected_type) {
printf("detected_type: %i does not match passed type: %i for filename: %s\n",
detected_type, ctype, filename);
g_assert(0);
}
}

file = cr_open(filename, CR_CW_MODE_READ, ctype, &tmp_err);
g_assert(file);
g_assert(!tmp_err);
Expand Down Expand Up @@ -281,6 +330,15 @@ test_cr_read_with_autodetection(void)
FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN);
test_helper_cw_input(FILE_COMPRESSED_1_XZ, CR_CW_AUTO_DETECT_COMPRESSION,
FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN);

#ifdef WITH_ZSTD
// Zstd

test_helper_cw_input(FILE_COMPRESSED_0_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION,
FILE_COMPRESSED_0_CONTENT, FILE_COMPRESSED_0_CONTENT_LEN);
test_helper_cw_input(FILE_COMPRESSED_1_ZSTD, CR_CW_AUTO_DETECT_COMPRESSION,
FILE_COMPRESSED_1_CONTENT, FILE_COMPRESSED_1_CONTENT_LEN);
#endif // WITH_ZSTD
}


Expand Down Expand Up @@ -447,6 +505,29 @@ outputtest_cw_output(Outputtest *outputtest,
test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
FILE_COMPRESSED_1_CONTENT_LEN);

#ifdef WITH_ZSTD
// Zstd

test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename,
CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
FILE_COMPRESSED_0_CONTENT_LEN);
test_helper_cw_output(OUTPUT_TYPE_WRITE, outputtest->tmp_filename,
CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
FILE_COMPRESSED_1_CONTENT_LEN);
test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename,
CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
FILE_COMPRESSED_0_CONTENT_LEN);
test_helper_cw_output(OUTPUT_TYPE_PUTS, outputtest->tmp_filename,
CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
FILE_COMPRESSED_1_CONTENT_LEN);
test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
FILE_COMPRESSED_0_CONTENT_LEN);
test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
CR_CW_ZSTD_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
FILE_COMPRESSED_1_CONTENT_LEN);
#endif // WITH_ZSTD
}


Expand Down Expand Up @@ -508,6 +589,15 @@ test_cr_error_handling(void)
g_error_free(tmp_err);
tmp_err = NULL;

#ifdef WITH_ZSTD
f = cr_open("/", CR_CW_MODE_WRITE, CR_CW_ZSTD_COMPRESSION, &tmp_err);
g_assert(!f);
g_assert(tmp_err);
g_assert_cmpint(tmp_err->code, ==, CRE_IO);
g_error_free(tmp_err);
tmp_err = NULL;
#endif // WITH_ZSTD

// Opening plain text file as compressed

char buf[256];
Expand Down Expand Up @@ -550,6 +640,21 @@ test_cr_error_handling(void)
ret = cr_close(f, &tmp_err);
g_assert_cmpint(ret, ==, CRE_OK);
g_assert(!tmp_err);

#ifdef WITH_ZSTD
f = cr_open(FILE_COMPRESSED_1_PLAIN, CR_CW_MODE_READ,
CR_CW_ZSTD_COMPRESSION, &tmp_err);
g_assert(f);
ret = cr_read(f, buf, 256, &tmp_err);
g_assert_cmpint(ret, ==, -1);
g_assert(tmp_err);
g_assert_cmpint(tmp_err->code, ==, CRE_ZSTD);
g_error_free(tmp_err);
tmp_err = NULL;
ret = cr_close(f, &tmp_err);
g_assert_cmpint(ret, ==, CRE_OK);
g_assert(!tmp_err);
#endif // WITH_ZSTD
}


Expand Down Expand Up @@ -670,6 +775,33 @@ test_contentstating_singlewrite(Outputtest *outputtest,
g_assert_cmpstr(stat->checksum, ==, content_sha256);
cr_contentstat_free(stat, &tmp_err);
g_assert(!tmp_err);

#ifdef WITH_ZSTD
// zstd compression
stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err);
g_assert(stat);
g_assert(!tmp_err);

f = cr_sopen(outputtest->tmp_filename,
CR_CW_MODE_WRITE,
CR_CW_ZSTD_COMPRESSION,
stat,
&tmp_err);
g_assert(f);
g_assert(!tmp_err);

ret = cr_write(f, content, content_len, &tmp_err);
g_assert_cmpint(ret, ==, content_len);
g_assert(!tmp_err);

cr_close(f, &tmp_err);
g_assert(!tmp_err);

g_assert_cmpint(stat->size, ==, content_len);
g_assert_cmpstr(stat->checksum, ==, content_sha256);
cr_contentstat_free(stat, &tmp_err);
g_assert(!tmp_err);
#endif // WITH_ZSTD
}

static void
Expand Down Expand Up @@ -715,6 +847,38 @@ test_contentstating_multiwrite(Outputtest *outputtest,
g_assert_cmpstr(stat->checksum, ==, content_sha256);
cr_contentstat_free(stat, &tmp_err);
g_assert(!tmp_err);

#ifdef WITH_ZSTD
// Zstd compression

stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err);
g_assert(stat);
g_assert(!tmp_err);

f = cr_sopen(outputtest->tmp_filename,
CR_CW_MODE_WRITE,
CR_CW_ZSTD_COMPRESSION,
stat,
&tmp_err);
g_assert(f);
g_assert(!tmp_err);

ret = cr_write(f, content, 10, &tmp_err);
g_assert_cmpint(ret, ==, 10);
g_assert(!tmp_err);

ret = cr_write(f, content+10, 29, &tmp_err);
g_assert_cmpint(ret, ==, 29);
g_assert(!tmp_err);

cr_close(f, &tmp_err);
g_assert(!tmp_err);

g_assert_cmpint(stat->size, ==, content_len);
g_assert_cmpstr(stat->checksum, ==, content_sha256);
cr_contentstat_free(stat, &tmp_err);
g_assert(!tmp_err);
#endif // WITH_ZSTD
}

static void
Expand Down
Binary file added tests/testdata/compressed_files/00_plain.foo5
View file
Binary file not shown.
Binary file added tests/testdata/compressed_files/00_plain.txt.zst
View file
Binary file not shown.
Binary file added tests/testdata/compressed_files/01_plain.foo5
View file
Binary file not shown.
Binary file added tests/testdata/compressed_files/01_plain.txt.zst
View file
Binary file not shown.

0 comments on commit e75ce31

Please sign in to comment.