Skip to content

Commit

Permalink
Expose Blosc internals and compressors for easier P/Invoke from Sprea…
Browse files Browse the repository at this point in the history
…ds. It's much easier to use Blosc for accessing raw compressors that building them separately.

On Windows, need these flags `-DCMAKE_CXX_FLAGS_RELEASE="/MT /O2 /Ob2 /DNDEBUG" -DCMAKE_C_FLAGS_RELEASE="/MT /O2 /Ob2 /DNDEBUG"`to link VCRUNTIME statically (/MT vs /MD).

These changes assume that LZ4/Ztsd/Zlib are present (was lazy to wrap them into #ifdef, TODO), but Snappy could be turned off to save space (-DDEACTIVATE_SNAPPY=ON).
  • Loading branch information
buybackoff committed Dec 5, 2018
1 parent eb37eb8 commit e34dddb
Show file tree
Hide file tree
Showing 4 changed files with 252 additions and 4 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
bench/bench
build/
cmake-build-debug
cmake-build-release
.idea

*.sw?
210 changes: 210 additions & 0 deletions blosc/blosc.c
Original file line number Diff line number Diff line change
Expand Up @@ -2233,3 +2233,213 @@ int blosc_free_resources(void)

return blosc_release_threadpool(g_global_context);
}


#if defined(HAVE_LZ4)
int compress_lz4(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return lz4_wrap_compress(input, input_length, output, maxout, 10 - clevel);
}

int decompress_lz4(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
size_t cbytes;
cbytes = LZ4_decompress_safe(input, output, compressed_length, (int)maxout);
if (cbytes <= 0) {
return 0;
}
return (int)cbytes;
//return lz4_wrap_decompress(input, compressed_length, output, maxout);
}

#endif

#if defined(HAVE_ZSTD)

int compress_zstd(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return zstd_wrap_compress(input, input_length, output, maxout, clevel);
}

int decompress_zstd(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
return zstd_wrap_decompress(input, compressed_length, output, maxout);
}

#endif

#if defined(HAVE_ZLIB)

int compress_zlib(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return zlib_wrap_compress(input, input_length, output, maxout, clevel);
}

int decompress_zlib(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
return zlib_wrap_decompress(input, compressed_length, output, maxout);
}


int compress_deflate(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = input_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = deflateInit2(&strm, clevel, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
if (err == Z_OK) {
err = deflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
deflateEnd(&strm);
return err;
}
}
else {
deflateEnd(&strm);
return err;
}

deflateEnd(&strm);
return ret;
}

int decompress_deflate(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = compressed_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = inflateInit2(&strm, -15); //15 window bits, and the +32 tells zlib to to detect if using gzip or zlib
if (err == Z_OK) {
err = inflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
inflateEnd(&strm);
return err;
}
}
else {
inflateEnd(&strm);
return err;
}

inflateEnd(&strm);
return ret;
}

int compress_gzip(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = input_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = deflateInit2(&strm, clevel, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
if (err == Z_OK) {
err = deflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
deflateEnd(&strm);
return err;
}
}
else {
deflateEnd(&strm);
return err;
}

deflateEnd(&strm);
return ret;
}

int decompress_gzip(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = compressed_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = inflateInit2(&strm, (15 + 32)); //15 window bits, and the +32 tells zlib to to detect if using gzip or zlib
if (err == Z_OK) {
err = inflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
inflateEnd(&strm);
return err;
}
}
else {
inflateEnd(&strm);
return err;
}

inflateEnd(&strm);
return ret;
}
#endif

int compress_noop(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return 0;
}

int decompress_noop(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
return 0;
}
35 changes: 35 additions & 0 deletions blosc/blosc.h
Original file line number Diff line number Diff line change
Expand Up @@ -504,6 +504,41 @@ BLOSC_EXPORT void blosc_set_blocksize(size_t blocksize);
*/
BLOSC_EXPORT void blosc_set_splitmode(int splitmode);

BLOSC_EXPORT int compress_lz4(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_lz4(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_zstd(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_zstd(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_zlib(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_zlib(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_deflate(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_deflate(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_gzip(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_gzip(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_noop(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_noop(const char* input, size_t compressed_length,
char* output, size_t maxout);

#ifdef __cplusplus
}
Expand Down
8 changes: 4 additions & 4 deletions blosc/shuffle.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ extern "C" {
calling the hardware-accelerated routines because this method is both cross-
platform and future-proof.
*/
BLOSC_NO_EXPORT void
BLOSC_EXPORT void
shuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* _src, const uint8_t* _dest);

BLOSC_NO_EXPORT int
BLOSC_EXPORT int
bitshuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* const _src, const uint8_t* _dest,
const uint8_t* _tmp);
Expand All @@ -50,12 +50,12 @@ bitshuffle(const size_t bytesoftype, const size_t blocksize,
calling the hardware-accelerated routines because this method is both cross-
platform and future-proof.
*/
BLOSC_NO_EXPORT void
BLOSC_EXPORT void
unshuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* _src, const uint8_t* _dest);


BLOSC_NO_EXPORT int
BLOSC_EXPORT int
bitunshuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* const _src, const uint8_t* _dest,
const uint8_t* _tmp);
Expand Down

0 comments on commit e34dddb

Please sign in to comment.