Skip to content
Permalink
Browse files

Expose Blosc internals and compressors for easier P/Invoke from Sprea…

…ds. It's much easier to use Blosc for accessing raw compressors that building them separately.

On Windows, need these flags `-DCMAKE_CXX_FLAGS_RELEASE="/MT /O2 /Ob2 /DNDEBUG" -DCMAKE_C_FLAGS_RELEASE="/MT /O2 /Ob2 /DNDEBUG"`to link VCRUNTIME statically (/MT vs /MD).

These changes assume that LZ4/Ztsd/Zlib are present (was lazy to wrap them into #ifdef, TODO), but Snappy could be turned off to save space (-DDEACTIVATE_SNAPPY=ON).
  • Loading branch information...
buybackoff committed Sep 10, 2018
1 parent eb37eb8 commit e34dddbad70aa2b270e2d991a9baee44c95ed175
Showing with 252 additions and 4 deletions.
  1. +3 −0 .gitignore
  2. +210 −0 blosc/blosc.c
  3. +35 −0 blosc/blosc.h
  4. +4 −4 blosc/shuffle.h
@@ -1,4 +1,7 @@
bench/bench
build/
cmake-build-debug
cmake-build-release
.idea

*.sw?
@@ -2233,3 +2233,213 @@ int blosc_free_resources(void)

return blosc_release_threadpool(g_global_context);
}


#if defined(HAVE_LZ4)
int compress_lz4(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return lz4_wrap_compress(input, input_length, output, maxout, 10 - clevel);
}

int decompress_lz4(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
size_t cbytes;
cbytes = LZ4_decompress_safe(input, output, compressed_length, (int)maxout);
if (cbytes <= 0) {
return 0;
}
return (int)cbytes;
//return lz4_wrap_decompress(input, compressed_length, output, maxout);
}

#endif

#if defined(HAVE_ZSTD)

int compress_zstd(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return zstd_wrap_compress(input, input_length, output, maxout, clevel);
}

int decompress_zstd(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
return zstd_wrap_decompress(input, compressed_length, output, maxout);
}

#endif

#if defined(HAVE_ZLIB)

int compress_zlib(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return zlib_wrap_compress(input, input_length, output, maxout, clevel);
}

int decompress_zlib(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
return zlib_wrap_decompress(input, compressed_length, output, maxout);
}


int compress_deflate(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = input_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = deflateInit2(&strm, clevel, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
if (err == Z_OK) {
err = deflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
deflateEnd(&strm);
return err;
}
}
else {
deflateEnd(&strm);
return err;
}

deflateEnd(&strm);
return ret;
}

int decompress_deflate(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = compressed_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = inflateInit2(&strm, -15); //15 window bits, and the +32 tells zlib to to detect if using gzip or zlib
if (err == Z_OK) {
err = inflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
inflateEnd(&strm);
return err;
}
}
else {
inflateEnd(&strm);
return err;
}

inflateEnd(&strm);
return ret;
}

int compress_gzip(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = input_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = deflateInit2(&strm, clevel, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
if (err == Z_OK) {
err = deflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
deflateEnd(&strm);
return err;
}
}
else {
deflateEnd(&strm);
return err;
}

deflateEnd(&strm);
return ret;
}

int decompress_gzip(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
z_stream strm = {0};
strm.total_in = strm.avail_in = compressed_length;
strm.total_out = strm.avail_out = maxout;
strm.next_in = (Bytef *) input;
strm.next_out = (Bytef *) output;

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

int err = -1;
int ret = -1;

err = inflateInit2(&strm, (15 + 32)); //15 window bits, and the +32 tells zlib to to detect if using gzip or zlib
if (err == Z_OK) {
err = inflate(&strm, Z_FINISH);
if (err == Z_STREAM_END) {
ret = strm.total_out;
}
else {
inflateEnd(&strm);
return err;
}
}
else {
inflateEnd(&strm);
return err;
}

inflateEnd(&strm);
return ret;
}
#endif

int compress_noop(const char* input, size_t input_length,
char* output, size_t maxout, int clevel)
{
return 0;
}

int decompress_noop(const char* input, size_t compressed_length,
char* output, size_t maxout)
{
return 0;
}
@@ -504,6 +504,41 @@ BLOSC_EXPORT void blosc_set_blocksize(size_t blocksize);
*/
BLOSC_EXPORT void blosc_set_splitmode(int splitmode);

BLOSC_EXPORT int compress_lz4(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_lz4(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_zstd(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_zstd(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_zlib(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_zlib(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_deflate(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_deflate(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_gzip(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_gzip(const char* input, size_t compressed_length,
char* output, size_t maxout);

BLOSC_EXPORT int compress_noop(const char* input, size_t input_length,
char* output, size_t maxout, int clevel);

BLOSC_EXPORT int decompress_noop(const char* input, size_t compressed_length,
char* output, size_t maxout);

#ifdef __cplusplus
}
@@ -31,11 +31,11 @@ extern "C" {
calling the hardware-accelerated routines because this method is both cross-
platform and future-proof.
*/
BLOSC_NO_EXPORT void
BLOSC_EXPORT void
shuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* _src, const uint8_t* _dest);

BLOSC_NO_EXPORT int
BLOSC_EXPORT int
bitshuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* const _src, const uint8_t* _dest,
const uint8_t* _tmp);
@@ -50,12 +50,12 @@ bitshuffle(const size_t bytesoftype, const size_t blocksize,
calling the hardware-accelerated routines because this method is both cross-
platform and future-proof.
*/
BLOSC_NO_EXPORT void
BLOSC_EXPORT void
unshuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* _src, const uint8_t* _dest);


BLOSC_NO_EXPORT int
BLOSC_EXPORT int
bitunshuffle(const size_t bytesoftype, const size_t blocksize,
const uint8_t* const _src, const uint8_t* _dest,
const uint8_t* _tmp);

0 comments on commit e34dddb

Please sign in to comment.
You can’t perform that action at this time.