Skip to content

Commit e34dddb

Browse files
buybackoffbuybackoff
committed
Expose Blosc internals and compressors for easier P/Invoke from Spreads. It's much easier to use Blosc for accessing raw compressors that building them separately.
On Windows, need these flags `-DCMAKE_CXX_FLAGS_RELEASE="/MT /O2 /Ob2 /DNDEBUG" -DCMAKE_C_FLAGS_RELEASE="/MT /O2 /Ob2 /DNDEBUG"`to link VCRUNTIME statically (/MT vs /MD). These changes assume that LZ4/Ztsd/Zlib are present (was lazy to wrap them into #ifdef, TODO), but Snappy could be turned off to save space (-DDEACTIVATE_SNAPPY=ON).
1 parent eb37eb8 commit e34dddb

File tree

4 files changed

+252
-4
lines changed

4 files changed

+252
-4
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
bench/bench
22
build/
3+
cmake-build-debug
4+
cmake-build-release
5+
.idea
36

47
*.sw?

‎blosc/blosc.c‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2233,3 +2233,213 @@ int blosc_free_resources(void)
22332233

22342234
return blosc_release_threadpool(g_global_context);
22352235
}
2236+
2237+
2238+
#if defined(HAVE_LZ4)
2239+
int compress_lz4(const char* input, size_t input_length,
2240+
char* output, size_t maxout, int clevel)
2241+
{
2242+
return lz4_wrap_compress(input, input_length, output, maxout, 10 - clevel);
2243+
}
2244+
2245+
int decompress_lz4(const char* input, size_t compressed_length,
2246+
char* output, size_t maxout)
2247+
{
2248+
size_t cbytes;
2249+
cbytes = LZ4_decompress_safe(input, output, compressed_length, (int)maxout);
2250+
if (cbytes <= 0) {
2251+
return 0;
2252+
}
2253+
return (int)cbytes;
2254+
//return lz4_wrap_decompress(input, compressed_length, output, maxout);
2255+
}
2256+
2257+
#endif
2258+
2259+
#if defined(HAVE_ZSTD)
2260+
2261+
int compress_zstd(const char* input, size_t input_length,
2262+
char* output, size_t maxout, int clevel)
2263+
{
2264+
return zstd_wrap_compress(input, input_length, output, maxout, clevel);
2265+
}
2266+
2267+
int decompress_zstd(const char* input, size_t compressed_length,
2268+
char* output, size_t maxout)
2269+
{
2270+
return zstd_wrap_decompress(input, compressed_length, output, maxout);
2271+
}
2272+
2273+
#endif
2274+
2275+
#if defined(HAVE_ZLIB)
2276+
2277+
int compress_zlib(const char* input, size_t input_length,
2278+
char* output, size_t maxout, int clevel)
2279+
{
2280+
return zlib_wrap_compress(input, input_length, output, maxout, clevel);
2281+
}
2282+
2283+
int decompress_zlib(const char* input, size_t compressed_length,
2284+
char* output, size_t maxout)
2285+
{
2286+
return zlib_wrap_decompress(input, compressed_length, output, maxout);
2287+
}
2288+
2289+
2290+
int compress_deflate(const char* input, size_t input_length,
2291+
char* output, size_t maxout, int clevel)
2292+
{
2293+
z_stream strm = {0};
2294+
strm.total_in = strm.avail_in = input_length;
2295+
strm.total_out = strm.avail_out = maxout;
2296+
strm.next_in = (Bytef *) input;
2297+
strm.next_out = (Bytef *) output;
2298+
2299+
strm.zalloc = Z_NULL;
2300+
strm.zfree = Z_NULL;
2301+
strm.opaque = Z_NULL;
2302+
2303+
int err = -1;
2304+
int ret = -1;
2305+
2306+
err = deflateInit2(&strm, clevel, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
2307+
if (err == Z_OK) {
2308+
err = deflate(&strm, Z_FINISH);
2309+
if (err == Z_STREAM_END) {
2310+
ret = strm.total_out;
2311+
}
2312+
else {
2313+
deflateEnd(&strm);
2314+
return err;
2315+
}
2316+
}
2317+
else {
2318+
deflateEnd(&strm);
2319+
return err;
2320+
}
2321+
2322+
deflateEnd(&strm);
2323+
return ret;
2324+
}
2325+
2326+
int decompress_deflate(const char* input, size_t compressed_length,
2327+
char* output, size_t maxout)
2328+
{
2329+
z_stream strm = {0};
2330+
strm.total_in = strm.avail_in = compressed_length;
2331+
strm.total_out = strm.avail_out = maxout;
2332+
strm.next_in = (Bytef *) input;
2333+
strm.next_out = (Bytef *) output;
2334+
2335+
strm.zalloc = Z_NULL;
2336+
strm.zfree = Z_NULL;
2337+
strm.opaque = Z_NULL;
2338+
2339+
int err = -1;
2340+
int ret = -1;
2341+
2342+
err = inflateInit2(&strm, -15); //15 window bits, and the +32 tells zlib to to detect if using gzip or zlib
2343+
if (err == Z_OK) {
2344+
err = inflate(&strm, Z_FINISH);
2345+
if (err == Z_STREAM_END) {
2346+
ret = strm.total_out;
2347+
}
2348+
else {
2349+
inflateEnd(&strm);
2350+
return err;
2351+
}
2352+
}
2353+
else {
2354+
inflateEnd(&strm);
2355+
return err;
2356+
}
2357+
2358+
inflateEnd(&strm);
2359+
return ret;
2360+
}
2361+
2362+
int compress_gzip(const char* input, size_t input_length,
2363+
char* output, size_t maxout, int clevel)
2364+
{
2365+
z_stream strm = {0};
2366+
strm.total_in = strm.avail_in = input_length;
2367+
strm.total_out = strm.avail_out = maxout;
2368+
strm.next_in = (Bytef *) input;
2369+
strm.next_out = (Bytef *) output;
2370+
2371+
strm.zalloc = Z_NULL;
2372+
strm.zfree = Z_NULL;
2373+
strm.opaque = Z_NULL;
2374+
2375+
int err = -1;
2376+
int ret = -1;
2377+
2378+
err = deflateInit2(&strm, clevel, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
2379+
if (err == Z_OK) {
2380+
err = deflate(&strm, Z_FINISH);
2381+
if (err == Z_STREAM_END) {
2382+
ret = strm.total_out;
2383+
}
2384+
else {
2385+
deflateEnd(&strm);
2386+
return err;
2387+
}
2388+
}
2389+
else {
2390+
deflateEnd(&strm);
2391+
return err;
2392+
}
2393+
2394+
deflateEnd(&strm);
2395+
return ret;
2396+
}
2397+
2398+
int decompress_gzip(const char* input, size_t compressed_length,
2399+
char* output, size_t maxout)
2400+
{
2401+
z_stream strm = {0};
2402+
strm.total_in = strm.avail_in = compressed_length;
2403+
strm.total_out = strm.avail_out = maxout;
2404+
strm.next_in = (Bytef *) input;
2405+
strm.next_out = (Bytef *) output;
2406+
2407+
strm.zalloc = Z_NULL;
2408+
strm.zfree = Z_NULL;
2409+
strm.opaque = Z_NULL;
2410+
2411+
int err = -1;
2412+
int ret = -1;
2413+
2414+
err = inflateInit2(&strm, (15 + 32)); //15 window bits, and the +32 tells zlib to to detect if using gzip or zlib
2415+
if (err == Z_OK) {
2416+
err = inflate(&strm, Z_FINISH);
2417+
if (err == Z_STREAM_END) {
2418+
ret = strm.total_out;
2419+
}
2420+
else {
2421+
inflateEnd(&strm);
2422+
return err;
2423+
}
2424+
}
2425+
else {
2426+
inflateEnd(&strm);
2427+
return err;
2428+
}
2429+
2430+
inflateEnd(&strm);
2431+
return ret;
2432+
}
2433+
#endif
2434+
2435+
int compress_noop(const char* input, size_t input_length,
2436+
char* output, size_t maxout, int clevel)
2437+
{
2438+
return 0;
2439+
}
2440+
2441+
int decompress_noop(const char* input, size_t compressed_length,
2442+
char* output, size_t maxout)
2443+
{
2444+
return 0;
2445+
}

‎blosc/blosc.h‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,41 @@ BLOSC_EXPORT void blosc_set_blocksize(size_t blocksize);
504504
*/
505505
BLOSC_EXPORT void blosc_set_splitmode(int splitmode);
506506

507+
BLOSC_EXPORT int compress_lz4(const char* input, size_t input_length,
508+
char* output, size_t maxout, int clevel);
509+
510+
BLOSC_EXPORT int decompress_lz4(const char* input, size_t compressed_length,
511+
char* output, size_t maxout);
512+
513+
BLOSC_EXPORT int compress_zstd(const char* input, size_t input_length,
514+
char* output, size_t maxout, int clevel);
515+
516+
BLOSC_EXPORT int decompress_zstd(const char* input, size_t compressed_length,
517+
char* output, size_t maxout);
518+
519+
BLOSC_EXPORT int compress_zlib(const char* input, size_t input_length,
520+
char* output, size_t maxout, int clevel);
521+
522+
BLOSC_EXPORT int decompress_zlib(const char* input, size_t compressed_length,
523+
char* output, size_t maxout);
524+
525+
BLOSC_EXPORT int compress_deflate(const char* input, size_t input_length,
526+
char* output, size_t maxout, int clevel);
527+
528+
BLOSC_EXPORT int decompress_deflate(const char* input, size_t compressed_length,
529+
char* output, size_t maxout);
530+
531+
BLOSC_EXPORT int compress_gzip(const char* input, size_t input_length,
532+
char* output, size_t maxout, int clevel);
533+
534+
BLOSC_EXPORT int decompress_gzip(const char* input, size_t compressed_length,
535+
char* output, size_t maxout);
536+
537+
BLOSC_EXPORT int compress_noop(const char* input, size_t input_length,
538+
char* output, size_t maxout, int clevel);
539+
540+
BLOSC_EXPORT int decompress_noop(const char* input, size_t compressed_length,
541+
char* output, size_t maxout);
507542

508543
#ifdef __cplusplus
509544
}

‎blosc/shuffle.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,11 @@ extern "C" {
3131
calling the hardware-accelerated routines because this method is both cross-
3232
platform and future-proof.
3333
*/
34-
BLOSC_NO_EXPORT void
34+
BLOSC_EXPORT void
3535
shuffle(const size_t bytesoftype, const size_t blocksize,
3636
const uint8_t* _src, const uint8_t* _dest);
3737

38-
BLOSC_NO_EXPORT int
38+
BLOSC_EXPORT int
3939
bitshuffle(const size_t bytesoftype, const size_t blocksize,
4040
const uint8_t* const _src, const uint8_t* _dest,
4141
const uint8_t* _tmp);
@@ -50,12 +50,12 @@ bitshuffle(const size_t bytesoftype, const size_t blocksize,
5050
calling the hardware-accelerated routines because this method is both cross-
5151
platform and future-proof.
5252
*/
53-
BLOSC_NO_EXPORT void
53+
BLOSC_EXPORT void
5454
unshuffle(const size_t bytesoftype, const size_t blocksize,
5555
const uint8_t* _src, const uint8_t* _dest);
5656

5757

58-
BLOSC_NO_EXPORT int
58+
BLOSC_EXPORT int
5959
bitunshuffle(const size_t bytesoftype, const size_t blocksize,
6060
const uint8_t* const _src, const uint8_t* _dest,
6161
const uint8_t* _tmp);

0 commit comments

Comments
 (0)