diff --git a/.gitignore b/.gitignore index db46b006ef..3669dd533e 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,7 @@ docs/_test_images/ # Ignore Bazel generated files bazel-* + +cmake-build-debug/ + +.idea/ diff --git a/src/bin/exrheader/main.cpp b/src/bin/exrheader/main.cpp index 2f25edd44c..b1e9ecbb63 100644 --- a/src/bin/exrheader/main.cpp +++ b/src/bin/exrheader/main.cpp @@ -65,6 +65,8 @@ printCompression (Compression c) case DWAB_COMPRESSION: cout << "dwa, medium scanline blocks"; break; + case ZSTD_COMPRESSION: cout << "zstd"; break; + default: cout << int (c); break; } } diff --git a/src/examples/deepExamples.cpp b/src/examples/deepExamples.cpp index 9abd61d030..b28a0cf3bd 100644 --- a/src/examples/deepExamples.cpp +++ b/src/examples/deepExamples.cpp @@ -50,7 +50,7 @@ readDeepScanlineFile ( // - allocate the memory requred to store the samples // - read the pixels from the file // - + DeepScanLineInputFile file (filename); const Header& header = file.header (); @@ -118,20 +118,18 @@ readDeepScanlineFile ( } } -unsigned int getPixelSampleCount (int i, int j) +unsigned int +getPixelSampleCount (int i, int j) { // Dummy code creating deep data from a flat image return 1; } Array2D testDataZ; -Array2D testDataA; +Array2D testDataA; -void getPixelSampleData( - int i, - int j, - Array2D& dataZ, - Array2D& dataA) +void +getPixelSampleData (int i, int j, Array2D& dataZ, Array2D& dataA) { // Dummy code creating deep data from a flat image dataZ[i][j][0] = testDataZ[i][j]; @@ -147,7 +145,8 @@ writeDeepScanlineFile ( Array2D& dataA, - Array2D& sampleCount) + Array2D& sampleCount, + Compression compression = Compression::ZIPS_COMPRESSION) { // @@ -160,7 +159,7 @@ writeDeepScanlineFile ( // - describe the memory layout of the A and Z pixels // - store the pixels in the file // - + int height = dataWindow.max.y - dataWindow.min.y + 1; int width = dataWindow.max.x - dataWindow.min.x + 1; @@ -169,7 +168,7 @@ writeDeepScanlineFile ( header.channels ().insert ("Z", Channel (FLOAT)); header.channels ().insert ("A", Channel (HALF)); header.setType (DEEPSCANLINE); - header.compression () = ZIPS_COMPRESSION; + header.compression () = compression; DeepScanLineOutputFile file (filename, header); @@ -211,7 +210,7 @@ writeDeepScanlineFile ( dataZ[i][j] = new float[sampleCount[i][j]]; dataA[i][j] = new half[sampleCount[i][j]]; // Generate data for dataZ and dataA. - getPixelSampleData(i, j, dataZ, dataA); + getPixelSampleData (i, j, dataZ, dataA); } file.writePixels (1); @@ -227,30 +226,56 @@ writeDeepScanlineFile ( } } - -void deepExamples() +void +deepExamples () { int w = 800; int h = 600; - + Box2i window; - window.min.setValue(0, 0); - window.max.setValue(w - 1, h - 1); - - Array2D dataZ; - dataZ.resizeErase(h, w); - - Array2D dataA; - dataA.resizeErase(h, w); - + window.min.setValue (0, 0); + window.max.setValue (w - 1, h - 1); + + Array2D dataZ; + dataZ.resizeErase (h, w); + + Array2D dataA; + dataA.resizeErase (h, w); + Array2D sampleCount; - sampleCount.resizeErase(h, w); - + sampleCount.resizeErase (h, w); + // Create an image to be used as a source for deep data - testDataA.resizeErase(h, w); - testDataZ.resizeErase(h, w); - drawImage2(testDataA, testDataZ, w, h); - - writeDeepScanlineFile("test.deep.exr", window, window, dataZ, dataA, sampleCount); - readDeepScanlineFile ("test.deep.exr", window, window, dataZ, dataA, sampleCount); + testDataA.resizeErase (h, w); + testDataZ.resizeErase (h, w); + drawImage2 (testDataA, testDataZ, w, h); + + { + writeDeepScanlineFile ( + "test.deep.exr", + window, + window, + dataZ, + dataA, + sampleCount, + Compression::ZSTD_COMPRESSION); + } + { + writeDeepScanlineFile ( + "test.zips.exr", + window, + window, + dataZ, + dataA, + sampleCount, + Compression::ZIPS_COMPRESSION); + } + { + readDeepScanlineFile ( + "test.deep.exr", window, window, dataZ, dataA, sampleCount); + } + { + readDeepScanlineFile ( + "test.zips.exr", window, window, dataZ, dataA, sampleCount); + } } diff --git a/src/examples/deepTiledExamples.cpp b/src/examples/deepTiledExamples.cpp index d6ec256361..3e58600c09 100644 --- a/src/examples/deepTiledExamples.cpp +++ b/src/examples/deepTiledExamples.cpp @@ -163,7 +163,8 @@ writeDeepTiledFile ( Box2i displayWindow, Box2i dataWindow, int tileSizeX, - int tileSizeY) + int tileSizeY, + Compression compression = Compression::ZIPS_COMPRESSION) { // // Write a deep image with only a A (alpha) and a Z (depth) channel, @@ -183,7 +184,7 @@ writeDeepTiledFile ( header.channels ().insert ("Z", Channel (FLOAT)); header.channels ().insert ("A", Channel (HALF)); header.setType (DEEPTILE); - header.compression () = ZIPS_COMPRESSION; + header.compression () = compression; header.setTileDescription ( TileDescription (tileSizeX, tileSizeY, ONE_LEVEL)); @@ -273,6 +274,8 @@ void deepTiledExamples() testDataZ.resizeErase(h, w); drawImage2(testDataA, testDataZ, w, h); - writeDeepTiledFile("testTiled.deep.exr", window, window, tileSizeX, tileSizeY); - readDeepTiledFile ("testTiled.deep.exr", window, window, dataZ, dataA, sampleCount); + writeDeepTiledFile("testTiled.deep.zip.exr", window, window, tileSizeX, tileSizeY); + readDeepTiledFile ("testTiled.deep.zip.exr", window, window, dataZ, dataA, sampleCount); + writeDeepTiledFile("testTiled.deep.zstd.exr", window, window, tileSizeX, tileSizeY, Compression::ZSTD_COMPRESSION); + readDeepTiledFile ("testTiled.deep.zstd.exr", window, window, dataZ, dataA, sampleCount); } diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index b58e0bace1..45a809f386 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -29,6 +29,7 @@ openexr_define_library(OpenEXR ImfTiledMisc.h ImfZip.h ImfZipCompressor.h + ImfZstdCompressor.h b44ExpLogTable.h dwaLookups.h ImfAcesFile.cpp @@ -122,6 +123,7 @@ openexr_define_library(OpenEXR ImfWav.cpp ImfZip.cpp ImfZipCompressor.cpp + ImfZstdCompressor.cpp HEADERS ImfAcesFile.h ImfArray.h @@ -220,3 +222,7 @@ openexr_define_library(OpenEXR OpenEXR::IlmThread OpenEXR::OpenEXRCore ) + +target_include_directories(OpenEXR PUBLIC "/home/vladal/bin/include/") +target_link_directories(OpenEXR PUBLIC "/home/vladal/bin/lib") +target_link_libraries(OpenEXR PUBLIC "dl" "blosc2") \ No newline at end of file diff --git a/src/lib/OpenEXR/ImfCompression.h b/src/lib/OpenEXR/ImfCompression.h index fbc0fa24ab..e8f3386bf9 100644 --- a/src/lib/OpenEXR/ImfCompression.h +++ b/src/lib/OpenEXR/ImfCompression.h @@ -45,6 +45,8 @@ enum IMF_EXPORT_ENUM Compression // wise and faster to decode full frames // than DWAA_COMPRESSION. + ZSTD_COMPRESSION = 10, + NUM_COMPRESSION_METHODS // number of different compression methods }; diff --git a/src/lib/OpenEXR/ImfCompressionAttribute.cpp b/src/lib/OpenEXR/ImfCompressionAttribute.cpp index b0e93db479..a443eba8bb 100644 --- a/src/lib/OpenEXR/ImfCompressionAttribute.cpp +++ b/src/lib/OpenEXR/ImfCompressionAttribute.cpp @@ -57,7 +57,8 @@ CompressionAttribute::readValueFrom ( tmp != ZIPS_COMPRESSION && tmp != ZIP_COMPRESSION && tmp != PIZ_COMPRESSION && tmp != PXR24_COMPRESSION && tmp != B44_COMPRESSION && tmp != B44A_COMPRESSION && - tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION) + tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION && + tmp != ZSTD_COMPRESSION) { tmp = NUM_COMPRESSION_METHODS; } diff --git a/src/lib/OpenEXR/ImfCompressor.cpp b/src/lib/OpenEXR/ImfCompressor.cpp index 4d9e99a33a..794195f82c 100644 --- a/src/lib/OpenEXR/ImfCompressor.cpp +++ b/src/lib/OpenEXR/ImfCompressor.cpp @@ -18,6 +18,7 @@ #include "ImfPxr24Compressor.h" #include "ImfRleCompressor.h" #include "ImfZipCompressor.h" +#include "ImfZstdCompressor.h" OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER @@ -63,7 +64,8 @@ isValidCompression (Compression c) case B44_COMPRESSION: case B44A_COMPRESSION: case DWAA_COMPRESSION: - case DWAB_COMPRESSION: return true; + case DWAB_COMPRESSION: + case ZSTD_COMPRESSION: return true; default: return false; } @@ -89,7 +91,8 @@ isValidDeepCompression (Compression c) { case NO_COMPRESSION: case RLE_COMPRESSION: - case ZIPS_COMPRESSION: return true; + case ZIPS_COMPRESSION: + case ZSTD_COMPRESSION: return true; default: return false; } } @@ -141,6 +144,8 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr) 256, DwaCompressor::STATIC_HUFFMAN); + case ZSTD_COMPRESSION: + return new ZstdCompressor (hdr, maxScanLineSize, 32); default: return 0; } } @@ -162,6 +167,7 @@ numLinesInBuffer (Compression comp) case B44_COMPRESSION: case B44A_COMPRESSION: case DWAA_COMPRESSION: return 32; + case ZSTD_COMPRESSION: return 32;// FIXME case DWAB_COMPRESSION: return 256; default: throw IEX_NAMESPACE::ArgExc ("Unknown compression type"); @@ -182,6 +188,9 @@ newTileCompressor ( case ZIP_COMPRESSION: return new ZipCompressor (hdr, tileLineSize, numTileLines); + case ZSTD_COMPRESSION: + + return new ZstdCompressor (hdr, tileLineSize, numTileLines); case PIZ_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfHeader.cpp b/src/lib/OpenEXR/ImfHeader.cpp index be7f3c9664..cebec77aca 100644 --- a/src/lib/OpenEXR/ImfHeader.cpp +++ b/src/lib/OpenEXR/ImfHeader.cpp @@ -70,9 +70,11 @@ struct CompressionRecord { exr_get_default_zip_compression_level(&zip_level); exr_get_default_dwa_compression_quality(&dwa_level); + exr_get_default_zstd_compression_level(&zstd_level); } int zip_level; float dwa_level; + int zstd_level; }; // NB: This is extra complicated than one would normally write to // handle scenario that seems to happen on MacOS/Windows (probably @@ -696,6 +698,18 @@ Header::zipCompressionLevel () const return retrieveCompressionRecord (this).zip_level; } +int& +Header::zstdCompressionLevel () +{ + return retrieveCompressionRecord (this).zstd_level; +} + +int +Header::zstdCompressionLevel () const +{ + return retrieveCompressionRecord (this).zstd_level; +} + float& Header::dwaCompressionLevel () { diff --git a/src/lib/OpenEXR/ImfHeader.h b/src/lib/OpenEXR/ImfHeader.h index e09782b89c..4aab3a1b64 100644 --- a/src/lib/OpenEXR/ImfHeader.h +++ b/src/lib/OpenEXR/ImfHeader.h @@ -285,6 +285,11 @@ class IMF_EXPORT_TYPE Header float& dwaCompressionLevel (); IMF_EXPORT float dwaCompressionLevel () const; + IMF_EXPORT + int& zstdCompressionLevel (); + IMF_EXPORT + int zstdCompressionLevel () const; + IMF_EXPORT //----------------------------------------------------- // Access to required attributes for multipart files diff --git a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp index a5c17e3781..b9ed6d5d29 100644 --- a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp +++ b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp @@ -547,6 +547,7 @@ MultiPartInputFile::Data::chunkOffsetReconstruction ( // (TODO) fix this so that it doesn't need to be revised for future compression types. switch (parts[i]->header.compression ()) { + case ZSTD_COMPRESSION: rowsizes[i] = 32; break; case DWAB_COMPRESSION: rowsizes[i] = 256; break; case PIZ_COMPRESSION: case B44_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp new file mode 100644 index 0000000000..43fa4c640c --- /dev/null +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -0,0 +1,143 @@ +#include +#include "ImfZstdCompressor.h" + +#include "blosc2.h" +#include "IlmThreadPool.h" +#include "ImfChannelList.h" +#include "ImfMisc.h" +namespace +{ +class BloscInit +{ +public: + static void Init () { getInstance (); } + BloscInit (const BloscInit&) = delete; + BloscInit& operator= (const BloscInit&) = delete; + +private: + BloscInit () { blosc2_init (); } + ~BloscInit () { blosc2_destroy (); } + static BloscInit& getInstance () + { + static BloscInit instance; + return instance; + } +}; +} // namespace + +OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER +ZstdCompressor::ZstdCompressor ( + const Header& hdr, size_t maxScanlineSize, size_t numScanLines) + : Compressor (hdr) + , _maxScanlineSize (maxScanlineSize) + , _numScanLines (numScanLines) + , _outBuffer (nullptr, &free) + , _schunk (nullptr, &blosc2_schunk_free) +{} + +int +ZstdCompressor::numScanLines () const +{ + return _numScanLines; // Needs to be in sync with ImfCompressor::numLinesInBuffer +} + +int +ZstdCompressor::compress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) +{ + int typeSize = std::numeric_limits::min (); + for (auto it = header ().channels ().begin (); + it != header ().channels ().end (); + ++it) + { + // BLOSC prefilter is affected by the typesize. Initializing to max will ensure that a channel is not split in 2 and filtered separately. + // probably compression can be improved for non-deep images by compressing every channel separately with the correct typeSize + // (much harder to do for Deep-Data). + typeSize = std::max (typeSize, Imf::pixelTypeSize (it.channel ().type)); + } + + auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); + auto data = malloc (Xdr::size () + ret); + auto write = (char*) data; + + Xdr::write (write, Versions::LATEST); + + memcpy (write, outPtr, ret); + outPtr = (char*) data; + + _outBuffer = raw_ptr ((char*) data, &free); + + return ret; +} + +int +ZstdCompressor::uncompress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) +{ + auto read = (const char*) inPtr; + int v; + Xdr::read (read, v); + if (v == Versions::SINGLE_BLOB) + { + return BLOSC_uncompress_impl_single_blob ( + read, inSize - Xdr::size (), outPtr); + } + else + { + throw Iex::InputExc ("Unsupported ZstdCompressor version"); + } +} + +int +ZstdCompressor::BLOSC_compress_impl ( + const char* inPtr, int inSize, int typeSize, const char*& out) +{ + BloscInit::Init (); + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + + cparams.typesize = typeSize; + // clevel 9 is about a 20% increase in compression compared to 5. + // Decompression speed is unchanged. + cparams.clevel = header ().zstdCompressionLevel (); + cparams.nthreads = 1; + cparams.compcode = BLOSC_ZSTD; // Codec + cparams.splitmode = + BLOSC_NEVER_SPLIT; // Split => multithreading, not split better compression + + blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; + storage.cparams = &cparams; + storage.contiguous = true; + + _schunk = schunk_ptr (blosc2_schunk_new (&storage), &blosc2_schunk_free); + + auto in = const_cast (inPtr); + blosc2_schunk_append_buffer (_schunk.get (), in, inSize); + + uint8_t* buffer; + bool shouldFree = true; + auto size = blosc2_schunk_to_buffer (_schunk.get (), &buffer, &shouldFree); + out = (char*) buffer; + if (shouldFree) { _outBuffer = raw_ptr ((char*) buffer, &free); } + return size; +} + +int +ZstdCompressor::BLOSC_uncompress_impl_single_blob ( + const char* inPtr, int inSize, const char*& out) +{ + auto in = const_cast (inPtr); + _schunk = schunk_ptr ( + blosc2_schunk_from_buffer ( + reinterpret_cast (in), inSize, true), + &blosc2_schunk_free); + + auto buffSize = _maxScanlineSize * numScanLines (); + _outBuffer = + Imf::ZstdCompressor::raw_ptr ((char*) malloc (buffSize), &free); + auto size = blosc2_schunk_decompress_chunk ( + _schunk.get (), 0, _outBuffer.get (), buffSize); + out = _outBuffer.get (); + return size; +} + +OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXR/ImfZstdCompressor.h b/src/lib/OpenEXR/ImfZstdCompressor.h new file mode 100644 index 0000000000..f1c9110dd0 --- /dev/null +++ b/src/lib/OpenEXR/ImfZstdCompressor.h @@ -0,0 +1,39 @@ +#pragma once +#include +#include "ImfNamespace.h" +#include "ImfCompressor.h" +#include "ImfHeader.h" +#include "blosc2.h" + +OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER +class ZstdCompressor : public Compressor +{ +public: + ZstdCompressor ( + const Header& hdr, size_t maxScanLines, size_t numScanLines); + +private: + using schunk_ptr = + std::unique_ptr; + using raw_ptr = std::unique_ptr; + raw_ptr _outBuffer; + schunk_ptr _schunk; + size_t _maxScanlineSize; + size_t _numScanLines; + int numScanLines () const override; // max + int compress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) override; + int uncompress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) override; + int BLOSC_compress_impl ( + const char* inPtr, int inSize, int typeSize, const char*& out); + int BLOSC_uncompress_impl_single_blob ( + const char* inPtr, int inSize, const char*& out); + enum Versions : int + { + SINGLE_BLOB = 1, + LATEST = SINGLE_BLOB + }; +}; + +OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXRCore/base.c b/src/lib/OpenEXRCore/base.c index 513e73ea2c..8659a6b4fb 100644 --- a/src/lib/OpenEXRCore/base.c +++ b/src/lib/OpenEXRCore/base.c @@ -208,3 +208,20 @@ exr_get_default_dwa_compression_quality (float* q) { if (q) *q = sDefaultDwaLevel; } + +// 9 is 20% more expensive to compress. Decompression time remains constant. +static int sDefaultZstdLevel = 5; + +void +exr_set_default_zstd_compression_level (int q) +{ + if (q < 0) q = 0; + if (q > 9) q = 9; + sDefaultZstdLevel = q; +} + +void +exr_get_default_zstd_compression_level (int* q) +{ + if (q) *q = sDefaultZstdLevel; +} \ No newline at end of file diff --git a/src/lib/OpenEXRCore/openexr_base.h b/src/lib/OpenEXRCore/openexr_base.h index 8df304235b..a145323aa2 100644 --- a/src/lib/OpenEXRCore/openexr_base.h +++ b/src/lib/OpenEXRCore/openexr_base.h @@ -136,6 +136,17 @@ EXR_EXPORT void exr_set_default_dwa_compression_quality (float q); */ EXR_EXPORT void exr_get_default_dwa_compression_quality (float* q); +/** @brief Assigns a default zstd compression level. + * + * This value may be controlled separately on each part, but this + * global control determines the initial value. + */ +EXR_EXPORT void exr_set_default_zstd_compression_level (int l); + +/** @brief Retrieve the global default zstd compression value + */ +EXR_EXPORT void exr_get_default_zstd_compression_level (int* l); + /** @} */ /**