Merge pull request #1078 from PyTables/direct-chunking-b2ndarray

Fix broken b2nd optimized slice assembly and tests

This fixes the assembly of slices obtained via Blosc2 ND optimized slicing, which was using `memcpy` from the outer dimension of each chunk slice instead of the inner one. The new code avoids the manual assembly of the slice altogether by leaving the job to `b2nd_copy_buffer`, which was published in C-Blosc2 2.11.0 (thus the dependencies on C-Blosc2 and python-blosc2 are updated too).

A new unit test `tables.test_carray.Blosc2Ndim3MinChunkOptTestCase` has been added that would trigger the error in case of the bug, to avoid regressions. Also, this fixes other unit tests that had been added for b2nd optimized slicing but were not enabled.

Finally, `tables.test_carray.Blosc2NDNoChunkshape` has been added to check the compatibility with arrays that contain b2nd chunks but do not include the extra filter parameters with the chunk rank and shape (e.g. because they were created with code other than `hdf5-blosc2`, see #1072).

RELEASE_NOTES.rst

@@ -13,6 +13,9 @@ Changes from 3.9.1 to 3.9.2
 
 XXX version-specific blurb XXX
 
+- Require python-blosc2 >= 2.3.0 or c-blosc2 >= 2.11.0 (support for the
+  `b2nd_copy_buffer` function).
+
 
 Improvements
 ------------

doc/source/usersguide/installation.rst

@@ -70,10 +70,10 @@ If you don't, fetch and install them before proceeding.
   use an external version of sources using the :envvar:`BLOSC_DIR` environment
   variable or the `--blosc` flag of the :file:`setup.py`)
 * Either
-   * python-blosc2_ >= 2.2.8, this is the Python wheel containing *both* the
-     C-Blosc2 libs and headers (>= 2.10.4), as well as the Python wrapper for
+   * python-blosc2_ >= 2.3.0, this is the Python wheel containing *both* the
+     C-Blosc2 libs and headers (>= 2.11.0), as well as the Python wrapper for
      Blosc2 (not currently used, but it might be in the future), or
-   * A standalone installation of the c-blosc2_ library (>= 2.10.4) including
+   * A standalone installation of the c-blosc2_ library (>= 2.11.0) including
      the headers.  The latter are usually provided by Linux distribtions in a
      package named `blosc2-devel`, `libblosc2-dev`, or similar.
 

pyproject.toml

@@ -12,7 +12,7 @@ requires = [
     # Included here for seamless wheel builds.
     # Packagers can choose to replace it by externally provided
     # c-blosc2 library and headers
-    "blosc2 >=2.2.8",
+    "blosc2 >=2.3.0",
 ]
 build-backend = "setuptools.build_meta"
 # build-backend = "mesonpy"  # and replace ``setuptools`` above
@@ -72,7 +72,7 @@ dependencies = [
     "numexpr >= 2.6.2",
     "packaging",
     "py-cpuinfo",
-    "blosc2 >= 2.2.8",
+    "blosc2 >= 2.3.0",
 ]
 
 

requirements.txt

@@ -4,4 +4,4 @@ numpy>=1.19.0
 numexpr>=2.6.2
 packaging
 py-cpuinfo
-blosc2>=2.2.8
+blosc2>=2.3.0

src/H5ARRAY-opt.c

@@ -65,7 +65,7 @@ herr_t get_blosc2_slice(char *filename,
                         const int rank,
                         hsize_t *slice_start,
                         hsize_t *slice_stop,
-                        const void *slice_data)
+                        void *slice_data)
 {
   herr_t retval = -1;
   hid_t space_id = -1;
@@ -77,14 +77,12 @@ herr_t get_blosc2_slice(char *filename,
   hsize_t *array_shape = NULL;
   int64_t *slice_shape = NULL;
   int64_t *chunks_in_array = NULL;
-  int64_t *slice_strides = NULL;
   int64_t *chunks_in_array_strides = NULL;
   int64_t *slice_chunks_start = NULL;
   int64_t *slice_chunks_shape = NULL;
   int64_t *slice_chunk_pos = NULL;
   hsize_t *chunk_start = NULL;
   hsize_t *chunk_stop = NULL;
-  int64_t *chunk_slice_strides = NULL;
   int64_t *start_in_stored_chunk = NULL;
   int64_t *stop_in_stored_chunk = NULL;
   int64_t *chunk_slice_shape = NULL;
@@ -125,13 +123,10 @@ herr_t get_blosc2_slice(char *filename,
                           + ((array_shape[i] % chunk_shape[i]) ? 1 : 0));
   }
 
-  /* Compute slice and chunk strides */
-  slice_strides = (int64_t *)(malloc(rank * sizeof(int64_t)));  // in items
+  /* Compute chunk strides */
   chunks_in_array_strides = (int64_t *)(malloc(rank * sizeof(int64_t)));  // in chunks
-  slice_strides[rank - 1] = 1;
   chunks_in_array_strides[rank - 1] = 1;
   for (int i = rank - 2; i >= 0; --i) {
-    slice_strides[i] = slice_strides[i + 1] * slice_shape[i + 1];
     chunks_in_array_strides[i] = chunks_in_array_strides[i + 1] * chunks_in_array[i + 1];
   }
 
@@ -156,7 +151,6 @@ herr_t get_blosc2_slice(char *filename,
   slice_chunk_pos = (int64_t *)(malloc(rank * sizeof(int64_t)));  // in chunks
   chunk_start = (hsize_t *)(malloc(rank * sizeof(hsize_t)));  // in items
   chunk_stop = (hsize_t *)(malloc(rank * sizeof(hsize_t)));  // in items
-  chunk_slice_strides = (int64_t *)(malloc(rank * sizeof(int64_t)));  // in items
   start_in_stored_chunk = (int64_t *)(malloc(rank * sizeof(int64_t)));  // in items
   stop_in_stored_chunk = (int64_t *)(malloc(rank * sizeof(int64_t)));  // in items
   chunk_slice_shape = (int64_t *)(malloc(rank * sizeof(int64_t)));  // in items
@@ -216,22 +210,11 @@ herr_t get_blosc2_slice(char *filename,
                     rv - 50);
 
     /* Copy from temp chunk slice to slice data */
-    chunk_slice_strides[rank - 1] = 1;
-    for (int i = rank - 2; i >= 0; --i) {
-      chunk_slice_strides[i] = chunk_slice_strides[i + 1] * chunk_slice_shape[i + 1];
-    }
-    int64_t chunk_slice_start_idx = -1;
-    blosc2_multidim_to_unidim((int64_t*)(chunk_slice_start), rank, slice_strides, &chunk_slice_start_idx);
-    uint8_t *chunk_line = (uint8_t*)(slice_data) + (chunk_slice_start_idx * typesize);
-    uint8_t *chunk_slice_line = chunk_slice_data;
-    for (int i = chunk_slice_start[0]; i < chunk_slice_stop[0]; i++) {
-        /* As the temporary chunk slice has no other chunks around it,
-           its main stride is the number of items to be copied per chunk line. */
-        memcpy(chunk_line, chunk_slice_line, chunk_slice_strides[0] * typesize);
-
-        chunk_line += slice_strides[0] * typesize;
-        chunk_slice_line += chunk_slice_strides[0] * typesize;
-    }
+    const int64_t zero_coords[B2ND_MAX_DIM] = {0};
+    b2nd_copy_buffer(rank, typesize,
+                     chunk_slice_data, chunk_slice_shape,
+                     zero_coords, chunk_slice_shape,
+                     slice_data, slice_shape, chunk_slice_start);
 
     assert(chunk_slice_data);
     free(chunk_slice_data);
@@ -251,14 +234,12 @@ herr_t get_blosc2_slice(char *filename,
   if (chunk_slice_shape) free(chunk_slice_shape);
   if (stop_in_stored_chunk) free(stop_in_stored_chunk);
   if (start_in_stored_chunk) free(start_in_stored_chunk);
-  if (chunk_slice_strides) free(chunk_slice_strides);
   if (chunk_stop) free(chunk_stop);
   if (chunk_start) free(chunk_start);
   if (slice_chunk_pos) free(slice_chunk_pos);
   if (slice_chunks_shape) free(slice_chunks_shape);
   if (slice_chunks_start) free(slice_chunks_start);
   if (chunks_in_array_strides) free(chunks_in_array_strides);
-  if (slice_strides) free(slice_strides);
   if (chunks_in_array) free(chunks_in_array);
   if (slice_shape) free(slice_shape);
   if (array_shape) free(array_shape);

tables/req_versions.py

@@ -13,4 +13,4 @@
 # These are library versions, not the python modules
 min_hdf5_version = Version('1.10.5')
 min_blosc_version = Version('1.11.1')
-min_blosc2_version = Version('2.10.4')
+min_blosc2_version = Version('2.11.0')

tables/tests/test_carray.py

@@ -838,6 +838,31 @@ class Blosc2PastLastChunkOptTestCase(Blosc2PastLastChunkTestCase):
     byteorder = sys.byteorder
 
 
+# Minimal test which can be figured out manually::
+#
+#     z  Data: 1   Chunk0:   Chunk1: 1   Slice:
+#    /        /|\                    |\
+#   |\       0 5 3       0           5 3        5
+#   x y      |X X|       |\           \|       / \
+#            4 2 7       4 2           7      4   7
+#             \|/         \|                   \ /
+#              6           6                    6
+#
+#                  Chunk0 & Slice: 4   Chunk1 & Slice: 5
+#                                   \                   \
+#                                    6                   7
+@common.unittest.skipIf(not common.blosc2_avail,
+                        'BLOSC2 compression library not available')
+class Blosc2Ndim3MinChunkOptTestCase(BasicTestCase):
+    shape = (2, 2, 2)
+    compress = 1
+    complib = "blosc2"
+    chunkshape = (2, 2, 1)
+    byteorder = sys.byteorder
+    type = 'int8'
+    slices = (slice(1, 2), slice(0, 2), slice(0, 2))
+
+
 @common.unittest.skipIf(not common.blosc2_avail,
                         'BLOSC2 compression library not available')
 class Blosc2Ndim3ChunkOptTestCase(BasicTestCase):
@@ -852,7 +877,7 @@ class Blosc2Ndim3ChunkOptTestCase(BasicTestCase):
 
 @common.unittest.skipIf(not common.blosc2_avail,
                         'BLOSC2 compression library not available')
-class Blosc2Ndim3ChunkOptTestCase(BasicTestCase):
+class Blosc2Ndim4ChunkOptTestCase(BasicTestCase):
     shape = (13, 13, 13, 3)
     compress = 1
     complib = "blosc2"
@@ -862,6 +887,75 @@ class Blosc2Ndim3ChunkOptTestCase(BasicTestCase):
     slices = (slice(0, 8), slice(7, 13), slice(3, 12), slice(1, 3))
 
 
+# The file used in the test below is created with this script,
+# producing a chunked array that lacks chunk rank/shape in filter args.
+# It is a reduced version of ``examples/direct-chunk-shape.py``,
+# check there for more info and the assemblage of the data array.
+# An h5py release is used which contains a version of hdf5-blosc2
+# that does not include chunk rank/shape in filter arguments.
+#
+# ::
+#
+#   import blosc2
+#   import h5py
+#   import hdf5plugin
+#   import numpy
+#
+#   assert(hdf5plugin.version_info < (4, 2, 1))
+#
+#   fparams = hdf5plugin.Blosc2(cname='zstd', clevel=1,
+#                               filters=hdf5plugin.Blosc2.SHUFFLE)
+#   cparams = {
+#       "codec": blosc2.Codec.ZSTD,
+#       "clevel": 1,
+#       "filters": [blosc2.Filter.SHUFFLE],
+#   }
+#
+#   achunk = numpy.arange(4 * 4, dtype='int8').reshape((4, 4))
+#   adata = numpy.zeros((6, 6), dtype=achunk.dtype)
+#   adata[0:4, 0:4] = achunk[:, :]
+#   adata[0:4, 4:6] = achunk[:, 0:2]
+#   adata[4:6, 0:4] = achunk[0:2, :]
+#   adata[4:6, 4:6] = achunk[0:2, 0:2]
+#
+#   h5f = h5py.File("b2nd-no-chunkshape.h5", "w")
+#   dataset = h5f.create_dataset(
+#       "data", adata.shape, dtype=adata.dtype, chunks=achunk.shape,
+#       **fparams)
+#   b2chunk = blosc2.asarray(achunk,
+#                            chunks=achunk.shape, blocks=achunk.shape,
+#                            cparams=cparams)
+#   b2frame = b2chunk._schunk.to_cframe()
+#   dataset.id.write_direct_chunk((0, 0), b2frame)
+#   dataset.id.write_direct_chunk((0, 4), b2frame)
+#   dataset.id.write_direct_chunk((4, 0), b2frame)
+#   dataset.id.write_direct_chunk((4, 4), b2frame)
+#   h5f.close()
+@common.unittest.skipIf(not common.blosc2_avail,
+                        'BLOSC2 compression library not available')
+class Blosc2NDNoChunkshape(common.TestFileMixin,
+                           common.PyTablesTestCase):
+    h5fname = common.test_filename('b2nd-no-chunkshape.h5')
+
+    adata = np.array(
+        [[ 0,  1,  2,  3,     0,  1],
+         [ 4,  5,  6,  7,     4,  5],
+         [ 8,  9, 10, 11,     8,  9],
+         [12, 13, 14, 15,    12, 13],
+
+         [ 0,  1,  2,  3,     0,  1],
+         [ 4,  5,  6,  7,     4,  5]],
+        dtype='int8')
+
+    def test_data_opt(self):
+        array = self.h5file.get_node('/data')
+        self.assertTrue(common.areArraysEqual(array[:], self.adata[:]))
+
+    def test_data_filter(self):
+        array = self.h5file.get_node('/data')
+        self.assertTrue(common.areArraysEqual(array[::2], self.adata[::2]))
+
+
 @common.unittest.skipIf(not common.lzo_avail,
                         'LZO compression library not available')
 class LZOComprTestCase(BasicTestCase):
@@ -2843,6 +2937,10 @@ def suite():
         theSuite.addTest(common.unittest.makeSuite(Blosc2CrossChunkOptTestCase))
         theSuite.addTest(common.unittest.makeSuite(Blosc2PastLastChunkTestCase))
         theSuite.addTest(common.unittest.makeSuite(Blosc2PastLastChunkOptTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2Ndim3MinChunkOptTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2Ndim3ChunkOptTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2Ndim4ChunkOptTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2NDNoChunkshape))
         theSuite.addTest(common.unittest.makeSuite(LZOComprTestCase))
         theSuite.addTest(common.unittest.makeSuite(LZOShuffleTestCase))
         theSuite.addTest(common.unittest.makeSuite(Bzip2ComprTestCase))

tables/tests/test_earray.py

@@ -189,7 +189,9 @@ def test01_iterEArray(self):
                 print("self, earray:", self.compress, earray.filters.complevel)
             self.assertEqual(earray.filters.complevel, self.compress)
             if self.compress > 0 and tb.which_lib_version(self.complib):
-                self.assertEqual(earray.filters.complib, self.complib)
+                # Some libraries like Blosc support different compressors,
+                # specified after ":".
+                self.assertEqual(earray.filters.complib.split(':')[0], self.complib)
             if self.shuffle != earray.filters.shuffle and common.verbose:
                 print("Error in shuffle. Class:", self.__class__.__name__)
                 print("self, earray:", self.shuffle, earray.filters.shuffle)
@@ -873,7 +875,7 @@ class Blosc2SlicesOptEArrayTestCase(BasicTestCase):
     compress = 1
     complib = "blosc2"
     type = 'int32'
-    shape = (13, 13, 13)
+    shape = (0, 13, 13)
     chunkshape = (4, 4, 4)
     nappends = 20
     slices = (slice(None, None), slice(2, 10), slice(0, 10))
@@ -893,10 +895,11 @@ class Blosc2ComprTestCase(BasicTestCase):
 @common.unittest.skipIf(not common.blosc2_avail,
                         'BLOSC2 compression library not available')
 class Blosc2CrossChunkTestCase(BasicTestCase):
-    shape = (10, 10)
+    shape = (0, 10)
     compress = 1  # sss
     complib = "blosc2"
     chunkshape = (4, 4)
+    nappends = 10
     start = 3
     stop = 6
     step = 3
@@ -909,13 +912,27 @@ class Blosc2CrossChunkOptTestCase(Blosc2CrossChunkTestCase):
     byteorder = sys.byteorder
 
 
+@common.unittest.skipIf(not common.blosc2_avail,
+                        'BLOSC2 compression library not available')
+class Blosc2InnerCrossChunkTestCase(Blosc2CrossChunkTestCase):
+    shape = (10, 0)
+
+
+@common.unittest.skipIf(not common.blosc2_avail,
+                        'BLOSC2 compression library not available')
+class Blosc2InnerCrossChunkOptTestCase(Blosc2InnerCrossChunkTestCase):
+    step = 1  # optimized
+    byteorder = sys.byteorder
+
+
 @common.unittest.skipIf(not common.blosc2_avail,
                         'BLOSC2 compression library not available')
 class Blosc2PastLastChunkTestCase(BasicTestCase):
-    shape = (10, 10)
+    shape = (0, 10)
     compress = 1  # sss
     complib = "blosc2"
     chunkshape = (4, 4)
+    nappends = 10
     start = 8
     stop = 100
     step = 3
@@ -2884,6 +2901,13 @@ def suite():
         theSuite.addTest(common.unittest.makeSuite(ZlibShuffleTestCase))
         theSuite.addTest(common.unittest.makeSuite(BloscComprTestCase))
         theSuite.addTest(common.unittest.makeSuite(BloscShuffleTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2SlicesOptEArrayTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2ComprTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2CrossChunkTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2CrossChunkOptTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2InnerCrossChunkTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2InnerCrossChunkOptTestCase))
+        theSuite.addTest(common.unittest.makeSuite(Blosc2PastLastChunkTestCase))
         theSuite.addTest(common.unittest.makeSuite(LZOComprTestCase))
         theSuite.addTest(common.unittest.makeSuite(LZOShuffleTestCase))
         theSuite.addTest(common.unittest.makeSuite(Bzip2ComprTestCase))