Add Changelog and document reason for changes.

hyperspy · Jan 23, 2024 · 68b72bf · 68b72bf
1 parent 995818c
commit 68b72bf
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/rsciio/_hierarchical.py b/rsciio/_hierarchical.py
@@ -263,13 +263,12 @@ def _read_array(group, dataset_key):
             key = "ragged_shapes"
         if key in group:
             ragged_shape = group[key]
-            # if the data is chunked saved array we must first
-            # cast to a numpy array to avoid multiple calls to
-            # _decode_chunk in zarr (or h5py)
+            # Use same chunks as data so that apply_gufunc doesn't rechunk
+            # Reduces the transfer of data between workers which
+            # significantly improves performance for distributed loading
             data = da.from_array(data, chunks=data.chunks)
-            shape = da.from_array(
-                ragged_shape, chunks=data.chunks
-            )  # same chunks as data
+            shape = da.from_array(ragged_shape, chunks=data.chunks)
+
             data = da.apply_gufunc(unflatten_data, "(),()->()", data, shape)
         return data
 

diff --git a/upcoming_changes/211.bugfix.rst b/upcoming_changes/211.bugfix.rst
@@ -0,0 +1 @@
+Fix saving ragged arrays of vectors from/to a chunked ``hspy`` and ``zspy`` store.  Greatly increases the speed of saving and loading ragged arrays from chunked datasets.