support for large files using dask. Squeeze for singletons

src/napari_h5/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.0.7"
+__version__ = "0.0.8"
 
 from ._reader import get_reader
 from ._writer import multi_layer_writer, single_layer_writer

src/napari_h5/_reader.py

@@ -71,13 +71,12 @@ def reader_function(path):
     import pathlib
     import h5py
 
-    data_list = None
+    data_list = []
     for p0 in paths:
-        with h5py.File(str(p0),'r') as f:
-            data_list = []
-            # If not ask user to help choosing data to display
 
-            #data0 = np.array(f['data']) #Loads 'data' by default
+        try:
+            f=h5py.File(str(p0),'r')
+                # If not ask user to help choosing data to display
 
             #Get keys available
             fkeys = list(f.keys())
@@ -94,18 +93,30 @@ def reader_function(path):
                         fname_stem = f_path.stem
                         add_kwargs = {'name':fname_stem+" "+k0}
                         layer_type = "image"
-
-                        data0 = np.array(i0) #Loads 'data' by default
-                        data_list.append( (data0, add_kwargs, layer_type) )
-
-                if len(data_list)==0:
-                    data_list=None
-
-    # optional kwargs for the corresponding viewer.add_* method
-
-
-    #layer_type = "image"  # optional, default is "image"
-    #return [(data, add_kwargs, layer_type)]
+
+                        data0=None
+
+                        if i0.nbytes<=1e9:
+                            data0 = np.array(i0)
+                            # Check for singleton dimensions
+                            if data0.ndim>2:
+                                data0=np.squeeze(data0)
+                        else:
+                            #Opens as dask
+                            print("Data is >1e bytes. Opening to dask object.")
+                            import dask.array as da
+                            data0 = da.from_array(i0)
+                            if data0.ndim>2:
+                                data0=da.squeeze(data0)
+
+                        if not data0 is None:
+                            data_list.append( (data0, add_kwargs, layer_type) )
+
+        except Exception as e:
+            print(f"Failed to open {str(p0)}")
+
+    if len(data_list)==0:
+        data_list=None
 
     return data_list
 

src/napari_h5/_writer.py

@@ -12,6 +12,7 @@
 
 import h5py
 import numpy as np
+import dask.array as da
 
 if TYPE_CHECKING:
     DataType = Union[Any, Sequence[Any]]
@@ -22,8 +23,8 @@
 def single_layer_writer(path: str, data: Any, attributes: dict) -> List[str]:
     """Writes a single image layer"""
 
-    if not isinstance(data, np.ndarray):
-        raise ValueError("data is not ndarray. Not saving.")
+    if not isinstance(data, np.ndarray) or isinstance(data,da.Array):
+        raise ValueError("data is not ndarray nor dask.Array. Not saving.")
 
     with h5py.File(path,'w') as f:
         f.create_dataset('data', data=data)
@@ -49,7 +50,7 @@ def multi_layer_writer(path: str, layer_data: List[FullLayerData]) -> List[str]:
         datatype0, layerattibs0, layername0 = ld0
 
         if layername0=='image' or layername0=='labels':
-            if isinstance(datatype0,np.ndarray):
+            if isinstance(datatype0,np.ndarray) or isinstance(datatype0,da.Array):
                 path1 = Path.joinpath(path0.parent, path0.stem + f"_{i:03d}"+ path0.suffix)
 
                 #print(f"data i:{i} , path1:{path1}")