feat: new file format for zipped hologram tif files

CHANGELOG

@@ -1,4 +1,5 @@
 0.1.4
+ - feat: new file format for zipped hologram tif files
  - feat: add "storage_type" property describing which type of data
    is stored originally in a dataset
  - feat: add hologram file formats: HyperSpy and tif-based 

examples/convert_txt2tif.py

@@ -20,6 +20,8 @@
 ignore_endswith = ['.bmp', '.npy', '.opj', '.png', '.pptx', '.py', '.svg',
                    '.tif', '.txt', '_RIdist', '_parameter', '_parameter_old',
                    '_phase', 'n_array', 'n_array_real', '~']
+# uncomment this line to keep background hologram files
+ignore_endswith += ['_bg']
 
 
 def get_paths(folder, ignore_endswith=ignore_endswith):
@@ -53,6 +55,6 @@ def get_paths(folder, ignore_endswith=ignore_endswith):
     for ff in files:
         # convert image data to uint8 (most image sensors)
         hol = np.loadtxt(str(ff), dtype=np.uint8)
-        tifout = str(pout / (ff.name[:-10] + ".tif"))
+        tifout = str(pout / (ff.name + ".tif"))
         # compress image data
         tifffile.imsave(tifout, hol, compress=9)

qpformat/file_formats/__init__.py

@@ -5,6 +5,7 @@
 from .dataset import SeriesData, hash_obj
 from .series_hdf5_hyperspy import SeriesHdf5HyperSpy
 from .series_hdf5_qpimage import SeriesHdf5Qpimage
+from .series_zip_tif_holo import SeriesZipTifHolo
 from .series_zip_tif_phasics import SeriesZipTifPhasics
 from .single_hdf5_qpimage import SingleHdf5Qpimage
 from .single_npy_numpy import SingleNpyNumpy
@@ -144,6 +145,7 @@ class UnknownFileFormatError(BaseException):
            SeriesHdf5HyperSpy,
            SeriesHdf5Qpimage,
            SeriesZipTifPhasics,
+           SeriesZipTifHolo,  # after phasics, b/c phasics has extra keywords
            ]
 
 # convenience dictionary

qpformat/file_formats/series_zip_tif_holo.py

@@ -0,0 +1,77 @@
+import io
+import functools
+import zipfile
+
+from .dataset import SeriesData
+from .single_tif_holo import SingleTifHolo
+
+
+class SeriesZipTifHolo(SeriesData):
+    storage_type = "hologram"
+
+    def __init__(self, *args, **kwargs):
+        super(SeriesZipTifHolo, self).__init__(*args, **kwargs)
+        self._files = None
+        self._dataset = None
+
+    def __len__(self):
+        return len(self.files)
+
+    def _get_dataset(self, idx):
+        if self._dataset is None:
+            self._dataset = [None] * len(self)
+        if self._dataset[idx] is None:
+            # Use ``zipfile.ZipFile.open`` to return an open file
+            zf = zipfile.ZipFile(self.path)
+            pt = zf.open(self.files[idx])
+            fd = io.BytesIO(pt.read())
+            self._dataset[idx] = SingleTifHolo(path=fd,
+                                               meta_data=self.meta_data,
+                                               holo_kw=self.holo_kw)
+        return self._dataset[idx]
+
+    @staticmethod
+    @functools.lru_cache(maxsize=32)
+    def _index_files(path):
+        """Search zip file for tif files"""
+        with zipfile.ZipFile(path) as zf:
+            names = sorted(zf.namelist())
+            names = [nn for nn in names if nn.endswith(".tif")]
+            phasefiles = []
+            for name in names:
+                with zf.open(name) as pt:
+                    fd = io.BytesIO(pt.read())
+                    if SingleTifHolo.verify(fd):
+                        phasefiles.append(name)
+            return phasefiles
+
+    @property
+    def files(self):
+        if self._files is None:
+            self._files = SeriesZipTifHolo._index_files(self.path)
+        return self._files
+
+    def get_qpimage_raw(self, idx):
+        """Return QPImage without background correction"""
+        ds = self._get_dataset(idx)
+        return ds.get_qpimage_raw()
+
+    @staticmethod
+    def verify(path):
+        """Verify hologram zip tif file format"""
+        valid = False
+        try:
+            zf = zipfile.ZipFile(path)
+        except (zipfile.BadZipfile, IsADirectoryError):
+            pass
+        else:
+            names = sorted(zf.namelist())
+            names = [nn for nn in names if nn.endswith(".tif")]
+            for name in names:
+                with zf.open(name) as pt:
+                    fd = io.BytesIO(pt.read())
+                    if SingleTifHolo.verify(fd):
+                        valid = True
+                        break
+            zf.close()
+        return valid