feat(HeE2eReader): read E2E volumes

docs/formats.md

@@ -6,6 +6,103 @@
 
 ## E2E (Heidelberg)
 
+###
+
+The general structure of an E2E file looks like this:
+
++ version
++ header
++ List of chunks where each chunk:
+    + has a header
+    + has a list of folders where each folder:
+        + has a header
+        + has a data container with:
+            + a header
+            + an item
+
+Each chunk contains exactly 512 folders. A chunk can contain folders with data of different patients, studies, series,slices and types but each folder contains data for a single (patient, study, series, slice, type) combination which is given in the folder header as well as the data container header.
+
+The first chunk starts with 10 assumingly study specific folders (only patient and study are given), followed by assumingly Series specific folders (slice id is not given). Interestingly there are some type ids repeating here, including the laterality type. The only difference between the folder headers with the identical types is the value of the `unknown2` field which takes the values 0, 1 or 65535. The `unknown` field of the folder header is always 0. `unknown4` might be some kind of unique identifier since in the test data there are no duplicates while `unknown3` might be some kind of a foreign key.
+
+Then the folders for every slice of all series follow. There might be multiple folders of the layer type for each slice, each holding a different layer for the respective B-scan. After the last slice 6 patient specific folders follow where only the patient_id is given and finally there is a folder of type 9011 which is the last folder in the last chunk before the filler folders start.
+
+We are currently parsing only a small portion of the available folders, to extract image and meta data. We extract data of the following types and sizes:
+
+Type: 9 - Size: 131   # Patient data
+Type: 11 - Size: 27  # Laterality data
+Type: 10004 - Size: 428  # Bscan meta data
+Type: 10019 - Size: ~3108  # Layer data
+Type: 1073741824 - Size: ~761876  # Image data
+
+### The mysteries of the E2E format
+There are several data folder types `HeE2eReader` currently does not support. If you believe the data you are looking for is in here, you can access the data using the hidden `_unknown_folders` attribute of the `HeE2eReader`. You will get a dictionary with keys in the format (PatientID, StudyID, SeriesID, SliceID, Type). The values are the parsed objects. You can access the unparsed binary data via the `.data_container.item` attribute. Very small entries might not have a `data_container` and can be `None` instead.
+
+From here it is up to you to figure out the meaning of the data. I would appreciate if you share your findings.
+
+Here is a list of all the data currently not understood by the `HeE2eReader`. Whenever one of the IDs is maxint the respective data has to be more general. The size given here might vary depending on the respective data.
+
+**Study fields (Series is maxint):**
+
+    Type: 9000 - Size: 264
+    Type: 9001 - Size: 776
+    Type: 58 - Size: 91
+    Type: 7 - Size: 68
+    Type: 1000 - Size: 51
+    Type: 53 - Size: 51
+    Type: 13 - Size: 200 (# Containtes OCT + HRA string)
+    Type: 10 - Size: 91
+    Type: 30 - Size: 2
+
+**Series fields (Slice is -1)**
+
+    Type: 9006 - Size: 520
+    Type: 9005 - Size: 264
+    Type: 9007 - Size: 520
+    Type: 9008 - Size: 520
+    Type: 59 - Size: 27
+    Type: 1003 - Size: 17
+    Type: 1000 - Size: 51
+    Type: 62 - Size: 228
+    Type: 10013 - Size: 2212
+    Type: 10005 - Size: 24
+    Type: 10009 - Size: 4
+    Type: 10025 - Size: 100 # Slodata
+    Type: 61 - Size: 4
+    Type: 10010 - Size: 4112
+    Type: 54 - Size: 97
+    Type: 1001 - Size: 56
+    Type: 3 - Size: 96
+    Type: 2 - Size: 2274 (Preview? contains letters (JFIF))
+    Type: 10011 - Size: 4
+    Type: 1008 - Size: 8
+    Type: 1073751824 - Size: 51220
+    Type: 1073751825 - Size: 51220
+    Type: 1073751826 - Size: 24596
+
+
+
+**Patient fields (study and series are maxint)**
+
+    Type: 52 - Size: 97
+    Type: 9010 - Size: 269064
+    Type: 31 - Size: 217
+    Type: 17 - Size: 2
+    Type: 29 - Size: 2
+
+**General fields (patient study and series are maxint)**
+
+    Type: 9011 - Size: 64664 # Last element in the last chunk before fillers fill the chunk
+    Type: 0 - Size: 0 # Filler at the end of the parsed file the last chunk is filled with folders of this type until there are 512 folders in the chunk
+
+**Bscan fields:**
+
+    Type: 5 - Size: 59
+    Type: 39 - Size: 497
+    Type: 3 - Size: 96
+    Type: 2 - Size: 2074 (Preview? contains letters (JFIF))
+    Type: 10012 - Size: 100
+    Type: 40 - Size: 28
+
 ## Topcon
 
 ## Zeiss

pyproject.toml

@@ -32,8 +32,8 @@ scikit-image = "^0.19.1"
 imagecodecs = "^2021.11.20"
 matplotlib = "^3.5.1"
 nptyping = "^2.3.1"
-construct = {extras = ["numpy"], version = "^2.10.68"}
 mkdocs-literate-nav = "^0.5.0"
+construct-typing = "^0.5.5"
 
 
 

src/eyepy/__init__.py

@@ -17,6 +17,7 @@
 from eyepy.core import EyeVolumeVoxelAnnotation
 from eyepy.io import import_bscan_folder
 from eyepy.io import import_duke_mat
+from eyepy.io import import_heyex_e2e
 from eyepy.io import import_heyex_vol
 from eyepy.io import import_heyex_xml
 from eyepy.io import import_retouch

src/eyepy/core/eyevolume.py

@@ -93,7 +93,8 @@ def save(self, path):
             np.save(tmpdirname / "raw_volume.npy", self._raw_data)
             with open(tmpdirname / "meta.json", "w") as meta_file:
                 if self.meta["intensity_transform"] == "custom":
-                    warnings.warn("Custom intensity transforms can not be saved.")
+                    warnings.warn(
+                        "Custom intensity transforms can not be saved.")
                     self.meta["intensity_transform"] = "default"
                 json.dump(self.meta.as_dict(), meta_file)
 
@@ -135,10 +136,12 @@ def save(self, path):
                     np.stack([p.data for p in self.localizer._area_maps]),
                 )
                 with open(pixels_path / "meta.json", "w") as meta_file:
-                    json.dump([p.meta for p in self.localizer._area_maps], meta_file)
+                    json.dump([p.meta for p in self.localizer._area_maps],
+                              meta_file)
 
             # Zip and copy to location
-            name = shutil.make_archive(Path(path).stem, "zip", tmpdirname)
+            name = shutil.make_archive(str(tmpdirname / Path(path).stem),
+                                       "zip", tmpdirname)
             shutil.move(name, path)
 
     @classmethod
@@ -203,13 +206,13 @@ def load(cls, path):
                     pixels_meta = json.load(meta_file)
 
                 for i, pixel_meta in enumerate(pixels_meta):
-                    localizer.add_area_annotation(pixel_annotations[i], pixel_meta)
+                    localizer.add_area_annotation(pixel_annotations[i],
+                                                  pixel_meta)
 
             from eyepy.io.utils import _compute_localizer_oct_transform
 
             transformation = _compute_localizer_oct_transform(
-                volume_meta, localizer_meta, data.shape
-            )
+                volume_meta, localizer_meta, data.shape)
 
             ev = cls(
                 data=data,
@@ -227,16 +230,16 @@ def load(cls, path):
 
     def _default_meta(self, volume):
         bscan_meta = [
-            EyeBscanMeta(
-                start_pos=(0, i), end_pos=((volume.shape[2] - 1), i), pos_unit="pixel"
-            )
+            EyeBscanMeta(start_pos=(0, i),
+                         end_pos=((volume.shape[2] - 1), i),
+                         pos_unit="pixel")
             for i in range(volume.shape[0] - 1, -1, -1)
         ]
         meta = EyeVolumeMeta(
-            scale_x=np.nan,
-            scale_y=np.nan,
-            scale_z=np.nan,
-            scale_unit="",
+            scale_x=1.0,
+            scale_y=1.0,
+            scale_z=1.0,
+            scale_unit="pixel",
             intensity_transform="default",
             bscan_meta=bscan_meta,
         )
@@ -252,33 +255,29 @@ def _default_localizer(self, data):
         )
         localizer = EyeEnface(
             image,
-            meta=EyeEnfaceMeta(
-                scale_x=self.scale_x, scale_y=self.scale_x, scale_unit=self.scale_unit
-            ),
+            meta=EyeEnfaceMeta(scale_x=self.scale_x,
+                               scale_y=self.scale_x,
+                               scale_unit=self.scale_unit),
         )
         return localizer
 
     def _estimate_transform(self):
         # Compute a transform to map a 2D projection of the volume to a square
         # Points in oct space
-        src = np.array(
-            [
-                [0, 0],  # Top left
-                [0, self.size_x - 1],  # Top right
-                [self.size_z - 1, 0],  # Bottom left
-                [self.size_z - 1, self.size_x - 1],
-            ]
-        )  # Bottom right
+        src = np.array([
+            [0, 0],  # Top left
+            [0, self.size_x - 1],  # Top right
+            [self.size_z - 1, 0],  # Bottom left
+            [self.size_z - 1, self.size_x - 1],
+        ])  # Bottom right
 
         # Respective points in enface space
-        dst = np.array(
-            [
-                (0, 0),  # Top left
-                (0, self.size_x - 1),  # Top right
-                (self.size_x - 1, 0),  # Bottom left
-                (self.size_x - 1, self.size_x - 1),
-            ]
-        )  # Bottom right
+        dst = np.array([
+            (0, 0),  # Top left
+            (0, self.size_x - 1),  # Top right
+            (self.size_x - 1, 0),  # Bottom left
+            (self.size_x - 1, self.size_x - 1),
+        ])  # Bottom right
 
         # Switch from x/y coordinates to row/column coordinates for src and dst
         src = np.flip(src, axis=1)
@@ -297,8 +296,8 @@ def __getitem__(self, index: slice) -> List[EyeBscan]:
         ...
 
     def __getitem__(
-        self, index: Union[SupportsIndex, slice]
-    ) -> Union[List[EyeBscan], EyeBscan]:
+            self, index: Union[SupportsIndex,
+                               slice]) -> Union[List[EyeBscan], EyeBscan]:
         """
 
         Args:
@@ -346,8 +345,7 @@ def set_intensity_transform(self, func: Union[str, Callable]):
                 self._data = None
             elif func == "custom":
                 logger.warning(
-                    "Custom intensity transforms can not be loaded currently"
-                )
+                    "Custom intensity transforms can not be loaded currently")
             else:
                 logger.warning(
                     f"Provided intensity transform name {func} is not known. Valid names are 'vol' or 'default'. You can also pass your own function."
@@ -379,7 +377,8 @@ def shape(self):
 
     @shape.setter
     def shape(self, value):
-        raise AttributeError("Shape can not be set since it is derived from the data")
+        raise AttributeError(
+            "Shape can not be set since it is derived from the data")
 
     @property
     def scale(self):
@@ -641,7 +640,9 @@ def plot(
         for name in projections:
             if not name in projection_kwargs.keys():
                 projection_kwargs[name] = {}
-            self.volume_maps[name].plot(ax=ax, region=region, **projection_kwargs[name])
+            self.volume_maps[name].plot(ax=ax,
+                                        region=region,
+                                        **projection_kwargs[name])
 
         if line_kwargs is None:
             line_kwargs = config.line_kwargs
@@ -656,10 +657,13 @@ def plot(
                 line_kwargs=line_kwargs,
             )
         if bscan_region:
-            self._plot_bscan_region(region=region, ax=ax, line_kwargs=line_kwargs)
+            self._plot_bscan_region(region=region,
+                                    ax=ax,
+                                    line_kwargs=line_kwargs)
 
         if quantification:
-            self.volume_maps[quantification].plot_quantification(region=region, ax=ax)
+            self.volume_maps[quantification].plot_quantification(region=region,
+                                                                 ax=ax)
 
     def _plot_bscan_positions(
         self,