Merge pull request #2835 from nem1234/RELEASE_next_minor

Bugfix around partly swept frame

doc/user_guide/io.rst

@@ -1762,6 +1762,13 @@ Extra loading arguments
   dataset, it can be integer or a tuple of length 2 to define ``x`` and ``y``
   separetely and it must be a mutiple of the size of the navigation dimension.
   (default 1).
+- ``only_valid_data`` : for ``pts`` file only, ignore incomplete and partly
+  acquired last frame, which typically occurs when the acquisition was
+  interrupted. When loading incomplete data (``only_valid_data=False``),
+  the missing data are filled with zeros. If ``sum_frames=True``, this argument
+  will be ignored to enforce consistent sum over the mapped area. 
+  (default True).
+
 
 Example of loading data downsampled, and with energy range cropped with the
 original navigation dimension 512 x 512 and the EDS range 40 keV over 4096

hyperspy/io_plugins/jeol.py

@@ -184,7 +184,8 @@ def read_img(filename, scale=None, **kwargs):
 
 
 def read_pts(filename, scale=None, rebin_energy=1, sum_frames=True,
-             SI_dtype=np.uint8, cutoff_at_kV=None, downsample=1, **kwargs):
+             SI_dtype=np.uint8, cutoff_at_kV=None, downsample=1,
+             only_valid_data=True, **kwargs):
     fd = open(filename, "br")
     file_magic = np.fromfile(fd, "<I", 1)[0]
 
@@ -292,9 +293,11 @@ def check_multiple(factor, number, string):
         pixel_time = meas_data_header["Doc"]["DwellTime(msec)"]
 
         data_shape = [height, width, channel_number]
+        sweep = meas_data_header["Doc"]["Sweep"]
         if not sum_frames:
-            sweep = meas_data_header["Doc"]["Sweep"]
             data_shape.insert(0, sweep)
+            if not only_valid_data:
+                data_shape[0] += 1 # for partly swept frame
             axes.insert(0, {
                 "index_in_array": 0,
                 "name": "Frame",
@@ -306,9 +309,19 @@ def check_multiple(factor, number, string):
 
         data = np.zeros(data_shape, dtype=SI_dtype)
         datacube_reader = readcube if sum_frames else readcube_frames
-        data = datacube_reader(rawdata, data, rebin_energy, channel_number,
+        data, swept = datacube_reader(rawdata, data, sweep, only_valid_data,
+                                      rebin_energy, channel_number,
                                width_norm, height_norm, np.iinfo(SI_dtype).max)
 
+        if not sum_frames and not only_valid_data:
+            if  (sweep == swept):
+                data = data[:sweep]
+            else:
+                axes[0]["size"] = swept
+
+        if sweep < swept and only_valid_data:
+            _logger.info("The last frame (sweep) is incomplete because the acquisition stopped during this frame. The partially acquired frame is ignored. Use 'sum_frames=False, only_valid_data=False' to read all frames individually, including the last partially completed frame.")
+
         hv = meas_data_header["MeasCond"]["AccKV"]
         if hv <= 30.0:
             mode = "SEM"
@@ -439,13 +452,21 @@ def parsejeol(fd):
 
 
 @numba.njit(cache=True)
-def readcube(rawdata, hypermap, rebin_energy, channel_number, width_norm,
-             height_norm, max_value):  # pragma: no cover
+def readcube(rawdata, hypermap, sweep, only_valid_data, rebin_energy, 
+            channel_number, width_norm, height_norm, max_value):  # pragma: no cover
+    frame_idx = 0
+    previous_y = 0
     for value in rawdata:
         if value >= 32768 and value < 36864:
             x = int((value - 32768) / width_norm)
         elif value >= 36864 and value < 40960:
             y = int((value - 36864) / height_norm)
+            if y < previous_y:
+                frame_idx += 1
+                if frame_idx >= sweep:
+                    # skip incomplete_frame
+                    break 
+            previous_y = y
         elif value >= 45056 and value < 49152:
             z = int((value - 45056) / rebin_energy)
             if z < channel_number:
@@ -454,11 +475,11 @@ def readcube(rawdata, hypermap, rebin_energy, channel_number, width_norm,
                     raise ValueError("The range of the dtype is too small, "
                                      "use `SI_dtype` to set a dtype with "
                                      "higher range.")
-    return hypermap
+    return hypermap, frame_idx + 1
 
 
 @numba.njit(cache=True)
-def readcube_frames(rawdata, hypermap, rebin_energy, channel_number,
+def readcube_frames(rawdata, hypermap, sweep, only_valid_data, rebin_energy, channel_number,
              width_norm, height_norm, max_value):  # pragma: no cover
     """
     We need to create a separate function, because numba.njit doesn't play well
@@ -473,6 +494,11 @@ def readcube_frames(rawdata, hypermap, rebin_energy, channel_number,
             y = int((value - 36864) / height_norm)
             if y < previous_y:
                 frame_idx += 1
+                if frame_idx == sweep: # incomplete frame exist
+                    if only_valid_data:
+                        break  # ignore
+                if frame_idx > sweep:
+                    raise ValueError("The frame number is too large")
             previous_y = y
         elif value >= 45056 and value < 49152:
             z = int((value - 45056) / rebin_energy)
@@ -482,7 +508,7 @@ def readcube_frames(rawdata, hypermap, rebin_energy, channel_number,
                     raise ValueError("The range of the dtype is too small, "
                                      "use `SI_dtype` to set a dtype with "
                                      "higher range.")
-    return hypermap
+    return hypermap, frame_idx + 1
 
 
 def read_eds(filename, **kwargs):