Merge branch 'development'

README.md

@@ -43,7 +43,7 @@ The main goals for this library are:
 * filter a subset of channels from original mdf file
 * cut measurement to specified time interval
 * convert to different mdf version
-* export to pandas, Excel, HDF5, Matlab (v4, v5 and v7.3), CSV and parquet
+* export to HDF5, Matlab (v4, v5 and v7.3), CSV and parquet
 * merge multiple files sharing the same internal structure
 * read and save mdf version 4.10 files containing zipped data blocks
 * space optimizations for saved files (no duplicated blocks)
@@ -167,7 +167,6 @@ asammdf uses the following libraries
 optional dependencies needed for exports
 
 * h5py : for HDF5 export
-* xlsxwriter : for Excel export
 * scipy : for Matlab v4 and v5 .mat export
 * hdf5storage : for Matlab v7.3 .mat export
 * fastparquet : for parquet export

appveyor.yml

@@ -25,7 +25,6 @@ install:
   - cmd: "%PYTHON%\\python.exe -m pip install --upgrade pip"
   - cmd: "%PYTHON%\\python.exe -m pip install --upgrade setuptools"
   - cmd: "%PYTHON%\\python.exe -m pip install wheel"
-  - cmd: "%PYTHON%\\python.exe -m pip install https://github.com/ebroecker/canmatrix/archive/development.zip"
   - cmd: "%PYTHON%\\python.exe -m pip install -r pip_requirements_tests.txt"
   - cmd: "if \"%PY%\" == \"37-x64\" (%PYTHON%\\python.exe -m pip install PyQt5 h5py xlsxwriter scipy hdf5storage fastparquet cChardet psutil lxml)"
   - cmd: "if \"%PY%\" == \"37-x64\" (%PYTHON%\\python.exe -m pip install https://github.com/pyinstaller/pyinstaller/archive/develop.zip)"
@@ -38,7 +37,6 @@ install:
   - sh: "sudo apt-get --yes install python3-dev python3-pip python3-setuptools"
   - sh: "python3 -m pip install --upgrade pip --user"
   - sh: "python3 -m pip install --upgrade setuptools --user"
-  - sh: "python3 -m pip install https://github.com/ebroecker/canmatrix/archive/development.zip --user"
   - sh: "python3 -m pip install wheel --user"
   - sh: "python3 -m pip install -r pip_requirements_tests.txt --user"
   - sh: "python3 -m pip install PyQt5 h5py xlsxwriter scipy hdf5storage fastparquet cChardet psutil --user"

asammdf/blocks/mdf_v4.py

@@ -703,23 +703,18 @@ def _read_channels(
             if self._remove_source_from_channel_names:
                 channel.name = channel.name.split(path_separator)[0]
 
-            value = channel
-            display_name = channel.display_name
-            name = channel.name
-
             entry = (dg_cntr, ch_cntr)
             self._ch_map[ch_addr] = entry
 
-            channels.append(value)
+            channels.append(channel)
             if channel_composition:
                 composition.append(entry)
 
-            self.channels_db.add(display_name, entry)
-            self.channels_db.add(name, entry)
+            self.channels_db.add(channel.display_name, entry)
+            self.channels_db.add(channel.name, entry)
 
             # signal data
-            address = channel.data_block_addr
-            grp.signal_data.append(address)
+            grp.signal_data.append(channel.data_block_addr)
 
             if channel.channel_type in MASTER_CHANNELS:
                 self.masters_db[dg_cntr] = ch_cntr

asammdf/blocks/v4_blocks.py

@@ -2599,6 +2599,8 @@ def convert(self, values):
             nr = self.val_param_nr
             raw_vals = [self[f"val_{i}"] for i in range(nr)]
 
+            ret = np.array([None]*len(values), dtype='O')
+
             phys = []
             for i in range(nr):
                 try:
@@ -2620,25 +2622,46 @@ def convert(self, values):
                 except TypeError:
                     default = b""
 
-            new_values = []
-            for i, val in enumerate(values):
-                try:
-                    idx = raw_vals.index(val)
-                    item = phys[idx]
-                except ValueError:
-                    item = default
+            x = sorted(zip(raw_vals, phys))
+            raw_vals = np.array(
+                [e[0] for e in x], dtype='<i8'
+            )
+            phys = [e[1] for e in x]
 
-                if isinstance(item, bytes):
-                    new_values.append(item)
-                else:
-                    new_values.append(item.convert(values[i: i+1])[0])
+            idx1 = np.searchsorted(raw_vals, values, side="right") - 1
+            idx2 = np.searchsorted(raw_vals, values, side="left")
+
+            idx = np.argwhere(idx1 != idx2).flatten()
 
-            if all(isinstance(v, bytes) for v in new_values):
-                values = np.array(new_values)
+            if isinstance(default, bytes):
+                ret[idx] = default
             else:
-                values = np.array(
-                    [np.nan if isinstance(v, bytes) else v for v in new_values]
-                )
+                ret[idx] = default.convert(values[idx])
+
+            idx = np.argwhere(idx1 == idx2).flatten()
+            if len(idx):
+                indexes = idx1[idx]
+                unique = np.unique(indexes)
+                for val in unique:
+
+                    item = phys[val]
+                    idx_ = np.argwhere(indexes == val).flatten()
+                    if isinstance(item, bytes):
+                        ret[idx[idx_]] = item
+                    else:
+                        ret[idx[idx_]] = item.convert(values[idx[idx_]])
+
+            if all(isinstance(v, bytes) for v in ret):
+                ret = ret.astype(bytes)
+            else:
+                try:
+                    ret = ret.astype('<f8')
+                except:
+                    ret = np.array(
+                        [np.nan if isinstance(v, bytes) else v for v in ret]
+                    )
+
+            values = ret
 
         elif conversion_type == v4c.CONVERSION_TYPE_RTABX:
             nr = self.val_param_nr // 2
@@ -2664,52 +2687,55 @@ def convert(self, values):
                 except TypeError:
                     default = b""
 
-            lower = np.array([self[f"lower_{i}"] for i in range(nr)])
-            upper = np.array([self[f"upper_{i}"] for i in range(nr)])
+            lower = [self[f"lower_{i}"] for i in range(nr)]
+            upper = [self[f"upper_{i}"] for i in range(nr)]
+
+            x = sorted(zip(lower, upper, phys))
+            lower = np.array(
+                [e[0] for e in x], dtype='<i8'
+            )
+            upper = np.array(
+                [e[1] for e in x], dtype='<i8'
+            )
+            phys = [e[2] for e in x]
 
-            all_values = phys + [default]
+            ret = np.array([None]*len(values), dtype='O')
 
             idx1 = np.searchsorted(lower, values, side="right") - 1
             idx2 = np.searchsorted(upper, values, side="left")
 
-            idx_ne = np.nonzero(idx1 != idx2)[0]
-            idx_eq = np.nonzero(idx1 == idx2)[0]
+            idx_ne = np.argwhere(idx1 != idx2).flatten()
+            idx_eq = np.argwhere(idx1 == idx2).flatten()
 
-            if all(isinstance(val, bytes) for val in all_values):
-                phys = np.array(phys)
-                all_values = np.array(all_values)
+            if isinstance(default, bytes):
+                ret[idx_ne] = default
+            else:
+                ret[idx_ne] = default.convert(values[idx_ne])
 
-                new_values = np.zeros(len(values), dtype=all_values.dtype)
 
-                if len(idx_ne):
-                    new_values[idx_ne] = default
-                if len(idx_eq):
-                    new_values[idx_eq] = phys[idx1[idx_eq]]
+            if len(idx_eq):
+                indexes = idx1[idx_eq]
+                unique = np.unique(indexes)
+                for val in unique:
 
-                values = new_values
-            else:
-                new_values = []
-                for i, val in enumerate(values):
-                    if i in idx_ne:
-                        item = default
-                    else:
-                        item = phys[idx1[i]]
+                    item = phys[val]
+                    idx_ = np.argwhere(indexes == val).flatten()
 
                     if isinstance(item, bytes):
-                        new_values.append(item)
+                        ret[idx_eq[idx_]] = item
                     else:
-                        try:
-                            new_values.append(item.convert(values[i : i + 1])[0])
-                        except:
-                            print(item, default, phys)
-                            1/0
-
-                if all(isinstance(v, bytes) for v in new_values):
-                    values = np.array(new_values)
-                else:
-                    values = np.array(
-                        [np.nan if isinstance(v, bytes) else v for v in new_values]
+                        ret[idx_eq[idx_]] = item.convert(values[idx_eq[idx_]])
+
+            if all(isinstance(v, bytes) for v in ret):
+                ret = ret.astype(bytes)
+            else:
+                try:
+                    ret = ret.astype('<f8')
+                except:
+                    ret = np.array(
+                        [np.nan if isinstance(v, bytes) else v for v in ret]
                     )
+            values = ret
 
         elif conversion_type == v4c.CONVERSION_TYPE_TTAB:
             nr = self.val_param_nr - 1