Merge pull request #25 from kouwenhovenlab/master

sync from lab repo.

plottr/data/datadict.py

@@ -10,7 +10,7 @@
 
 import numpy as np
 
-from plottr.utils import num
+from plottr.utils import num, misc
 
 __author__ = 'Wolfgang Pfaff'
 __license__ = 'MIT'
@@ -475,30 +475,9 @@ def reorder_axes_indices(self, name: str,
                  new order.
 
         """
-        # check if the given indices are each unique
-        used = []
-        for n, i in pos.items():
-            if i in used:
-                raise ValueError('Order indices have to be unique.')
-            used.append(i)
-
-        axlist = self[name]['axes']
-        neworder = [None for a in axlist]
-        oldorder = list(range(len(axlist)))
-
-        for n, newidx in pos.items():
-            neworder[newidx] = axlist.index(n)
-
-        for i in neworder:
-            if i in oldorder:
-                del oldorder[oldorder.index(i)]
-
-        for i in range(len(neworder)):
-            if neworder[i] is None:
-                neworder[i] = oldorder[0]
-                del oldorder[0]
-
-        return tuple(neworder), [self[name]['axes'][i] for i in neworder]
+        axlist = self.axes(name)
+        order = misc.reorder_indices_from_new_positions(axlist, **pos)
+        return order, [axlist[i] for i in order]
 
     def reorder_axes(self, data_names: Union[str, List[str], None] = None,
                      **pos: int) -> 'DataDictBase':
@@ -790,20 +769,32 @@ class MeshgridDataDict(DataDictBase):
     the axes can be obtained from np.meshgrid (hence the name of the class).
 
     Example: a simple uniform 3x2 grid might look like this; x and y are the
-    coordinates of the grid, and z is a function of the two:
-
-    x = [[0, 0],
-         [1, 1],
-         [2, 2]]
-
-    y = [[0, 1],
-         [0, 1],
-         [0, 1]]
-
-    z = x * y =
-        [[0, 0],
-         [0, 1],
-         [0, 2]]
+    coordinates of the grid, and z is a function of the two::
+
+        x = [[0, 0],
+             [1, 1],
+             [2, 2]]
+
+        y = [[0, 1],
+             [0, 1],
+             [0, 1]]
+
+        z = x * y =
+            [[0, 0],
+             [0, 1],
+             [0, 2]]
+
+    Note: Internally we will typically assume that the nested axes are
+    ordered from slow to fast, i.e., dimension 1 is the most outer axis, and
+    dimension N of an N-dimensional array the most inner (i.e., the fastest
+    changing one). This guarantees, for example, that the default implementation
+    of np.reshape has the expected outcome. If, for some reason, the specified
+    axes are not in that order (e.g., we might have ``z`` with
+    ``axes = ['x', 'y']``, but ``x`` is the fast axis in the data).
+    In such a case, the guideline is that at creation of the meshgrid, the data
+    should be transposed such that it conforms correctly to the order as given
+    in the ``axis = [...]`` specification of the data.
+    The function ``datadict_to_meshgrid`` provides options for that.
     """
 
     def shape(self) -> Union[None, Tuple[int]]:
@@ -889,10 +880,9 @@ def reorder_axes(self, **pos) -> 'MeshgridDataDict':
 # Tools for converting between different data types
 
 def guess_shape_from_datadict(data: DataDict) -> \
-        Dict[str, Union[Tuple[int], None]]:
+        Dict[str, Union[None, Tuple[List[str], Tuple[int]]]]:
     """
-    Try to guess the shape of the datadict dependents from the unique values of
-    their axes.
+    Try to guess the shape of the datadict dependents from the axes values.
 
     :param data: dataset to examine.
     :return: a dictionary with the dependents as keys, and inferred shapes as
@@ -901,66 +891,70 @@ def guess_shape_from_datadict(data: DataDict) -> \
 
     shapes = {}
     for d in data.dependents():
-        shp = []
-        axes = data.axes(d)
-        for a in axes:
-            # need to make sure we remove invalids before determining unique
-            # vals.
-            cleaned_data = data.data_vals(a)
-            cleaned_data = cleaned_data[cleaned_data != None]
-            try:
-                cleaned_data = cleaned_data[~np.isnan(cleaned_data)]
-            except TypeError:
-                # means it's not float. that's ok.
-                pass
-
-            shp.append(np.unique(cleaned_data).size)
-
-        if np.prod(shp) != data.data_vals(d).size:
-            shapes[d] = None
-        else:
-            shapes[d] = tuple(shp)
+        axnames = data.axes(d)
+        axes = {a: data.data_vals(a) for a in axnames}
+        shapes[d] = num.guess_grid_from_sweep_direction(**axes)
 
     return shapes
 
 
 def datadict_to_meshgrid(data: DataDict,
-                         target_shape: Union[Tuple[int, ...], None] = None) \
+                         target_shape: Union[Tuple[int, ...], None] = None,
+                         inner_axis_order: Union[None, List[str]] = None) \
         -> MeshgridDataDict:
     """
     Try to make a meshgrid from a dataset.
 
     :param data: input DataDict.
     :param target_shape: target shape. if ``None`` we use
                          ``guess_shape_from_datadict`` to infer.
+    :param inner_axis_order: if axes of the datadict are not specified in the
+                             'C' order (1st the slowest, last the fastest axis)
+                             then the 'true' inner order can be specified as
+                             a list of axes names, which has to match the
+                             specified axes in all but order.
+                             The data is then transposed to conform to the
+                             specified order.
     :return: the generated ``MeshgridDataDict``.
     """
-    # TODO: support for cues inside the data set about the shape.
-    # TODO: maybe it could make sense to include a method to sort the
-    #  meshgrid axes.
 
     # if the data is empty, return empty MeshgridData
     if len([k for k, _ in data.data_items()]) == 0:
         return MeshgridDataDict()
 
-    # guess what the shape likely is.
     if not data.axes_are_compatible():
         raise ValueError('Non-compatible axes, cannot grid that.')
 
+    # guess what the shape likely is.
     if target_shape is None:
-        shps = guess_shape_from_datadict(data)
-        if len(set(shps.values())) > 1:
+        shp_specs = guess_shape_from_datadict(data)
+        shps = [shape for (order, shape) in shp_specs.values()]
+        if len(set(shps)) > 1:
             raise ValueError('Cannot determine unique shape for all data.')
 
-        target_shape = list(shps.values())[0]
-        if target_shape is None:
+        ret = list(shp_specs.values())[0]
+        if ret is None:
             raise ValueError('Shape could not be inferred.')
 
+        # the guess-function returns both axis order as well as shape.
+        inner_axis_order, target_shape = ret
+
+    # construct new data
     newdata = MeshgridDataDict(**data.structure(add_shape=False))
+    axlist = data.axes(data.dependents()[0])
+
     for k, v in data.data_items():
-        newdata[k]['values'] = num.array1d_to_meshgrid(v['values'],
-                                                       target_shape,
-                                                       copy=True)
+        vals = num.array1d_to_meshgrid(v['values'], target_shape, copy=True)
+
+        # if an inner axis order is given, we transpose to transform from that
+        # to the specified order.
+        if inner_axis_order is not None:
+            transpose_idxs = misc.reorder_indices(
+                inner_axis_order, axlist)
+            vals = vals.transpose(transpose_idxs)
+
+        newdata[k]['values'] = vals
+
     newdata = newdata.sanitize()
     newdata.validate()
     return newdata

plottr/utils/misc.py

@@ -0,0 +1,56 @@
+"""misc.py
+
+Various utility functions.
+"""
+
+from typing import List, Tuple
+
+
+def reorder_indices(lst: List, target: List) -> Tuple[int]:
+    """
+    Determine how to bring a list with unique entries to a different order.
+
+    Supports only lists of strings.
+
+    :param lst: input list
+    :param target: list in the desired order
+    :return: the indices that will reorder the input to obtain the target.
+    :raises: ``ValueError`` for invalid inputs.
+    """
+    if set([type(i) for i in lst]) != {str}:
+        raise ValueError('Only lists of strings are supported')
+    if len(set(lst)) < len(lst):
+        raise ValueError('Input list elements are not unique.')
+    if set(lst) != set(target) or len(lst) != len(target):
+        raise ValueError('Contents of input and target do not match.')
+
+    idxs = []
+    for elt in target:
+        idxs.append(lst.index(elt))
+
+    return tuple(idxs)
+
+
+def reorder_indices_from_new_positions(lst: List[str], **pos: int) \
+        -> Tuple[int]:
+    """
+    Determine how to bring a list with unique entries to a different order.
+
+    :param lst: input list (of strings)
+    :param pos: new positions in the format ``element = new_position``.
+                non-specified elements will be adjusted automatically.
+    :return: the indices that will reorder the input to obtain the target.
+    :raises: ``ValueError`` for invalid inputs.
+    """
+    if set([type(i) for i in lst]) != {str}:
+        raise ValueError('Only lists of strings are supported')
+    if len(set(lst)) < len(lst):
+        raise ValueError('Input list elements are not unique.')
+
+    target = lst.copy()
+    for item, newidx in pos.items():
+        oldidx = target.index(item)
+        del target[oldidx]
+        target.insert(newidx, item)
+
+    return reorder_indices(lst, target)

plottr/utils/num.py

@@ -2,7 +2,7 @@
 
 Tools for numerical operations.
 """
-from typing import Sequence, Tuple
+from typing import Sequence, Tuple, Union, List
 import numpy as np
 
 
@@ -93,3 +93,101 @@ def array1d_to_meshgrid(arr: Sequence, target_shape: Tuple[int],
         arr = np.append(arr, fill)
 
     return arr.reshape(target_shape)
+
+
+def find_direction_period(vals: np.ndarray, ignore_last: bool = False) \
+        -> Union[None, int]:
+    """
+    Find the period with which the values in an array change direction.
+
+    :param vals: the axes values (1d array)
+    :param ignore_last: if True, we'll ignore the last value when determining
+                        if the period is unique (useful for incomplete data),
+    :return: None if we could not determine a unique period.
+             The period, i.e., the number of elements after which
+             the more common direction is changed.
+    """
+    direction = np.sign(vals[1:] - vals[:-1])
+    ups = np.where(direction == 1)[0]
+    downs = np.where(direction == -1)[0]
+
+    if len(ups) > len(downs):
+        switches = downs
+    else:
+        switches = ups
+
+    if len(switches) == 0:
+        return vals.size
+    elif len(switches) == 1:
+        if switches[0] >= (vals.size / 2.) - 1:
+            return switches[0] + 1
+        else:
+            return None
+
+    if switches[-1] < vals.size - 1:
+        switches = np.append(switches, vals.size-1)
+    periods = (switches[1:] - switches[:-1])
+
+    if ignore_last and periods[-1] < periods[0]:
+        periods = periods[:-1]
+
+    if len(set(periods)) > 1:
+        return None
+    elif len(periods) == 0:
+        return vals.size
+    else:
+        return int(periods[0])
+
+
+def guess_grid_from_sweep_direction(**axes: np.ndarray) \
+        -> Union[None, Tuple[List[str], Tuple[int]]]:
+    """
+    Try to determine order and shape of a set of axes data
+    (such as flattened meshgrid data).
+
+    Analyzes the periodicity (in sweep direction) of the given set of axes
+    values, and use that information to infer the shape of the dataset,
+    and the order of the axes, given from slowest to fastest.
+
+    :param axes: all axes values as keyword args, given as 1d numpy arrays.
+    :return: None, if we cannot infer a shape that makes sense.
+             Sorted list of axes names, and shape tuple for the dataset.
+    :raises: `ValueError` for incorrect input
+    """
+    periods = []
+    names = []
+    size = None
+
+    if len(axes) < 1:
+        raise ValueError("Empty input.")
+
+    for name, vals in axes.items():
+        if len(np.array(vals).shape) > 1:
+            raise ValueError(
+                f"Expect 1-dimensional axis data, not {np.array(vals).shape}")
+        if size is None:
+            size = np.array(vals).size
+        else:
+            if size != np.array(vals).size:
+                raise ValueError("Non-matching array sizes.")
+
+        period = find_direction_period(vals)
+        if period is not None:
+            periods.append(period)
+            names.append(name)
+        else:
+            return None
+
+    order = np.argsort(periods)
+    periods = np.array(periods)[order]
+    names = np.array(names)[order]
+
+    divisor = 1
+    for i, p in enumerate(periods.copy()):
+        periods[i] //= divisor
+        divisor *= int(periods[i])
+
+    if np.prod(periods) != size or divisor != size:
+        return None
+
+    return names[::-1].tolist(), tuple(periods[::-1])