[MRG] Array Annotations (improved)

doc/source/core.rst

@@ -251,3 +251,23 @@ limitations on the data types of annotations: they must be "simple" types or
 containers (lists, dicts, tuples, NumPy arrays) of simple types, where the simple types
 are ``integer``, ``float``, ``complex``, ``Quantity``, ``string``, ``date``, ``time`` and
 ``datetime``.
+
+Array Annotations
+-----------------
+
+Next to "regular" annotations there is also a way to annotate arrays of values
+in order to create annotations with one value per data point. Using this feature,
+called Array Annotations, the consistency of those annotations with the actual data
+is ensured.
+Apart from adding those on object construction, Array Annotations can also be added
+using the :meth:`array_annotate` method provided by all Neo data objects, e.g.::
+
+    >>> sptr = SpikeTrain(times=[1, 2, 3]*pq.s, t_stop=3*pq.s)
+    >>> sptr.array_annotate(index=[0, 1, 2], relevant=[True, False, True])
+    >>> print(sptr.array_annotations)
+    {'index': array([0, 1, 2]), 'relevant': array([ True, False,  True])}
+
+Since Array Annotations may be written to a file or database, there are some
+limitations on the data types of arrays: they must be 1-dimensional (i.e. not nested)
+and contain the same types as annotations:
+ ``integer``, ``float``, ``complex``, ``Quantity``, ``string``, ``date``, ``time`` and ``datetime``.

neo/core/analogsignal.py

@@ -27,6 +27,7 @@
 import quantities as pq
 
 from neo.core.baseneo import BaseNeo, MergeError, merge_annotations
+from neo.core.dataobject import DataObject
 from neo.core.channelindex import ChannelIndex
 from copy import copy, deepcopy
 
@@ -54,14 +55,17 @@ def _get_sampling_rate(sampling_rate, sampling_period):
 
 def _new_AnalogSignalArray(cls, signal, units=None, dtype=None, copy=True, t_start=0 * pq.s,
                            sampling_rate=None, sampling_period=None, name=None, file_origin=None,
-                           description=None, annotations=None, channel_index=None, segment=None):
+                           description=None, array_annotations=None, annotations=None,
+                           channel_index=None, segment=None):
     '''
     A function to map AnalogSignal.__new__ to function that
         does not do the unit checking. This is needed for pickle to work.
     '''
-    obj = cls(signal=signal, units=units, dtype=dtype, copy=copy, t_start=t_start,
-              sampling_rate=sampling_rate, sampling_period=sampling_period, name=name,
-              file_origin=file_origin, description=description, **annotations)
+    obj = cls(signal=signal, units=units, dtype=dtype, copy=copy,
+              t_start=t_start, sampling_rate=sampling_rate,
+              sampling_period=sampling_period, name=name,
+              file_origin=file_origin, description=description,
+              array_annotations=array_annotations, **annotations)
     obj.channel_index = channel_index
     obj.segment = segment
     return obj
@@ -116,6 +120,8 @@ class AnalogSignal(BaseSignal):
     *Optional attributes/properties*:
         :dtype: (numpy dtype or str) Override the dtype of the signal array.
         :copy: (bool) True by default.
+        :array_annotations: (dict) Dict mapping strings to numpy arrays containing annotations \
+                                   for all data points
 
     Note: Any other additional arguments are assumed to be user-specific
     metadata and stored in :attr:`annotations`.
@@ -161,7 +167,7 @@ class AnalogSignal(BaseSignal):
 
     def __new__(cls, signal, units=None, dtype=None, copy=True, t_start=0 * pq.s,
                 sampling_rate=None, sampling_period=None, name=None, file_origin=None,
-                description=None, **annotations):
+                description=None, array_annotations=None, **annotations):
         '''
         Constructs new :class:`AnalogSignal` from data.
 
@@ -188,7 +194,7 @@ def __new__(cls, signal, units=None, dtype=None, copy=True, t_start=0 * pq.s,
 
     def __init__(self, signal, units=None, dtype=None, copy=True, t_start=0 * pq.s,
                  sampling_rate=None, sampling_period=None, name=None, file_origin=None,
-                 description=None, **annotations):
+                 description=None, array_annotations=None, **annotations):
         '''
         Initializes a newly constructed :class:`AnalogSignal` instance.
         '''
@@ -199,8 +205,8 @@ def __init__(self, signal, units=None, dtype=None, copy=True, t_start=0 * pq.s,
 
         # Calls parent __init__, which grabs universally recommended
         # attributes and sets up self.annotations
-        BaseNeo.__init__(self, name=name, file_origin=file_origin, description=description,
-                         **annotations)
+        DataObject.__init__(self, name=name, file_origin=file_origin, description=description,
+                            array_annotations=array_annotations, **annotations)
 
     def __reduce__(self):
         '''
@@ -210,8 +216,8 @@ def __reduce__(self):
         return _new_AnalogSignalArray, (self.__class__, np.array(self), self.units, self.dtype,
                                         True, self.t_start, self.sampling_rate,
                                         self.sampling_period, self.name, self.file_origin,
-                                        self.description, self.annotations, self.channel_index,
-                                        self.segment)
+                                        self.description, self.array_annotations,
+                                        self.annotations, self.channel_index, self.segment)
 
     def _array_finalize_spec(self, obj):
         '''
@@ -286,10 +292,12 @@ def __getitem__(self, i):
                     obj = obj.reshape(-1, 1)
                 if self.channel_index:
                     obj.channel_index = self.channel_index.__getitem__(k)
+                obj.array_annotate(**deepcopy(self.array_annotations_at_index(k)))
         elif isinstance(i, slice):
             obj = super(AnalogSignal, self).__getitem__(i)
             if i.start:
                 obj.t_start = self.t_start + i.start * self.sampling_period
+            obj.array_annotations = deepcopy(self.array_annotations)
         elif isinstance(i, np.ndarray):
             # Indexing of an AnalogSignal is only consistent if the resulting number of
             # samples is the same for each trace. The time axis for these samples is not
@@ -485,6 +493,13 @@ def time_slice(self, t_start, t_stop):
         # we're going to send the list of indicies so that we get *copy* of the
         # sliced data
         obj = super(AnalogSignal, self).__getitem__(np.arange(i, j, 1))
+
+        # If there is any data remaining, there will be data for every channel
+        # In this case, array_annotations need to stay available
+        # super.__getitem__ cannot do this, so it needs to be done here
+        if len(obj) > 0:
+            obj.array_annotations = self.array_annotations
+
         obj.t_start = self.t_start + i * self.sampling_period
 
         return obj

neo/core/basesignal.py

@@ -10,36 +10,38 @@
 http://docs.scipy.org/doc/numpy/user/basics.subclassing.html
 
 In brief:
-* Constructor :meth:`__new__` for :class:`BaseSignal` doesn't exist. 
+* Constructor :meth:`__new__` for :class:`BaseSignal` doesn't exist.
 Only child objects :class:`AnalogSignal` and :class:`IrregularlySampledSignal`
 can be created.
 '''
 
 # needed for Python 3 compatibility
 from __future__ import absolute_import, division, print_function
 
+import copy
 import logging
 
 import numpy as np
 import quantities as pq
 
 from neo.core.baseneo import BaseNeo, MergeError, merge_annotations
+from neo.core.dataobject import DataObject, ArrayDict
 from neo.core.channelindex import ChannelIndex
 
 logger = logging.getLogger("Neo")
 
 
-class BaseSignal(BaseNeo, pq.Quantity):
+class BaseSignal(DataObject):
     '''
     This is the base class from which all signal objects inherit:
     :class:`AnalogSignal` and :class:`IrregularlySampledSignal`.
 
     This class contains all common methods of both child classes.
     It uses the following child class attributes:
 
-        :_necessary_attrs: a list of the attributes that the class must have. 
+        :_necessary_attrs: a list of the attributes that the class must have.
 
-        :_recommended_attrs: a list of the attributes that the class may 
+        :_recommended_attrs: a list of the attributes that the class may
         optionally have.
     '''
 
@@ -58,16 +60,21 @@ def __array_finalize__(self, obj):
 
         User-specified values are only relevant for construction from
         constructor, and these are set in __new__ in the child object.
-        Then they are just copied over here. Default values for the 
+        Then they are just copied over here. Default values for the
         specific attributes for subclasses (:class:`AnalogSignal`
-        and :class:`IrregularlySampledSignal`) are set in 
+        and :class:`IrregularlySampledSignal`) are set in
         :meth:`_array_finalize_spec`
         '''
         super(BaseSignal, self).__array_finalize__(obj)
         self._array_finalize_spec(obj)
 
         # The additional arguments
         self.annotations = getattr(obj, 'annotations', {})
+        # Add empty array annotations, because they cannot always be copied,
+        # but do not overwrite existing ones from slicing etc.
+        # This ensures the attribute exists
+        if not hasattr(self, 'array_annotations'):
+            self.array_annotations = ArrayDict(self._get_arr_ann_length())
 
         # Globally recommended attributes
         self.name = getattr(obj, 'name', None)
@@ -83,18 +90,24 @@ def _rescale(self, signal, units=None):
         '''
         Check that units are present, and rescale the signal if necessary.
         This is called whenever a new signal is
-        created from the constructor. See :meth:`__new__' in  
+        created from the constructor. See :meth:`__new__' in
         :class:`AnalogSignal` and :class:`IrregularlySampledSignal`
         '''
         if units is None:
             if not hasattr(signal, "units"):
                 raise ValueError("Units must be specified")
         elif isinstance(signal, pq.Quantity):
-            # could improve this test, what if units is a string?
-            if units != signal.units:
+            # This test always returns True, i.e. rescaling is always executed if one of the units
+            # is a pq.CompoundUnit. This is fine because rescaling is correct anyway.
+            if pq.quantity.validate_dimensionality(units) != signal.dimensionality:
                 signal = signal.rescale(units)
         return signal
 
+    def rescale(self, units):
+        obj = super(BaseSignal, self).rescale(units)
+        obj.channel_index = self.channel_index
+        return obj
+
     def __getslice__(self, i, j):
         '''
         Get a slice from :attr:`i` to :attr:`j`.attr[0]
@@ -118,6 +131,9 @@ def _apply_operator(self, other, op, *args):
         f = getattr(super(BaseSignal, self), op)
         new_signal = f(other, *args)
         new_signal._copy_data_complement(self)
+        # _copy_data_complement can't always copy array annotations,
+        # so this needs to be done locally
+        new_signal.array_annotations = copy.deepcopy(self.array_annotations)
         return new_signal
 
     def _get_required_attributes(self, signal, units):
@@ -133,49 +149,32 @@ def _get_required_attributes(self, signal, units):
         required_attributes['units'] = units
         return required_attributes
 
-    def rescale(self, units):
-        '''
-        Return a copy of the signal converted to the specified
-        units
-        '''
-        to_dims = pq.quantity.validate_dimensionality(units)
-        if self.dimensionality == to_dims:
-            to_u = self.units
-            signal = np.array(self)
-        else:
-            to_u = pq.Quantity(1.0, to_dims)
-            from_u = pq.Quantity(1.0, self.dimensionality)
-            try:
-                cf = pq.quantity.get_conversion_factor(from_u, to_u)
-            except AssertionError:
-                raise ValueError('Unable to convert between units of "%s" \
-                                 and "%s"' % (from_u._dimensionality,
-                                              to_u._dimensionality))
-            signal = cf * self.magnitude
-        required_attributes = self._get_required_attributes(signal, to_u)
-        new = self.__class__(**required_attributes)
-        new._copy_data_complement(self)
-        new.channel_index = self.channel_index
-        new.segment = self.segment
-        new.annotations.update(self.annotations)
-        return new
-
-    def duplicate_with_new_array(self, signal):
+    def duplicate_with_new_data(self, signal, units=None):
         '''
         Create a new signal with the same metadata but different data.
         Required attributes of the signal are used.
+        Note: Array annotations can not be copied here because length of data can change
         '''
+        if units is None:
+            units = self.units
+        # else:
+        #     units = pq.quantity.validate_dimensionality(units)
+
         # signal is the new signal
-        required_attributes = self._get_required_attributes(signal, self.units)
+        required_attributes = self._get_required_attributes(signal, units)
         new = self.__class__(**required_attributes)
         new._copy_data_complement(self)
         new.annotations.update(self.annotations)
+        # Note: Array annotations are not copied here, because it is not ensured
+        # that the same number of signals is used and they would possibly make no sense
+        # when combined with another signal
         return new
 
     def _copy_data_complement(self, other):
         '''
         Copy the metadata from another signal.
         Required and recommended attributes of the signal are used.
+        Note: Array annotations can not be copied here because length of data can change
         '''
         all_attr = {self._recommended_attrs, self._necessary_attrs}
         for sub_at in all_attr:
@@ -184,6 +183,9 @@ def _copy_data_complement(self, other):
                     setattr(self, attr[0], getattr(other, attr[0], None))
         setattr(self, 'annotations', getattr(other, 'annotations', None))
 
+        # Note: Array annotations cannot be copied because length of data can be changed  # here
+        #  which would cause inconsistencies
+
     def __rsub__(self, other, *args):
         '''
         Backwards subtraction (other-self)
@@ -223,23 +225,6 @@ def __div__(self, other, *args):
     __radd__ = __add__
     __rmul__ = __sub__
 
-    def as_array(self, units=None):
-        """
-        Return the signal as a plain NumPy array.
-
-        If `units` is specified, first rescale to those units.
-        """
-        if units:
-            return self.rescale(units).magnitude
-        else:
-            return self.magnitude
-
-    def as_quantity(self):
-        """
-        Return the signal as a quantities array.
-        """
-        return self.view(pq.Quantity)
-
     def merge(self, other):
         '''
         Merge another signal into this one.
@@ -279,26 +264,25 @@ def merge(self, other):
                 kwargs[name] = attr_self
             else:
                 kwargs[name] = "merge(%s, %s)" % (attr_self, attr_other)
-        merged_annotations = merge_annotations(self.annotations,
-                                               other.annotations)
+        merged_annotations = merge_annotations(self.annotations, other.annotations)
         kwargs.update(merged_annotations)
-        signal = self.__class__(stack, units=self.units, dtype=self.dtype,
-                                copy=False, t_start=self.t_start,
-                                sampling_rate=self.sampling_rate,
-                                **kwargs)
+
+        kwargs['array_annotations'] = self._merge_array_annotations(other)
+
+        signal = self.__class__(stack, units=self.units, dtype=self.dtype, copy=False,
+                                t_start=self.t_start, sampling_rate=self.sampling_rate, **kwargs)
         signal.segment = self.segment
 
         if hasattr(self, "lazy_shape"):
             signal.lazy_shape = merged_lazy_shape
 
         # merge channel_index (move to ChannelIndex.merge()?)
         if self.channel_index and other.channel_index:
-            signal.channel_index = ChannelIndex(
-                index=np.arange(signal.shape[1]),
-                channel_ids=np.hstack([self.channel_index.channel_ids,
-                                       other.channel_index.channel_ids]),
-                channel_names=np.hstack([self.channel_index.channel_names,
-                                         other.channel_index.channel_names]))
+            signal.channel_index = ChannelIndex(index=np.arange(signal.shape[1]),
+                channel_ids=np.hstack(
+                    [self.channel_index.channel_ids, other.channel_index.channel_ids]),
+                channel_names=np.hstack(
+                    [self.channel_index.channel_names, other.channel_index.channel_names]))
         else:
             signal.channel_index = ChannelIndex(index=np.arange(signal.shape[1]))