Merge e6b9320 into 8f164ed

holoviews/core/dimension.py

@@ -6,17 +6,12 @@
 import re
 from operator import itemgetter
 
-try:
-    from cyordereddict import OrderedDict
-except:
-    from collections import OrderedDict
-
 import numpy as np
 import param
 
 from ..core.util import (basestring, sanitize_identifier,
                          group_sanitizer, label_sanitizer, max_range,
-                         find_range, dimension_sanitizer)
+                         find_range, dimension_sanitizer, OrderedDict)
 from .options import Store, StoreOptions
 from .pprint import PrettyPrinter
 

holoviews/core/element.py

@@ -284,13 +284,17 @@ def __init__(self, data=None, **params):
         if isinstance(data, list) and all(np.isscalar(el) for el in data):
             data = (((k,), (v,)) for k, v in enumerate(data))
 
-        if not isinstance(data, NdElement) and isinstance(data, Element):
-            mapping = data.mapping()
+        if isinstance(data, Element):
+            params = dict(get_param_values(data), **params)
+            if isinstance(data, NdElement):
+                mapping = data.mapping()
+                data = mapping.data
+            else:
+                data = data.data
             if 'kdims' not in params:
                 params['kdims'] = mapping.kdims
             if 'vdims' not in params:
                 params['vdims'] = mapping.vdims
-            data = mapping.data
 
         kdims = params.get('kdims', self.kdims)
         if (data is not None and not isinstance(data, NdMapping)
@@ -348,8 +352,11 @@ def reindex(self, kdims=None, vdims=None, force=False):
 
 
     def _add_item(self, key, value, sort=True, update=True):
-        value = (value,) if np.isscalar(value) else tuple(value)
-        if len(value) != len(self.vdims):
+        if np.isscalar(value):
+            value = (value,)
+        elif not isinstance(value, NdElement):
+            value = tuple(value)
+        if len(value) != len(self.vdims) and not isinstance(value, NdElement):
             raise ValueError("%s values must match value dimensions"
                              % type(self).__name__)
         super(NdElement, self)._add_item(key, value, sort, update)
@@ -458,14 +465,6 @@ def sample(self, samples=[]):
         return self.clone(sample_data)
 
 
-    def _item_check(self, dim_vals, data):
-        if isinstance(data, tuple):
-            for el in data:
-                self._item_check(dim_vals, el)
-            return
-        super(NdElement, self)._item_check(dim_vals, data)
-
-
     def aggregate(self, dimensions, function, **kwargs):
         """
         Allows aggregating.
@@ -474,7 +473,7 @@ def aggregate(self, dimensions, function, **kwargs):
         grouped = self.groupby(dimensions) if len(dimensions) else HoloMap({(): self}, kdims=[])
         for k, group in grouped.data.items():
             reduced = []
-            for vdim in group.vdims:
+            for vdim in self.vdims:
                 data = group[vdim.name]
                 if isinstance(function, np.ufunc):
                     reduced.append(function.reduce(data, **kwargs))

holoviews/core/ndmapping.py

@@ -11,7 +11,7 @@
 
 import param
 
-from . import traversal
+from . import traversal, util
 from .dimension import OrderedDict, Dimension, Dimensioned, ViewableElement
 from .util import unique_iterator, sanitize_identifier, dimension_sort, group_select, iterative_select, basestring, wrap_tuple, process_ellipses
 
@@ -275,22 +275,13 @@ def groupby(self, dimensions, container_type=None, group_type=None, **kwargs):
         if self.ndims == 1:
             self.warning('Cannot split Map with only one dimension.')
             return self
-
-        dimensions = [self.get_dimension(d).name for d in dimensions]
         container_type = container_type if container_type else type(self)
         group_type = group_type if group_type else type(self)
-        dims, inds = zip(*((self.get_dimension(dim), self.get_dimension_index(dim))
-                         for dim in dimensions))
-        inames, idims = zip(*((dim.name, dim) for dim in self.kdims
-                              if not dim.name in dimensions))
-        selects = unique_iterator(itemgetter(*inds)(key) if len(inds) > 1 else (key[inds[0]],)
-                                  for key in self.data.keys())
+        dimensions = [self.get_dimension(d) for d in dimensions]
+        sort = not self._sorted
         with item_check(False):
-            selects = group_select(list(selects))
-            groups = [(k, group_type((v.reindex(inames) if isinstance(v, NdMapping)
-                                      else [((), (v,))]), **kwargs))
-                      for k, v in iterative_select(self, dimensions, selects)]
-            return container_type(groups, kdims=dims)
+            return util.ndmapping_groupby(self, dimensions, container_type,
+                                          group_type, sort=sort, **kwargs)
 
 
     def add_dimension(self, dimension, dim_pos, dim_val, vdim=False, **kwargs):

holoviews/core/util.py

@@ -8,6 +8,11 @@
 import numpy as np
 import param
 
+try:
+    from cyordereddict import OrderedDict
+except:
+    from collections import OrderedDict
+
 try:
     import pandas as pd
 except ImportError:
@@ -710,3 +715,76 @@ def get_param_values(data):
 def wrap_tuple(unwrapped):
     """ Wraps any non-tuple types in a tuple """
     return (unwrapped if isinstance(unwrapped, tuple) else (unwrapped,))
+
+
+def get_unique_keys(ndmapping, dimensions):
+    inds = [ndmapping.get_dimension_index(dim) for dim in dimensions]
+    getter = operator.itemgetter(*inds)
+    return unique_iterator(getter(key) if len(inds) > 1 else (key[inds[0]],)
+                           for key in ndmapping.data.keys())
+
+
+def unpack_group(group, getter):
+    for k, v in group.iterrows():
+        obj = v.values[0]
+        key = getter(k)
+        if hasattr(obj, 'kdims'):
+            yield (key, obj)
+        else:
+            obj = tuple(v)
+            yield (wrap_tuple(key), obj)
+
+
+
+
+class ndmapping_groupby(param.ParameterizedFunction):
+    """
+    Apply a groupby operation to an NdMapping, using pandas to improve
+    performance (if available).
+    """
+
+    def __call__(self, ndmapping, dimensions, container_type,
+                 group_type, sort=False, **kwargs):
+        try:
+            import pandas
+            groupby = self.groupby_pandas
+        except:
+            groupby = self.groupby_python
+        return groupby(ndmapping, dimensions, container_type,
+                       group_type, sort=sort, **kwargs)
+
+    @param.parameterized.bothmethod
+    def groupby_pandas(self_or_cls, ndmapping, dimensions, container_type,
+                       group_type, sort=False, **kwargs):
+        if 'kdims' in kwargs:
+            idims = [ndmapping.get_dimension(d) for d in kwargs['kdims']]
+        else:
+            idims = [dim for dim in ndmapping.kdims if dim not in dimensions]
+
+        all_dims = [d.name for d in ndmapping.kdims]
+        inds = [ndmapping.get_dimension_index(dim) for dim in idims]
+        getter = operator.itemgetter(*inds) if inds else lambda x: tuple()
+
+        multi_index = pd.MultiIndex.from_tuples(ndmapping.keys(), names=all_dims)
+        df = pd.DataFrame(ndmapping.values(), index=multi_index)
+
+        kwargs = dict(dict(get_param_values(ndmapping), kdims=idims), **kwargs)
+        groups = ((wrap_tuple(k), group_type(OrderedDict(unpack_group(group, getter)), **kwargs))
+                   for k, group in df.groupby(level=[d.name for d in dimensions]))
+
+        if sort:
+            selects = list(get_unique_keys(ndmapping, dimensions))
+            groups = sorted(groups, key=lambda x: selects.index(x[0]))
+        return container_type(groups, kdims=dimensions)
+
+    @param.parameterized.bothmethod
+    def groupby_python(self_or_cls, ndmapping, dimensions, container_type,
+                       group_type, sort=False, **kwargs):
+        idims = [dim for dim in ndmapping.kdims if dim not in dimensions]
+        dim_names = [dim.name for dim in dimensions]
+        selects = get_unique_keys(ndmapping, dimensions)
+        selects = group_select(list(selects))
+        groups = [(k, group_type((v.reindex(idims) if hasattr(v, 'kdims')
+                                  else [((), (v,))]), **kwargs))
+                  for k, v in iterative_select(ndmapping, dim_names, selects)]
+        return container_type(groups, kdims=dimensions)