diff --git a/.travis.yml b/.travis.yml index 97cf8a37a2..94b98eefec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: - conda update -q conda # Useful for debugging any issues with conda - conda info -a - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION scipy numpy freetype nose bokeh pandas jupyter ipython=4.2.0 param matplotlib=1.5.1 xarray + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION scipy numpy freetype nose bokeh pandas jupyter ipython=4.2.0 param matplotlib=1.5.1 xarray datashader - source activate test-environment - conda install -c conda-forge -c scitools iris sip=4.18 plotly - if [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py index bbfdc7e949..dc20b0bf6e 100644 --- a/holoviews/core/data/xarray.py +++ b/holoviews/core/data/xarray.py @@ -37,7 +37,17 @@ def init(cls, eltype, data, kdims, vdims): kdim_param = element_params['kdims'] vdim_param = element_params['vdims'] - if not isinstance(data, xr.Dataset): + if isinstance (data, xr.DataArray): + if data.name: + vdim = Dimension(data.name) + elif vdims: + vdim = vdims[0] + elif len(vdim_param.default) == 1: + vdim = vdim_param.default[0] + vdims = [vdim] + kdims = [Dimension(d) for d in data.dims[::-1]] + data = xr.Dataset({vdim.name: data}) + elif not isinstance(data, xr.Dataset): if kdims is None: kdims = kdim_param.default if vdims is None: diff --git a/holoviews/core/operation.py b/holoviews/core/operation.py index 652d9c5112..6262264d11 100644 --- a/holoviews/core/operation.py +++ b/holoviews/core/operation.py @@ -70,80 +70,6 @@ def get_overlay_bounds(cls, overlay): raise ValueError("Extents across the overlay are inconsistent") -class DynamicOperation(Operation): - """ - Dynamically applies an operation to the elements of a HoloMap - or DynamicMap. Will return a DynamicMap wrapping the original - map object, which will lazily evaluate when a key is requested. - The _process method should be overridden in subclasses to apply - a specific operation, DynamicOperation itself applies a no-op, - making the DynamicOperation baseclass useful for converting - existing HoloMaps to a DynamicMap. - """ - - def __call__(self, map_obj, **params): - self.p = param.ParamOverrides(self, params) - callback = self._dynamic_operation(map_obj) - if isinstance(map_obj, DynamicMap): - return map_obj.clone(callback=callback, shared_data=False) - else: - return self._make_dynamic(map_obj, callback) - - - def _process(self, element): - return element - - - def _dynamic_operation(self, map_obj): - """ - Generate function to dynamically apply the operation. - Wraps an existing HoloMap or DynamicMap. - """ - if not isinstance(map_obj, DynamicMap): - def dynamic_operation(*key): - return self._process(map_obj[key]) - return dynamic_operation - - def dynamic_operation(*key): - key = key[0] if map_obj.mode == 'open' else key - _, el = util.get_dynamic_item(map_obj, map_obj.kdims, key) - return self._process(el) - - return dynamic_operation - - - def _make_dynamic(self, hmap, dynamic_fn): - """ - Accepts a HoloMap and a dynamic callback function creating - an equivalent DynamicMap from the HoloMap. - """ - dim_values = zip(*hmap.data.keys()) - params = util.get_param_values(hmap) - kdims = [d(values=list(set(values))) for d, values in - zip(hmap.kdims, dim_values)] - return DynamicMap(dynamic_fn, **dict(params, kdims=kdims)) - - - -class DynamicFunction(DynamicOperation): - """ - Dynamically applies a function to the Elements in a DynamicMap - or HoloMap. Must supply a HoloMap or DynamicMap type and will - return another DynamicMap type, which will apply the supplied - function with the supplied kwargs whenever a value is requested - from the map. - """ - - function = param.Callable(default=lambda x: x, doc=""" - Function to apply to DynamicMap items dynamically.""") - - kwargs = param.Dict(default={}, doc=""" - Keyword arguments passed to the function.""") - - def _process(self, element): - return self.p.function(element, **self.p.kwargs) - - class ElementOperation(Operation): """ @@ -152,6 +78,12 @@ class ElementOperation(Operation): input, a processed holomap is returned as output where the individual elements have been transformed accordingly. An ElementOperation may turn overlays in new elements or vice versa. + + An ElementOperation can be set to be dynamic, which will return a + DynamicMap with a callback that will apply the operation + dynamically. An ElementOperation may also supply a list of Stream + classes on a streams parameter, which can allow dynamic control + over the parameters on the operation. """ dynamic = param.ObjectSelector(default='default', @@ -171,7 +103,6 @@ class ElementOperation(Operation): first component is a Normalization.ranges list and the second component is Normalization.keys. """) - def _process(self, view, key=None): """ Process a single input element and outputs new single element @@ -197,16 +128,19 @@ def __call__(self, element, **params): isinstance(element, DynamicMap)) or self.p.dynamic is True) - if isinstance(element, ViewableElement): - processed = self._process(element) - elif isinstance(element, GridSpace): + if isinstance(element, GridSpace): # Initialize an empty axis layout grid_data = ((pos, self(cell, **params)) for pos, cell in element.items()) processed = GridSpace(grid_data, label=element.label, kdims=element.kdims) elif dynamic: - processed = DynamicFunction(element, function=self, kwargs=params) + from ..util import Dynamic + streams = getattr(self, 'streams', []) + processed = Dynamic(element, streams=streams, + operation=self, kwargs=params) + elif isinstance(element, ViewableElement): + processed = self._process(element) elif isinstance(element, DynamicMap): if any((not d.values) for d in element.kdims): raise ValueError('Applying a non-dynamic operation requires ' diff --git a/holoviews/core/overlay.py b/holoviews/core/overlay.py index 0c2b425323..0a611da9ca 100644 --- a/holoviews/core/overlay.py +++ b/holoviews/core/overlay.py @@ -24,10 +24,10 @@ class Overlayable(object): def __mul__(self, other): if type(other).__name__ == 'DynamicMap': - from .operation import DynamicFunction + from ..util import Dynamic def dynamic_mul(element): return self * element - return DynamicFunction(other, function=dynamic_mul) + return Dynamic(other, operation=dynamic_mul) if isinstance(other, UniformNdMapping) and not isinstance(other, CompositeOverlay): items = [(k, self * v) for (k, v) in other.items()] return other.clone(items) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index 952725f50d..84a1e78ec4 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -204,10 +204,10 @@ def __mul__(self, other): return self.clone(items, kdims=dimensions, label=self._label, group=self._group) elif isinstance(other, self.data_type): if isinstance(self, DynamicMap): - from .operation import DynamicFunction + from ..util import Dynamic def dynamic_mul(element): return element * other - return DynamicFunction(self, function=dynamic_mul) + return Dynamic(self, operation=dynamic_mul) items = [(k, v * other) for (k, v) in self.data.items()] return self.clone(items, label=self._label, group=self._group) else: diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py new file mode 100644 index 0000000000..a116c54a7a --- /dev/null +++ b/holoviews/operation/datashader.py @@ -0,0 +1,285 @@ +from __future__ import absolute_import + +from collections import Callable, Iterable + +import param +import numpy as np +import pandas as pd +import xarray as xr +import datashader as ds +import datashader.transfer_functions as tf + +from datashader.core import bypixel +from datashader.pandas import pandas_pipeline +from datashape.dispatch import dispatch +from datashape import discover as dsdiscover + +from ..core import (ElementOperation, Element, Dimension, NdOverlay, + Overlay, CompositeOverlay, Dataset) +from ..core.data import ArrayInterface, PandasInterface +from ..core.util import get_param_values, basestring +from ..element import GridImage, Path, Curve, Contours, RGB +from ..streams import RangeXY + + +@dispatch(Element) +def discover(dataset): + """ + Allows datashader to correctly discover the dtypes of the data + in a holoviews Element. + """ + if isinstance(dataset.interface, (PandasInterface, ArrayInterface)): + return dsdiscover(dataset.data) + else: + return dsdiscover(dataset.dframe()) + + +@bypixel.pipeline.register(Element) +def dataset_pipeline(dataset, schema, canvas, glyph, summary): + """ + Defines how to apply a datashader pipeline to a holoviews Element, + using multidispatch. Returns an Image type with the appropriate + bounds and dimensions. Passing the returned Image to datashader + transfer functions is not yet supported. + """ + x0, x1 = canvas.x_range + y0, y1 = canvas.y_range + kdims = [dataset.get_dimension(d) for d in (glyph.x, glyph.y)] + + column = summary.column + if column and isinstance(summary, ds.count_cat): + name = '%s Count' % summary.column + else: + name = column + vdims = [dataset.get_dimension(column)(name) if column + else Dimension('Count')] + + aggregate = pandas_pipeline(dataset.dframe(), schema, canvas, + glyph, summary) + aggregate = aggregate.rename({'x_axis': kdims[0].name, + 'y_axis': kdims[1].name}) + + params = dict(get_param_values(dataset), kdims=kdims, + datatype=['xarray'], vdims=vdims) + + if aggregate.ndim == 2: + return GridImage(aggregate, **params) + else: + return NdOverlay({c: GridImage(aggregate.sel(**{column: c}), + **params) + for c in aggregate.coords[column].data}, + kdims=[dataset.get_dimension(column)]) + + +class Aggregate(ElementOperation): + """ + Aggregate implements 2D binning for any valid HoloViews Element + type using datashader. I.e., this operation turns a HoloViews + Element or overlay of Elements into an hv.Image or an overlay of + hv.Images by rasterizing it, which provides a fixed-sized + representation independent of the original dataset size. + + By default it will simply count the number of values in each bin + but other aggregators can be supplied implementing mean, max, min + and other reduction operations. + + The bins of the aggregate are defined by the width and height and + the x_range and y_range. If x_sampling or y_sampling are supplied + the operation will ensure that a bin is no smaller than theminimum + sampling distance by reducing the width and height when the zoomed + in beyond the minimum sampling distance. + """ + + aggregator = param.ClassSelector(class_=ds.reductions.Reduction, + default=ds.count()) + + dynamic = param.Boolean(default=True, doc=""" + Enables dynamic processing by default.""") + + height = param.Integer(default=800, doc=""" + The height of the aggregated image in pixels.""") + + width = param.Integer(default=600, doc=""" + The width of the aggregated image in pixels.""") + + x_range = param.NumericTuple(default=None, length=2, doc=""" + The x_range as a tuple of min and max x-value. Auto-ranges + if set to None.""") + + y_range = param.NumericTuple(default=None, length=2, doc=""" + The x_range as a tuple of min and max y-value. Auto-ranges + if set to None.""") + + x_sampling = param.Number(default=None, doc=""" + Specifies the smallest allowed sampling interval along the y-axis.""") + + y_sampling = param.Number(default=None, doc=""" + Specifies the smallest allowed sampling interval along the y-axis.""") + + streams = param.List(default=[RangeXY], doc=""" + List of streams that are applied if dynamic=True, allowing + for dynamic interaction with the plot.""") + + @classmethod + def get_agg_data(cls, obj, category=None): + """ + Reduces any Overlay or NdOverlay of Elements into a single + xarray Dataset that can be aggregated. + """ + paths = [] + kdims = obj.kdims + vdims = obj.vdims + x, y = obj.dimensions(label=True)[:2] + if isinstance(obj, Path): + glyph = 'line' + for p in obj.data: + df = pd.DataFrame(p, columns=obj.dimensions('key', True)) + if isinstance(obj, Contours) and obj.vdims and obj.level: + df[obj.vdims[0].name] = p.level + paths.append(df) + elif isinstance(obj, CompositeOverlay): + for key, el in obj.data.items(): + x, y, element, glyph = cls.get_agg_data(el) + df = element.dframe() + if isinstance(obj, NdOverlay): + df = df.assign(**dict(zip(obj.dimensions('key', True), key))) + paths.append(df) + kdims += element.kdims + vdims = element.vdims + elif isinstance(obj, Element): + glyph = 'line' if isinstance(obj, Curve) else 'points' + paths.append(obj.dframe()) + if glyph == 'line': + empty = paths[0][:1].copy() + empty.loc[0, :] = (np.NaN,) * empty.shape[1] + paths = [elem for path in paths for elem in (path, empty)][:-1] + df = pd.concat(paths).reset_index(drop=True) + if category and df[category].dtype.name != 'category': + df[category] = df[category].astype('category') + return x, y, Dataset(df, kdims=kdims, vdims=vdims), glyph + + + def _process(self, element, key=None): + agg_fn = self.p.aggregator + category = agg_fn.column if isinstance(agg_fn, ds.count_cat) else None + x, y, data, glyph = self.get_agg_data(element, category) + + xstart, xend = self.p.x_range if self.p.x_range else data.range(x) + ystart, yend = self.p.y_range if self.p.y_range else data.range(y) + + # Compute highest allowed sampling density + width, height = self.p.width, self.p.height + if self.p.x_sampling: + x_range = xend - xstart + width = int(min([(x_range/self.p.x_sampling), width])) + if self.p.y_sampling: + y_range = yend - ystart + height = int(min([(y_range/self.p.y_sampling), height])) + + cvs = ds.Canvas(plot_width=width, plot_height=height, + x_range=(xstart, xend), y_range=(ystart, yend)) + return getattr(cvs, glyph)(data, x, y, self.p.aggregator) + + + +class Shade(ElementOperation): + """ + Shade applies a normalization function followed by colormapping to + an Image or NdOverlay of Images, returning an RGB Element. + The data must be in the form of a 2D or 3D DataArray, but NdOverlays + of 2D Images will be automatically converted to a 3D array. + + In the 2D case data is normalized and colormapped, while a 3D + array representing categorical aggregates will be supplied a color + key for each category. The colormap (cmap) may be supplied as an + Iterable or a Callable. + """ + + cmap = param.ClassSelector(class_=(Iterable, Callable), doc=""" + Iterable or callable which returns colors as hex colors. + Callable type must allow mapping colors between 0 and 1.""") + + normalization = param.ClassSelector(default='eq_hist', + class_=(basestring, Callable), + doc=""" + The normalization operation applied before colormapping. + Valid options include 'linear', 'log', 'eq_hist', 'cbrt', + and any valid transfer function that accepts data, mask, nbins + arguments.""") + + @classmethod + def concatenate(cls, overlay): + """ + Concatenates an NdOverlay of GridImage types into a single 3D + xarray Dataset. + """ + if not isinstance(overlay, NdOverlay): + raise ValueError('Only NdOverlays can be concatenated') + xarr = xr.concat([v.data.T for v in overlay.values()], + dim=overlay.kdims[0].name) + params = dict(get_param_values(overlay.last), + vdims=overlay.last.vdims, + kdims=overlay.kdims+overlay.last.kdims) + return Dataset(xarr.T, **params) + + + @classmethod + def uint32_to_uint8(cls, img): + """ + Cast uint32 RGB image to 4 uint8 channels. + """ + return np.flipud(img.view(dtype=np.uint8).reshape(img.shape + (4,))) + + + def _process(self, element, key=None): + if isinstance(element, NdOverlay): + bounds = element.last.bounds + element = self.concatenate(element) + else: + bounds = element.bounds + + array = element.data[element.vdims[0].name] + kdims = element.kdims + + # Compute shading options depending on whether + # it is a categorical or regular aggregate + shade_opts = dict(how=self.p.normalization) + if element.ndims > 2: + kdims = element.kdims[1:] + categories = array.shape[-1] + if not self.p.cmap: + pass + elif isinstance(self.p.cmap, Iterator): + shade_opts['color_key'] = [c for i, c in + zip(range(categories), self.p.cmap)] + else: + shade_opts['color_key'] = [self.p.cmap(s) for s in + np.linspace(0, 1, categories)] + elif not self.p.cmap: + pass + elif isinstance(self.p.cmap, Callable): + shade_opts['cmap'] = [self.p.cmap(s) for s in np.linspace(0, 1, 256)] + else: + shade_opts['cmap'] = self.p.cmap + + + img = tf.shade(array, **shade_opts) + params = dict(get_param_values(element), kdims=kdims, + bounds=bounds, vdims=RGB.vdims[:]) + return RGB(self.uint32_to_uint8(img.data), **params) + + + +class Datashade(Aggregate, Shade): + """ + Applies the Aggregate and Shade operations, aggregating all + elements in the supplied object and then applying normalization + and colormapping the aggregated data returning RGB elements. + + See Aggregate and Shade operations for more details. + """ + + def _process(self, element, key=None): + aggregate = Aggregate._process(self, element, key) + shaded = Shade._process(self, aggregate, key) + return shaded diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py index d255ddae5d..3c691f688d 100644 --- a/holoviews/plotting/bokeh/element.py +++ b/holoviews/plotting/bokeh/element.py @@ -28,7 +28,7 @@ from ...core.options import abbreviated_exception from ...core import util from ...element import RGB -from ...streams import Stream +from ...streams import Stream, RangeXY, RangeX, RangeY from ..plot import GenericElementPlot, GenericOverlayPlot from ..util import dynamic_update from .plot import BokehPlot @@ -660,8 +660,24 @@ def current_handles(self): handles.append(plot.title) if self.current_frame: - if self.framewise or isinstance(self.hmap, DynamicMap): - handles += [plot.x_range, plot.y_range] + if self.framewise: + rangex, rangey = True, True + elif isinstance(self.hmap, DynamicMap): + rangex, rangey = True, True + for stream in self.hmap.streams: + if isinstance(stream, RangeXY): + rangex, rangey = False, False + break + elif isinstance(stream, RangeX): + rangex = False + elif isinstance(stream, RangeY): + rangey = False + else: + rangex, rangey = False, False + if rangex: + handles += [plot.x_range] + if rangey: + handles += [plot.y_range] return handles diff --git a/holoviews/streams.py b/holoviews/streams.py index 53f18df704..48922c119c 100644 --- a/holoviews/streams.py +++ b/holoviews/streams.py @@ -223,10 +223,10 @@ class RangeXY(Stream): Axis ranges along x- and y-axis in data coordinates. """ - x_range = param.NumericTuple(default=(0, 1), constant=True, doc=""" + x_range = param.NumericTuple(default=None, length=2, constant=True, doc=""" Range of the x-axis of a plot in data coordinates""") - y_range = param.NumericTuple(default=(0, 1), constant=True, doc=""" + y_range = param.NumericTuple(default=None, length=2, constant=True, doc=""" Range of the y-axis of a plot in data coordinates""") @@ -235,7 +235,7 @@ class RangeX(Stream): Axis range along x-axis in data coordinates. """ - x_range = param.NumericTuple(default=(0, 1), constant=True, doc=""" + x_range = param.NumericTuple(default=None, length=2, constant=True, doc=""" Range of the x-axis of a plot in data coordinates""") @@ -244,7 +244,7 @@ class RangeY(Stream): Axis range along y-axis in data coordinates. """ - y_range = param.NumericTuple(default=(0, 1), constant=True, doc=""" + y_range = param.NumericTuple(default=None, length=2, constant=True, doc=""" Range of the y-axis of a plot in data coordinates""") diff --git a/holoviews/util.py b/holoviews/util.py new file mode 100644 index 0000000000..3446030a87 --- /dev/null +++ b/holoviews/util.py @@ -0,0 +1,86 @@ +import param + +from .core import DynamicMap, ViewableElement +from .core.operation import ElementOperation +from .core.util import Aliases +from .core import util + + +class Dynamic(param.ParameterizedFunction): + """ + Dynamically applies a callable to the Elements in any HoloViews + object. Will return a DynamicMap wrapping the original map object, + which will lazily evaluate when a key is requested. By default + Dynamic applies a no-op, making it useful for converting HoloMaps + to a DynamicMap. + + Any supplied kwargs will be passed to the callable and any streams + will be instantiated on the returned DynamicMap. + """ + + operation = param.Callable(default=lambda x: x, doc=""" + Operation or user-defined callable to apply dynamically""") + + kwargs = param.Dict(default={}, doc=""" + Keyword arguments passed to the function.""") + + streams = param.List(default=[], doc=""" + List of streams to attach to the returned DynamicMap""") + + def __call__(self, map_obj, **params): + self.p = param.ParamOverrides(self, params) + callback = self._dynamic_operation(map_obj) + if isinstance(map_obj, DynamicMap): + dmap = map_obj.clone(callback=callback, shared_data=False) + else: + dmap = self._make_dynamic(map_obj, callback) + if isinstance(self.p.operation, ElementOperation): + streams = [] + for s in self.p.streams: + stream = s() + stream.update(**{k: self.p.operation.p.get(k) for k, v in + stream.contents.items()}) + streams.append(stream) + return dmap.clone(streams=streams) + return dmap + + + def _process(self, element, key=None): + if isinstance(self.p.operation, ElementOperation): + return self.p.operation.process_element(element, key, **self.p.kwargs) + else: + return self.p.operation(element, **self.p.kwargs) + + + def _dynamic_operation(self, map_obj): + """ + Generate function to dynamically apply the operation. + Wraps an existing HoloMap or DynamicMap. + """ + if not isinstance(map_obj, DynamicMap): + def dynamic_operation(*key, **kwargs): + self.p.kwargs.update(kwargs) + return self._process(map_obj[key], key) + return dynamic_operation + + def dynamic_operation(*key, **kwargs): + key = key[0] if map_obj.mode == 'open' else key + self.p.kwargs.update(kwargs) + _, el = util.get_dynamic_item(map_obj, map_obj.kdims, key) + return self._process(el, key) + + return dynamic_operation + + + def _make_dynamic(self, hmap, dynamic_fn): + """ + Accepts a HoloMap and a dynamic callback function creating + an equivalent DynamicMap from the HoloMap. + """ + if isinstance(hmap, ViewableElement): + return DynamicMap(dynamic_fn, kdims=[]) + dim_values = zip(*hmap.data.keys()) + params = util.get_param_values(hmap) + kdims = [d(values=list(set(values))) for d, values in + zip(hmap.kdims, dim_values)] + return DynamicMap(dynamic_fn, **dict(params, kdims=kdims)) diff --git a/tests/testdynamic.py b/tests/testdynamic.py index 162203c5db..20c27bf09f 100644 --- a/tests/testdynamic.py +++ b/tests/testdynamic.py @@ -1,6 +1,6 @@ import numpy as np from holoviews import Dimension, DynamicMap, Image, HoloMap -from holoviews.core.operation import DynamicFunction +from holoviews.util import Dynamic from holoviews.element.comparison import ComparisonTestCase frequencies = np.linspace(0.5,2.0,5) @@ -82,19 +82,19 @@ def test_sampled_bounded_resample(self): class DynamicTestOperation(ComparisonTestCase): - def test_dynamic_function(self): + def test_dynamic_operation(self): fn = lambda i: Image(sine_array(0,i)) dmap=DynamicMap(fn, sampled=True) - dmap_with_fn = DynamicFunction(dmap, function=lambda x: x.clone(x.data*2)) + dmap_with_fn = Dynamic(dmap, operation=lambda x: x.clone(x.data*2)) self.assertEqual(dmap_with_fn[5], Image(sine_array(0,5)*2)) - def test_dynamic_function_with_kwargs(self): + def test_dynamic_operation_with_kwargs(self): fn = lambda i: Image(sine_array(0,i)) dmap=DynamicMap(fn, sampled=True) def fn(x, multiplier=2): return x.clone(x.data*multiplier) - dmap_with_fn = DynamicFunction(dmap, function=fn, kwargs=dict(multiplier=3)) + dmap_with_fn = Dynamic(dmap, operation=fn, kwargs=dict(multiplier=3)) self.assertEqual(dmap_with_fn[5], Image(sine_array(0,5)*3))