From 0f5ad5dc3dcdf7e5bb1afdd4a63213070cbb335b Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 22 Dec 2017 12:04:03 +0000 Subject: [PATCH] GridInterface bin support for Histogram and QuadMesh (#2160) --- .../demos/bokeh/irregular_quadmesh.ipynb | 84 +++++++ .../demos/matplotlib/irregular_quadmesh.ipynb | 85 +++++++ .../reference/elements/bokeh/QuadMesh.ipynb | 70 +++++- .../elements/matplotlib/QuadMesh.ipynb | 70 +++++- holoviews/core/data/__init__.py | 5 +- holoviews/core/data/array.py | 4 +- holoviews/core/data/dictionary.py | 5 +- holoviews/core/data/grid.py | 169 ++++++++++++-- holoviews/core/data/image.py | 6 +- holoviews/core/data/interface.py | 2 + holoviews/core/data/iris.py | 6 + holoviews/core/data/xarray.py | 81 +++++-- holoviews/core/util.py | 14 ++ holoviews/element/chart.py | 131 +++-------- holoviews/element/comparison.py | 9 +- holoviews/element/raster.py | 213 ++++++----------- holoviews/element/util.py | 13 -- holoviews/operation/datashader.py | 33 ++- holoviews/plotting/bokeh/chart.py | 21 +- holoviews/plotting/bokeh/raster.py | 69 ++++-- holoviews/plotting/bokeh/util.py | 19 ++ holoviews/plotting/mpl/chart.py | 10 +- holoviews/plotting/mpl/raster.py | 30 +-- tests/testbinneddatasets.py | 215 ++++++++++++++++++ tests/testcomparisonchart.py | 16 +- tests/testcoreutils.py | 25 +- tests/testdataset.py | 16 +- tests/testdatashader.py | 31 ++- tests/testelementconstructors.py | 12 +- tests/testelementindexing.py | 136 ----------- tests/testelementutils.py | 26 --- tests/testellipsis.py | 11 +- tests/testraster.py | 24 +- 33 files changed, 1089 insertions(+), 572 deletions(-) create mode 100644 examples/gallery/demos/bokeh/irregular_quadmesh.ipynb create mode 100644 examples/gallery/demos/matplotlib/irregular_quadmesh.ipynb create mode 100644 tests/testbinneddatasets.py delete mode 100644 tests/testelementindexing.py delete mode 100644 tests/testelementutils.py diff --git a/examples/gallery/demos/bokeh/irregular_quadmesh.ipynb b/examples/gallery/demos/bokeh/irregular_quadmesh.ipynb new file mode 100644 index 0000000000..76f468d127 --- /dev/null +++ b/examples/gallery/demos/bokeh/irregular_quadmesh.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Most examples work across multiple plotting backends, this example is also available for:\n", + "\n", + "* [Matplotlib irregular_quadmesh](../matplotlib/irregular_quadmesh.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import holoviews as hv\n", + "hv.extension('bokeh')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Declaring data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "n = 20\n", + "coords = np.linspace(-1.5, 1.5, n)\n", + "X,Y = np.meshgrid(coords, coords);\n", + "Qx = np.cos(Y) - np.cos(X)\n", + "Qz = np.sin(Y) + np.sin(X)\n", + "Z = np.sqrt(X**2 + Y**2)\n", + "\n", + "qmesh = hv.QuadMesh((Qx, Qz, Z))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "qmesh" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/gallery/demos/matplotlib/irregular_quadmesh.ipynb b/examples/gallery/demos/matplotlib/irregular_quadmesh.ipynb new file mode 100644 index 0000000000..3e7d02c130 --- /dev/null +++ b/examples/gallery/demos/matplotlib/irregular_quadmesh.ipynb @@ -0,0 +1,85 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Most examples work across multiple plotting backends, this example is also available for:\n", + "\n", + "* [Bokeh irregular_quadmesh](../bokeh/irregular_quadmesh.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import holoviews as hv\n", + "hv.extension('matplotlib')\n", + "%output fig='svg'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Declaring data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "n = 20\n", + "coords = np.linspace(-1.5, 1.5, n)\n", + "X,Y = np.meshgrid(coords, coords);\n", + "Qx = np.cos(Y) - np.cos(X)\n", + "Qz = np.sin(Y) + np.sin(X)\n", + "Z = np.sqrt(X**2 + Y**2)\n", + "\n", + "qmesh = hv.QuadMesh((Qx, Qz, Z))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "qmesh" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/reference/elements/bokeh/QuadMesh.ipynb b/examples/reference/elements/bokeh/QuadMesh.ipynb index 223404b858..6dc8b10a0f 100644 --- a/examples/reference/elements/bokeh/QuadMesh.ipynb +++ b/examples/reference/elements/bokeh/QuadMesh.ipynb @@ -41,6 +41,9 @@ "xs = np.logspace(1, 3, n)\n", "ys = np.linspace(1, 10, n)\n", "zs = np.arange((n-1)**2).reshape(n-1, n-1)\n", + "print('Shape of x-coordinates:', xs.shape)\n", + "print('Shape of y-coordinates:', ys.shape)\n", + "print('Shape of value array:', zs.shape)\n", "hv.QuadMesh((xs, ys, zs))" ] }, @@ -106,12 +109,77 @@ "\n", "For full documentation and the available style and plot options, use ``hv.help(hv.QuadMesh).``" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Irregular meshes\n", + "\n", + "In addition to axis aligned meshes like those we worked with above, a ``QuadMesh`` may also be used to represent irregular or unstructured meshes. In this example we will create an irregular mesh consisting of 2D X, Y and Z arrays defining the position and value of each simplex in the mesh:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "n=20\n", + "coords = np.linspace(-1.5,1.5,n)\n", + "X,Y = np.meshgrid(coords, coords);\n", + "Qx = np.cos(Y) - np.cos(X)\n", + "Qy = np.sin(Y) + np.sin(X)\n", + "Z = np.sqrt(X**2 + Y**2)\n", + "\n", + "print('Shape of x-coordinates:', Qx.shape)\n", + "print('Shape of y-coordinates:', Qy.shape)\n", + "print('Shape of value array:', Z.shape)\n", + "\n", + "qmesh = hv.QuadMesh((Qx, Qy, Z))\n", + "qmesh" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To illustrate irregular meshes a bit further we will randomly jitter the mesh coordinates along both dimensions, demonstrating that ``QuadMesh`` may be used to represent completely arbitrary meshes. It may also be used to represent overlapping meshes, however the behavior during slicing and other operations may not be well defined in such cases." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(13)\n", + "xs, ys = np.meshgrid(np.linspace(-20, 20, 10), np.linspace(0, 30, 8))\n", + "xs += xs/10 + np.random.rand(*xs.shape)*4\n", + "ys += ys/10 + np.random.rand(*ys.shape)*4\n", + "\n", + "zs = np.arange(80).reshape(8, 10)\n", + "hv.QuadMesh((xs, ys, zs))" + ] } ], "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "pygments_lexer": "ipython3" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" } }, "nbformat": 4, diff --git a/examples/reference/elements/matplotlib/QuadMesh.ipynb b/examples/reference/elements/matplotlib/QuadMesh.ipynb index 5b30391528..8f178cebb7 100644 --- a/examples/reference/elements/matplotlib/QuadMesh.ipynb +++ b/examples/reference/elements/matplotlib/QuadMesh.ipynb @@ -41,6 +41,9 @@ "xs = np.logspace(1, 3, n)\n", "ys = np.linspace(1, 10, n)\n", "zs = np.arange((n-1)**2).reshape(n-1, n-1)\n", + "print('Shape of x-coordinates:', xs.shape)\n", + "print('Shape of y-coordinates:', ys.shape)\n", + "print('Shape of value array:', zs.shape)\n", "hv.QuadMesh((xs, ys, zs))" ] }, @@ -88,12 +91,77 @@ "\n", "For full documentation and the available style and plot options, use ``hv.help(hv.QuadMesh).``" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Irregular meshes\n", + "\n", + "In addition to axis aligned meshes like those we worked with above, a ``QuadMesh`` may also be used to represent irregular or unstructured meshes. In this example we will create an irregular mesh consisting of 2D X, Y and Z arrays defining the position and value of each simplex in the mesh:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "n=20\n", + "coords = np.linspace(-1.5,1.5,n)\n", + "X,Y = np.meshgrid(coords, coords);\n", + "Qx = np.cos(Y) - np.cos(X)\n", + "Qy = np.sin(Y) + np.sin(X)\n", + "Z = np.sqrt(X**2 + Y**2)\n", + "\n", + "print('Shape of x-coordinates:', Qx.shape)\n", + "print('Shape of y-coordinates:', Qy.shape)\n", + "print('Shape of value array:', Z.shape)\n", + "\n", + "qmesh = hv.QuadMesh((Qx, Qy, Z))\n", + "qmesh" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To illustrate irregular meshes a bit further we will randomly jitter the mesh coordinates along both dimensions, demonstrating that ``QuadMesh`` may be used to represent completely arbitrary meshes. It may also be used to represent overlapping meshes, however the behavior during slicing and other operations may not be well defined in such cases." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(13)\n", + "xs, ys = np.meshgrid(np.linspace(-20, 20, 10), np.linspace(0, 30, 8))\n", + "xs += xs/10 + np.random.rand(*xs.shape)*4\n", + "ys += ys/10 + np.random.rand(*ys.shape)*4\n", + "\n", + "zs = np.arange(80).reshape(8, 10)\n", + "hv.QuadMesh((xs, ys, zs))" + ] } ], "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "pygments_lexer": "ipython3" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" } }, "nbformat": 4, diff --git a/holoviews/core/data/__init__.py b/holoviews/core/data/__init__.py index 9f23a5db13..02e86d533a 100644 --- a/holoviews/core/data/__init__.py +++ b/holoviews/core/data/__init__.py @@ -175,6 +175,9 @@ class Dataset(Element): # Define a class used to transform Datasets into other Element types _conversion_interface = DataConversion + # Whether the key dimensions are specified as bins + _binned = False + _vdim_reductions = {} _kdim_reductions = {} @@ -361,7 +364,7 @@ def __getitem__(self, slices): value_select = slices[self.ndims] elif len(slices) == self.ndims+1 and isinstance(slices[self.ndims], (Dimension,str)): - raise Exception("%r is not an available value dimension" % slices[self.ndims]) + raise IndexError("%r is not an available value dimension" % slices[self.ndims]) else: selection = dict(zip(self.dimensions(label=True), slices)) data = self.select(**selection) diff --git a/holoviews/core/data/array.py b/holoviews/core/data/array.py index ac3f9fdc2c..973a7ba060 100644 --- a/holoviews/core/data/array.py +++ b/holoviews/core/data/array.py @@ -41,7 +41,9 @@ def init(cls, eltype, data, kdims, vdims): for k, v in dict_data)) data = np.column_stack(dataset) elif isinstance(data, tuple): - data = [d if isinstance(d, np.ndarray) else np.array(d) for d in data] + data = [np.asarray(d) for d in data] + if any(arr.ndim > 1 for arr in data): + raise ValueError('ArrayInterface expects data to be of flat shape.') if cls.expanded(data): data = np.column_stack(data) else: diff --git a/holoviews/core/data/dictionary.py b/holoviews/core/data/dictionary.py index ea272a9591..b8ae1d0093 100644 --- a/holoviews/core/data/dictionary.py +++ b/holoviews/core/data/dictionary.py @@ -95,7 +95,10 @@ def init(cls, eltype, data, kdims, vdims): for i, sd in enumerate(d): unpacked.append((sd, vals[:, i])) else: - unpacked.append((d, vals if np.isscalar(vals) else np.asarray(vals))) + vals = vals if np.isscalar(vals) else np.asarray(vals) + if not np.isscalar(vals) and not vals.ndim == 1: + raise ValueError('DictInterface expects data for each column to be flat.') + unpacked.append((d, vals)) if not cls.expanded([d[1] for d in unpacked if not np.isscalar(d[1])]): raise ValueError('DictInterface expects data to be of uniform shape.') if isinstance(data, odict_types): diff --git a/holoviews/core/data/grid.py b/holoviews/core/data/grid.py index e57299554d..740e7d1f5c 100644 --- a/holoviews/core/data/grid.py +++ b/holoviews/core/data/grid.py @@ -55,6 +55,18 @@ def init(cls, eltype, data, kdims, vdims): d for d in kdims + vdims] if isinstance(data, tuple): data = {d: v for d, v in zip(dimensions, data)} + elif isinstance(data, list) and data == []: + data = OrderedDict([(d, []) for d in dimensions]) + elif not any(isinstance(data, tuple(t for t in interface.types if t is not None)) + for interface in cls.interfaces.values()): + data = {k: v for k, v in zip(dimensions, zip(*data))} + elif isinstance(data, np.ndarray): + if data.ndim == 1: + if eltype._auto_indexable_1d and len(kdims)+len(vdims)>1: + data = np.column_stack([np.arange(len(data)), data]) + else: + data = np.atleast_2d(data).T + data = {k: data[:,i] for i,k in enumerate(dimensions)} elif isinstance(data, list) and data == []: data = {d: np.array([]) for d in dimensions[:ndims]} data.update({d: np.empty((0,) * ndims) for d in dimensions[ndims:]}) @@ -72,16 +84,30 @@ def init(cls, eltype, data, kdims, vdims): kdim_names = [d.name if isinstance(d, Dimension) else d for d in kdims] vdim_names = [d.name if isinstance(d, Dimension) else d for d in vdims] expected = tuple([len(data[kd]) for kd in kdim_names]) + shapes = tuple([data[kd].shape for kd in kdim_names]) for vdim in vdim_names: shape = data[vdim].shape error = DataError if len(shape) > 1 else ValueError - if shape != expected[::-1] and not (not expected and shape == (1,)): + if (not expected and shape == (1,)) or (len(set((shape,)+shapes)) == 1 and len(shape) > 1): + # If empty or an irregular mesh + pass + elif len(shape) != len(expected): + raise error('The shape of the %s value array does not ' + 'match the expected dimensionality indicated ' + 'by the key dimensions. Expected %d-D array, ' + 'found %d-D array.' % (vdim, len(expected), len(shape))) + elif any((s!=e and (s+1)!=e) for s, e in zip(shape, expected[::-1])): raise error('Key dimension values and value array %s ' 'shapes do not match. Expected shape %s, ' 'actual shape: %s' % (vdim, expected[::-1], shape), cls) return data, {'kdims':kdims, 'vdims':vdims}, {} + @classmethod + def irregular(cls, dataset, dim): + return dataset.data[dim.name if isinstance(dim, Dimension) else dim].ndim > 1 + + @classmethod def isscalar(cls, dataset, dim): return np.unique(cls.values(dataset, dim, expanded=False)) == 1 @@ -103,30 +129,69 @@ def dimension_type(cls, dataset, dim): @classmethod def shape(cls, dataset, gridded=False): + shape = dataset.data[dataset.vdims[0].name].shape if gridded: - return dataset.data[dataset.vdims[0].name].shape + return shape else: - return (cls.length(dataset), len(dataset.dimensions())) + return (np.product(shape), len(dataset.dimensions())) @classmethod def length(cls, dataset): - return np.product([len(dataset.data[d.name]) for d in dataset.kdims]) + return cls.shape(dataset)[0] + + + @classmethod + def _infer_interval_breaks(cls, coord, axis=0): + """ + >>> GridInterface._infer_interval_breaks(np.arange(5)) + array([-0.5, 0.5, 1.5, 2.5, 3.5, 4.5]) + >>> GridInterface._infer_interval_breaks([[0, 1], [3, 4]], axis=1) + array([[-0.5, 0.5, 1.5], + [ 2.5, 3.5, 4.5]]) + """ + coord = np.asarray(coord) + deltas = 0.5 * np.diff(coord, axis=axis) + first = np.take(coord, [0], axis=axis) - np.take(deltas, [0], axis=axis) + last = np.take(coord, [-1], axis=axis) + np.take(deltas, [-1], axis=axis) + trim_last = tuple(slice(None, -1) if n == axis else slice(None) + for n in range(coord.ndim)) + return np.concatenate([first, coord[trim_last] + deltas, last], axis=axis) @classmethod - def coords(cls, dataset, dim, ordered=False, expanded=False): + def coords(cls, dataset, dim, ordered=False, expanded=False, edges=False): """ Returns the coordinates along a dimension. Ordered ensures coordinates are in ascending order and expanded creates ND-array matching the dimensionality of the dataset. """ dim = dataset.get_dimension(dim, strict=True) - if expanded: - return util.expand_grid_coords(dataset, dim) + irregular = cls.irregular(dataset, dim) + if irregular or expanded: + if irregular: + data = dataset.data[dim.name] + else: + data = util.expand_grid_coords(dataset, dim) + if edges and data.shape == dataset.data[dataset.vdims[0].name].shape: + data = cls._infer_interval_breaks(data, axis=1) + data = cls._infer_interval_breaks(data, axis=0) + return data + data = dataset.data[dim.name] if ordered and np.all(data[1:] < data[:-1]): data = data[::-1] + shape = cls.shape(dataset, True) + if dim in dataset.kdims: + idx = dataset.get_dimension_index(dim) + isedges = (dim in dataset.kdims and len(shape) == dataset.ndims + and len(data) == (shape[dataset.ndims-idx-1]+1)) + else: + isedges = False + if edges and not isedges: + data = cls._infer_interval_breaks(data) + elif not edges and isedges: + data = np.convolve(data, [0.5, 0.5], 'valid') return data @@ -197,7 +262,7 @@ def ndloc(cls, dataset, indices): selected = {} adjusted_inds = [] all_scalar = True - for kd, ind in zip(dataset.kdims[::-1], indices): + for i, (kd, ind) in enumerate(zip(dataset.kdims[::-1], indices)): coords = cls.coords(dataset, kd.name, True) if np.isscalar(ind): ind = [ind] @@ -210,19 +275,21 @@ def ndloc(cls, dataset, indices): coords = cls.coords(dataset, kd.name) selected[kd.name] = coords all_scalar = False - for vd in dataset.vdims: - arr = dataset.dimension_values(vd, flat=False) + for d in dataset.dimensions(): + if d in dataset.kdims and not cls.irregular(dataset, d): + continue + arr = dataset.dimension_values(d, flat=False) if all_scalar and len(dataset.vdims) == 1: return arr[tuple(ind[0] for ind in adjusted_inds)] - selected[vd.name] = arr[tuple(adjusted_inds)] + selected[d.name] = arr[tuple(adjusted_inds)] return tuple(selected[d.name] for d in dataset.dimensions()) @classmethod def values(cls, dataset, dim, expanded=True, flat=True): dim = dataset.get_dimension(dim, strict=True) - if dim in dataset.vdims: - data = dataset.data.get(dim.name) + if dim in dataset.vdims or dataset.data[dim.name].ndim > 1: + data = dataset.data[dim.name] data = cls.canonicalize(dataset, data) return data.T.flatten() if flat else data elif expanded: @@ -238,6 +305,12 @@ def groupby(cls, dataset, dim_names, container_type, group_type, **kwargs): dimensions = [dataset.get_dimension(d, strict=True) for d in dim_names] kdims = [kdim for kdim in dataset.kdims if kdim not in dimensions] + invalid = [d for d in dimensions if dataset.data[d.name].ndim > 1] + if invalid: + if len(invalid) == 1: invalid = "'%s'" % invalid[0] + raise ValueError("Cannot groupby irregularly sampled dimension(s) %s." + % invalid) + # Update the kwargs appropriately for Element group types group_kwargs = {} group_type = dict if group_type == 'raw' else group_type @@ -249,7 +322,7 @@ def groupby(cls, dataset, dim_names, container_type, group_type, **kwargs): drop_dim = any(d not in group_kwargs['kdims'] for d in kdims) # Find all the keys along supplied dimensions - keys = [dataset.data[d.name] for d in dimensions] + keys = [cls.coords(dataset, d.name) for d in dimensions] # Iterate over the unique entries applying selection masks grouped_data = [] @@ -303,12 +376,14 @@ def key_select_mask(cls, dataset, values, ind): mask = None else: index_mask = values == ind - if dataset.ndims == 1 and np.sum(index_mask) == 0: + if (dataset.ndims == 1 or dataset._binned) and np.sum(index_mask) == 0: data_index = np.argmin(np.abs(values - ind)) - mask = np.zeros(len(dataset), dtype=np.bool) + mask = np.zeros(len(values), dtype=np.bool) mask[data_index] = True else: mask = index_mask + if mask is None: + mask = np.ones(values.shape, dtype=bool) return mask @@ -321,24 +396,54 @@ def select(cls, dataset, selection_mask=None, **selection): 'convert to expanded format before slicing.') indexed = cls.indexed(dataset, selection) - selection = [(d, selection.get(d.name, selection.get(d.label))) - for d in dimensions] + full_selection = [(d, selection.get(d.name, selection.get(d.label))) + for d in dimensions] data = {} value_select = [] - for dim, ind in selection: - values = cls.values(dataset, dim, False) + for i, (dim, ind) in enumerate(full_selection): + irregular = cls.irregular(dataset, dim) + values = cls.coords(dataset, dim, irregular) mask = cls.key_select_mask(dataset, values, ind) - if mask is None: - mask = np.ones(values.shape, dtype=bool) + if irregular: + if np.isscalar(ind) or isinstance(ind, (set, list)): + raise IndexError("Indexing not supported for irregularly " + "sampled data. %s value along %s dimension." + "must be a slice or 2D boolean mask." + % (ind, dim)) + mask = mask.max(axis=i) + elif dataset._binned: + edges = cls.coords(dataset, dim, False, edges=True) + inds = np.argwhere(mask) + if np.isscalar(ind): + emin, emax = edges.min(), edges.max() + if ind < emin: + raise IndexError("Index %s less than lower bound " + "of %s for %s dimension." % (ind, emin, dim)) + elif ind >= emax: + raise IndexError("Index %s more than or equal to upper bound " + "of %s for %s dimension." % (ind, emax, dim)) + idx = max([np.digitize([ind], edges)[0]-1, 0]) + mask = np.zeros(len(values), dtype=np.bool) + mask[idx] = True + values = edges[idx:idx+2] + elif len(inds): + values = edges[inds.min(): inds.max()+2] + else: + values = edges[0:0] else: values = values[mask] + values, mask = np.asarray(values), np.asarray(mask) value_select.append(mask) data[dim.name] = np.array([values]) if np.isscalar(values) else values + int_inds = [np.argwhere(v) for v in value_select][::-1] index = np.ix_(*[np.atleast_1d(np.squeeze(ind)) if ind.ndim > 1 else np.atleast_1d(ind) for ind in int_inds]) + for kdim in dataset.kdims: + if cls.irregular(dataset, dim): + data[kdim.name] = np.asarray(data[kdim.name])[index] for vdim in dataset.vdims: - data[vdim.name] = dataset.data[vdim.name][index] + data[vdim.name] = np.asarray(dataset.data[vdim.name])[index] if indexed: if len(dataset.vdims) == 1: @@ -465,5 +570,23 @@ def iloc(cls, dataset, index): return new_data[0][0] return tuple(new_data) + @classmethod + def range(cls, dataset, dimension): + if dataset._binned and dimension in dataset.kdims: + expanded = cls.irregular(dataset, dimension) + column = cls.coords(dataset, dimension, expanded=expanded, edges=True) + else: + column = dataset.dimension_values(dimension) + if dataset.get_dimension_type(dimension) is np.datetime64: + return column.min(), column.max() + elif len(column) == 0: + return np.NaN, np.NaN + else: + try: + return (np.nanmin(column), np.nanmax(column)) + except TypeError: + column.sort() + return column[0], column[-1] + Interface.register(GridInterface) diff --git a/holoviews/core/data/image.py b/holoviews/core/data/image.py index b73e7275fd..97193c26de 100644 --- a/holoviews/core/data/image.py +++ b/holoviews/core/data/image.py @@ -47,7 +47,7 @@ def init(cls, eltype, data, kdims, vdims): data = data[::-1, :] expected = (len(ys), len(xs)) shape = data.shape[:2] - error = DataError if len(shape) > 1 else ValueError + error = DataError if len(shape) > 1 and not eltype._binned else ValueError if shape != expected and not (not expected and shape == (1,)): raise error('Key dimension values and value array %s ' 'shapes do not match. Expected shape %s, ' @@ -58,6 +58,10 @@ def init(cls, eltype, data, kdims, vdims): return data, {'kdims':kdims, 'vdims':vdims}, kwargs + @classmethod + def irregular(cls, dataset, dim): + "ImageInterface does not support irregular data" + return False @classmethod def shape(cls, dataset, gridded=False): diff --git a/holoviews/core/data/interface.py b/holoviews/core/data/interface.py index 38ebfd0431..923cfbce42 100644 --- a/holoviews/core/data/interface.py +++ b/holoviews/core/data/interface.py @@ -283,6 +283,8 @@ def range(cls, dataset, dimension): column = dataset.dimension_values(dimension) if dataset.get_dimension_type(dimension) is np.datetime64: return column.min(), column.max() + elif len(column) == 0: + return np.NaN, np.NaN else: try: return (np.nanmin(column), np.nanmax(column)) diff --git a/holoviews/core/data/iris.py b/holoviews/core/data/iris.py index caebdb26b4..96bd1fd10d 100644 --- a/holoviews/core/data/iris.py +++ b/holoviews/core/data/iris.py @@ -132,6 +132,12 @@ def validate(cls, dataset, vdims=True): raise DataError("Iris cubes do not support more than one value dimension", cls) + @classmethod + def irregular(cls, dataset, dim): + "CubeInterface does not support irregular data" + return False + + @classmethod def shape(cls, dataset, gridded=False): if gridded: diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py index bd1487135a..a6211a774d 100644 --- a/holoviews/core/data/xarray.py +++ b/holoviews/core/data/xarray.py @@ -101,7 +101,11 @@ def init(cls, eltype, data, kdims, vdims): if isinstance(data[name].data, np.ndarray)] kdims = [d if isinstance(d, Dimension) else Dimension(d) for d in kdims] - not_found = [d for d in kdims if d.name not in data.coords] + not_found = [] + for d in kdims: + if not any(d.name == k or (isinstance(v, xr.DataArray) and d.name in v.dims) + for k, v in data.coords.items()): + not_found.append(d) if not isinstance(data, xr.Dataset): raise TypeError('Data must be be an xarray Dataset type.') elif not_found: @@ -114,16 +118,17 @@ def init(cls, eltype, data, kdims, vdims): @classmethod def range(cls, dataset, dimension): dim = dataset.get_dimension(dimension, strict=True).name - if dim in dataset.data: + if dataset._binned and dimension in dataset.kdims: + data = cls.coords(dataset, dim, edges=True) + dmin, dmax = np.nanmin(data), np.nanmax(data) + else: data = dataset.data[dim] dmin, dmax = data.min().data, data.max().data if dask and isinstance(dmin, dask.array.Array): dmin, dmax = dmin.compute(), dmax.compute() - dmin = dmin if np.isscalar(dmin) else dmin.item() - dmax = dmax if np.isscalar(dmax) else dmax.item() - return dmin, dmax - else: - return np.NaN, np.NaN + dmin = dmin if np.isscalar(dmin) else dmin.item() + dmax = dmax if np.isscalar(dmax) else dmax.item() + return dmin, dmax @classmethod @@ -132,6 +137,12 @@ def groupby(cls, dataset, dimensions, container_type, group_type, **kwargs): element_dims = [kdim for kdim in dataset.kdims if kdim not in index_dims] + invalid = [d for d in index_dims if dataset.data[d.name].ndim > 1] + if invalid: + if len(invalid) == 1: invalid = "'%s'" % invalid[0] + raise ValueError("Cannot groupby irregularly sampled dimension(s) %s." + % invalid) + group_kwargs = {} if group_type != 'raw' and issubclass(group_type, Element): group_kwargs = dict(util.get_param_values(dataset), @@ -167,12 +178,35 @@ def groupby(cls, dataset, dimensions, container_type, group_type, **kwargs): @classmethod - def coords(cls, dataset, dim, ordered=False, expanded=False): - if expanded: - return util.expand_grid_coords(dataset, dim) + def coords(cls, dataset, dimension, ordered=False, expanded=False, edges=False): + dim = dataset.get_dimension(dimension) + dim = dimension if dim is None else dim.name + irregular = cls.irregular(dataset, dim) + if irregular or expanded: + if irregular: + data = dataset.data[dim] + else: + data = util.expand_grid_coords(dataset, dim) + if edges: + data = cls._infer_interval_breaks(data, axis=1) + data = cls._infer_interval_breaks(data, axis=0) + return data + data = np.atleast_1d(dataset.data[dim].data) if ordered and data.shape and np.all(data[1:] < data[:-1]): data = data[::-1] + shape = cls.shape(dataset, True) + + if dim in dataset.kdims: + idx = dataset.get_dimension_index(dim) + isedges = (dim in dataset.kdims and len(shape) == dataset.ndims + and len(data) == (shape[dataset.ndims-idx-1]+1)) + else: + isedges = False + if edges and not isedges: + data = cls._infer_interval_breaks(data) + elif not edges and isedges: + data = np.convolve(data, [0.5, 0.5], 'valid') return data @@ -180,11 +214,13 @@ def coords(cls, dataset, dim, ordered=False, expanded=False): def values(cls, dataset, dim, expanded=True, flat=True): dim = dataset.get_dimension(dim, strict=True) data = dataset.data[dim.name].data - if dim in dataset.vdims: + irregular = cls.irregular(dataset, dim) if dim in dataset.kdims else False + if dim in dataset.vdims or irregular: coord_dims = list(dataset.data[dim.name].dims) if dask and isinstance(data, dask.array.Array): data = data.compute() - data = cls.canonicalize(dataset, data, coord_dims=coord_dims) + if not irregular and not any(cls.irregular(dataset, d) for d in dataset.kdims): + data = cls.canonicalize(dataset, data, coord_dims=coord_dims) return data.T.flatten() if flat else data elif expanded: data = cls.coords(dataset, dim.name, expanded=True) @@ -213,10 +249,19 @@ def unpack_scalar(cls, dataset, data): @classmethod def ndloc(cls, dataset, indices): - kdims = [d.name for d in dataset.kdims[::-1]] + kdims = [d for d in dataset.kdims[::-1]] adjusted_indices = [] + slice_dims = [] for kd, ind in zip(kdims, indices): - coords = cls.coords(dataset, kd, False) + if cls.irregular(dataset, kd): + coords = [c for c in dataset.data.coords if c not in dataset.data.dims] + dim = dataset.data[kd.name].dims[coords.index(kd.name)] + shape = dataset.data[kd.name].shape[coords.index(kd.name)] + coords = np.arange(shape) + else: + coords = cls.coords(dataset, kd, False) + dim = kd.name + slice_dims.append(dim) ncoords = len(coords) if np.all(coords[1:] < coords[:-1]): if np.isscalar(ind): @@ -235,7 +280,7 @@ def ndloc(cls, dataset, indices): ind = np.where(ind)[0] adjusted_indices.append(ind) - isel = dict(zip(kdims, adjusted_indices)) + isel = dict(zip(slice_dims, adjusted_indices)) all_scalar = all(map(np.isscalar, indices)) if all_scalar and len(dataset.vdims) == 1: return dataset.data[dataset.vdims[0].name].isel(**isel).values.item() @@ -283,8 +328,12 @@ def sort(cls, dataset, by=[], reverse=False): @classmethod def select(cls, dataset, selection_mask=None, **selection): validated = {} + irregular = False for k, v in selection.items(): - dim = dataset.get_dimension(k, strict=True).name + dim = dataset.get_dimension(k, strict=True) + if cls.irregular(dataset, dim): + return GridInterface.select(dataset, selection_mask, **selection) + dim = dim.name if isinstance(v, slice): v = (v.start, v.stop) if isinstance(v, set): diff --git a/holoviews/core/util.py b/holoviews/core/util.py index 361fe10286..3a0b1b991f 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -1578,3 +1578,17 @@ def search_indices(values, source): """ orig_indices = source.argsort() return orig_indices[np.searchsorted(source[orig_indices], values)] + + +def compute_edges(edges): + """ + Computes edges as midpoints of the bin centers. The first and + last boundaries are equidistant from the first and last midpoints + respectively. + """ + edges = np.asarray(edges) + if edges.dtype.kind == 'i': + edges = edges.astype('f') + midpoints = (edges[:-1] + edges[1:])/2.0 + boundaries = (2*edges[0] - midpoints[0], 2*edges[-1] - midpoints[-1]) + return np.concatenate([boundaries[:1], midpoints, boundaries[-1:]]) diff --git a/holoviews/element/chart.py b/holoviews/element/chart.py index d2d04cea66..08d1ae7c64 100644 --- a/holoviews/element/chart.py +++ b/holoviews/element/chart.py @@ -4,7 +4,8 @@ from ..core import util from ..core import Dimension, Dataset, Element2D -from .util import compute_edges +from ..core.data import GridInterface + class Chart(Dataset, Element2D): """ @@ -156,126 +157,58 @@ class BoxWhisker(Chart): _auto_indexable_1d = False -class Histogram(Element2D): +class Histogram(Chart): """ Histogram contains a number of bins, which are defined by the upper and lower bounds of their edges and the computed bin values. """ + datatype = param.List(default=['grid']) + + group = param.String(default='Histogram', constant=True) + kdims = param.List(default=[Dimension('x')], bounds=(1,1), doc=""" Dimensions on Element2Ds determine the number of indexable dimensions.""") - group = param.String(default='Histogram', constant=True) - vdims = param.List(default=[Dimension('Frequency')], bounds=(1,1)) - def __init__(self, values, edges=None, **params): + _binned = True + + def __init__(self, data, edges=None, **params): if edges is not None: self.warning("Histogram edges should be supplied as a tuple " "along with the values, passing the edges will " "be deprecated in holoviews 2.0.") - self.values, self.edges, settings = self._process_data(values, edges) - settings.update(params) - super(Histogram, self).__init__((self.values, self.edges), **settings) + data = (edges, data) + elif isinstance(data, tuple) and len(data) == 2 and len(data[0])+1 == len(data[1]): + data = data[::-1] + super(Histogram, self).__init__(data, **params) - def __getitem__(self, key): - """ - Implements slicing or indexing of the Histogram - """ - if key in self.dimensions(): return self.dimension_values(key) - if key is () or key is Ellipsis: return self # May no longer be necessary - key = util.process_ellipses(self, key) - if not isinstance(key, tuple): pass - elif len(key) == self.ndims + 1: - if key[-1] != slice(None) and (key[-1] not in self.vdims): - raise KeyError("%r is the only selectable value dimension" % - self.vdims[0].name) - key = key[0] - elif len(key) == self.ndims + 1: key = key[0] - else: - raise KeyError("Histogram cannot slice more than %d dimension." - % len(self.kdims)+1) - - centers = [(float(l)+r)/2 for (l,r) in zip(self.edges, self.edges[1:])] - if isinstance(key, slice): - start, stop = key.start, key.stop - if [start, stop] == [None,None]: return self - start_idx, stop_idx = None,None - if start is not None: - start_idx = np.digitize([start], centers, right=True)[0] - if stop is not None: - stop_idx = np.digitize([stop], centers, right=True)[0] - - slice_end = stop_idx+1 if stop_idx is not None else None - slice_values = self.values[start_idx:stop_idx] - slice_edges = self.edges[start_idx: slice_end] - - extents = (min(slice_edges), self.extents[1], - max(slice_edges), self.extents[3]) - return self.clone((slice_values, slice_edges), extents=extents) - else: - if not (self.edges.min() <= key < self.edges.max()): - raise KeyError("Key value %s is out of the histogram bounds" % key) - idx = np.digitize([key], self.edges)[0] - return self.values[idx-1 if idx>0 else idx] - - - - def _process_data(self, values, edges): - """ - Ensure that edges are specified as left and right edges of the - histogram bins rather than bin centers. - """ - settings = {} - (values, edges) = values if isinstance(values, tuple) else (values, edges) - if isinstance(values, Chart): - settings = dict(values.get_param_values(onlychanged=True)) - edges = values.dimension_values(0) - values = values.dimension_values(1) - elif isinstance(values, np.ndarray) and len(values.shape) == 2: - edges = values[:, 0] - values = values[:, 1] - elif all(isinstance(el, tuple) for el in values): - edges, values = zip(*values) - else: - values = np.array(values) - if edges is None: - edges = np.arange(len(values), dtype=np.float) - else: - edges = np.array(edges, dtype=np.float) - - if len(edges) == len(values): - edges = compute_edges(edges) - return values, edges, settings + def __setstate__(self, state): + """ + Ensures old-style Histogram types without an interface can be unpickled. - def range(self, dimension, data_range=True): - if self.get_dimension_index(dimension) == 0 and data_range: - dim = self.get_dimension(dimension) - lower, upper = np.min(self.edges), np.max(self.edges) - return util.dimension_range(lower, upper, dim) - else: - return super(Histogram, self).range(dimension, data_range) - - - def dimension_values(self, dim): - dim = self.get_dimension(dim, strict=True).name - if dim in self.vdims: - return self.values - elif dim in self.kdims: - return np.convolve(self.edges, np.ones((2,))/2, mode='valid') - else: - return super(Histogram, self).dimension_values(dim) - + Note: Deprecate as part of 2.0 + """ + if 'interface' not in state: + self.interface = GridInterface + x, y = state['_kdims_param_value'][0], state['_vdims_param_value'][0] + state['data'] = {x.name: state['data'][1], y.name: state['data'][0]} + super(Dataset, self).__setstate__(state) - def sample(self, samples=[], **sample_values): - raise NotImplementedError('Cannot sample a Histogram.') + @property + def values(self): + "Property to access the Histogram values provided for backward compatibility" + return self.dimension_values(1) - def reduce(self, dimensions=None, function=None, **reduce_map): - raise NotImplementedError('Reduction of Histogram not implemented.') + @property + def edges(self): + "Property to access the Histogram edges provided for backward compatibility" + return self.interface.coords(self, self.kdims[0], edges=True) class Points(Chart): diff --git a/holoviews/element/comparison.py b/holoviews/element/comparison.py index 6066527948..f1d7321886 100644 --- a/holoviews/element/comparison.py +++ b/holoviews/element/comparison.py @@ -541,9 +541,7 @@ def compare_trisurface(cls, el1, el2, msg='TriSurface'): @classmethod def compare_histogram(cls, el1, el2, msg='Histogram'): - cls.compare_dimensioned(el1, el2) - cls.compare_arrays(el1.edges, el2.edges, ' '.join([msg, 'edges'])) - cls.compare_arrays(el1.values, el2.values, ' '.join([msg, 'values'])) + cls.compare_dataset(el1, el2, msg) @classmethod def compare_points(cls, el1, el2, msg='Points'): @@ -598,10 +596,7 @@ def compare_raster(cls, el1, el2, msg='Raster'): @classmethod def compare_quadmesh(cls, el1, el2, msg='QuadMesh'): - cls.compare_dimensioned(el1, el2) - cls.compare_arrays(el1.data[0], el2.data[0], ' '.join([msg, 'x-data'])) - cls.compare_arrays(el1.data[1], el2.data[1], ' '.join([msg, 'y-data'])) - cls.compare_arrays(el1.data[2], el2.data[2], ' '.join([msg, 'z-data'])) + cls.compare_dataset(el1, el2, msg) @classmethod def compare_heatmap(cls, el1, el2, msg='HeatMap'): diff --git a/holoviews/element/raster.py b/holoviews/element/raster.py index 84fc683fab..007b066faf 100644 --- a/holoviews/element/raster.py +++ b/holoviews/element/raster.py @@ -1,17 +1,20 @@ +import sys from operator import itemgetter + import numpy as np import colorsys import param from ..core import util -from ..core.data import ImageInterface +from ..core.data import ImageInterface, GridInterface from ..core import Dimension, Element2D, Overlay, Dataset from ..core.boundingregion import BoundingRegion, BoundingBox from ..core.sheetcoords import SheetCoordinateSystem, Slice from ..core.util import dimension_range, compute_density, datetime_types from .chart import Curve +from .graphs import TriMesh from .tabular import Table -from .util import compute_edges, compute_slice_bounds, categorical_aggregate2d +from .util import compute_slice_bounds, categorical_aggregate2d class Raster(Element2D): @@ -259,6 +262,19 @@ def __init__(self, data, kdims=None, vdims=None, bounds=None, extents=None, raise ValueError("Input array has shape %r but %d value dimensions defined" % (self.shape, len(self.vdims))) + xvals = np.unique(np.diff(self.dimension_values(0, expanded=False))) + if len(xvals) > 1 and np.abs(xvals.min()-xvals.max()) > sys.float_info.epsilon*100: + raise ValueError("%s dimension %s is not evenly sampled, " + "please use the QuadMesh element for " + "unevenly or irregularly sampled data." % + (type(self).__name__, self.get_dimension(0))) + yvals = np.unique(np.diff(self.dimension_values(1, expanded=False))) + if len(yvals) > 1 and np.abs(yvals.min()-yvals.max()) > sys.float_info.epsilon*100: + raise ValueError("%s dimension %s is not evenly sampled, " + "please use the QuadMesh element for " + "unevenly or irregularly sampled data." % + (type(self).__name__, self.get_dimension(1))) + def __setstate__(self, state): """ @@ -602,7 +618,7 @@ def rgb(self): **params) -class QuadMesh(Raster): +class QuadMesh(Dataset, Element2D): """ QuadMesh is a Raster type to hold x- and y- bin values with associated values. The x- and y-values of the QuadMesh @@ -616,6 +632,8 @@ class QuadMesh(Raster): 2D arrays for the x-/y-coordinates and grid values. """ + datatype = param.List(default=['grid', 'xarray']) + group = param.String(default="QuadMesh", constant=True) kdims = param.List(default=[Dimension('x'), Dimension('y')], @@ -623,156 +641,57 @@ class QuadMesh(Raster): vdims = param.List(default=[Dimension('z')], bounds=(1,1)) - def __init__(self, data, kdims=None, vdims=None, **params): - data, kwargs = self._process_data(data) - params = dict(kwargs, **params) - if kdims is not None: - params['kdims'] = kdims - if vdims is not None: - params['vdims'] = vdims - Element2D.__init__(self, data, **params) - self.data = self._validate_data(self.data) - self._grid = self.data[0].ndim == 1 - - - @property - def depth(self): return 1 - - - def _process_data(self, data): - if isinstance(data, Image): - x = data.dimension_values(0, expanded=False) - y = data.dimension_values(1, expanded=False) - zarray = data.dimension_values(2, flat=False) - params = util.get_param_values(data) - else: - data = tuple(np.asarray(el) for el in data) - x, y, zarray = data - params = {} - ys, xs = zarray.shape - if x.ndim == 1 and len(x) == xs: - x = compute_edges(x) - if y.ndim == 1 and len(y) == ys: - y = compute_edges(y) - return (x, y, zarray), params - - - @property - def _zdata(self): - return self.data[2] - - - def _validate_data(self, data): - x, y, z = data - if not z.ndim == 2: - raise ValueError("Z-values must be 2D array") - - ys, xs = z.shape - shape_errors = [] - if x.ndim == 1 and xs+1 != len(x): - shape_errors.append('x') - if x.ndim == 1 and ys+1 != len(y): - shape_errors.append('y') - if shape_errors: - raise ValueError("%s-edges must match shape of z-array." % - '/'.join(shape_errors)) - return data + _binned = True + def __setstate__(self, state): + """ + Ensures old-style QuadMesh types without an interface can be unpickled. - def __getitem__(self, slices): - if slices in self.dimensions(): return self.dimension_values(slices) - slices = util.process_ellipses(self,slices) - if not self._grid: - raise KeyError("Indexing of non-grid based QuadMesh" - "currently not supported") - slices = util.wrap_tuple(slices) - if len(slices) == 1: - slices = slices+(slice(None),) - if len(slices) > (2 + self.depth): - raise KeyError("Can only slice %d dimensions" % (2 + self.depth)) - elif len(slices) == 3 and slices[-1] not in [self.vdims[0].name, slice(None)]: - raise KeyError("%r is the only selectable value dimension" % self.vdims[0].name) - slices = slices[:2] - if not isinstance(slices, tuple): slices = (slices, slice(None)) - slc_types = [isinstance(sl, slice) for sl in slices] - if not any(slc_types): - indices = [] - for idx, data in zip(slices, self.data[:self.ndims]): - dig = np.digitize([idx], data) - indices.append(dig-1 if dig else dig) - return self.data[2][tuple(indices[::-1])][0] - else: - sliced_data, indices = [], [] - for slc, data in zip(slices, self.data[:self.ndims]): - if isinstance(slc, slice): - low, high = slc.start, slc.stop - lidx = (None if low is None else - max((np.digitize([low], data)-1, 0))[0]) - hidx = (None if high is None else - np.digitize([high], data)[0]) - sliced_data.append(data[lidx:hidx]) - indices.append(slice(lidx, (hidx if hidx is None else hidx-1))) - else: - index = (np.digitize([slc], data)-1)[0] - sliced_data.append(data[index:index+2]) - indices.append(index) - z = np.atleast_2d(self.data[2][tuple(indices[::-1])]) - if not all(slc_types) and not slc_types[0]: - z = z.T - return self.clone(tuple(sliced_data+[z])) + Note: Deprecate as part of 2.0 + """ + if 'interface' not in state: + self.interface = GridInterface + x, y = state['_kdims_param_value'] + z = state['_vdims_param_value'][0] + data = state['data'] + state['data'] = {x.name: data[0], y.name: data[1], z.name: data[2]} + super(Dataset, self).__setstate__(state) - @classmethod - def collapse_data(cls, data_list, function, kdims=None, **kwargs): + def trimesh(self): """ - Allows collapsing the data of a number of QuadMesh - Elements with a function. + Converts a QuadMesh into a TriMesh. """ - if not all(data[0].ndim == 1 for data in data_list): - raise Exception("Collapsing of non-grid based QuadMesh" - "currently not supported") - xs, ys, zs = zip(data_list) - if isinstance(function, np.ufunc): - z = function.reduce(zs) - else: - z = function(np.dstack(zs), axis=-1, **kwargs) - return xs[0], ys[0], z - - - def _coord2matrix(self, coord): - return tuple((np.digitize([coord[i]], self.data[i])-1)[0] - for i in [1, 0]) - - - def range(self, dimension, data_range=True): - idx = self.get_dimension_index(dimension) - dim = self.get_dimension(dimension) - if idx in [0, 1, 2] and data_range: - data = self.data[idx] - lower, upper = np.nanmin(data), np.nanmax(data) - return dimension_range(lower, upper, dim) - return super(QuadMesh, self).range(dimension, data_range) - - - def dimension_values(self, dimension, expanded=True, flat=True): - idx = self.get_dimension_index(dimension) - data = self.data[idx] - if idx in [0, 1]: - # Handle grid - if not self._grid: - return data.flatten() - odim = self.data[2].shape[idx] if expanded else 1 - vals = np.tile(np.convolve(data, np.ones((2,))/2, mode='valid'), odim) - if idx: - return np.sort(vals) - else: - return vals - elif idx == 2: - # Value dimension - return data.flatten() if flat else data - else: - # Handle constant dimensions - return super(QuadMesh, self).dimension_values(idx) + # Generate vertices + xs = self.interface.coords(self, 'x', edges=True) + ys = self.interface.coords(self, 'y', edges=True) + if xs.ndim == 1: + xs, ys = (np.tile(xs[:, np.newaxis], len(ys)).T, + np.tile(ys[:, np.newaxis], len(xs))) + vertices = (xs.T.flatten(), ys.T.flatten()) + + # Generate triangle simplexes + s0 = self.interface.shape(self, gridded=True)[0] + t1 = np.arange(len(self)) + js = (t1//s0) + t1s = js*(s0+1)+t1%s0 + t2s = t1s+1 + t3s = (js+1)*(s0+1)+t1%s0 + t4s = t2s + t5s = t3s + t6s = t3s+1 + t1 = np.concatenate([t1s, t6s]) + t2 = np.concatenate([t2s, t5s]) + t3 = np.concatenate([t3s, t4s]) + ts = (t1, t2, t3) + for vd in self.vdims: + zs = self.dimension_values(2) + ts = ts + (np.concatenate([zs, zs]),) + + # Construct TriMesh + params = {k: v for k, v in util.get_param_values(self).items() + if k != 'kdims'} + return TriMesh(((ts,), vertices), **params) diff --git a/holoviews/element/util.py b/holoviews/element/util.py index 94bab31bb7..313362e9b2 100644 --- a/holoviews/element/util.py +++ b/holoviews/element/util.py @@ -27,19 +27,6 @@ xr = None -def compute_edges(edges): - """ - Computes edges as midpoints of the bin centers. - The first and last boundaries are equidistant from the first and last - midpoints respectively. - """ - edges = np.asarray(edges) - if edges.dtype.kind == 'i': - edges = edges.astype('f') - midpoints = (edges[:-1] + edges[1:])/2.0 - boundaries = (2*edges[0] - midpoints[0], 2*edges[-1] - midpoints[-1]) - return np.concatenate([boundaries[:1], midpoints, boundaries[-1:]]) - def split_path(path): """ diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py index 33f95ff08c..cbbd7b485a 100644 --- a/holoviews/operation/datashader.py +++ b/holoviews/operation/datashader.py @@ -27,7 +27,8 @@ from ..core.data import PandasInterface, XArrayInterface from ..core.sheetcoords import BoundingBox from ..core.util import get_param_values, basestring, datetime_types, dt_to_int -from ..element import Image, Path, Curve, RGB, Graph, TriMesh, Points, Scatter, Dataset +from ..element import (Image, Path, Curve, RGB, Graph, TriMesh, Points, + Scatter, Dataset, QuadMesh) from ..streams import RangeXY, PlotSize @@ -535,7 +536,10 @@ def _precompute(self, element): def _process(self, element, key=None): - x, y = element.nodes.kdims[:2] + if isinstance(element, TriMesh): + x, y = element.nodes.kdims[:2] + else: + x, y = element.kdims info = self._get_sampling(element, x, y) (x_range, y_range), _, (width, height), (xtype, ytype) = info cvs = ds.Canvas(plot_width=width, plot_height=height, @@ -551,7 +555,7 @@ def _process(self, element, key=None): pts = precomputed['vertices'] mesh = precomputed['mesh'] if self.p.precompute: - self._precomputed[element._plot_id] = precomputed + self._precomputed = {element._plot_id: precomputed} vdim = element.vdims[0] if element.vdims else element.nodes.vdims[0] interpolate = bool(self.p.interpolation) @@ -563,6 +567,18 @@ def _process(self, element, key=None): +class quadmesh_rasterize(trimesh_rasterize): + """ + Rasterize the QuadMesh element using the supplied aggregator. + Simply converts to a TriMesh and let's trimesh_rasterize + handle the actual rasterization. + """ + + def _precompute(self, element): + return super(quadmesh_rasterize, self)._precompute(element.trimesh()) + + + class rasterize(ResamplingOperation): """ Rasterize is a high-level operation which will rasterize any @@ -605,6 +621,14 @@ def _process(self, element, key=None): element = element.map(trirasterize, TriMesh) self._precomputed = trirasterize._precomputed + # Rasterize QuadMesh + quad_params = dict({k: v for k, v in self.p.items() + if k in aggregate.params()}, dynamic=False) + quadrasterize = quadmesh_rasterize.instance(**quad_params) + quadrasterize._precomputed = self._precomputed + element = element.map(quadrasterize, QuadMesh) + self._precomputed = quadrasterize._precomputed + # Rasterize NdOverlay of objects agg_params = dict({k: v for k, v in self.p.items() if k in aggregate.params()}, dynamic=False) @@ -614,13 +638,12 @@ def _process(self, element, key=None): issubclass(x.type, Dataset) and not issubclass(x.type, Image)) element = element.map(dsrasterize, predicate) - self._precomputed = trirasterize._precomputed # Rasterize other Dataset types predicate = lambda x: (isinstance(x, Dataset) and (not isinstance(x, Image) or x in imgs)) element = element.map(dsrasterize, predicate) - self._precomputed = trirasterize._precomputed + self._precomputed = dsrasterize._precomputed return element diff --git a/holoviews/plotting/bokeh/chart.py b/holoviews/plotting/bokeh/chart.py index 05fc65cad3..482d5d6627 100644 --- a/holoviews/plotting/bokeh/chart.py +++ b/holoviews/plotting/bokeh/chart.py @@ -337,9 +337,11 @@ def get_data(self, element, ranges, style): if self.static_source: data = dict(top=[], left=[], right=[]) else: - data = dict(top=element.values, left=element.edges[:-1], - right=element.edges[1:]) - self._get_hover_data(data, element) + x = element.kdims[0] + values = element.dimension_values(1) + edges = element.interface.coords(element, x, edges=True) + data = dict(top=values, left=edges[:-1], right=edges[1:]) + self._get_hover_data(data, element) return (data, mapping, style) def get_extents(self, element, ranges): @@ -374,17 +376,7 @@ class SideHistogramPlot(ColorbarPlot, HistogramPlot): """ def get_data(self, element, ranges, style): - if self.invert_axes: - mapping = dict(top='right', bottom='left', left=0, right='top') - else: - mapping = dict(top='top', bottom=0, left='left', right='right') - - if self.static_source: - data = dict(top=[], left=[], right=[]) - else: - data = dict(top=element.values, left=element.edges[:-1], - right=element.edges[1:]) - + data, mapping, style = HistogramPlot.get_data(self, element, ranges, style) color_dims = [d for d in self.adjoined.traverse(lambda x: x.handles.get('color_dim')) if d is not None] dim = color_dims[0] if color_dims else None @@ -393,7 +385,6 @@ def get_data(self, element, ranges, style): data[dim.name] = [] if self.static_source else element.dimension_values(dim) mapping['fill_color'] = {'field': dim.name, 'transform': cmapper} - self._get_hover_data(data, element) return (data, mapping, style) diff --git a/holoviews/plotting/bokeh/raster.py b/holoviews/plotting/bokeh/raster.py index d9b29f57b5..177b55c55d 100644 --- a/holoviews/plotting/bokeh/raster.py +++ b/holoviews/plotting/bokeh/raster.py @@ -6,7 +6,7 @@ from ...element import Raster from ..renderer import SkipRendering from .element import ElementPlot, ColorbarPlot, line_properties, fill_properties - +from .util import mpl_to_bokeh, colormesh class RasterPlot(ColorbarPlot): @@ -170,27 +170,52 @@ class QuadMeshPlot(ColorbarPlot): style_opts = ['cmap', 'color'] + line_properties + fill_properties def get_data(self, element, ranges, style): - x, y, z = element.dimensions(label=True) + x, y, z = element.dimensions()[:3] if self.invert_axes: x, y = y, x - cmapper = self._get_colormapper(element.vdims[0], element, ranges, style) + cmapper = self._get_colormapper(z, element, ranges, style) + cmapper = {'field': z.name, 'transform': cmapper} + + irregular = element.interface.irregular(element, x) + if irregular: + mapping = dict(xs='xs', ys='ys', fill_color=cmapper) + else: + mapping = {'x': x.name, 'y': y.name, 'fill_color': cmapper, + 'width': 'widths', 'height': 'heights'} + if self.static_source: - return {}, {'x': x, 'y': y, 'fill_color': {'field': z, 'transform': cmapper}}, style - - if len(set(v.shape for v in element.data)) == 1: - raise SkipRendering("Bokeh QuadMeshPlot only supports rectangular meshes") - zdata = element.data[2] - xvals = element.dimension_values(0, False) - yvals = element.dimension_values(1, False) - widths = np.diff(element.data[0]) - heights = np.diff(element.data[1]) - if self.invert_axes: - zvals = zdata.flatten() - xvals, yvals, widths, heights = yvals, xvals, heights, widths + return {}, mapping, style + + zdata = element.dimension_values(z, flat=False) + if irregular: + dims = element.kdims + if self.invert_axes: dims = dims[::-1] + X, Y = [element.interface.coords(element, d, expanded=True, edges=True) + for d in dims] + X, Y = colormesh(X, Y) + zvals = zdata.T.flatten() if self.invert_axes else zdata.flatten() + data = {'xs': list(X), 'ys': list(Y), z.name: zvals} + else: + xvals = element.dimension_values(x, expanded=True, flat=False) + yvals = element.dimension_values(y, expanded=True, flat=False) + xc, yc = (element.interface.coords(element, x, edges=True), + element.interface.coords(element, y, edges=True)) + widths, heights = np.diff(xc), np.diff(yc) + xs, ys = xvals.flatten(), yvals.flatten() + ws, hs = cartesian_product([widths, heights], copy=True) + zvals = zdata.flatten() if self.invert_axes else zdata.T.flatten() + data = {x.name: xs, y.name: ys, z.name: zvals, + 'widths': ws, 'heights': hs} + return data, mapping, style + + + def _init_glyph(self, plot, mapping, properties): + """ + Returns a Bokeh glyph object. + """ + properties = mpl_to_bokeh(properties) + properties = dict(properties, **mapping) + if 'xs' in mapping: + renderer = plot.patches(**properties) else: - zvals = zdata.T.flatten() - xs, ys = cartesian_product([xvals, yvals], copy=True) - ws, hs = cartesian_product([widths, heights], copy=True) - data = {x: xs, y: ys, z: zvals, 'widths': ws, 'heights': hs} - return (data, {'x': x, 'y': y, - 'fill_color': {'field': z, 'transform': cmapper}, - 'height': 'heights', 'width': 'widths'}, style) + renderer = plot.rect(**properties) + return renderer, renderer.glyph diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py index a4b3e25260..41316e01d5 100644 --- a/holoviews/plotting/bokeh/util.py +++ b/holoviews/plotting/bokeh/util.py @@ -622,3 +622,22 @@ def order_fn(glyph): return ((draw_order.index(matches[0]), glyph) if matches else (1e9+keys.index(glyph), glyph)) return sorted(keys, key=order_fn) + + +def colormesh(X, Y): + """ + Generates line paths for a quadmesh given 2D arrays of X and Y + coordinates. + """ + X1 = X[0:-1, 0:-1].ravel() + Y1 = Y[0:-1, 0:-1].ravel() + X2 = X[1:, 0:-1].ravel() + Y2 = Y[1:, 0:-1].ravel() + X3 = X[1:, 1:].ravel() + Y3 = Y[1:, 1:].ravel() + X4 = X[0:-1, 1:].ravel() + Y4 = Y[0:-1, 1:].ravel() + + X = np.column_stack([X1, X2, X3, X4, X1]) + Y = np.column_stack([Y1, Y2, Y3, Y4, Y1]) + return X, Y diff --git a/holoviews/plotting/mpl/chart.py b/holoviews/plotting/mpl/chart.py index e7e1433fef..5cce4baf08 100644 --- a/holoviews/plotting/mpl/chart.py +++ b/holoviews/plotting/mpl/chart.py @@ -282,11 +282,13 @@ def _process_hist(self, hist): Get data from histogram, including bin_ranges and values. """ self.cyclic = hist.get_dimension(0).cyclic - edges = hist.edges[:-1] - hist_vals = np.array(hist.values) - widths = np.diff(hist.edges) + x = hist.kdims[0] + edges = hist.interface.coords(hist, x, edges=True) + values = hist.dimension_values(1) + hist_vals = np.array(values) + widths = np.diff(edges) lims = hist.range(0) + hist.range(1) - return edges, hist_vals, widths, lims + return edges[:-1], hist_vals, widths, lims def _compute_ticks(self, element, edges, widths, lims): diff --git a/holoviews/plotting/mpl/raster.py b/holoviews/plotting/mpl/raster.py index adf85e18f8..e4d5bc4e5c 100644 --- a/holoviews/plotting/mpl/raster.py +++ b/holoviews/plotting/mpl/raster.py @@ -231,39 +231,43 @@ class QuadMeshPlot(ColorbarPlot): _plot_methods = dict(single='pcolormesh') def get_data(self, element, ranges, style): - data = np.ma.array(element.data[2], - mask=np.logical_not(np.isfinite(element.data[2]))) - coords = list(element.data[:2]) + zdata = element.dimension_values(2, flat=False) + data = np.ma.array(zdata, mask=np.logical_not(np.isfinite(zdata))) + expanded = element.interface.irregular(element, element.kdims[0]) + edges = style.get('shading') != 'gouraud' + coords = [element.interface.coords(element, d, ordered=True, + expanded=expanded, edges=edges) + for d in element.kdims] if self.invert_axes: coords = coords[::-1] data = data.T cmesh_data = coords + [data] - style['locs'] = np.concatenate(element.data[:2]) + if expanded: + style['locs'] = np.concatenate(coords) vdim = element.vdims[0] self._norm_kwargs(element, ranges, style, vdim) return tuple(cmesh_data), style, {} def init_artists(self, ax, plot_args, plot_kwargs): - locs = plot_kwargs.pop('locs') + locs = plot_kwargs.pop('locs', None) artist = ax.pcolormesh(*plot_args, **plot_kwargs) return {'artist': artist, 'locs': locs} def update_handles(self, key, axis, element, ranges, style): cmesh = self.handles['artist'] - locs = np.concatenate(element.data[:2]) - - if (locs != self.handles['locs']).any(): - return super(QuadMeshPlot, self).update_handles(key, axis, element, - ranges, style) - else: - data, style, axis_kwargs = self.get_data(element, ranges, style) - cmesh.set_array(data[-1]) + data, style, axis_kwargs = self.get_data(element, ranges, style) + locs = style.get('locs') + old_locs = self.handles.get('locs') + if None in (locs, old_locs) or (locs == old_locs).all(): + cmesh.set_array(data[-1].flatten()) cmesh.set_clim((style['vmin'], style['vmax'])) if 'norm' in style: cmesh.norm = style['norm'] return axis_kwargs + return super(QuadMeshPlot, self).update_handles(key, axis, element, + ranges, style) class RasterGridPlot(GridPlot, OverlayPlot): diff --git a/tests/testbinneddatasets.py b/tests/testbinneddatasets.py new file mode 100644 index 0000000000..bbbeec5b0b --- /dev/null +++ b/tests/testbinneddatasets.py @@ -0,0 +1,215 @@ +""" +Tests for binned interfaces including GridInterface and XArrayInterface +""" + +from unittest import SkipTest + +import numpy as np + +from holoviews.core.spaces import HoloMap +from holoviews.core.data import Dataset +from holoviews.element import Histogram, QuadMesh +from holoviews.element.comparison import ComparisonTestCase + + +class Binned1DTest(ComparisonTestCase): + + def setUp(self): + self.values = np.arange(10) + self.edges = np.arange(11) + self.dataset1d = Histogram((self.edges, self.values)) + + def test_slice_all(self): + sliced = self.dataset1d[:] + self.assertEqual(sliced.values, self.values) + self.assertEqual(sliced.edges, self.edges) + + def test_slice_exclusive_upper(self): + "Exclusive upper boundary semantics for bin centers" + sliced = self.dataset1d[:6.5] + self.assertEqual(sliced.values, np.arange(6)) + self.assertEqual(sliced.edges, np.arange(7)) + + def test_slice_exclusive_upper_exceeded(self): + "Slightly above the boundary in the previous test" + sliced = self.dataset1d[:6.55] + self.assertEqual(sliced.values, np.arange(7)) + self.assertEqual(sliced.edges, np.arange(8)) + + def test_slice_inclusive_lower(self): + "Inclusive lower boundary semantics for bin centers" + sliced = self.dataset1d[3.5:] + self.assertEqual(sliced.values, np.arange(3, 10)) + self.assertEqual(sliced.edges, np.arange(3, 11)) + + def test_slice_inclusive_lower_undershot(self): + "Inclusive lower boundary semantics for bin centers" + sliced = self.dataset1d[3.45:] + self.assertEqual(sliced.values, np.arange(3, 10)) + self.assertEqual(sliced.edges, np.arange(3, 11)) + + def test_slice_bounded(self): + sliced = self.dataset1d[3.5:6.5] + self.assertEqual(sliced.values, np.arange(3, 6)) + self.assertEqual(sliced.edges, np.arange(3, 7)) + + def test_slice_lower_out_of_bounds(self): + sliced = self.dataset1d[-3:] + self.assertEqual(sliced.values, self.values) + self.assertEqual(sliced.edges, self.edges) + + def test_slice_upper_out_of_bounds(self): + sliced = self.dataset1d[:12] + self.assertEqual(sliced.values, self.values) + self.assertEqual(sliced.edges, self.edges) + + def test_slice_both_out_of_bounds(self): + sliced = self.dataset1d[-3:13] + self.assertEqual(sliced.values, self.values) + self.assertEqual(sliced.edges, self.edges) + + def test_scalar_index(self): + self.assertEqual(self.dataset1d[4.5], 4) + self.assertEqual(self.dataset1d[3.7], 3) + self.assertEqual(self.dataset1d[9.9], 9) + + def test_scalar_index_boundary(self): + """ + Scalar at boundary indexes next bin. + (exclusive upper boundary for current bin) + """ + self.assertEqual(self.dataset1d[4], 4) + self.assertEqual(self.dataset1d[5], 5) + + def test_scalar_lowest_index(self): + self.assertEqual(self.dataset1d[0], 0) + + def test_scalar_lowest_index_out_of_bounds(self): + with self.assertRaises(IndexError): + self.dataset1d[-1] + + def test_scalar_highest_index_out_of_bounds(self): + with self.assertRaises(IndexError): + self.dataset1d[10] + + def test_groupby_kdim(self): + grouped = self.dataset1d.groupby('x', group_type=Dataset) + holomap = HoloMap({self.edges[i:i+2].mean(): Dataset([(i,)], vdims=['Frequency']) + for i in range(10)}, kdims=['x']) + self.assertEqual(grouped, holomap) + + +class Binned2DTest(ComparisonTestCase): + + def setUp(self): + n = 4 + self.xs = np.logspace(1, 3, n) + self.ys = np.linspace(1, 10, n) + self.zs = np.arange((n-1)**2).reshape(n-1, n-1) + self.dataset2d = QuadMesh((self.xs, self.ys, self.zs)) + + def test_qmesh_index_lower_left(self): + self.assertEqual(self.dataset2d[10, 1], 0) + + def test_qmesh_index_lower_right(self): + self.assertEqual(self.dataset2d[800, 3.9], 2) + + def test_qmesh_index_top_left(self): + self.assertEqual(self.dataset2d[10, 9.9], 6) + + def test_qmesh_index_top_right(self): + self.assertEqual(self.dataset2d[216, 7], 8) + + def test_qmesh_index_xcoords(self): + sliced = QuadMesh((self.xs[2:4], self.ys, self.zs[:, 2:3])) + self.assertEqual(self.dataset2d[300, :], sliced) + + def test_qmesh_index_ycoords(self): + sliced = QuadMesh((self.xs, self.ys[-2:], self.zs[-1:, :])) + self.assertEqual(self.dataset2d[:, 7], sliced) + + def test_qmesh_slice_xcoords(self): + sliced = QuadMesh((self.xs[1:], self.ys, self.zs[:, 1:])) + self.assertEqual(self.dataset2d[100:1000, :], sliced) + + def test_qmesh_slice_ycoords(self): + sliced = QuadMesh((self.xs, self.ys[:-1], self.zs[:-1, :])) + self.assertEqual(self.dataset2d[:, 2:7], sliced) + + def test_qmesh_slice_xcoords_ycoords(self): + sliced = QuadMesh((self.xs[1:], self.ys[:-1], self.zs[:-1, 1:])) + self.assertEqual(self.dataset2d[100:1000, 2:7], sliced) + + def test_groupby_xdim(self): + grouped = self.dataset2d.groupby('x', group_type=Dataset) + holomap = HoloMap({self.xs[i:i+2].mean(): Dataset((self.ys, self.zs[:, i]), 'y', 'z') + for i in range(3)}, kdims=['x']) + self.assertEqual(grouped, holomap) + + def test_groupby_ydim(self): + grouped = self.dataset2d.groupby('y', group_type=Dataset) + holomap = HoloMap({self.ys[i:i+2].mean(): Dataset((self.xs, self.zs[i]), 'x', 'z') + for i in range(3)}, kdims=['y']) + self.assertEqual(grouped, holomap) + + + +class Irregular2DBinsTest(ComparisonTestCase): + + def setUp(self): + lon, lat = np.meshgrid(np.linspace(-20, 20, 5), np.linspace(0, 30, 4)) + lon += lat/10 + lat += lon/10 + self.xs = lon + self.ys = lat + self.zs = np.arange(20).reshape(4, 5) + + def test_construct_from_dict(self): + dataset = Dataset((self.xs, self.ys, self.zs), ['x', 'y'], 'z') + self.assertEqual(dataset.dimension_values('x'), self.xs.T.flatten()) + self.assertEqual(dataset.dimension_values('y'), self.ys.T.flatten()) + self.assertEqual(dataset.dimension_values('z'), self.zs.T.flatten()) + + def test_construct_from_xarray(self): + try: + import xarray as xr + except: + raise SkipError("Test requires xarray") + da = xr.DataArray(self.zs, dims=['y', 'x'], + coords = {'lat': (('y', 'x'), self.ys), + 'lon': (('y', 'x'), self.xs)}, name='z') + dataset = Dataset(da, ['lon', 'lat'], 'z') + self.assertEqual(dataset.dimension_values('lon'), self.xs.T.flatten()) + self.assertEqual(dataset.dimension_values('lat'), self.ys.T.flatten()) + self.assertEqual(dataset.dimension_values('z'), self.zs.T.flatten()) + + def test_construct_3d_from_xarray(self): + try: + import xarray as xr + except: + raise SkipError("Test requires xarray") + zs = np.arange(40).reshape(2, 4, 5) + da = xr.DataArray(zs, dims=['z', 'y', 'x'], + coords = {'lat': (('y', 'x'), self.ys), + 'lon': (('y', 'x'), self.xs), + 'z': [0, 1]}, name='A') + dataset = Dataset(da, ['lon', 'lat', 'z'], 'A') + self.assertEqual(dataset.dimension_values('lon'), self.xs.T.flatten()) + self.assertEqual(dataset.dimension_values('lat'), self.ys.T.flatten()) + self.assertEqual(dataset.dimension_values('z', expanded=False), np.array([0, 1])) + self.assertEqual(dataset.dimension_values('A'), zs.T.flatten()) + + def test_groupby_3d_from_xarray(self): + try: + import xarray as xr + except: + raise SkipError("Test requires xarray") + zs = np.arange(40).reshape(2, 4, 5) + da = xr.DataArray(zs, dims=['z', 'y', 'x'], + coords = {'lat': (('y', 'x'), self.ys), + 'lon': (('y', 'x'), self.xs), + 'z': [0, 1]}, name='A') + grouped = Dataset(da, ['lon', 'lat', 'z'], 'A').groupby('z') + hmap = HoloMap({0: Dataset((self.xs, self.ys, zs[0]), ['lon', 'lat'], 'A'), + 1: Dataset((self.xs, self.ys, zs[1]), ['lon', 'lat'], 'A')}, kdims='z') + self.assertEqual(grouped, hmap) diff --git a/tests/testcomparisonchart.py b/tests/testcomparisonchart.py index cc0e783dd2..ea51a0dfb1 100644 --- a/tests/testcomparisonchart.py +++ b/tests/testcomparisonchart.py @@ -89,26 +89,16 @@ def test_histograms_equal_2(self): self.assertEqual(self.hist2, self.hist2) def test_histograms_unequal_1(self): - try: + with self.assertRaises(AssertionError): self.assertEqual(self.hist1, self.hist2) - except AssertionError as e: - if not str(e).startswith("Histogram edges not almost equal to 6 decimals"): - raise self.failureException("Histogram edge data mismatch error not raised.") def test_histograms_unequal_2(self): - try: + with self.assertRaises(AssertionError): self.assertEqual(self.hist1, self.hist3) - except AssertionError as e: - if not str(e).startswith("Histogram edges not almost equal to 6 decimals"): - raise self.failureException("Histogram edge data mismatch error not raised.") def test_histograms_unequal_3(self): - try: + with self.assertRaises(AssertionError): self.assertEqual(self.hist1, self.hist4) - except AssertionError as e: - if not str(e).startswith("Histogram values not almost equal to 6 decimals"): - raise self.failureException("Histogram value data mismatch error not raised.") - diff --git a/tests/testcoreutils.py b/tests/testcoreutils.py index 5dfa222a0d..e8952595a2 100644 --- a/tests/testcoreutils.py +++ b/tests/testcoreutils.py @@ -17,7 +17,7 @@ from holoviews.core.util import ( sanitize_identifier_fn, find_range, max_range, wrap_tuple_streams, deephash, merge_dimensions, get_path, make_path_unique, compute_density, - date_range, dt_to_int + date_range, dt_to_int, compute_edges ) from holoviews import Dimension, Element from holoviews.streams import PointerXY @@ -591,3 +591,26 @@ def test_date_range_1_sec(self): drange = date_range(start, end, 10) self.assertEqual(drange[0], start+np.timedelta64(50, 'ms')) self.assertEqual(drange[-1], end-np.timedelta64(50, 'ms')) + + +class TestComputeEdges(ComparisonTestCase): + """ + Tests for compute_edges function. + """ + + def setUp(self): + self.array1 = [.5, 1.5, 2.5] + self.array2 = [.5, 1.0000001, 1.5] + self.array3 = [1, 2, 4] + + def test_simple_edges(self): + self.assertEqual(compute_edges(self.array1), + np.array([0, 1, 2, 3])) + + def test_close_edges(self): + self.assertEqual(compute_edges(self.array2), + np.array([0.25, 0.75, 1.25, 1.75])) + + def test_uneven_edges(self): + self.assertEqual(compute_edges(self.array3), + np.array([0.5, 1.5, 3.0, 5.0])) diff --git a/tests/testdataset.py b/tests/testdataset.py index 0f9028eca5..d320c41458 100644 --- a/tests/testdataset.py +++ b/tests/testdataset.py @@ -1264,14 +1264,6 @@ def setUp(self): self.init_grid_data() self.init_column_data() - def test_dataset_array_init_hm(self): - "Tests support for arrays (homogeneous)" - exception = "None of the available storage backends "\ - "were able to support the supplied data format." - with self.assertRaisesRegexp(Exception, exception): - Dataset(np.column_stack([self.xs, self.xs_2]), - kdims=['x'], vdims=['x2']) - def test_dataset_dataframe_init_hm(self): "Tests support for homogeneous DataFrames" if pd is None: @@ -1458,6 +1450,10 @@ def setUp(self): self.init_column_data() self.init_grid_data() + def test_dataset_array_init_hm(self): + "Tests support for arrays (homogeneous)" + raise SkipTest("Not supported") + # Disabled tests for NotImplemented methods def test_dataset_add_dimensions_values_hm(self): raise SkipTest("Not supported") @@ -1544,6 +1540,10 @@ def test_xarray_dataset_with_scalar_dim_canonicalize(self): expected = np.array([[0, 1], [2, 3], [4, 5]]) self.assertEqual(canonical, expected) + def test_dataset_array_init_hm(self): + "Tests support for arrays (homogeneous)" + raise SkipTest("Not supported") + # Disabled tests for NotImplemented methods def test_dataset_add_dimensions_values_hm(self): raise SkipTest("Not supported") diff --git a/tests/testdatashader.py b/tests/testdatashader.py index 565cd2a743..1296c43881 100644 --- a/tests/testdatashader.py +++ b/tests/testdatashader.py @@ -2,12 +2,13 @@ from nose.plugins.attrib import attr import numpy as np -from holoviews import Curve, Points, Image, Dataset, RGB, Path, Graph +from holoviews import Curve, Points, Image, Dataset, RGB, Path, Graph, TriMesh, QuadMesh from holoviews.element.comparison import ComparisonTestCase try: + import datashader as ds from holoviews.operation.datashader import ( - aggregate, regrid, ds_version, stack, directly_connect_edges + aggregate, regrid, ds_version, stack, directly_connect_edges, rasterize ) except: ds_version = None @@ -150,6 +151,32 @@ def test_regrid_disabled_expand(self): self.assertEqual(regridded, img) +@attr(optional=1) +class DatashaderRasterizeTests(ComparisonTestCase): + """ + Tests for datashader aggregation + """ + + def setUp(self): + if ds_version is None or ds_version <= '0.6.4': + raise SkipTest('Regridding operations require datashader>=0.7.0') + + def test_rasterize_trimesh(self): + simplices = [(0, 1, 2, 0.5), (3, 2, 1, 1.5)] + vertices = [(0., 0.), (0., 1.), (1., 0), (1, 1)] + trimesh = TriMesh((simplices, vertices), vdims=['z']) + img = rasterize(trimesh, width=3, height=3, dynamic=False, aggregator=ds.mean('z')) + image = Image(np.array([[1.5, 1.5, np.NaN], [0.5, 1.5, np.NaN], [np.NaN, np.NaN, np.NaN]]), + bounds=(0, 0, 1, 1)) + self.assertEqual(img, image) + + def test_rasterize_quadmesh(self): + qmesh = QuadMesh(([0, 1], [0, 1], np.array([[0, 1], [2, 3]]))) + img = rasterize(qmesh, width=3, height=3, dynamic=False, aggregator=ds.mean('z')) + image = Image(np.array([[2., 3., np.NaN], [0, 1, np.NaN], [np.NaN, np.NaN, np.NaN]]), + bounds=(-.5, -.5, 1.5, 1.5)) + self.assertEqual(img, image) + @attr(optional=1) class DatashaderStackTests(ComparisonTestCase): diff --git a/tests/testelementconstructors.py b/tests/testelementconstructors.py index 3bbafeb609..e748a8f7a5 100644 --- a/tests/testelementconstructors.py +++ b/tests/testelementconstructors.py @@ -50,13 +50,17 @@ def test_hist_yvalues_construct(self): def test_hist_curve_construct(self): hist = Histogram(Curve(([0.1, 0.3, 0.5], [2.1, 2.2, 3.3]))) - self.assertEqual(hist.data[0], np.array([2.1, 2.2, 3.3])) - self.assertEqual(hist.data[1], np.array([0, 0.2, 0.4, 0.6])) + values = hist.dimension_values(1) + edges = hist.interface.coords(hist, hist.kdims[0], edges=True) + self.assertEqual(values, np.array([2.1, 2.2, 3.3])) + self.assertEqual(edges, np.array([0, 0.2, 0.4, 0.6])) def test_hist_curve_int_edges_construct(self): hist = Histogram(Curve(range(3))) - self.assertEqual(hist.data[0], np.arange(3)) - self.assertEqual(hist.data[1], np.array([-.5, .5, 1.5, 2.5])) + values = hist.dimension_values(1) + edges = hist.interface.coords(hist, hist.kdims[0], edges=True) + self.assertEqual(values, np.arange(3)) + self.assertEqual(edges, np.array([-.5, .5, 1.5, 2.5])) def test_heatmap_construct(self): hmap = HeatMap([('A', 'a', 1), ('B', 'b', 2)]) diff --git a/tests/testelementindexing.py b/tests/testelementindexing.py deleted file mode 100644 index 835a072bb3..0000000000 --- a/tests/testelementindexing.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Test cases for both indexing and slicing of elements -""" -import numpy as np -from holoviews import Histogram, QuadMesh -from holoviews.element.comparison import ComparisonTestCase - - -class HistogramIndexingTest(ComparisonTestCase): - - def setUp(self): - self.values = [i for i in range(10)] - self.edges = [i for i in range(11)] - self.hist=Histogram(self.values, self.edges) - - def test_slice_all(self): - sliced = self.hist[:] - self.assertEqual(np.all(sliced.values == self.values), True) - self.assertEqual(np.all(sliced.edges == self.edges), True) - - def test_slice_exclusive_upper(self): - "Exclusive upper boundary semantics for bin centers" - sliced = self.hist[:6.5] - self.assertEqual(np.all(sliced.values == [0, 1, 2, 3, 4, 5]), True) - self.assertEqual(np.all(sliced.edges == [0, 1, 2, 3, 4, 5, 6]), True) - - def test_slice_exclusive_upper_exceeded(self): - "Slightly above the boundary in the previous test" - sliced = self.hist[:6.55] - self.assertEqual(np.all(sliced.values == [0, 1, 2, 3, 4, 5, 6]), True) - self.assertEqual(np.all(sliced.edges == [0, 1, 2, 3, 4, 5, 6, 7]), True) - - def test_slice_inclusive_lower(self): - "Inclusive lower boundary semantics for bin centers" - sliced = self.hist[3.5:] - self.assertEqual(np.all(sliced.values == [3, 4, 5, 6, 7, 8, 9]), True) - self.assertEqual(np.all(sliced.edges == [3, 4, 5, 6, 7, 8, 9, 10]), True) - - def test_slice_inclusive_lower_undershot(self): - "Inclusive lower boundary semantics for bin centers" - sliced = self.hist[3.45:] - self.assertEqual(np.all(sliced.values == [3, 4, 5, 6, 7, 8, 9]), True) - self.assertEqual(np.all(sliced.edges == [3, 4, 5, 6, 7, 8, 9, 10]), True) - - def test_slice_bounded(self): - sliced = self.hist[3.5:6.5] - self.assertEqual(np.all(sliced.values == [3, 4, 5]), True) - self.assertEqual(np.all(sliced.edges == [3, 4, 5, 6]), True) - - def test_slice_lower_out_of_bounds(self): - sliced = self.hist[-3:] - self.assertEqual(np.all(sliced.values == self.values), True) - self.assertEqual(np.all(sliced.edges == self.edges), True) - - def test_slice_upper_out_of_bounds(self): - sliced = self.hist[:12] - self.assertEqual(np.all(sliced.values == self.values), True) - self.assertEqual(np.all(sliced.edges == self.edges), True) - - def test_slice_both_out_of_bounds(self): - sliced = self.hist[-3:13] - self.assertEqual(np.all(sliced.values == self.values), True) - self.assertEqual(np.all(sliced.edges == self.edges), True) - - def test_scalar_index(self): - self.assertEqual(self.hist[4.5], 4) - self.assertEqual(self.hist[3.7], 3) - self.assertEqual(self.hist[9.9], 9) - - def test_scalar_index_boundary(self): - """ - Scalar at boundary indexes next bin. - (exclusive upper boundary for current bin) - """ - self.assertEqual(self.hist[4], 4) - self.assertEqual(self.hist[5], 5) - - def test_scalar_lowest_index(self): - self.assertEqual(self.hist[0], 0) - - def test_scalar_lowest_index_out_of_bounds(self): - try: - self.hist[-0.1] - except Exception as e: - if not str(e).startswith("'Key value -0.1 is out of the histogram bounds"): - raise AssertionError("Out of bound exception not generated") - - def test_scalar_highest_index_out_of_bounds(self): - try: - self.hist[10] - except Exception as e: - if not str(e).startswith("'Key value 10 is out of the histogram bounds"): - raise AssertionError("Out of bound exception not generated") - - -class QuadMeshIndexingTest(ComparisonTestCase): - - - def setUp(self): - n = 4 - self.xs = np.logspace(1, 3, n) - self.ys = np.linspace(1, 10, n) - self.zs = np.arange((n-1)**2).reshape(n-1, n-1) - self.qmesh = QuadMesh((self.xs, self.ys, self.zs)) - - def test_qmesh_index_lower_left(self): - self.assertEqual(self.qmesh[0, 0], 0) - - def test_qmesh_index_lower_right(self): - self.assertEqual(self.qmesh[800, 3.9], 2) - - def test_qmesh_index_top_left(self): - self.assertEqual(self.qmesh[10, 9.9], 6) - - def test_qmesh_index_top_right(self): - self.assertEqual(self.qmesh[216, 7], 8) - - def test_qmesh_index_xcoords(self): - sliced = QuadMesh((self.xs[2:4], self.ys, self.zs[:, 2:3])) - self.assertEqual(self.qmesh[300, :], sliced) - - def test_qmesh_index_ycoords(self): - sliced = QuadMesh((self.xs, self.ys[-2:], self.zs[-1:, :])) - self.assertEqual(self.qmesh[:, 7], sliced) - - def test_qmesh_slice_xcoords(self): - sliced = QuadMesh((self.xs[1:], self.ys, self.zs[:, 1:])) - self.assertEqual(self.qmesh[100:1000, :], sliced) - - def test_qmesh_slice_ycoords(self): - sliced = QuadMesh((self.xs, self.ys[:-1], self.zs[:-1, :])) - self.assertEqual(self.qmesh[:, 2:7], sliced) - - def test_qmesh_slice_xcoords_ycoords(self): - sliced = QuadMesh((self.xs[1:], self.ys[:-1], self.zs[:-1, 1:])) - self.assertEqual(self.qmesh[100:1000, 2:7], sliced) diff --git a/tests/testelementutils.py b/tests/testelementutils.py deleted file mode 100644 index afa7ccc97a..0000000000 --- a/tests/testelementutils.py +++ /dev/null @@ -1,26 +0,0 @@ -import numpy as np - -from holoviews.element.util import compute_edges -from holoviews.element.comparison import ComparisonTestCase - -class TestComputeEdges(ComparisonTestCase): - """ - Tests for compute_edges function. - """ - - def setUp(self): - self.array1 = [.5, 1.5, 2.5] - self.array2 = [.5, 1.0000001, 1.5] - self.array3 = [1, 2, 4] - - def test_simple_edges(self): - self.assertEqual(compute_edges(self.array1), - np.array([0, 1, 2, 3])) - - def test_close_edges(self): - self.assertEqual(compute_edges(self.array2), - np.array([0.25, 0.75, 1.25, 1.75])) - - def test_uneven_edges(self): - self.assertEqual(compute_edges(self.array3), - np.array([0.5, 1.5, 3.0, 5.0])) diff --git a/tests/testellipsis.py b/tests/testellipsis.py index 843e48c0b0..14e917d36f 100644 --- a/tests/testellipsis.py +++ b/tests/testellipsis.py @@ -38,23 +38,18 @@ def test_points_ellipsis_slice_y(self): def test_histogram_ellipsis_slice_value(self): frequencies, edges = np.histogram(range(20), 20) sliced = hv.Histogram(frequencies, edges)[..., 'Frequency'] - self.assertEqual(len(sliced.data[0]), 20) + self.assertEqual(len(sliced.dimension_values(0)), 20) def test_histogram_ellipsis_slice_range(self): frequencies, edges = np.histogram(range(20), 20) sliced = hv.Histogram(frequencies, edges)[0:5, ...] - self.assertEqual(len(sliced.data[0]), 5) + self.assertEqual(len(sliced.dimension_values(0)), 5) def test_histogram_ellipsis_slice_value_missing(self): frequencies, edges = np.histogram(range(20), 20) - try: + with self.assertRaises(IndexError): hv.Histogram(frequencies, edges)[..., 'Non-existent'] - raise AssertionError("No assertion raised") - except Exception as e: - if str(e) != repr("'Frequency' is the only selectable value dimension"): - raise AssertionError("Incorrect exception raised.") - class TestEllipsisTable(ComparisonTestCase): diff --git a/tests/testraster.py b/tests/testraster.py index ec387c988e..db658749c5 100644 --- a/tests/testraster.py +++ b/tests/testraster.py @@ -3,7 +3,7 @@ """ import numpy as np -from holoviews.element import Raster, Image, Curve, QuadMesh +from holoviews.element import Raster, Image, Curve, QuadMesh, TriMesh from holoviews.element.comparison import ComparisonTestCase class TestRaster(ComparisonTestCase): @@ -67,11 +67,27 @@ def setUp(self): def test_cast_image_to_quadmesh(self): img = Image(self.array1, kdims=['a', 'b'], vdims=['c'], group='A', label='B') qmesh = QuadMesh(img) - self.assertEqual(qmesh.data[0], np.array([-0.5, -0.166667, 0.166667, 0.5])) - self.assertEqual(qmesh.data[1], np.array([-0.5, 0, 0.5])) - self.assertEqual(qmesh.data[2], self.array1[::-1]) + self.assertEqual(qmesh.dimension_values(0, False), np.array([-0.333333, 0., 0.333333])) + self.assertEqual(qmesh.dimension_values(1, False), np.array([-0.25, 0.25])) + self.assertEqual(qmesh.dimension_values(2, flat=False), self.array1[::-1]) self.assertEqual(qmesh.kdims, img.kdims) self.assertEqual(qmesh.vdims, img.vdims) self.assertEqual(qmesh.group, img.group) self.assertEqual(qmesh.label, img.label) + def test_quadmesh_to_trimesh(self): + qmesh = QuadMesh(([0, 1], [0, 1], np.array([[0, 1], [2, 3]]))) + trimesh = qmesh.trimesh() + simplices = np.array([[0, 1, 3, 0], + [1, 2, 4, 2], + [3, 4, 6, 1], + [4, 5, 7, 3], + [4, 3, 1, 0], + [5, 4, 2, 2], + [7, 6, 4, 1], + [8, 7, 5, 3]]) + vertices = np.array([(-0.5, -0.5), (-0.5, 0.5), (-0.5, 1.5), + (0.5, -0.5), (0.5, 0.5), (0.5, 1.5), + (1.5, -0.5), (1.5, 0.5), (1.5, 1.5)]) + self.assertEqual(trimesh.array(), simplices) + self.assertEqual(trimesh.nodes.array([0, 1]), vertices)