From 378914805cdbc378dfbfb3e57f3eb400fe746e28 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Thu, 12 Sep 2019 16:32:08 -0500 Subject: [PATCH 1/4] Remove deprecated sharedict module --- dask/array/tests/test_atop.py | 33 ---------- dask/sharedict.py | 121 ---------------------------------- dask/tests/test_sharedict.py | 84 ----------------------- 3 files changed, 238 deletions(-) delete mode 100644 dask/sharedict.py delete mode 100644 dask/tests/test_sharedict.py diff --git a/dask/array/tests/test_atop.py b/dask/array/tests/test_atop.py index 03f429745b0..53ba5b070ce 100644 --- a/dask/array/tests/test_atop.py +++ b/dask/array/tests/test_atop.py @@ -13,7 +13,6 @@ rewrite_blockwise, optimize_blockwise, index_subs, - blockwise, ) from dask.array.utils import assert_eq from dask.array.numpy_compat import _numpy_116 @@ -552,38 +551,6 @@ def test_validate_top_inputs(): assert "i" in str(info.value) -def test_gh_4176(): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - from dask.sharedict import ShareDict - - def foo(A): - return A[None, ...] - - A = da.ones(shape=(10, 20, 4), chunks=(2, 5, 4)) - - name = "D" - - dsk = blockwise( - foo, - name, - ("nsrc", "ntime", "nbl", "npol"), - A.name, - ("ntime", "nbl", "npol"), - new_axes={"nsrc": 1}, - numblocks={a.name: a.numblocks for a in (A,)}, - ) - - array_dsk = ShareDict() - array_dsk.update(dsk) - array_dsk.update(A.__dask_graph__()) - - chunks = ((1,),) + A.chunks - - D = da.Array(array_dsk, name, chunks, dtype=A.dtype) - D.sum(axis=0).compute() - - def test_dont_merge_before_reductions(): x = da.ones(10, chunks=(5,)) y = da.blockwise(inc, "i", x, "i", dtype=x.dtype) diff --git a/dask/sharedict.py b/dask/sharedict.py deleted file mode 100644 index ce82f916ca3..00000000000 --- a/dask/sharedict.py +++ /dev/null @@ -1,121 +0,0 @@ -from toolz import concat, unique, count - -from .compatibility import Mapping - -import warnings - -warnings.warn( - "ShareDict has been deprecated in favor of HighLevelGraph " - "and will be removed in future versions", - stacklevel=2, -) - - -class ShareDict(Mapping): - """ A Mapping composed of other Mappings - - This is a union of other disjoint mappings. It allows the combination of - many dicts into a single dict-like object without creating copies of the - underlying dicts. It provides cheap ``update``, ``len`` and ``__iter__`` - operations as well as a fairly cheap ``__getitem__`` operation (linear in - the number of constituent mappings). - - This class is optimized for Dask's use, and may not be generally useful. - Users may want to consider the standard ``collections.ChainMap`` data - structure. - - This class makes the following assumptions: - - 1. Constituent mappings are disjoint. No key is in more than one mapping. - 2. Constituent mappings will not be modified - - Note that ShareDict does not enforce these assumptions. It is up to the - user to guarantee them. - - Examples - -------- - >>> a = {'x': 1, 'y': 2} - >>> b = {'z': 3} - >>> s = ShareDict() - >>> s.update(a) - >>> s.update(b) - - >>> dict(s) # doctest: +SKIP - {'x': 1, 'y': 2, 'z': 3} - - These dictionaries are stored within an internal dictionary of dictionaries - - >>> list(s.dicts.values()) # doctest: +SKIP - [{'x': 1, 'y': 2}, {'z': 3}] - - By default these are named by their object id. However, you can also - provide explicit names. - - >>> s = ShareDict() - >>> s.update_with_key(a, key='a') - >>> s.update_with_key(b, key='b') - >>> s.dicts # doctest: +SKIP - {'a': {'x': 1, 'y': 2}, 'b': {'z': 3}} - """ - - def __init__(self, dicts=None, dependencies=None): - self.dicts = dicts or dict() - self.dependencies = dependencies or dict() - - assert set(self.dependencies) == set(self.dicts) - - def update_with_key(self, arg, key=None, dependencies=None): - if type(arg) is ShareDict: - assert key is None or key in arg.dicts - self.dicts.update(arg.dicts) - self.dependencies.update(arg.dependencies) - return - - if key is None: - key = id(arg) - - if dependencies: - assert isinstance(dependencies, (tuple, list, set)) - self.dependencies[key] = set(dependencies) - - assert isinstance(arg, Mapping) - if arg: - self.dicts[key] = arg - - def update(self, arg): - self.update_with_key(arg) - - def __getitem__(self, key): - for d in self.dicts.values(): - if key in d: - return d[key] - raise KeyError(key) - - def __len__(self): - return count(iter(self)) - - def items(self): - seen = set() - for d in self.dicts.values(): - for key in d: - if key not in seen: - seen.add(key) - yield (key, d[key]) - - def __iter__(self): - return unique(concat(self.dicts.values())) - - -def merge(*dicts, **kwargs): - dependencies = kwargs.pop("dependencies", None) - assert not kwargs - # assert dependencies is not None - result = ShareDict() - for d in dicts: - if isinstance(d, tuple): - key, d = d - result.update_with_key(d, key=key) - else: - result.update_with_key(d) - result.dependencies.update(dependencies or {}) - return result diff --git a/dask/tests/test_sharedict.py b/dask/tests/test_sharedict.py deleted file mode 100644 index c718253088f..00000000000 --- a/dask/tests/test_sharedict.py +++ /dev/null @@ -1,84 +0,0 @@ -import warnings -import pytest -from toolz import merge - -from dask.compatibility import Mapping - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - from dask.sharedict import ShareDict - - -a = {"x": 1, "y": 2} -b = {"z": 3} -c = {"w": 2} - - -def test_core(): - s = ShareDict() - assert isinstance(s, Mapping) - - s.update(a) - s.update(b) - - assert s["x"] == 1 - with pytest.raises(KeyError): - s["abc"] - - with pytest.raises((NotImplementedError, TypeError)): - s["abc"] = 123 - - -def test_structure(): - s = ShareDict() - s.update(a) - s.update(b) - s.update(c) - - assert all(any(d is x for d in s.dicts.values()) for x in [a, b, c]) - - -@pytest.mark.skip -def test_structure_2(): - s = ShareDict() - s.update_with_key(a, key="a") - s.update_with_key(b, key="b") - s.update_with_key(c, key="c") - - assert s.order == ["a", "b", "c"] - - s.update_with_key(b, key="b") - - assert s.order == ["a", "c", "b"] - - -def test_keys_items(): - s = ShareDict() - s.update_with_key(a, key="a") - s.update_with_key(b, key="b") - s.update_with_key(c, key="c") - - d = merge(a, b, c) - - for fn in [dict, set, len]: - assert fn(s) == fn(d) - - for fn in [lambda x: x.values(), lambda x: x.keys(), lambda x: x.items()]: - assert set(fn(s)) == set(fn(d)) - - -def test_update_with_sharedict(): - s = ShareDict() - s.update_with_key(a, key="a") - s.update_with_key(b, key="b") - s.update_with_key(c, key="c") - - d = {"z": 5} - - s2 = ShareDict() - s2.update_with_key(a, key="a") - s2.update_with_key(d, key="d") - - s.update(s2) - - assert s.dicts["a"] is s.dicts["a"] From 96d163ab6aea687343849d3e51d438b0aae86199 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Thu, 12 Sep 2019 16:32:58 -0500 Subject: [PATCH 2/4] Remove dask.array.ghost module Just raised an ImportError for the last year. --- dask/array/ghost.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 dask/array/ghost.py diff --git a/dask/array/ghost.py b/dask/array/ghost.py deleted file mode 100644 index 3f0233b7f38..00000000000 --- a/dask/array/ghost.py +++ /dev/null @@ -1 +0,0 @@ -raise ImportError("The dask.array.ghost module has moved to dask.array.overlap") From 3106eade1318b8bac66be3ddbf00aa33dc4b7120 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Thu, 12 Sep 2019 16:49:11 -0500 Subject: [PATCH 3/4] Remove deprecated set_options function --- dask/__init__.py | 1 - dask/context.py | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/dask/__init__.py b/dask/__init__.py index 77c53da0933..05877a93094 100644 --- a/dask/__init__.py +++ b/dask/__init__.py @@ -2,7 +2,6 @@ from . import config, datasets from .core import istask -from .context import set_options from .local import get_sync as get try: diff --git a/dask/context.py b/dask/context.py index c04266e786d..0547f7903ad 100644 --- a/dask/context.py +++ b/dask/context.py @@ -13,18 +13,6 @@ thread_state = threading.local() -def set_options(*args, **kwargs): - """ Deprecated: see dask.config.set instead """ - raise TypeError( - "The dask.set_options function has been deprecated.\n" - "Please use dask.config.set instead\n\n" - " Before: with dask.set_options(foo='bar'):\n" - " ...\n" - " After: with dask.config.set(foo='bar'):\n" - " ..." - ) - - def globalmethod(default=None, key=None, falsey=None): """ Allow function to be taken over by globals From a188a522f79273307f0313c846a76686926e6836 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Thu, 12 Sep 2019 16:55:24 -0500 Subject: [PATCH 4/4] Remove deprecated `freq` arg to rolling Also apply black. --- dask/array/tests/test_atop.py | 7 +------ dask/dataframe/core.py | 9 +-------- dask/dataframe/rolling.py | 13 +------------ 3 files changed, 3 insertions(+), 26 deletions(-) diff --git a/dask/array/tests/test_atop.py b/dask/array/tests/test_atop.py index 53ba5b070ce..564381ae883 100644 --- a/dask/array/tests/test_atop.py +++ b/dask/array/tests/test_atop.py @@ -8,12 +8,7 @@ import dask import dask.array as da from dask.highlevelgraph import HighLevelGraph -from dask.blockwise import ( - Blockwise, - rewrite_blockwise, - optimize_blockwise, - index_subs, -) +from dask.blockwise import Blockwise, rewrite_blockwise, optimize_blockwise, index_subs from dask.array.utils import assert_eq from dask.array.numpy_compat import _numpy_116 from dask.utils_test import inc, dec diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py index ec0be535398..fef6383fac4 100644 --- a/dask/dataframe/core.py +++ b/dask/dataframe/core.py @@ -1339,9 +1339,7 @@ def _get_binary_operator(cls, op, inv=False): else: return lambda self, other: elemwise(op, self, other) - def rolling( - self, window, min_periods=None, freq=None, center=False, win_type=None, axis=0 - ): + def rolling(self, window, min_periods=None, center=False, win_type=None, axis=0): """Provides rolling transformations. Parameters @@ -1370,10 +1368,6 @@ def rolling( Returns ------- a Rolling object on which to call a method to compute a statistic - - Notes - ----- - The `freq` argument is not supported. """ from dask.dataframe.rolling import Rolling @@ -1391,7 +1385,6 @@ def rolling( self, window=window, min_periods=min_periods, - freq=freq, center=center, win_type=win_type, axis=axis, diff --git a/dask/dataframe/rolling.py b/dask/dataframe/rolling.py index d04ef49f28a..6f48e979539 100644 --- a/dask/dataframe/rolling.py +++ b/dask/dataframe/rolling.py @@ -264,19 +264,8 @@ class Rolling(object): """Provides rolling window calculations.""" def __init__( - self, - obj, - window=None, - min_periods=None, - freq=None, - center=False, - win_type=None, - axis=0, + self, obj, window=None, min_periods=None, center=False, win_type=None, axis=0 ): - if freq is not None: - msg = "The deprecated freq argument is not supported." - raise NotImplementedError(msg) - self.obj = obj # dataframe or series self.window = window self.min_periods = min_periods