Skip to content

Commit

Permalink
Fix compute_meta recursion in blockwise (dask#5048)
Browse files Browse the repository at this point in the history
  • Loading branch information
pentschev authored and mrocklin committed Jul 5, 2019
1 parent ce253ba commit bc82231
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 7 deletions.
4 changes: 2 additions & 2 deletions dask/dataframe/core.py
Expand Up @@ -4759,7 +4759,7 @@ def map_partitions(func, *args, **kwargs):
}
graph = HighLevelGraph.from_collections(name, layer, dependencies=args)
return Scalar(graph, name, meta)
elif not (has_parallel_type(meta) or is_arraylike(meta)):
elif not (has_parallel_type(meta) or is_arraylike(meta) and meta.shape):
# If `meta` is not a pandas object, the concatenated results will be a
# different type
meta = make_meta(_concat([meta]), index=meta_index)
Expand Down Expand Up @@ -5841,7 +5841,7 @@ def new_dd_object(dsk, name, meta, divisions):
"""
if has_parallel_type(meta):
return get_parallel_type(meta)(dsk, name, meta, divisions)
elif is_arraylike(meta):
elif is_arraylike(meta) and meta.shape:
import dask.array as da

chunks = ((np.nan,) * (len(divisions) - 1),) + tuple(
Expand Down
2 changes: 1 addition & 1 deletion dask/dataframe/utils.py
Expand Up @@ -317,7 +317,7 @@ def make_meta_object(x, index=None):
"""
if hasattr(x, "_meta"):
return x._meta
elif is_arraylike(x):
elif is_arraylike(x) and x.shape:
return x[:0]

if index is not None:
Expand Down
13 changes: 13 additions & 0 deletions dask/tests/test_utils.py
Expand Up @@ -27,6 +27,7 @@
derived_from,
parse_timedelta,
parse_bytes,
is_arraylike,
)
from dask.utils_test import inc
from dask.highlevelgraph import HighLevelGraph
Expand Down Expand Up @@ -506,3 +507,15 @@ def test_parse_timedelta():
assert parse_timedelta("1", default="seconds") == 1
assert parse_timedelta("1", default="ms") == 0.001
assert parse_timedelta(1, default="ms") == 0.001


def test_is_arraylike():
assert is_arraylike(0) is False
assert is_arraylike(()) is False
assert is_arraylike((0)) is False
assert is_arraylike([]) is False
assert is_arraylike([0]) is False

assert is_arraylike(np.empty(())) is True
assert is_arraylike(np.empty((0,))) is True
assert is_arraylike(np.empty((0, 0))) is True
8 changes: 4 additions & 4 deletions dask/utils.py
Expand Up @@ -1051,8 +1051,9 @@ def is_arraylike(x):
Examples
--------
>>> import numpy as np
>>> x = np.ones(5)
>>> is_arraylike(x)
>>> is_arraylike(np.ones(5))
True
>>> is_arraylike(np.ones(()))
True
>>> is_arraylike(5)
False
Expand All @@ -1061,10 +1062,9 @@ def is_arraylike(x):
"""
from .base import is_dask_collection

return (
return bool(
hasattr(x, "shape")
and isinstance(x.shape, tuple)
and x.shape
and hasattr(x, "dtype")
and not any(is_dask_collection(n) for n in x.shape)
)
Expand Down

0 comments on commit bc82231

Please sign in to comment.