From 9e5f6b7e7ec3cc8ffc72852fb2ad4034485c41bc Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 21 Aug 2019 09:49:47 +0200 Subject: [PATCH 01/26] added dtype kwarg (doesn't do anything) + tests --- numba/targets/arraymath.py | 24 ++++++++++++++++++++++++ numba/tests/test_array_methods.py | 20 ++++++++++++++++---- numba/tests/test_array_reductions.py | 19 +++++++++++++++++++ numba/types/functions.py | 1 + numba/typing/arraydecl.py | 27 ++++++++++++++++++++++----- numba/typing/npydecl.py | 15 ++++++++++++--- 6 files changed, 94 insertions(+), 12 deletions(-) diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py index 5135254dda1..0e06572dff4 100644 --- a/numba/targets/arraymath.py +++ b/numba/targets/arraymath.py @@ -177,6 +177,30 @@ def _array_sum_axis_nop(arr, v): return arr +# @lower_builtin(np.sum, types.Array, types.intp, types.DTypeSpec) +# @lower_builtin(np.sum, types.Array, types.IntegerLiteral, types.DTypeSpec) +# @lower_builtin("array.sum", types.Array, types.intp, types.DTypeSpec) +# @lower_builtin("array.sum", types.Array, types.IntegerLiteral, types.DTypeSpec) + +@lower_builtin(np.sum, types.Array, types.DTypeSpec) +@lower_builtin(np.sum, types.Array, types.DTypeSpec) +@lower_builtin("array.sum", types.Array, types.DTypeSpec) +@lower_builtin("array.sum", types.Array, types.DTypeSpec) +def array_sum_axis(context, builder, sig, args): + zero = sig.return_type(0) + + def array_sum_impl(arr): + c = zero + for v in np.nditer(arr): + c += v.item() + return c + + res = context.compile_internal(builder, array_sum_impl, sig, args, + locals=dict(c=sig.return_type)) + return impl_ret_borrowed(context, builder, sig.return_type, res) + + + @lower_builtin(np.sum, types.Array, types.intp) @lower_builtin(np.sum, types.Array, types.IntegerLiteral) @lower_builtin("array.sum", types.Array, types.intp) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index d2704817c20..676e5d33ecc 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -148,9 +148,12 @@ def array_itemset(a, v): def array_sum(a, *args): return a.sum(*args) -def array_sum_kws(a, axis): +def array_sum_axis_kws(a, axis): return a.sum(axis=axis) +def array_sum_dtype_kws(a, dtype): + return a.sum(dtype=dtype) + def array_sum_const_multi(arr, axis): # use np.sum with different constant args multiple times to check # for internal compile cache to see if constant-specialization is @@ -779,8 +782,8 @@ def test_sum(self): # OK self.assertPreciseEqual(pyfunc(a, 0), cfunc(a, 0)) - def test_sum_kws(self): - pyfunc = array_sum_kws + def test_sum_axis_kws(self): + pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) # OK a = np.ones((7, 6, 5, 4, 3)) @@ -788,9 +791,18 @@ def test_sum_kws(self): # OK self.assertPreciseEqual(pyfunc(a, axis=2), cfunc(a, axis=2)) + def test_sum_dtype_kws(self): + pyfunc = array_sum_dtype_kws + cfunc = jit(nopython=True)(pyfunc) + dtype = np.float64 + # OK + a = np.ones((7, 6, 5, 4, 3)) + self.assertPreciseEqual(pyfunc(a, dtype=dtype), cfunc(a, dtype=dtype)) + + def test_sum_1d_kws(self): # check 1d reduces to scalar - pyfunc = array_sum_kws + pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) a = np.arange(10.) self.assertPreciseEqual(pyfunc(a, axis=0), cfunc(a, axis=0)) diff --git a/numba/tests/test_array_reductions.py b/numba/tests/test_array_reductions.py index de2ab8eac6d..96093051ab0 100644 --- a/numba/tests/test_array_reductions.py +++ b/numba/tests/test_array_reductions.py @@ -47,6 +47,9 @@ def array_sum(arr): def array_sum_global(arr): return np.sum(arr) +def array_sum_global_dtype(arr, dtype): + return np.sum(arr) + def array_prod(arr): return arr.prod() @@ -577,6 +580,22 @@ def test_array_sum_global(self): self.assertEqual(np.sum(arr), cfunc(arr)) + def test_array_sum_global_dtype(self): + arr = np.arange(10, dtype=np.int32) + arrty = typeof(arr) + dtype = np.float64 + dtypety = typeof(dtype) + + self.assertEqual(arrty.ndim, 1) + self.assertEqual(arrty.layout, 'C') + + cres = compile_isolated(array_sum_global_dtype, [arrty, dtypety]) + cfunc = cres.entry_point + + self.assertEqual(type(np.sum(arr, dtype=dtype)), + type(cfunc(arr, dtype))) + self.assertEqual(np.sum(arr, dtype=dtype), cfunc(arr, dtype)) + def test_array_prod_int_1d(self): arr = np.arange(10, dtype=np.int32) + 1 arrty = typeof(arr) diff --git a/numba/types/functions.py b/numba/types/functions.py index be0d38942fb..4c66c1edd80 100644 --- a/numba/types/functions.py +++ b/numba/types/functions.py @@ -82,6 +82,7 @@ class BaseFunction(Callable): """ def __init__(self, template): + if isinstance(template, (list, tuple)): self.templates = tuple(template) keys = set(temp.key for temp in self.templates) diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py index 4a684f6b260..8873a2ad129 100644 --- a/numba/typing/arraydecl.py +++ b/numba/typing/arraydecl.py @@ -612,22 +612,30 @@ def sum_expand(self, args, kws): sum can be called with or without an axis parameter. """ pysig = None - if kws: + if 'axis' in kws and not 'dtype' in kws: def sum_stub(axis): pass pysig = utils.pysignature(sum_stub) # rewrite args args = list(args) + [kws['axis']] - kws = None + # kws = None + + if 'dtype' in kws and not 'axis' in kws: + def sum_stub(dtype): + pass + pysig = utils.pysignature(sum_stub) + # rewrite args + args = list(args) + [kws['dtype']] + # kws = None args_len = len(args) assert args_len <= 1 if args_len == 0: - # No axis parameter so the return type of the summation is a scalar + # No axis or dtype parameter so the return type of the summation is a scalar # of the type of the array. out = signature(_expand_integer(self.this.dtype), *args, recvr=self.this) - else: - # There is an axis parameter + elif args_len == 1 and not 'dtype' in kws: + # There is an axis parameter, either arg or kwarg if self.this.ndim == 1: # 1d reduces to a scalar return_type = self.this.dtype @@ -637,6 +645,15 @@ def sum_stub(axis): return_type = types.Array(dtype=_expand_integer(self.this.dtype), ndim=self.this.ndim-1, layout='C') out = signature(return_type, *args, recvr=self.this) + elif args_len == 1 and 'dtype' in kws: + # No axis parameter so the return type of the summation is a scalar + # of the dtype parameter. + from .npydecl import _parse_dtype + dtype, = args + dtype = _parse_dtype(dtype) + out = signature(_expand_integer(dtype), *args, recvr=self.this) + else: + pass return out.replace(pysig=pysig) def generic_expand_cumulative(self, args, kws): diff --git a/numba/typing/npydecl.py b/numba/typing/npydecl.py index bde8225600c..77292c40785 100644 --- a/numba/typing/npydecl.py +++ b/numba/typing/npydecl.py @@ -378,9 +378,18 @@ def generic(self, args, kws): pysig = None if kws: if self.method_name == 'sum': - def sum_stub(arr, axis): - pass - pysig = utils.pysignature(sum_stub) + if 'axis' in kws and not 'dtype' in kws: + def sum_stub(arr, axis): + pass + pysig = utils.pysignature(sum_stub) + if 'dtype' in kws and not 'axis' in kws: + def sum_stub(arr, dtype): + pass + pysig = utils.pysignature(sum_stub) + if 'dtype' in kws and 'axis' in kws: + def sum_stub(arr, axis, dtype): + pass + pysig = utils.pysignature(sum_stub) elif self.method_name == 'argsort': def argsort_stub(arr, kind='quicksort'): pass From cf899bf7b3905557903b280999a7d4fcc511aeb5 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 21 Aug 2019 13:15:08 +0200 Subject: [PATCH 02/26] fixed impl - basic implementation passes tests --- numba/targets/arraymath.py | 2 +- numba/tests/test_array_methods.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py index 0e06572dff4..7d780a7810c 100644 --- a/numba/targets/arraymath.py +++ b/numba/targets/arraymath.py @@ -189,7 +189,7 @@ def _array_sum_axis_nop(arr, v): def array_sum_axis(context, builder, sig, args): zero = sig.return_type(0) - def array_sum_impl(arr): + def array_sum_impl(arr, dtype): c = zero for v in np.nditer(arr): c += v.item() diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 676e5d33ecc..17bd32ac3e9 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -799,6 +799,14 @@ def test_sum_dtype_kws(self): a = np.ones((7, 6, 5, 4, 3)) self.assertPreciseEqual(pyfunc(a, dtype=dtype), cfunc(a, dtype=dtype)) + def test_sum_dtype_kws_negative(self): + pyfunc = array_sum_dtype_kws + cfunc = jit(nopython=True)(pyfunc) + dtype = np.float64 + # OK + a = np.ones((7, 6, 5, 4, 3)) + self.assertFalse(type(pyfunc(a, dtype=np.int32))==cfunc(a, dtype=dtype)) + def test_sum_1d_kws(self): # check 1d reduces to scalar From de3d1c055721577cfcd079408faba031215e8fba Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 21 Aug 2019 15:14:40 +0200 Subject: [PATCH 03/26] format fixes and test removal --- numba/tests/test_array_methods.py | 1 - numba/tests/test_array_reductions.py | 19 ------------------- 2 files changed, 20 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 17bd32ac3e9..0d49a456779 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -807,7 +807,6 @@ def test_sum_dtype_kws_negative(self): a = np.ones((7, 6, 5, 4, 3)) self.assertFalse(type(pyfunc(a, dtype=np.int32))==cfunc(a, dtype=dtype)) - def test_sum_1d_kws(self): # check 1d reduces to scalar pyfunc = array_sum_axis_kws diff --git a/numba/tests/test_array_reductions.py b/numba/tests/test_array_reductions.py index 96093051ab0..de2ab8eac6d 100644 --- a/numba/tests/test_array_reductions.py +++ b/numba/tests/test_array_reductions.py @@ -47,9 +47,6 @@ def array_sum(arr): def array_sum_global(arr): return np.sum(arr) -def array_sum_global_dtype(arr, dtype): - return np.sum(arr) - def array_prod(arr): return arr.prod() @@ -580,22 +577,6 @@ def test_array_sum_global(self): self.assertEqual(np.sum(arr), cfunc(arr)) - def test_array_sum_global_dtype(self): - arr = np.arange(10, dtype=np.int32) - arrty = typeof(arr) - dtype = np.float64 - dtypety = typeof(dtype) - - self.assertEqual(arrty.ndim, 1) - self.assertEqual(arrty.layout, 'C') - - cres = compile_isolated(array_sum_global_dtype, [arrty, dtypety]) - cfunc = cres.entry_point - - self.assertEqual(type(np.sum(arr, dtype=dtype)), - type(cfunc(arr, dtype))) - self.assertEqual(np.sum(arr, dtype=dtype), cfunc(arr, dtype)) - def test_array_prod_int_1d(self): arr = np.arange(10, dtype=np.int32) + 1 arrty = typeof(arr) From ee11c3dd285cd90a2b043e0e94ef0f08db3d8ef2 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Tue, 27 Aug 2019 21:28:18 +0200 Subject: [PATCH 04/26] added dtype-axis combination --- numba/targets/arraymath.py | 101 +++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 4 deletions(-) diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py index 7d780a7810c..19457f76fab 100644 --- a/numba/targets/arraymath.py +++ b/numba/targets/arraymath.py @@ -177,10 +177,103 @@ def _array_sum_axis_nop(arr, v): return arr -# @lower_builtin(np.sum, types.Array, types.intp, types.DTypeSpec) -# @lower_builtin(np.sum, types.Array, types.IntegerLiteral, types.DTypeSpec) -# @lower_builtin("array.sum", types.Array, types.intp, types.DTypeSpec) -# @lower_builtin("array.sum", types.Array, types.IntegerLiteral, types.DTypeSpec) +@lower_builtin(np.sum, types.Array, types.intp, types.DTypeSpec) +@lower_builtin(np.sum, types.Array, types.IntegerLiteral, types.DTypeSpec) +@lower_builtin("array.sum", types.Array, types.intp, types.DTypeSpec) +@lower_builtin("array.sum", types.Array, types.IntegerLiteral, types.DTypeSpec) +def array_sum_axis_dtype(context, builder, sig, args): + """ + The third parameter to gen_index_tuple that generates the indexing + tuples has to be a const so we can't just pass "axis" through since + that isn't const. We can check for specific values and have + different instances that do take consts. Supporting axis summation + only up to the fourth dimension for now. + """ + # typing/arraydecl.py:sum_expand defines the return type for sum with axis. + # It is one dimension less than the input array. + + retty = sig.return_type + zero = getattr(retty, 'dtype', retty)(0) + # if the return is scalar in type then "take" the 0th element of the + # 0d array accumulator as the return value + if getattr(retty, 'ndim', None) is None: + op = np.take + else: + op = _array_sum_axis_nop + [ty_array, ty_axis, ty_dtype] = sig.args + is_axis_const = False + const_axis_val = 0 + if isinstance(ty_axis, types.Literal): + # this special-cases for constant axis + const_axis_val = ty_axis.literal_value + # fix negative axis + if const_axis_val < 0: + const_axis_val = ty_array.ndim + const_axis_val + if const_axis_val < 0 or const_axis_val > ty_array.ndim: + raise ValueError("'axis' entry is out of bounds") + + ty_axis = context.typing_context.resolve_value_type(const_axis_val) + axis_val = context.get_constant(ty_axis, const_axis_val) + # rewrite arguments + args = args[0], axis_val, args[2] + # rewrite sig + sig = sig.replace(args=[ty_array, ty_axis, ty_dtype]) + is_axis_const = True + + def array_sum_impl_axis(arr, axis, dtype): + ndim = arr.ndim + + if not is_axis_const: + # Catch where axis is negative or greater than 3. + if axis < 0 or axis > 3: + raise ValueError("Numba does not support sum with axis " + "parameter outside the range 0 to 3.") + + # Catch the case where the user misspecifies the axis to be + # more than the number of the array's dimensions. + if axis >= ndim: + raise ValueError("axis is out of bounds for array") + + # Convert the shape of the input array to a list. + ashape = list(arr.shape) + # Get the length of the axis dimension. + axis_len = ashape[axis] + # Remove the axis dimension from the list of dimensional lengths. + ashape.pop(axis) + # Convert this shape list back to a tuple using above intrinsic. + ashape_without_axis = _create_tuple_result_shape(ashape, arr.shape) + # Tuple needed here to create output array with correct size. + result = np.full(ashape_without_axis, zero, type(zero)) + + # Iterate through the axis dimension. + for axis_index in range(axis_len): + if is_axis_const: + # constant specialized version works for any valid axis value + index_tuple_generic = _gen_index_tuple(arr.shape, axis_index, + const_axis_val) + result += arr[index_tuple_generic] + else: + # Generate a tuple used to index the input array. + # The tuple is ":" in all dimensions except the axis + # dimension where it is "axis_index". + if axis == 0: + index_tuple1 = _gen_index_tuple(arr.shape, axis_index, 0) + result += arr[index_tuple1] + elif axis == 1: + index_tuple2 = _gen_index_tuple(arr.shape, axis_index, 1) + result += arr[index_tuple2] + elif axis == 2: + index_tuple3 = _gen_index_tuple(arr.shape, axis_index, 2) + result += arr[index_tuple3] + elif axis == 3: + index_tuple4 = _gen_index_tuple(arr.shape, axis_index, 3) + result += arr[index_tuple4] + + return op(result, 0) + + res = context.compile_internal(builder, array_sum_impl_axis, sig, args) + return impl_ret_new_ref(context, builder, sig.return_type, res) + @lower_builtin(np.sum, types.Array, types.DTypeSpec) @lower_builtin(np.sum, types.Array, types.DTypeSpec) From 4d674faf4ce94344894043568a7269e8b5493278 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Tue, 27 Aug 2019 21:29:59 +0200 Subject: [PATCH 05/26] added dtype-axis tests --- numba/tests/test_array_methods.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 0d49a456779..744c8c6de5c 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -154,6 +154,9 @@ def array_sum_axis_kws(a, axis): def array_sum_dtype_kws(a, dtype): return a.sum(dtype=dtype) +def array_sum_axis_dtype_kws(a, dtype, axis): + return a.sum(axis=axis, dtype=dtype) + def array_sum_const_multi(arr, axis): # use np.sum with different constant args multiple times to check # for internal compile cache to see if constant-specialization is @@ -807,6 +810,18 @@ def test_sum_dtype_kws_negative(self): a = np.ones((7, 6, 5, 4, 3)) self.assertFalse(type(pyfunc(a, dtype=np.int32))==cfunc(a, dtype=dtype)) + def test_sum_axis_dtype_kws(self): + pyfunc = array_sum_axis_dtype_kws + cfunc = jit(nopython=True)(pyfunc) + dtype = np.float64 + # OK + a = np.ones((7, 6, 5, 4, 3)) + self.assertPreciseEqual(pyfunc(a, axis=1, dtype=dtype), + cfunc(a, axis=1, dtype=dtype)) + + self.assertPreciseEqual(pyfunc(a, axis=2, dtype=dtype), + cfunc(a, axis=2, dtype=dtype)) + def test_sum_1d_kws(self): # check 1d reduces to scalar pyfunc = array_sum_axis_kws From bbde75f62343ccfd25ba37fcb15df73ec6a6d8ae Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Tue, 27 Aug 2019 21:32:36 +0200 Subject: [PATCH 06/26] adjusting arraydecl (1 of 2) --- numba/typing/arraydecl.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py index 8873a2ad129..c6cb42e198d 100644 --- a/numba/typing/arraydecl.py +++ b/numba/typing/arraydecl.py @@ -627,8 +627,16 @@ def sum_stub(dtype): # rewrite args args = list(args) + [kws['dtype']] # kws = None + + if 'dtype' in kws and 'axis' in kws: + def sum_stub(axis, dtype): + pass + pysig = utils.pysignature(sum_stub) + # rewrite args + args = list(args) + [kws['axis'], kws['dtype']] + args_len = len(args) - assert args_len <= 1 + assert args_len <= 2 if args_len == 0: # No axis or dtype parameter so the return type of the summation is a scalar # of the type of the array. From 66d525cda10450c9fec593b85f3f2c82c81068db Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Tue, 27 Aug 2019 21:33:27 +0200 Subject: [PATCH 07/26] adjusting arraydecl (2 of 2) --- numba/typing/arraydecl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py index c6cb42e198d..562d0fbc8e9 100644 --- a/numba/typing/arraydecl.py +++ b/numba/typing/arraydecl.py @@ -660,6 +660,20 @@ def sum_stub(axis, dtype): dtype, = args dtype = _parse_dtype(dtype) out = signature(_expand_integer(dtype), *args, recvr=self.this) + + elif args_len == 2: + # There is an axis and dtype parameter, either arg or kwarg + from .npydecl import _parse_dtype + dtype = _parse_dtype(args[1]) + return_type = _expand_integer(dtype) + if self.this.ndim != 1: + # 1d reduces to a scalar, 2d and above reduce dim by 1 + # the return type of this summation is an array of dimension one + # less than the input array. + return_type = types.Array(dtype=return_type, + ndim=self.this.ndim-1, layout='C') + out = signature(return_type, *args, recvr=self.this) + else: pass return out.replace(pysig=pysig) From bd5014a9c1612b821acb28de5cf3cc11d307ac0b Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 28 Aug 2019 10:35:42 +0200 Subject: [PATCH 08/26] flake8 fixes --- numba/targets/arraymath.py | 3 +-- numba/typing/arraydecl.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py index 19457f76fab..0934da76b11 100644 --- a/numba/targets/arraymath.py +++ b/numba/targets/arraymath.py @@ -279,7 +279,7 @@ def array_sum_impl_axis(arr, axis, dtype): @lower_builtin(np.sum, types.Array, types.DTypeSpec) @lower_builtin("array.sum", types.Array, types.DTypeSpec) @lower_builtin("array.sum", types.Array, types.DTypeSpec) -def array_sum_axis(context, builder, sig, args): +def array_sum_dtype(context, builder, sig, args): zero = sig.return_type(0) def array_sum_impl(arr, dtype): @@ -293,7 +293,6 @@ def array_sum_impl(arr, dtype): return impl_ret_borrowed(context, builder, sig.return_type, res) - @lower_builtin(np.sum, types.Array, types.intp) @lower_builtin(np.sum, types.Array, types.IntegerLiteral) @lower_builtin("array.sum", types.Array, types.intp) diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py index 562d0fbc8e9..76fe521f85e 100644 --- a/numba/typing/arraydecl.py +++ b/numba/typing/arraydecl.py @@ -653,6 +653,7 @@ def sum_stub(axis, dtype): return_type = types.Array(dtype=_expand_integer(self.this.dtype), ndim=self.this.ndim-1, layout='C') out = signature(return_type, *args, recvr=self.this) + elif args_len == 1 and 'dtype' in kws: # No axis parameter so the return type of the summation is a scalar # of the dtype parameter. From 7c3d9bba0da563f36c6788c14c19a6729ae36c94 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 28 Aug 2019 19:02:36 +0200 Subject: [PATCH 09/26] factoring out common parts of sum implementations (with and without dtype arg) --- numba/targets/arraymath.py | 151 ++++++++++++++----------------------- 1 file changed, 56 insertions(+), 95 deletions(-) diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py index 0934da76b11..08072999f05 100644 --- a/numba/targets/arraymath.py +++ b/numba/targets/arraymath.py @@ -176,51 +176,8 @@ def array_sum_impl(arr): def _array_sum_axis_nop(arr, v): return arr - -@lower_builtin(np.sum, types.Array, types.intp, types.DTypeSpec) -@lower_builtin(np.sum, types.Array, types.IntegerLiteral, types.DTypeSpec) -@lower_builtin("array.sum", types.Array, types.intp, types.DTypeSpec) -@lower_builtin("array.sum", types.Array, types.IntegerLiteral, types.DTypeSpec) -def array_sum_axis_dtype(context, builder, sig, args): - """ - The third parameter to gen_index_tuple that generates the indexing - tuples has to be a const so we can't just pass "axis" through since - that isn't const. We can check for specific values and have - different instances that do take consts. Supporting axis summation - only up to the fourth dimension for now. - """ - # typing/arraydecl.py:sum_expand defines the return type for sum with axis. - # It is one dimension less than the input array. - - retty = sig.return_type - zero = getattr(retty, 'dtype', retty)(0) - # if the return is scalar in type then "take" the 0th element of the - # 0d array accumulator as the return value - if getattr(retty, 'ndim', None) is None: - op = np.take - else: - op = _array_sum_axis_nop - [ty_array, ty_axis, ty_dtype] = sig.args - is_axis_const = False - const_axis_val = 0 - if isinstance(ty_axis, types.Literal): - # this special-cases for constant axis - const_axis_val = ty_axis.literal_value - # fix negative axis - if const_axis_val < 0: - const_axis_val = ty_array.ndim + const_axis_val - if const_axis_val < 0 or const_axis_val > ty_array.ndim: - raise ValueError("'axis' entry is out of bounds") - - ty_axis = context.typing_context.resolve_value_type(const_axis_val) - axis_val = context.get_constant(ty_axis, const_axis_val) - # rewrite arguments - args = args[0], axis_val, args[2] - # rewrite sig - sig = sig.replace(args=[ty_array, ty_axis, ty_dtype]) - is_axis_const = True - - def array_sum_impl_axis(arr, axis, dtype): +def gen_sum_axis_impl(is_axis_const, const_axis_val, op, zero): + def inner(arr, axis): ndim = arr.ndim if not is_axis_const: @@ -268,8 +225,57 @@ def array_sum_impl_axis(arr, axis, dtype): elif axis == 3: index_tuple4 = _gen_index_tuple(arr.shape, axis_index, 3) result += arr[index_tuple4] - return op(result, 0) + return inner + +@lower_builtin(np.sum, types.Array, types.intp, types.DTypeSpec) +@lower_builtin(np.sum, types.Array, types.IntegerLiteral, types.DTypeSpec) +@lower_builtin("array.sum", types.Array, types.intp, types.DTypeSpec) +@lower_builtin("array.sum", types.Array, types.IntegerLiteral, types.DTypeSpec) +def array_sum_axis_dtype(context, builder, sig, args): + """ + The third parameter to gen_index_tuple that generates the indexing + tuples has to be a const so we can't just pass "axis" through since + that isn't const. We can check for specific values and have + different instances that do take consts. Supporting axis summation + only up to the fourth dimension for now. + """ + # typing/arraydecl.py:sum_expand defines the return type for sum with axis. + # It is one dimension less than the input array. + + retty = sig.return_type + zero = getattr(retty, 'dtype', retty)(0) + # if the return is scalar in type then "take" the 0th element of the + # 0d array accumulator as the return value + if getattr(retty, 'ndim', None) is None: + op = np.take + else: + op = _array_sum_axis_nop + [ty_array, ty_axis, ty_dtype] = sig.args + is_axis_const = False + const_axis_val = 0 + if isinstance(ty_axis, types.Literal): + # this special-cases for constant axis + const_axis_val = ty_axis.literal_value + # fix negative axis + if const_axis_val < 0: + const_axis_val = ty_array.ndim + const_axis_val + if const_axis_val < 0 or const_axis_val > ty_array.ndim: + raise ValueError("'axis' entry is out of bounds") + + ty_axis = context.typing_context.resolve_value_type(const_axis_val) + axis_val = context.get_constant(ty_axis, const_axis_val) + # rewrite arguments + args = args[0], axis_val, args[2] + # rewrite sig + sig = sig.replace(args=[ty_array, ty_axis, ty_dtype]) + is_axis_const = True + + gen_impl = gen_sum_axis_impl(is_axis_const, const_axis_val, op, zero) + compiled = register_jitable(gen_impl) + + def array_sum_impl_axis(arr, axis, dtype): + return compiled(arr, axis) res = context.compile_internal(builder, array_sum_impl_axis, sig, args) return impl_ret_new_ref(context, builder, sig.return_type, res) @@ -336,56 +342,11 @@ def array_sum_axis(context, builder, sig, args): sig = sig.replace(args=[ty_array, ty_axis]) is_axis_const = True - def array_sum_impl_axis(arr, axis): - ndim = arr.ndim - - if not is_axis_const: - # Catch where axis is negative or greater than 3. - if axis < 0 or axis > 3: - raise ValueError("Numba does not support sum with axis " - "parameter outside the range 0 to 3.") - - # Catch the case where the user misspecifies the axis to be - # more than the number of the array's dimensions. - if axis >= ndim: - raise ValueError("axis is out of bounds for array") + gen_impl = gen_sum_axis_impl(is_axis_const, const_axis_val, op, zero) + compiled = register_jitable(gen_impl) - # Convert the shape of the input array to a list. - ashape = list(arr.shape) - # Get the length of the axis dimension. - axis_len = ashape[axis] - # Remove the axis dimension from the list of dimensional lengths. - ashape.pop(axis) - # Convert this shape list back to a tuple using above intrinsic. - ashape_without_axis = _create_tuple_result_shape(ashape, arr.shape) - # Tuple needed here to create output array with correct size. - result = np.full(ashape_without_axis, zero, type(zero)) - - # Iterate through the axis dimension. - for axis_index in range(axis_len): - if is_axis_const: - # constant specialized version works for any valid axis value - index_tuple_generic = _gen_index_tuple(arr.shape, axis_index, - const_axis_val) - result += arr[index_tuple_generic] - else: - # Generate a tuple used to index the input array. - # The tuple is ":" in all dimensions except the axis - # dimension where it is "axis_index". - if axis == 0: - index_tuple1 = _gen_index_tuple(arr.shape, axis_index, 0) - result += arr[index_tuple1] - elif axis == 1: - index_tuple2 = _gen_index_tuple(arr.shape, axis_index, 1) - result += arr[index_tuple2] - elif axis == 2: - index_tuple3 = _gen_index_tuple(arr.shape, axis_index, 2) - result += arr[index_tuple3] - elif axis == 3: - index_tuple4 = _gen_index_tuple(arr.shape, axis_index, 3) - result += arr[index_tuple4] - - return op(result, 0) + def array_sum_impl_axis(arr, axis): + return compiled(arr, axis) res = context.compile_internal(builder, array_sum_impl_axis, sig, args) return impl_ret_new_ref(context, builder, sig.return_type, res) From 3abb4a896912231b74ec05cedcb39d3c93ce63f2 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Sat, 31 Aug 2019 11:41:21 +0200 Subject: [PATCH 10/26] added tests --- numba/tests/test_array_methods.py | 130 +++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 10 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 744c8c6de5c..caf52652ecb 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -157,6 +157,9 @@ def array_sum_dtype_kws(a, dtype): def array_sum_axis_dtype_kws(a, dtype, axis): return a.sum(axis=axis, dtype=dtype) +def array_sum_axis_dtype_pos(a, a1, a2): + return a.sum(a1, a2) + def array_sum_const_multi(arr, axis): # use np.sum with different constant args multiple times to check # for internal compile cache to see if constant-specialization is @@ -498,7 +501,6 @@ def check_arr(arr): arr = np.array([0]).reshape(()) check_arr(arr) - def test_array_transpose(self): self.check_layout_dependent_func(array_transpose) @@ -537,7 +539,7 @@ def check(shape): def test_np_frombuffer_allocated(self): self.check_np_frombuffer_allocated(np_frombuffer_allocated) - def test_np_frombuffer_allocated(self): + def test_np_frombuffer_allocated2(self): self.check_np_frombuffer_allocated(np_frombuffer_allocated_dtype) def check_nonzero(self, pyfunc): @@ -785,6 +787,23 @@ def test_sum(self): # OK self.assertPreciseEqual(pyfunc(a, 0), cfunc(a, 0)) + def test_sum2(self): + pyfunc = array_sum + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, + np.complex64, np.uint32, np.uint64, np.timedelta64] + all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.complex64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + for arr_list in all_test_arrays: + for arr in arr_list: + with self.subTest("Test np.sum with {} input ".format(arr.dtype)): + self.assertPreciseEqual(pyfunc(arr), cfunc(arr)) + def test_sum_axis_kws(self): pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) @@ -794,13 +813,55 @@ def test_sum_axis_kws(self): # OK self.assertPreciseEqual(pyfunc(a, axis=2), cfunc(a, axis=2)) + def test_sum_axis_kws2(self): + pyfunc = array_sum_axis_kws + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, + np.complex64, np.uint32, np.uint64, np.timedelta64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + for arr_list in all_test_arrays: + for arr in arr_list: + for axis in (0, 1, 2): + if axis > len(arr.shape)-1: + continue + with self.subTest("Testing np.sum(axis) with {} " + "input ".format(arr.dtype)): + self.assertPreciseEqual(pyfunc(arr, axis=axis), + cfunc(arr, axis=axis)) + def test_sum_dtype_kws(self): pyfunc = array_sum_dtype_kws cfunc = jit(nopython=True)(pyfunc) - dtype = np.float64 - # OK - a = np.ones((7, 6, 5, 4, 3)) - self.assertPreciseEqual(pyfunc(a, dtype=dtype), cfunc(a, dtype=dtype)) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, + np.uint64, np.timedelta64, np.complex64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + out_dtypes = {np.dtype('float64'): [np.float64], + np.dtype('float32'): [np.float64, np.float32], + np.dtype('int64'): [np.float64, np.int64, np.float32], + np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], + np.dtype('uint32'): [np.float64, np.int64, np.float32], + np.dtype('uint64'): [np.float64, np.unit64], + np.dtype('complex64'): [np.complex64], + np.dtype('timedelta64'): [np.timedelta64]} + + for arr_list in all_test_arrays: + for arr in arr_list: + for out_dtype in out_dtypes[arr.dtype]: + subtest_str = ("Testing np.sum with {} input and {} output" + .format(arr.dtype, out_dtype)) + with self.subTest(subtest_str): + self.assertPreciseEqual(pyfunc(arr, dtype=out_dtype), + cfunc(arr, dtype=out_dtype)) def test_sum_dtype_kws_negative(self): pyfunc = array_sum_dtype_kws @@ -808,7 +869,7 @@ def test_sum_dtype_kws_negative(self): dtype = np.float64 # OK a = np.ones((7, 6, 5, 4, 3)) - self.assertFalse(type(pyfunc(a, dtype=np.int32))==cfunc(a, dtype=dtype)) + self.assertFalse(type(pyfunc(a, dtype=np.int32)) == cfunc(a, dtype=dtype)) def test_sum_axis_dtype_kws(self): pyfunc = array_sum_axis_dtype_kws @@ -818,10 +879,59 @@ def test_sum_axis_dtype_kws(self): a = np.ones((7, 6, 5, 4, 3)) self.assertPreciseEqual(pyfunc(a, axis=1, dtype=dtype), cfunc(a, axis=1, dtype=dtype)) - self.assertPreciseEqual(pyfunc(a, axis=2, dtype=dtype), cfunc(a, axis=2, dtype=dtype)) + def test_sum_axis_dtype_kws2(self): + pyfunc = array_sum_axis_dtype_kws + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, + np.uint64, np.complex64] + + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + out_dtypes = {np.dtype('float64'): [np.float64], + np.dtype('float32'): [np.float64, np.float32], + np.dtype('int64'): [np.float64, np.int64, np.float32], + np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], + np.dtype('uint32'): [np.float64, np.int64, np.float32], + np.dtype('uint64'): [np.float64, np.uint64], + np.dtype('complex64'): [np.complex64], + np.dtype('timedelta64'): [np.timedelta64]} + + for arr_list in all_test_arrays: + for arr in arr_list: + for out_dtype in out_dtypes[arr.dtype]: + for axis in (0, 1, 2): + if axis > len(arr.shape) - 1: + continue + subtest_str = ("Testing np.sum with {} input and {} output " + .format(arr.dtype, out_dtype)) + with self.subTest(subtest_str): + py_res = pyfunc(arr, axis=axis, dtype=out_dtype) + nb_res = cfunc(arr, axis=axis, dtype=out_dtype) + self.assertPreciseEqual(py_res, nb_res) + + def test_sum_axis_dtype_pos_arg(self): + """ + testing that axis and dtype inputs work when passed as positional + :return: + """ + pyfunc = array_sum_axis_dtype_pos + cfunc = jit(nopython=True)(pyfunc) + dtype = np.float64 + # OK + a = np.ones((7, 6, 5, 4, 3)) + self.assertPreciseEqual(pyfunc(a, 1, dtype), + cfunc(a, 1, dtype)) + + self.assertPreciseEqual(pyfunc(a, 2, dtype), + cfunc(a, 2, dtype)) + def test_sum_1d_kws(self): # check 1d reduces to scalar pyfunc = array_sum_axis_kws @@ -928,7 +1038,7 @@ def check(arr, ind): test_indices.append(np.array([1, 5, 1, 11, 3])) test_indices.append(np.array([[1, 5, 1], [11, 3, 0]], order='F')) test_indices.append(np.array([[[1, 5, 1], [11, 3, 0]]])) - test_indices.append(np.array([[[[1, 5]], [[11, 0]],[[1, 2]]]])) + test_indices.append(np.array([[[[1, 5]], [[11, 0]], [[1, 2]]]])) test_indices.append([1, 5, 1, 11, 3]) test_indices.append((1, 5, 1)) test_indices.append(((1, 5, 1), (11, 3, 2))) @@ -948,7 +1058,7 @@ def check(arr, ind): [szA], [-szA - 1]] for x in illegal_indices: with self.assertRaises(IndexError): - cfunc(A, x) # oob raises + cfunc(A, x) # oob raises # check float indexing raises with self.assertRaises(TypingError): From 496eeca7c9527af9d8982359220d3b4d6d0f191c Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Fri, 6 Sep 2019 20:57:07 +0200 Subject: [PATCH 11/26] fixed issue with np.int32 arrays --- numba/tests/test_array_methods.py | 6 +++--- numba/typing/arraydecl.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index caf52652ecb..c47cf28eab5 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -838,7 +838,8 @@ def test_sum_dtype_kws(self): pyfunc = array_sum_dtype_kws cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, - np.uint64, np.timedelta64, np.complex64] + np.uint64, np.complex64] + # all_dtypes += [np.timedelta64] all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), np.ones(1, arr_dtype), @@ -850,7 +851,7 @@ def test_sum_dtype_kws(self): np.dtype('int64'): [np.float64, np.int64, np.float32], np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], np.dtype('uint32'): [np.float64, np.int64, np.float32], - np.dtype('uint64'): [np.float64, np.unit64], + np.dtype('uint64'): [np.float64, np.int64], np.dtype('complex64'): [np.complex64], np.dtype('timedelta64'): [np.timedelta64]} @@ -887,7 +888,6 @@ def test_sum_axis_dtype_kws2(self): cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, np.uint64, np.complex64] - all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), np.ones(1, arr_dtype), diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py index 76fe521f85e..fd0a60d5076 100644 --- a/numba/typing/arraydecl.py +++ b/numba/typing/arraydecl.py @@ -660,13 +660,13 @@ def sum_stub(axis, dtype): from .npydecl import _parse_dtype dtype, = args dtype = _parse_dtype(dtype) - out = signature(_expand_integer(dtype), *args, recvr=self.this) + out = signature(dtype, *args, recvr=self.this) elif args_len == 2: # There is an axis and dtype parameter, either arg or kwarg from .npydecl import _parse_dtype dtype = _parse_dtype(args[1]) - return_type = _expand_integer(dtype) + return_type = dtype if self.this.ndim != 1: # 1d reduces to a scalar, 2d and above reduce dim by 1 # the return type of this summation is an array of dimension one From affa1ec6767da3e8c3077497589386c07c873f59 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Fri, 6 Sep 2019 22:53:56 +0200 Subject: [PATCH 12/26] disabled timedelta test --- numba/tests/test_array_methods.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index c47cf28eab5..6c4da9ff27b 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -792,7 +792,6 @@ def test_sum2(self): cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.complex64, np.uint32, np.uint64, np.timedelta64] - all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.complex64] all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), np.ones(1, arr_dtype), @@ -816,8 +815,9 @@ def test_sum_axis_kws(self): def test_sum_axis_kws2(self): pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.int32, - np.complex64, np.uint32, np.uint64, np.timedelta64] + all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, + np.uint64, np.complex64] + # all_dtypes += [np.timedelta64] all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), np.ones(1, arr_dtype), From b63d8d61248a45592af68d4a219353e3c5493a7c Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Sat, 7 Sep 2019 09:00:14 +0200 Subject: [PATCH 13/26] flake8 fix --- numba/targets/arraymath.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py index 08072999f05..d6086f0f4bc 100644 --- a/numba/targets/arraymath.py +++ b/numba/targets/arraymath.py @@ -176,6 +176,7 @@ def array_sum_impl(arr): def _array_sum_axis_nop(arr, v): return arr + def gen_sum_axis_impl(is_axis_const, const_axis_val, op, zero): def inner(arr, axis): ndim = arr.ndim @@ -228,6 +229,7 @@ def inner(arr, axis): return op(result, 0) return inner + @lower_builtin(np.sum, types.Array, types.intp, types.DTypeSpec) @lower_builtin(np.sum, types.Array, types.IntegerLiteral, types.DTypeSpec) @lower_builtin("array.sum", types.Array, types.intp, types.DTypeSpec) From 5a3e219de3995bd097f1f60b9e9dcfe396f9208d Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Tue, 17 Sep 2019 18:59:13 +0200 Subject: [PATCH 14/26] updated documentation --- docs/source/reference/numpysupported.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/reference/numpysupported.rst b/docs/source/reference/numpysupported.rst index cc624c13704..6392899623c 100644 --- a/docs/source/reference/numpysupported.rst +++ b/docs/source/reference/numpysupported.rst @@ -210,12 +210,20 @@ The following methods of Numpy arrays are supported: * :meth:`~numpy.ndarray.repeat` (no axis argument) * :meth:`~numpy.ndarray.reshape` (only the 1-argument form) * :meth:`~numpy.ndarray.sort` (without arguments) -* :meth:`~numpy.ndarray.sum` (with or without the ``axis`` argument) +* :meth:`~numpy.ndarray.sum` (with or without the ``axis`` and/or ``dtype`` arguments) * If the ``axis`` argument is a compile-time constant, all valid values are supported. An out-of-range value will result in a ``LoweringError`` at compile-time. * If the ``axis`` argument is not a compile-time constant, only values from 0 to 3 are supported. An out-of-range value will result in a runtime exception. + * All numeric ``dtypes`` are supported as ``dtype`` parameter. ``timedelta`` arrays can + be used as input arrays but ``timedelta`` is not supported as ``dtype`` parameter. + * When ``dtype`` is given, it determines the type of the internal accumulator. When it is not, + the selection is made automatically based on the input array's ``dtype``, mostly following the same + rules as NumPy. However, on 64-bit Windows, Numba uses a 64-bit accumulator for integer inputs + (``int64`` for ``int32`` inputs and ``uint64`` for ``uint32`` inputs), while NumPy would use a 32-bit + accumulator in those cases. + * :meth:`~numpy.ndarray.transpose` * :meth:`~numpy.ndarray.view` (only the 1-argument form) From e0370f7852a3f0a294f522a386e1f83c77513d01 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Tue, 17 Sep 2019 19:00:37 +0200 Subject: [PATCH 15/26] split out tests for int32 and uint32 due to special numpy behaviour --- numba/tests/test_array_methods.py | 39 +++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 6c4da9ff27b..5467c04a3aa 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -815,8 +815,8 @@ def test_sum_axis_kws(self): def test_sum_axis_kws2(self): pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, - np.uint64, np.complex64] + all_dtypes = [np.float64, np.float32, np.int64, np.uint64, np.complex64] + # timedelta test cannot be enabled until issue #4540 is fixed # all_dtypes += [np.timedelta64] all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), @@ -834,11 +834,46 @@ def test_sum_axis_kws2(self): self.assertPreciseEqual(pyfunc(arr, axis=axis), cfunc(arr, axis=axis)) + def test_sum_axis_kws3(self): + """ uint32 and int32 must be tested separately because Numpy's current + behaviour is different in 64bits Windows (accumulates as int32) + and 64bits Linux (accumulates as int64), while Numba has decided to always + accumulate as int64, when the OS is 64bits. No testing has been done + for behaviours in 32 bits platforms. + + :return: + """ + pyfunc = array_sum_axis_kws + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.int32, np.uint32] + # expected return dtypes in Numba + out_dtypes = {np.dtype('int32'): np.int64, np.dtype('uint32'): np.uint64, + np.dtype('int64'): np.int64} + # timedelta test cannot be enabled until issue #4540 is fixed + # all_dtypes += [np.timedelta64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + for arr_list in all_test_arrays: + for arr in arr_list: + for axis in (0, 1, 2): + if axis > len(arr.shape)-1: + continue + with self.subTest("Testing np.sum(axis) with {} " + "input ".format(arr.dtype)): + self.assertPreciseEqual( + pyfunc(arr, axis=axis).astype(out_dtypes[arr.dtype]), + cfunc(arr, axis=axis)) + def test_sum_dtype_kws(self): pyfunc = array_sum_dtype_kws cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, np.uint64, np.complex64] + # timedelta test cannot be enabled until issue #4540 is fixed # all_dtypes += [np.timedelta64] all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), From 26ab11fa9373142648ce0c8f127e506b37a8ce24 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Tue, 17 Sep 2019 19:11:21 +0200 Subject: [PATCH 16/26] Merge branch 'master' into sum_with_dtype # Conflicts: # docs/source/reference/numpysupported.rst --- docs/source/reference/numpysupported.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/reference/numpysupported.rst b/docs/source/reference/numpysupported.rst index 1c333eac7ac..a9760dea164 100644 --- a/docs/source/reference/numpysupported.rst +++ b/docs/source/reference/numpysupported.rst @@ -210,9 +210,9 @@ The following methods of Numpy arrays are supported: * :meth:`~numpy.ndarray.repeat` (no axis argument) * :meth:`~numpy.ndarray.reshape` (only the 1-argument form) * :meth:`~numpy.ndarray.sort` (without arguments) -* :meth:`~numpy.ndarray.sum` (with or without the ``axis`` and/or ``dtype`` arguments. - ``axis`` only supports ``integer`` values) +* :meth:`~numpy.ndarray.sum` (with or without the ``axis`` and/or ``dtype`` arguments.) + * ``axis`` only supports ``integer`` values. * If the ``axis`` argument is a compile-time constant, all valid values are supported. An out-of-range value will result in a ``LoweringError`` at compile-time. * If the ``axis`` argument is not a compile-time constant, only values from 0 to 3 are supported. From d236a980a6ca1e9e2253c0130c2b13ee0fe97d88 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 18 Sep 2019 08:44:00 +0200 Subject: [PATCH 17/26] test fix for 32-bit machines --- numba/tests/test_array_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 5467c04a3aa..1a36773f7f2 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -866,7 +866,7 @@ def test_sum_axis_kws3(self): "input ".format(arr.dtype)): self.assertPreciseEqual( pyfunc(arr, axis=axis).astype(out_dtypes[arr.dtype]), - cfunc(arr, axis=axis)) + cfunc(arr, axis=axis).astype(out_dtypes[arr.dtype])) def test_sum_dtype_kws(self): pyfunc = array_sum_dtype_kws From 0ffe3818da3b3614e33bdedf3ab1b40da078edbd Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 18 Sep 2019 17:13:02 +0200 Subject: [PATCH 18/26] fix for scalar results --- numba/tests/test_array_methods.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 1a36773f7f2..a199b8cd4ec 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -864,9 +864,15 @@ def test_sum_axis_kws3(self): continue with self.subTest("Testing np.sum(axis) with {} " "input ".format(arr.dtype)): - self.assertPreciseEqual( - pyfunc(arr, axis=axis).astype(out_dtypes[arr.dtype]), - cfunc(arr, axis=axis).astype(out_dtypes[arr.dtype])) + npy_res = pyfunc(arr, axis=axis) + numba_res = cfunc(arr, axis=axis) + if isinstance(numba_res, np.ndarray): + self.assertPreciseEqual( + npy_res.astype(out_dtypes[arr.dtype]), + numba_res.astype(out_dtypes[arr.dtype])) + else: + # the results are scalars + self.assertEqual(npy_res, numba_res) def test_sum_dtype_kws(self): pyfunc = array_sum_dtype_kws From 18be7fb52a4fd60a9642122559d844885aa468eb Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 25 Sep 2019 10:45:12 +0200 Subject: [PATCH 19/26] fixes to address review --- numba/targets/arraymath.py | 34 +++++++++++-------------------- numba/tests/test_array_methods.py | 9 +++++--- numba/typing/arraydecl.py | 21 ++++++++++--------- numba/typing/npydecl.py | 6 +++--- 4 files changed, 32 insertions(+), 38 deletions(-) diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py index d6086f0f4bc..92a6b508592 100644 --- a/numba/targets/arraymath.py +++ b/numba/targets/arraymath.py @@ -179,6 +179,18 @@ def _array_sum_axis_nop(arr, v): def gen_sum_axis_impl(is_axis_const, const_axis_val, op, zero): def inner(arr, axis): + """ + function that performs sums over one specific axis + + The third parameter to gen_index_tuple that generates the indexing + tuples has to be a const so we can't just pass "axis" through since + that isn't const. We can check for specific values and have + different instances that do take consts. Supporting axis summation + only up to the fourth dimension for now. + + typing/arraydecl.py:sum_expand defines the return type for sum with axis. + It is one dimension less than the input array. + """ ndim = arr.ndim if not is_axis_const: @@ -235,16 +247,6 @@ def inner(arr, axis): @lower_builtin("array.sum", types.Array, types.intp, types.DTypeSpec) @lower_builtin("array.sum", types.Array, types.IntegerLiteral, types.DTypeSpec) def array_sum_axis_dtype(context, builder, sig, args): - """ - The third parameter to gen_index_tuple that generates the indexing - tuples has to be a const so we can't just pass "axis" through since - that isn't const. We can check for specific values and have - different instances that do take consts. Supporting axis summation - only up to the fourth dimension for now. - """ - # typing/arraydecl.py:sum_expand defines the return type for sum with axis. - # It is one dimension less than the input array. - retty = sig.return_type zero = getattr(retty, 'dtype', retty)(0) # if the return is scalar in type then "take" the 0th element of the @@ -284,8 +286,6 @@ def array_sum_impl_axis(arr, axis, dtype): @lower_builtin(np.sum, types.Array, types.DTypeSpec) -@lower_builtin(np.sum, types.Array, types.DTypeSpec) -@lower_builtin("array.sum", types.Array, types.DTypeSpec) @lower_builtin("array.sum", types.Array, types.DTypeSpec) def array_sum_dtype(context, builder, sig, args): zero = sig.return_type(0) @@ -306,16 +306,6 @@ def array_sum_impl(arr, dtype): @lower_builtin("array.sum", types.Array, types.intp) @lower_builtin("array.sum", types.Array, types.IntegerLiteral) def array_sum_axis(context, builder, sig, args): - """ - The third parameter to gen_index_tuple that generates the indexing - tuples has to be a const so we can't just pass "axis" through since - that isn't const. We can check for specific values and have - different instances that do take consts. Supporting axis summation - only up to the fourth dimension for now. - """ - # typing/arraydecl.py:sum_expand defines the return type for sum with axis. - # It is one dimension less than the input array. - retty = sig.return_type zero = getattr(retty, 'dtype', retty)(0) # if the return is scalar in type then "take" the 0th element of the diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index a199b8cd4ec..c0d0a0b1e4a 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -815,7 +815,8 @@ def test_sum_axis_kws(self): def test_sum_axis_kws2(self): pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.uint64, np.complex64] + all_dtypes = [np.float64, np.float32, np.int64, np.uint64, np.complex64, + np.complex128] # timedelta test cannot be enabled until issue #4540 is fixed # all_dtypes += [np.timedelta64] all_test_arrays = [ @@ -878,7 +879,7 @@ def test_sum_dtype_kws(self): pyfunc = array_sum_dtype_kws cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, - np.uint64, np.complex64] + np.uint64, np.complex64, np.complex128] # timedelta test cannot be enabled until issue #4540 is fixed # all_dtypes += [np.timedelta64] all_test_arrays = [ @@ -894,6 +895,7 @@ def test_sum_dtype_kws(self): np.dtype('uint32'): [np.float64, np.int64, np.float32], np.dtype('uint64'): [np.float64, np.int64], np.dtype('complex64'): [np.complex64], + np.dtype('complex128'): [np.complex128], np.dtype('timedelta64'): [np.timedelta64]} for arr_list in all_test_arrays: @@ -928,7 +930,7 @@ def test_sum_axis_dtype_kws2(self): pyfunc = array_sum_axis_dtype_kws cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, - np.uint64, np.complex64] + np.uint64, np.complex64, np.complex128] all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), np.ones(1, arr_dtype), @@ -942,6 +944,7 @@ def test_sum_axis_dtype_kws2(self): np.dtype('uint32'): [np.float64, np.int64, np.float32], np.dtype('uint64'): [np.float64, np.uint64], np.dtype('complex64'): [np.complex64], + np.dtype('complex128'): [np.complex128], np.dtype('timedelta64'): [np.timedelta64]} for arr_list in all_test_arrays: diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py index fd0a60d5076..118162f1b1e 100644 --- a/numba/typing/arraydecl.py +++ b/numba/typing/arraydecl.py @@ -597,38 +597,38 @@ def _expand_integer(ty): else: return ty + def generic_homog(self, args, kws): assert not args assert not kws return signature(self.this.dtype, recvr=self.this) + def generic_expand(self, args, kws): assert not args assert not kws return signature(_expand_integer(self.this.dtype), recvr=self.this) + def sum_expand(self, args, kws): """ - sum can be called with or without an axis parameter. + sum can be called with or without an axis parameter, and with or without + a dtype parameter """ pysig = None - if 'axis' in kws and not 'dtype' in kws: + if 'axis' in kws and 'dtype' not in kws: def sum_stub(axis): pass pysig = utils.pysignature(sum_stub) # rewrite args args = list(args) + [kws['axis']] - # kws = None - - if 'dtype' in kws and not 'axis' in kws: + elif 'dtype' in kws and 'axis' not in kws: def sum_stub(dtype): pass pysig = utils.pysignature(sum_stub) # rewrite args args = list(args) + [kws['dtype']] - # kws = None - - if 'dtype' in kws and 'axis' in kws: + elif 'dtype' in kws and 'axis' in kws: def sum_stub(axis, dtype): pass pysig = utils.pysignature(sum_stub) @@ -642,7 +642,7 @@ def sum_stub(axis, dtype): # of the type of the array. out = signature(_expand_integer(self.this.dtype), *args, recvr=self.this) - elif args_len == 1 and not 'dtype' in kws: + elif args_len == 1 and 'dtype' not in kws: # There is an axis parameter, either arg or kwarg if self.this.ndim == 1: # 1d reduces to a scalar @@ -674,11 +674,11 @@ def sum_stub(axis, dtype): return_type = types.Array(dtype=return_type, ndim=self.this.ndim-1, layout='C') out = signature(return_type, *args, recvr=self.this) - else: pass return out.replace(pysig=pysig) + def generic_expand_cumulative(self, args, kws): assert not args assert not kws @@ -687,6 +687,7 @@ def generic_expand_cumulative(self, args, kws): ndim=1, layout='C') return signature(return_type, recvr=self.this) + def generic_hetero_real(self, args, kws): assert not args assert not kws diff --git a/numba/typing/npydecl.py b/numba/typing/npydecl.py index 77292c40785..8cd3d89f840 100644 --- a/numba/typing/npydecl.py +++ b/numba/typing/npydecl.py @@ -378,15 +378,15 @@ def generic(self, args, kws): pysig = None if kws: if self.method_name == 'sum': - if 'axis' in kws and not 'dtype' in kws: + if 'axis' in kws and 'dtype' not in kws: def sum_stub(arr, axis): pass pysig = utils.pysignature(sum_stub) - if 'dtype' in kws and not 'axis' in kws: + elif 'dtype' in kws and 'axis' not in kws: def sum_stub(arr, dtype): pass pysig = utils.pysignature(sum_stub) - if 'dtype' in kws and 'axis' in kws: + elif 'dtype' in kws and 'axis' in kws: def sum_stub(arr, axis, dtype): pass pysig = utils.pysignature(sum_stub) From 89598012789a892ffc879dedc766d6d3318f3e8f Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 25 Sep 2019 20:24:00 +0200 Subject: [PATCH 20/26] fix docs char limit --- docs/source/reference/numpysupported.rst | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/source/reference/numpysupported.rst b/docs/source/reference/numpysupported.rst index a9760dea164..66860ecdb2e 100644 --- a/docs/source/reference/numpysupported.rst +++ b/docs/source/reference/numpysupported.rst @@ -210,20 +210,25 @@ The following methods of Numpy arrays are supported: * :meth:`~numpy.ndarray.repeat` (no axis argument) * :meth:`~numpy.ndarray.reshape` (only the 1-argument form) * :meth:`~numpy.ndarray.sort` (without arguments) -* :meth:`~numpy.ndarray.sum` (with or without the ``axis`` and/or ``dtype`` arguments.) +* :meth:`~numpy.ndarray.sum` (with or without the ``axis`` and/or ``dtype`` + arguments.) * ``axis`` only supports ``integer`` values. - * If the ``axis`` argument is a compile-time constant, all valid values are supported. + * If the ``axis`` argument is a compile-time constant, all valid values + are supported. An out-of-range value will result in a ``LoweringError`` at compile-time. - * If the ``axis`` argument is not a compile-time constant, only values from 0 to 3 are supported. + * If the ``axis`` argument is not a compile-time constant, only values + from 0 to 3 are supported. An out-of-range value will result in a runtime exception. - * All numeric ``dtypes`` are supported as ``dtype`` parameter. ``timedelta`` arrays can - be used as input arrays but ``timedelta`` is not supported as ``dtype`` parameter. - * When ``dtype`` is given, it determines the type of the internal accumulator. When it is not, - the selection is made automatically based on the input array's ``dtype``, mostly following the same - rules as NumPy. However, on 64-bit Windows, Numba uses a 64-bit accumulator for integer inputs - (``int64`` for ``int32`` inputs and ``uint64`` for ``uint32`` inputs), while NumPy would use a 32-bit - accumulator in those cases. + * All numeric ``dtypes`` are supported as ``dtype`` parameter. ``timedelta`` + arrays can be used as input arrays but ``timedelta`` is not supported + as ``dtype`` parameter. + * When ``dtype`` is given, it determines the type of the internal accumulator. + When it is not, the selection is made automatically based on the input + array's ``dtype``, mostly following the same rules as NumPy. However, + on 64-bit Windows, Numba uses a 64-bit accumulator for integer inputs + (``int64`` for ``int32`` inputs and ``uint64`` for ``uint32`` inputs), + while NumPy would use a 32-bit accumulator in those cases. * :meth:`~numpy.ndarray.transpose` From c25d0b50847d9fc071bf68edecf5d311282faa1a Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Wed, 25 Sep 2019 22:58:37 +0200 Subject: [PATCH 21/26] changes after second review --- numba/tests/test_array_methods.py | 45 +++++++++++-------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index c0d0a0b1e4a..6fe66b3e012 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -779,6 +779,9 @@ def check_err(a): check_err(np.array([])) def test_sum(self): + """ + test sum - basic + """ pyfunc = array_sum cfunc = jit(nopython=True)(pyfunc) # OK @@ -788,10 +791,12 @@ def test_sum(self): self.assertPreciseEqual(pyfunc(a, 0), cfunc(a, 0)) def test_sum2(self): + """ test sum over a whole range of dtypes, no axis or dtype parameter + """ pyfunc = array_sum cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, - np.complex64, np.uint32, np.uint64, np.timedelta64] + np.complex64, np.complex128, np.uint32, np.uint64, np.timedelta64] all_test_arrays = [ [np.ones((7, 6, 5, 4, 3), arr_dtype), np.ones(1, arr_dtype), @@ -804,6 +809,7 @@ def test_sum2(self): self.assertPreciseEqual(pyfunc(arr), cfunc(arr)) def test_sum_axis_kws(self): + """ test sum with axis parameter - basic """ pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) # OK @@ -813,6 +819,7 @@ def test_sum_axis_kws(self): self.assertPreciseEqual(pyfunc(a, axis=2), cfunc(a, axis=2)) def test_sum_axis_kws2(self): + """ test sum with axis parameter over a whole range of dtypes """ pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.uint64, np.complex64, @@ -836,13 +843,13 @@ def test_sum_axis_kws2(self): cfunc(arr, axis=axis)) def test_sum_axis_kws3(self): - """ uint32 and int32 must be tested separately because Numpy's current + """ testing uint32 and int32 separately + + uint32 and int32 must be tested separately because Numpy's current behaviour is different in 64bits Windows (accumulates as int32) and 64bits Linux (accumulates as int64), while Numba has decided to always accumulate as int64, when the OS is 64bits. No testing has been done for behaviours in 32 bits platforms. - - :return: """ pyfunc = array_sum_axis_kws cfunc = jit(nopython=True)(pyfunc) @@ -876,6 +883,7 @@ def test_sum_axis_kws3(self): self.assertEqual(npy_res, numba_res) def test_sum_dtype_kws(self): + """ test sum with dtype parameter over a whole range of dtypes """ pyfunc = array_sum_dtype_kws cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, @@ -894,7 +902,7 @@ def test_sum_dtype_kws(self): np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], np.dtype('uint32'): [np.float64, np.int64, np.float32], np.dtype('uint64'): [np.float64, np.int64], - np.dtype('complex64'): [np.complex64], + np.dtype('complex64'): [np.complex64, np.complex128], np.dtype('complex128'): [np.complex128], np.dtype('timedelta64'): [np.timedelta64]} @@ -907,26 +915,8 @@ def test_sum_dtype_kws(self): self.assertPreciseEqual(pyfunc(arr, dtype=out_dtype), cfunc(arr, dtype=out_dtype)) - def test_sum_dtype_kws_negative(self): - pyfunc = array_sum_dtype_kws - cfunc = jit(nopython=True)(pyfunc) - dtype = np.float64 - # OK - a = np.ones((7, 6, 5, 4, 3)) - self.assertFalse(type(pyfunc(a, dtype=np.int32)) == cfunc(a, dtype=dtype)) - def test_sum_axis_dtype_kws(self): - pyfunc = array_sum_axis_dtype_kws - cfunc = jit(nopython=True)(pyfunc) - dtype = np.float64 - # OK - a = np.ones((7, 6, 5, 4, 3)) - self.assertPreciseEqual(pyfunc(a, axis=1, dtype=dtype), - cfunc(a, axis=1, dtype=dtype)) - self.assertPreciseEqual(pyfunc(a, axis=2, dtype=dtype), - cfunc(a, axis=2, dtype=dtype)) - - def test_sum_axis_dtype_kws2(self): + """ test sum with axis and dtype parameters over a whole range of dtypes """ pyfunc = array_sum_axis_dtype_kws cfunc = jit(nopython=True)(pyfunc) all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, @@ -943,7 +933,7 @@ def test_sum_axis_dtype_kws2(self): np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], np.dtype('uint32'): [np.float64, np.int64, np.float32], np.dtype('uint64'): [np.float64, np.uint64], - np.dtype('complex64'): [np.complex64], + np.dtype('complex64'): [np.complex64, np.complex128], np.dtype('complex128'): [np.complex128], np.dtype('timedelta64'): [np.timedelta64]} @@ -961,10 +951,7 @@ def test_sum_axis_dtype_kws2(self): self.assertPreciseEqual(py_res, nb_res) def test_sum_axis_dtype_pos_arg(self): - """ - testing that axis and dtype inputs work when passed as positional - :return: - """ + """ testing that axis and dtype inputs work when passed as positional """ pyfunc = array_sum_axis_dtype_pos cfunc = jit(nopython=True)(pyfunc) dtype = np.float64 From 0726ed1879745b3c58d3d56ccb3dac6cee3281a7 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Thu, 26 Sep 2019 21:21:17 +0200 Subject: [PATCH 22/26] removing redundant tests --- numba/tests/test_array_methods.py | 1237 ----------------------------- 1 file changed, 1237 deletions(-) delete mode 100644 numba/tests/test_array_methods.py diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py deleted file mode 100644 index 6fe66b3e012..00000000000 --- a/numba/tests/test_array_methods.py +++ /dev/null @@ -1,1237 +0,0 @@ -from __future__ import division - -from itertools import product, cycle, permutations -import sys - -import numpy as np - -from numba import unittest_support as unittest -from numba import jit, typeof, types -from numba.compiler import compile_isolated -from numba.errors import TypingError, LoweringError -from numba.numpy_support import (as_dtype, strict_ufunc_typing, - version as numpy_version) -from .support import TestCase, CompilationCache, MemoryLeak, MemoryLeakMixin, tag -from .matmul_usecase import needs_blas - - -def np_around_array(arr, decimals, out): - np.around(arr, decimals, out) - -def np_around_binary(val, decimals): - return np.around(val, decimals) - -def np_around_unary(val): - return np.around(val) - -def np_round_array(arr, decimals, out): - np.round(arr, decimals, out) - -def np_round_binary(val, decimals): - return np.round(val, decimals) - -def np_round_unary(val): - return np.round(val) - -def _fixed_np_round(arr, decimals=0, out=None): - """ - A slightly bugfixed version of np.round(). - """ - if out is not None and arr.dtype.kind == 'c': - # workaround for https://github.com/numpy/numpy/issues/5779 - _fixed_np_round(arr.real, decimals, out.real) - _fixed_np_round(arr.imag, decimals, out.imag) - return out - else: - res = np.round(arr, decimals, out) - if out is None: - # workaround for https://github.com/numpy/numpy/issues/5780 - def fixup_signed_zero(arg, res): - if res == 0.0 and arg < 0: - return -np.abs(res) - else: - return res - if isinstance(arr, (complex, np.complexfloating)): - res = complex(fixup_signed_zero(arr.real, res.real), - fixup_signed_zero(arr.imag, res.imag)) - else: - res = fixup_signed_zero(arr, res) - return res - - -def array_T(arr): - return arr.T - -def array_transpose(arr): - return arr.transpose() - -def array_copy(arr): - return arr.copy() - -def np_copy(arr): - return np.copy(arr) - -def np_asfortranarray(arr): - return np.asfortranarray(arr) - -def np_ascontiguousarray(arr): - return np.ascontiguousarray(arr) - -def array_view(arr, newtype): - return arr.view(newtype) - -def array_take(arr, indices): - return arr.take(indices) - -def array_take_kws(arr, indices, axis): - return arr.take(indices, axis=axis) - -def array_fill(arr, val): - return arr.fill(val) - -# XXX Can't pass a dtype as a Dispatcher argument for now -def make_array_view(newtype): - def array_view(arr): - return arr.view(newtype) - return array_view - -def array_sliced_view(arr, ): - return arr[0:4].view(np.float32)[0] - -def make_array_astype(newtype): - def array_astype(arr): - return arr.astype(newtype) - return array_astype - - -def np_frombuffer(b): - """ - np.frombuffer() on a Python-allocated buffer. - """ - return np.frombuffer(b) - -def np_frombuffer_dtype(b): - return np.frombuffer(b, dtype=np.complex64) - -def np_frombuffer_allocated(shape): - """ - np.frombuffer() on a Numba-allocated buffer. - """ - arr = np.ones(shape, dtype=np.int32) - return np.frombuffer(arr) - -def np_frombuffer_allocated_dtype(shape): - arr = np.ones(shape, dtype=np.int32) - return np.frombuffer(arr, dtype=np.complex64) - -def identity_usecase(a, b): - return (a is b), (a is not b) - -def array_nonzero(a): - return a.nonzero() - -def np_nonzero(a): - return np.nonzero(a) - -def np_where_1(c): - return np.where(c) - -def np_where_3(c, x, y): - return np.where(c, x, y) - -def array_item(a): - return a.item() - -def array_itemset(a, v): - a.itemset(v) - -def array_sum(a, *args): - return a.sum(*args) - -def array_sum_axis_kws(a, axis): - return a.sum(axis=axis) - -def array_sum_dtype_kws(a, dtype): - return a.sum(dtype=dtype) - -def array_sum_axis_dtype_kws(a, dtype, axis): - return a.sum(axis=axis, dtype=dtype) - -def array_sum_axis_dtype_pos(a, a1, a2): - return a.sum(a1, a2) - -def array_sum_const_multi(arr, axis): - # use np.sum with different constant args multiple times to check - # for internal compile cache to see if constant-specialization is - # applied properly. - a = np.sum(arr, axis=4) - b = np.sum(arr, 3) - # the last invocation uses runtime-variable - c = np.sum(arr, axis) - # as method - d = arr.sum(axis=5) - # negative const axis - e = np.sum(arr, axis=-1) - return a, b, c, d, e - -def array_sum_const_axis_neg_one(a, axis): - # use .sum with -1 axis, this is for use with 1D arrays where the above - # "const_multi" variant would raise errors - return a.sum(axis=-1) - -def array_cumsum(a, *args): - return a.cumsum(*args) - -def array_cumsum_kws(a, axis): - return a.cumsum(axis=axis) - -def array_real(a): - return np.real(a) - -def array_imag(a): - return np.imag(a) - -def array_conj(a): - return a.conj() - -def array_conjugate(a): - return a.conjugate() - -def np_unique(a): - return np.unique(a) - - -def array_dot(a, b): - return a.dot(b) - -def array_dot_chain(a, b): - return a.dot(b).dot(b) - -def array_ctor(n, dtype): - return np.ones(n, dtype=dtype) - - -class TestArrayMethods(MemoryLeakMixin, TestCase): - """ - Test various array methods and array-related functions. - """ - - def setUp(self): - super(TestArrayMethods, self).setUp() - self.ccache = CompilationCache() - - def check_round_scalar(self, unary_pyfunc, binary_pyfunc): - base_values = [-3.0, -2.5, -2.25, -1.5, 1.5, 2.25, 2.5, 2.75] - complex_values = [x * (1 - 1j) for x in base_values] - int_values = [int(x) for x in base_values] - argtypes = (types.float64, types.float32, types.int32, - types.complex64, types.complex128) - argvalues = [base_values, base_values, int_values, - complex_values, complex_values] - - pyfunc = binary_pyfunc - for ty, values in zip(argtypes, argvalues): - cres = compile_isolated(pyfunc, (ty, types.int32)) - cfunc = cres.entry_point - for decimals in (1, 0, -1): - for v in values: - if decimals > 0: - v *= 10 - expected = _fixed_np_round(v, decimals) - got = cfunc(v, decimals) - self.assertPreciseEqual(got, expected) - - pyfunc = unary_pyfunc - for ty, values in zip(argtypes, argvalues): - cres = compile_isolated(pyfunc, (ty,)) - cfunc = cres.entry_point - for v in values: - expected = _fixed_np_round(v) - got = cfunc(v) - self.assertPreciseEqual(got, expected) - - def test_round_scalar(self): - self.check_round_scalar(np_round_unary, np_round_binary) - - def test_around_scalar(self): - self.check_round_scalar(np_around_unary, np_around_binary) - - def check_round_array(self, pyfunc): - def check_round(cfunc, values, inty, outty, decimals): - # Create input and output arrays of the right type - arr = values.astype(as_dtype(inty)) - out = np.zeros_like(arr).astype(as_dtype(outty)) - pyout = out.copy() - _fixed_np_round(arr, decimals, pyout) - self.memory_leak_setup() - cfunc(arr, decimals, out) - self.memory_leak_teardown() - np.testing.assert_allclose(out, pyout) - # Output shape mismatch - with self.assertRaises(ValueError) as raises: - cfunc(arr, decimals, out[1:]) - self.assertEqual(str(raises.exception), - "invalid output shape") - - def check_types(argtypes, outtypes, values): - for inty, outty in product(argtypes, outtypes): - cres = compile_isolated(pyfunc, - (types.Array(inty, 1, 'A'), - types.int32, - types.Array(outty, 1, 'A'))) - cfunc = cres.entry_point - check_round(cres.entry_point, values, inty, outty, 0) - check_round(cres.entry_point, values, inty, outty, 1) - if not isinstance(outty, types.Integer): - check_round(cres.entry_point, values * 10, inty, outty, -1) - else: - # Avoid Numpy bug when output is an int: - # https://github.com/numpy/numpy/issues/5777 - pass - - values = np.array([-3.0, -2.5, -2.25, -1.5, 1.5, 2.25, 2.5, 2.75]) - - if strict_ufunc_typing: - argtypes = (types.float64, types.float32) - else: - argtypes = (types.float64, types.float32, types.int32) - check_types(argtypes, argtypes, values) - - argtypes = (types.complex64, types.complex128) - check_types(argtypes, argtypes, values * (1 - 1j)) - - # Exceptions leak references - self.disable_leak_check() - - def test_round_array(self): - self.check_round_array(np_round_array) - - def test_around_array(self): - self.check_round_array(np_around_array) - - def test_array_view(self): - - def run(arr, dtype): - pyfunc = make_array_view(dtype) - cres = self.ccache.compile(pyfunc, (typeof(arr),)) - return cres.entry_point(arr) - def check(arr, dtype): - expected = arr.view(dtype) - self.memory_leak_setup() - got = run(arr, dtype) - self.assertPreciseEqual(got, expected) - del got - self.memory_leak_teardown() - def check_err(arr, dtype): - with self.assertRaises(ValueError) as raises: - run(arr, dtype) - self.assertEqual(str(raises.exception), - "new type not compatible with array") - - dt1 = np.dtype([('a', np.int8), ('b', np.int8)]) - dt2 = np.dtype([('u', np.int16), ('v', np.int8)]) - dt3 = np.dtype([('x', np.int16), ('y', np.int16)]) - - # C-contiguous - arr = np.arange(24, dtype=np.int8) - check(arr, np.dtype('int16')) - check(arr, np.int16) - check(arr, np.int8) - check(arr, np.float32) - check(arr, np.complex64) - check(arr, dt1) - check(arr, dt2) - check_err(arr, np.complex128) - - # Last dimension must have a compatible size - arr = arr.reshape((3, 8)) - check(arr, np.int8) - check(arr, np.float32) - check(arr, np.complex64) - check(arr, dt1) - check_err(arr, dt2) - check_err(arr, np.complex128) - - # F-contiguous - arr = np.arange(24, dtype=np.int8).reshape((3, 8)).T - check(arr, np.int8) - check(arr, np.float32) - check(arr, np.complex64) - check(arr, dt1) - check_err(arr, dt2) - check_err(arr, np.complex128) - - # Non-contiguous: only a type with the same itemsize can be used - arr = np.arange(16, dtype=np.int32)[::2] - check(arr, np.uint32) - check(arr, np.float32) - check(arr, dt3) - check_err(arr, np.int8) - check_err(arr, np.int16) - check_err(arr, np.int64) - check_err(arr, dt1) - check_err(arr, dt2) - - # Zero-dim array: only a type with the same itemsize can be used - arr = np.array([42], dtype=np.int32).reshape(()) - check(arr, np.uint32) - check(arr, np.float32) - check(arr, dt3) - check_err(arr, np.int8) - check_err(arr, np.int16) - check_err(arr, np.int64) - check_err(arr, dt1) - check_err(arr, dt2) - - # Exceptions leak references - self.disable_leak_check() - - def test_array_sliced_view(self): - """ - Test .view() on A layout array but has contiguous innermost dimension. - """ - pyfunc = array_sliced_view - cres = self.ccache.compile(pyfunc, (types.uint8[:],)) - cfunc = cres.entry_point - - orig = np.array([1.5, 2], dtype=np.float32) - byteary = orig.view(np.uint8) - - expect = pyfunc(byteary) - got = cfunc(byteary) - - self.assertEqual(expect, got) - - def test_array_astype(self): - - def run(arr, dtype): - pyfunc = make_array_astype(dtype) - cres = self.ccache.compile(pyfunc, (typeof(arr),)) - return cres.entry_point(arr) - def check(arr, dtype): - expected = arr.astype(dtype).copy(order='A') - got = run(arr, dtype) - self.assertPreciseEqual(got, expected) - - # C-contiguous - arr = np.arange(24, dtype=np.int8) - check(arr, np.dtype('int16')) - check(arr, np.int32) - check(arr, np.float32) - check(arr, np.complex128) - - # F-contiguous - arr = np.arange(24, dtype=np.int8).reshape((3, 8)).T - check(arr, np.float32) - - # Non-contiguous - arr = np.arange(16, dtype=np.int32)[::2] - check(arr, np.uint64) - - # check read only attr does not get copied - arr = np.arange(16, dtype=np.int32) - arr.flags.writeable = False - check(arr, np.int32) - - # Invalid conversion - dt = np.dtype([('x', np.int8)]) - with self.assertTypingError() as raises: - check(arr, dt) - self.assertIn('cannot convert from int32 to Record', - str(raises.exception)) - - def check_np_frombuffer(self, pyfunc): - def run(buf): - cres = self.ccache.compile(pyfunc, (typeof(buf),)) - return cres.entry_point(buf) - def check(buf): - old_refcnt = sys.getrefcount(buf) - expected = pyfunc(buf) - self.memory_leak_setup() - got = run(buf) - self.assertPreciseEqual(got, expected) - del expected - self.assertEqual(sys.getrefcount(buf), old_refcnt + 1) - del got - self.assertEqual(sys.getrefcount(buf), old_refcnt) - self.memory_leak_teardown() - - b = bytearray(range(16)) - check(b) - if sys.version_info >= (3,): - check(bytes(b)) - check(memoryview(b)) - check(np.arange(12)) - b = np.arange(12).reshape((3, 4)) - check(b) - - # Exceptions leak references - self.disable_leak_check() - - with self.assertRaises(ValueError) as raises: - run(bytearray(b"xxx")) - self.assertEqual("buffer size must be a multiple of element size", - str(raises.exception)) - - def test_np_frombuffer(self): - self.check_np_frombuffer(np_frombuffer) - - def test_np_frombuffer_dtype(self): - self.check_np_frombuffer(np_frombuffer_dtype) - - def check_layout_dependent_func(self, pyfunc, fac=np.arange, - check_sameness=True): - def is_same(a, b): - return a.ctypes.data == b.ctypes.data - def check_arr(arr): - cres = compile_isolated(pyfunc, (typeof(arr),)) - expected = pyfunc(arr) - got = cres.entry_point(arr) - self.assertPreciseEqual(expected, got) - if check_sameness: - self.assertEqual(is_same(expected, arr), is_same(got, arr)) - arr = fac(24) - check_arr(arr) - check_arr(arr.reshape((3, 8))) - check_arr(arr.reshape((3, 8)).T) - check_arr(arr.reshape((3, 8))[::2]) - check_arr(arr.reshape((2, 3, 4))) - check_arr(arr.reshape((2, 3, 4)).T) - check_arr(arr.reshape((2, 3, 4))[::2]) - arr = np.array([0]).reshape(()) - check_arr(arr) - - def test_array_transpose(self): - self.check_layout_dependent_func(array_transpose) - - @tag('important') - def test_array_T(self): - self.check_layout_dependent_func(array_T) - - @tag('important') - def test_array_copy(self): - self.check_layout_dependent_func(array_copy) - - def test_np_copy(self): - self.check_layout_dependent_func(np_copy) - - def test_np_asfortranarray(self): - self.check_layout_dependent_func(np_asfortranarray, - check_sameness=numpy_version >= (1, 8)) - - def test_np_ascontiguousarray(self): - self.check_layout_dependent_func(np_ascontiguousarray, - check_sameness=numpy_version > (1, 11)) - - def check_np_frombuffer_allocated(self, pyfunc): - def run(shape): - cres = self.ccache.compile(pyfunc, (typeof(shape),)) - return cres.entry_point(shape) - def check(shape): - expected = pyfunc(shape) - got = run(shape) - self.assertPreciseEqual(got, expected) - - check((16,)) - check((4, 4)) - check((1, 0, 1)) - - def test_np_frombuffer_allocated(self): - self.check_np_frombuffer_allocated(np_frombuffer_allocated) - - def test_np_frombuffer_allocated2(self): - self.check_np_frombuffer_allocated(np_frombuffer_allocated_dtype) - - def check_nonzero(self, pyfunc): - def fac(N): - np.random.seed(42) - arr = np.random.random(N) - arr[arr < 0.3] = 0.0 - arr[arr > 0.7] = float('nan') - return arr - - def check_arr(arr): - cres = compile_isolated(pyfunc, (typeof(arr),)) - expected = pyfunc(arr) - # NOTE: Numpy 1.9 returns readonly arrays for multidimensional - # arrays. Workaround this by copying the results. - expected = [a.copy() for a in expected] - self.assertPreciseEqual(cres.entry_point(arr), expected) - - arr = np.int16([1, 0, -1, 0]) - check_arr(arr) - arr = np.bool_([1, 0, 1]) - check_arr(arr) - - arr = fac(24) - check_arr(arr) - check_arr(arr.reshape((3, 8))) - check_arr(arr.reshape((3, 8)).T) - check_arr(arr.reshape((3, 8))[::2]) - check_arr(arr.reshape((2, 3, 4))) - check_arr(arr.reshape((2, 3, 4)).T) - check_arr(arr.reshape((2, 3, 4))[::2]) - for v in (0.0, 1.5, float('nan')): - arr = np.array([v]).reshape(()) - check_arr(arr) - - def test_array_nonzero(self): - self.check_nonzero(array_nonzero) - - def test_np_nonzero(self): - self.check_nonzero(np_nonzero) - - def test_np_where_1(self): - self.check_nonzero(np_where_1) - - def test_np_where_3(self): - pyfunc = np_where_3 - def fac(N): - np.random.seed(42) - arr = np.random.random(N) - arr[arr < 0.3] = 0.0 - arr[arr > 0.7] = float('nan') - return arr - - layouts = cycle(['C', 'F', 'A']) - _types = [np.int32, np.int64, np.float32, np.float64, np.complex64, - np.complex128] - - np.random.seed(42) - - def check_arr(arr, layout=False): - np.random.shuffle(_types) - if layout != False: - x = np.zeros_like(arr, dtype=_types[0], order=layout) - y = np.zeros_like(arr, dtype=_types[1], order=layout) - arr = arr.copy(order=layout) - else: - x = np.zeros_like(arr, dtype=_types[0], order=next(layouts)) - y = np.zeros_like(arr, dtype=_types[1], order=next(layouts)) - x.fill(4) - y.fill(9) - cres = compile_isolated(pyfunc, (typeof(arr), typeof(x), typeof(y))) - expected = pyfunc(arr, x, y) - got = cres.entry_point(arr, x, y) - # Contiguity of result varies accross Numpy versions, only - # check contents. NumPy 1.11+ seems to stabilize. - if numpy_version < (1, 11): - self.assertEqual(got.dtype, expected.dtype) - np.testing.assert_array_equal(got, expected) - else: - self.assertPreciseEqual(got, expected) - - def check_scal(scal): - x = 4 - y = 5 - np.random.shuffle(_types) - x = _types[0](4) - y = _types[1](5) - cres = compile_isolated(pyfunc, (typeof(scal), typeof(x), typeof(y))) - expected = pyfunc(scal, x, y) - got = cres.entry_point(scal, x, y) - self.assertPreciseEqual(got, expected) - - arr = np.int16([1, 0, -1, 0]) - check_arr(arr) - arr = np.bool_([1, 0, 1]) - check_arr(arr) - - arr = fac(24) - check_arr(arr) - check_arr(arr.reshape((3, 8))) - check_arr(arr.reshape((3, 8)).T) - check_arr(arr.reshape((3, 8))[::2]) - check_arr(arr.reshape((2, 3, 4))) - check_arr(arr.reshape((2, 3, 4)).T) - check_arr(arr.reshape((2, 3, 4))[::2]) - - check_arr(arr.reshape((2, 3, 4)), layout='F') - check_arr(arr.reshape((2, 3, 4)).T, layout='F') - check_arr(arr.reshape((2, 3, 4))[::2], layout='F') - - for v in (0.0, 1.5, float('nan')): - arr = np.array([v]).reshape(()) - check_arr(arr) - - for x in (0, 1, True, False, 2.5, 0j): - check_scal(x) - - def test_np_where_3_broadcast_x_y_scalar(self): - pyfunc = np_where_3 - cfunc = jit(nopython=True)(pyfunc) - - def check_ok(args): - expected = pyfunc(*args) - got = cfunc(*args) - self.assertPreciseEqual(got, expected) - - def a_variations(): - a = np.linspace(-2, 4, 20) - self.random.shuffle(a) - yield a - yield a.reshape(2, 5, 2) - yield a.reshape(4, 5, order='F') - yield a.reshape(2, 5, 2)[::-1] - - for a in a_variations(): - params = (a > 0, 0, 1) - check_ok(params) - - params = (a < 0, np.nan, 1 + 4j) - check_ok(params) - - params = (a > 1, True, False) - check_ok(params) - - def test_np_where_3_broadcast_x_or_y_scalar(self): - pyfunc = np_where_3 - cfunc = jit(nopython=True)(pyfunc) - - def check_ok(args): - condition, x, y = args - - expected = pyfunc(condition, x, y) - got = cfunc(condition, x, y) - self.assertPreciseEqual(got, expected) - - # swap x and y - expected = pyfunc(condition, y, x) - got = cfunc(condition, y, x) - self.assertPreciseEqual(got, expected) - - def array_permutations(): - x = np.arange(9).reshape(3, 3) - yield x - yield x * 1.1 - yield np.asfortranarray(x) - yield x[::-1] - yield np.linspace(-10, 10, 60).reshape(3, 4, 5) * 1j - - def scalar_permutations(): - yield 0 - yield 4.3 - yield np.nan - yield True - yield 8 + 4j - - for x in array_permutations(): - for y in scalar_permutations(): - x_mean = np.mean(x) - condition = x > x_mean - params = (condition, x, y) - check_ok(params) - - def test_item(self): - pyfunc = array_item - cfunc = jit(nopython=True)(pyfunc) - - def check_ok(arg): - expected = pyfunc(arg) - got = cfunc(arg) - self.assertPreciseEqual(got, expected) - - def check_err(arg): - with self.assertRaises(ValueError) as raises: - cfunc(arg) - self.assertIn("item(): can only convert an array of size 1 to a Python scalar", - str(raises.exception)) - - # Exceptions leak references - self.disable_leak_check() - - # Test on different kinds of scalars and 1-item arrays - check_ok(np.float32([1.5])) - check_ok(np.complex128([[1.5j]])) - check_ok(np.array(1.5)) - check_ok(np.bool_(True)) - check_ok(np.float32(1.5)) - - check_err(np.array([1, 2])) - check_err(np.array([])) - - def test_itemset(self): - pyfunc = array_itemset - cfunc = jit(nopython=True)(pyfunc) - - def check_ok(a, v): - expected = a.copy() - got = a.copy() - pyfunc(expected, v) - cfunc(got, v) - self.assertPreciseEqual(got, expected) - - def check_err(a): - with self.assertRaises(ValueError) as raises: - cfunc(a, 42) - self.assertIn("itemset(): can only write to an array of size 1", - str(raises.exception)) - - # Exceptions leak references - self.disable_leak_check() - - # Test on different kinds of 1-item arrays - check_ok(np.float32([1.5]), 42) - check_ok(np.complex128([[1.5j]]), 42) - check_ok(np.array(1.5), 42) - - check_err(np.array([1, 2])) - check_err(np.array([])) - - def test_sum(self): - """ - test sum - basic - """ - pyfunc = array_sum - cfunc = jit(nopython=True)(pyfunc) - # OK - a = np.ones((7, 6, 5, 4, 3)) - self.assertPreciseEqual(pyfunc(a), cfunc(a)) - # OK - self.assertPreciseEqual(pyfunc(a, 0), cfunc(a, 0)) - - def test_sum2(self): - """ test sum over a whole range of dtypes, no axis or dtype parameter - """ - pyfunc = array_sum - cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.int32, - np.complex64, np.complex128, np.uint32, np.uint64, np.timedelta64] - all_test_arrays = [ - [np.ones((7, 6, 5, 4, 3), arr_dtype), - np.ones(1, arr_dtype), - np.ones((7, 3), arr_dtype) * -5] - for arr_dtype in all_dtypes] - - for arr_list in all_test_arrays: - for arr in arr_list: - with self.subTest("Test np.sum with {} input ".format(arr.dtype)): - self.assertPreciseEqual(pyfunc(arr), cfunc(arr)) - - def test_sum_axis_kws(self): - """ test sum with axis parameter - basic """ - pyfunc = array_sum_axis_kws - cfunc = jit(nopython=True)(pyfunc) - # OK - a = np.ones((7, 6, 5, 4, 3)) - self.assertPreciseEqual(pyfunc(a, axis=1), cfunc(a, axis=1)) - # OK - self.assertPreciseEqual(pyfunc(a, axis=2), cfunc(a, axis=2)) - - def test_sum_axis_kws2(self): - """ test sum with axis parameter over a whole range of dtypes """ - pyfunc = array_sum_axis_kws - cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.uint64, np.complex64, - np.complex128] - # timedelta test cannot be enabled until issue #4540 is fixed - # all_dtypes += [np.timedelta64] - all_test_arrays = [ - [np.ones((7, 6, 5, 4, 3), arr_dtype), - np.ones(1, arr_dtype), - np.ones((7, 3), arr_dtype) * -5] - for arr_dtype in all_dtypes] - - for arr_list in all_test_arrays: - for arr in arr_list: - for axis in (0, 1, 2): - if axis > len(arr.shape)-1: - continue - with self.subTest("Testing np.sum(axis) with {} " - "input ".format(arr.dtype)): - self.assertPreciseEqual(pyfunc(arr, axis=axis), - cfunc(arr, axis=axis)) - - def test_sum_axis_kws3(self): - """ testing uint32 and int32 separately - - uint32 and int32 must be tested separately because Numpy's current - behaviour is different in 64bits Windows (accumulates as int32) - and 64bits Linux (accumulates as int64), while Numba has decided to always - accumulate as int64, when the OS is 64bits. No testing has been done - for behaviours in 32 bits platforms. - """ - pyfunc = array_sum_axis_kws - cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.int32, np.uint32] - # expected return dtypes in Numba - out_dtypes = {np.dtype('int32'): np.int64, np.dtype('uint32'): np.uint64, - np.dtype('int64'): np.int64} - # timedelta test cannot be enabled until issue #4540 is fixed - # all_dtypes += [np.timedelta64] - all_test_arrays = [ - [np.ones((7, 6, 5, 4, 3), arr_dtype), - np.ones(1, arr_dtype), - np.ones((7, 3), arr_dtype) * -5] - for arr_dtype in all_dtypes] - - for arr_list in all_test_arrays: - for arr in arr_list: - for axis in (0, 1, 2): - if axis > len(arr.shape)-1: - continue - with self.subTest("Testing np.sum(axis) with {} " - "input ".format(arr.dtype)): - npy_res = pyfunc(arr, axis=axis) - numba_res = cfunc(arr, axis=axis) - if isinstance(numba_res, np.ndarray): - self.assertPreciseEqual( - npy_res.astype(out_dtypes[arr.dtype]), - numba_res.astype(out_dtypes[arr.dtype])) - else: - # the results are scalars - self.assertEqual(npy_res, numba_res) - - def test_sum_dtype_kws(self): - """ test sum with dtype parameter over a whole range of dtypes """ - pyfunc = array_sum_dtype_kws - cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, - np.uint64, np.complex64, np.complex128] - # timedelta test cannot be enabled until issue #4540 is fixed - # all_dtypes += [np.timedelta64] - all_test_arrays = [ - [np.ones((7, 6, 5, 4, 3), arr_dtype), - np.ones(1, arr_dtype), - np.ones((7, 3), arr_dtype) * -5] - for arr_dtype in all_dtypes] - - out_dtypes = {np.dtype('float64'): [np.float64], - np.dtype('float32'): [np.float64, np.float32], - np.dtype('int64'): [np.float64, np.int64, np.float32], - np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], - np.dtype('uint32'): [np.float64, np.int64, np.float32], - np.dtype('uint64'): [np.float64, np.int64], - np.dtype('complex64'): [np.complex64, np.complex128], - np.dtype('complex128'): [np.complex128], - np.dtype('timedelta64'): [np.timedelta64]} - - for arr_list in all_test_arrays: - for arr in arr_list: - for out_dtype in out_dtypes[arr.dtype]: - subtest_str = ("Testing np.sum with {} input and {} output" - .format(arr.dtype, out_dtype)) - with self.subTest(subtest_str): - self.assertPreciseEqual(pyfunc(arr, dtype=out_dtype), - cfunc(arr, dtype=out_dtype)) - - def test_sum_axis_dtype_kws(self): - """ test sum with axis and dtype parameters over a whole range of dtypes """ - pyfunc = array_sum_axis_dtype_kws - cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, - np.uint64, np.complex64, np.complex128] - all_test_arrays = [ - [np.ones((7, 6, 5, 4, 3), arr_dtype), - np.ones(1, arr_dtype), - np.ones((7, 3), arr_dtype) * -5] - for arr_dtype in all_dtypes] - - out_dtypes = {np.dtype('float64'): [np.float64], - np.dtype('float32'): [np.float64, np.float32], - np.dtype('int64'): [np.float64, np.int64, np.float32], - np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], - np.dtype('uint32'): [np.float64, np.int64, np.float32], - np.dtype('uint64'): [np.float64, np.uint64], - np.dtype('complex64'): [np.complex64, np.complex128], - np.dtype('complex128'): [np.complex128], - np.dtype('timedelta64'): [np.timedelta64]} - - for arr_list in all_test_arrays: - for arr in arr_list: - for out_dtype in out_dtypes[arr.dtype]: - for axis in (0, 1, 2): - if axis > len(arr.shape) - 1: - continue - subtest_str = ("Testing np.sum with {} input and {} output " - .format(arr.dtype, out_dtype)) - with self.subTest(subtest_str): - py_res = pyfunc(arr, axis=axis, dtype=out_dtype) - nb_res = cfunc(arr, axis=axis, dtype=out_dtype) - self.assertPreciseEqual(py_res, nb_res) - - def test_sum_axis_dtype_pos_arg(self): - """ testing that axis and dtype inputs work when passed as positional """ - pyfunc = array_sum_axis_dtype_pos - cfunc = jit(nopython=True)(pyfunc) - dtype = np.float64 - # OK - a = np.ones((7, 6, 5, 4, 3)) - self.assertPreciseEqual(pyfunc(a, 1, dtype), - cfunc(a, 1, dtype)) - - self.assertPreciseEqual(pyfunc(a, 2, dtype), - cfunc(a, 2, dtype)) - - def test_sum_1d_kws(self): - # check 1d reduces to scalar - pyfunc = array_sum_axis_kws - cfunc = jit(nopython=True)(pyfunc) - a = np.arange(10.) - self.assertPreciseEqual(pyfunc(a, axis=0), cfunc(a, axis=0)) - pyfunc = array_sum_const_axis_neg_one - cfunc = jit(nopython=True)(pyfunc) - a = np.arange(10.) - self.assertPreciseEqual(pyfunc(a, axis=-1), cfunc(a, axis=-1)) - - def test_sum_const(self): - pyfunc = array_sum_const_multi - cfunc = jit(nopython=True)(pyfunc) - - arr = np.ones((3, 4, 5, 6, 7, 8)) - axis = 1 - self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) - axis = 2 - self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) - - def test_sum_exceptions(self): - # Exceptions leak references - self.disable_leak_check() - pyfunc = array_sum - cfunc = jit(nopython=True)(pyfunc) - - a = np.ones((7, 6, 5, 4, 3)) - b = np.ones((4, 3)) - # BAD: axis > dimensions - with self.assertRaises(ValueError): - cfunc(b, 2) - # BAD: negative axis - with self.assertRaises(ValueError): - cfunc(a, -1) - # BAD: axis greater than 3 - with self.assertRaises(ValueError): - cfunc(a, 4) - - def test_sum_const_negative(self): - # Exceptions leak references - self.disable_leak_check() - - @jit(nopython=True) - def foo(arr): - return arr.sum(axis=-3) - - # ndim == 4, axis == -3, OK - a = np.ones((1, 2, 3, 4)) - self.assertPreciseEqual(foo(a), foo.py_func(a)) - # ndim == 3, axis == -3, OK - a = np.ones((1, 2, 3)) - self.assertPreciseEqual(foo(a), foo.py_func(a)) - # ndim == 2, axis == -3, BAD - a = np.ones((1, 2)) - with self.assertRaises(LoweringError) as raises: - foo(a) - errmsg = "'axis' entry is out of bounds" - self.assertIn(errmsg, str(raises.exception)) - with self.assertRaises(ValueError) as raises: - foo.py_func(a) - # Numpy 1.13 has a different error message than prior numpy - # Just check for the "out of bounds" phrase in it. - self.assertIn("out of bounds", str(raises.exception)) - - def test_cumsum(self): - pyfunc = array_cumsum - cfunc = jit(nopython=True)(pyfunc) - # OK - a = np.ones((2, 3)) - self.assertPreciseEqual(pyfunc(a), cfunc(a)) - # BAD: with axis - with self.assertRaises(TypingError): - cfunc(a, 1) - # BAD: with kw axis - pyfunc = array_cumsum_kws - cfunc = jit(nopython=True)(pyfunc) - with self.assertRaises(TypingError): - cfunc(a, axis=1) - - def test_take(self): - pyfunc = array_take - cfunc = jit(nopython=True)(pyfunc) - - def check(arr, ind): - expected = pyfunc(arr, ind) - got = cfunc(arr, ind) - self.assertPreciseEqual(expected, got) - if hasattr(expected, 'order'): - self.assertEqual(expected.order == got.order) - - # need to check: - # 1. scalar index - # 2. 1d array index - # 3. nd array index, >2d and F order - # 4. reflected list - # 5. tuples - - test_indices = [] - test_indices.append(1) - test_indices.append(5) - test_indices.append(11) - test_indices.append(-2) - test_indices.append(np.array([1, 5, 1, 11, 3])) - test_indices.append(np.array([[1, 5, 1], [11, 3, 0]], order='F')) - test_indices.append(np.array([[[1, 5, 1], [11, 3, 0]]])) - test_indices.append(np.array([[[[1, 5]], [[11, 0]], [[1, 2]]]])) - test_indices.append([1, 5, 1, 11, 3]) - test_indices.append((1, 5, 1)) - test_indices.append(((1, 5, 1), (11, 3, 2))) - test_indices.append((((1,), (5,), (1,)), ((11,), (3,), (2,)))) - - layouts = cycle(['C', 'F', 'A']) - - for dt in [np.float64, np.int64, np.complex128]: - A = np.arange(12, dtype=dt).reshape((4, 3), order=next(layouts)) - for ind in test_indices: - check(A, ind) - - #check illegal access raises - A = np.arange(12, dtype=dt).reshape((4, 3), order=next(layouts)) - szA = A.size - illegal_indices = [szA, -szA - 1, np.array(szA), np.array(-szA - 1), - [szA], [-szA - 1]] - for x in illegal_indices: - with self.assertRaises(IndexError): - cfunc(A, x) # oob raises - - # check float indexing raises - with self.assertRaises(TypingError): - cfunc(A, [1.7]) - - # check unsupported arg raises - with self.assertRaises(TypingError): - take_kws = jit(nopython=True)(array_take_kws) - take_kws(A, 1, 1) - - # check kwarg unsupported raises - with self.assertRaises(TypingError): - take_kws = jit(nopython=True)(array_take_kws) - take_kws(A, 1, axis=1) - - #exceptions leak refs - self.disable_leak_check() - - def test_fill(self): - pyfunc = array_fill - cfunc = jit(nopython=True)(pyfunc) - def check(arr, val): - expected = np.copy(arr) - erv = pyfunc(expected, val) - self.assertTrue(erv is None) - got = np.copy(arr) - grv = cfunc(got, val) - self.assertTrue(grv is None) - # check mutation is the same - self.assertPreciseEqual(expected, got) - - # scalar - A = np.arange(1) - for x in [np.float64, np.bool_]: - check(A, x(10)) - - # 2d - A = np.arange(12).reshape(3, 4) - for x in [np.float64, np.bool_]: - check(A, x(10)) - - # 4d - A = np.arange(48, dtype=np.complex64).reshape(2, 3, 4, 2) - for x in [np.float64, np.complex128, np.bool_]: - check(A, x(10)) - - def test_real(self): - pyfunc = array_real - cfunc = jit(nopython=True)(pyfunc) - - x = np.linspace(-10, 10) - np.testing.assert_equal(pyfunc(x), cfunc(x)) - - x, y = np.meshgrid(x, x) - z = x + 1j*y - np.testing.assert_equal(pyfunc(z), cfunc(z)) - - def test_imag(self): - pyfunc = array_imag - cfunc = jit(nopython=True)(pyfunc) - - x = np.linspace(-10, 10) - np.testing.assert_equal(pyfunc(x), cfunc(x)) - - x, y = np.meshgrid(x, x) - z = x + 1j*y - np.testing.assert_equal(pyfunc(z), cfunc(z)) - - def test_conj(self): - for pyfunc in [array_conj, array_conjugate]: - cfunc = jit(nopython=True)(pyfunc) - - x = np.linspace(-10, 10) - np.testing.assert_equal(pyfunc(x), cfunc(x)) - - x, y = np.meshgrid(x, x) - z = x + 1j*y - np.testing.assert_equal(pyfunc(z), cfunc(z)) - - def test_unique(self): - pyfunc = np_unique - cfunc = jit(nopython=True)(pyfunc) - - def check(a): - np.testing.assert_equal(pyfunc(a), cfunc(a)) - - check(np.array([[1, 1, 3], [3, 4, 5]])) - check(np.array(np.zeros(5))) - check(np.array([[3.1, 3.1], [1.7, 2.29], [3.3, 1.7]])) - check(np.array([])) - - @needs_blas - def test_array_dot(self): - # just ensure that the dot impl dispatches correctly, do - # not test dot itself, this is done in test_linalg. - pyfunc = array_dot - cfunc = jit(nopython=True)(pyfunc) - a = np.arange(20.).reshape(4, 5) - b = np.arange(5.) - np.testing.assert_equal(pyfunc(a, b), cfunc(a, b)) - - # check that chaining works - pyfunc = array_dot_chain - cfunc = jit(nopython=True)(pyfunc) - a = np.arange(16.).reshape(4, 4) - np.testing.assert_equal(pyfunc(a, a), cfunc(a, a)) - - def test_array_ctor_with_dtype_arg(self): - # Test using np.dtype and np.generic (i.e. np.dtype.type) has args - pyfunc = array_ctor - cfunc = jit(nopython=True)(pyfunc) - n = 2 - args = n, np.int32 - np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) - args = n, np.dtype('int32') - np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) - args = n, np.float32 - np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) - args = n, np.dtype('f4') - np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) - - -class TestArrayComparisons(TestCase): - - def test_identity(self): - def check(a, b, expected): - cres = compile_isolated(pyfunc, (typeof(a), typeof(b))) - self.assertPreciseEqual(cres.entry_point(a, b), - (expected, not expected)) - - pyfunc = identity_usecase - - arr = np.zeros(10, dtype=np.int32).reshape((2, 5)) - check(arr, arr, True) - check(arr, arr[:], True) - check(arr, arr.copy(), False) - check(arr, arr.view('uint32'), False) - check(arr, arr.T, False) - check(arr, arr[:-1], False) - - # Other comparison operators ('==', etc.) are tested in test_ufuncs - - -if __name__ == '__main__': - unittest.main() From 50ac59031f54efbcad8aad848965dc6ce671a7b4 Mon Sep 17 00:00:00 2001 From: Stuart Archibald Date: Thu, 26 Sep 2019 20:38:47 +0100 Subject: [PATCH 23/26] Word wrap docs As title --- docs/source/reference/numpysupported.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/reference/numpysupported.rst b/docs/source/reference/numpysupported.rst index 66860ecdb2e..85c265773b2 100644 --- a/docs/source/reference/numpysupported.rst +++ b/docs/source/reference/numpysupported.rst @@ -220,15 +220,15 @@ The following methods of Numpy arrays are supported: * If the ``axis`` argument is not a compile-time constant, only values from 0 to 3 are supported. An out-of-range value will result in a runtime exception. - * All numeric ``dtypes`` are supported as ``dtype`` parameter. ``timedelta`` - arrays can be used as input arrays but ``timedelta`` is not supported - as ``dtype`` parameter. - * When ``dtype`` is given, it determines the type of the internal accumulator. - When it is not, the selection is made automatically based on the input - array's ``dtype``, mostly following the same rules as NumPy. However, - on 64-bit Windows, Numba uses a 64-bit accumulator for integer inputs - (``int64`` for ``int32`` inputs and ``uint64`` for ``uint32`` inputs), - while NumPy would use a 32-bit accumulator in those cases. + * All numeric ``dtypes`` are supported in the ``dtype`` parameter. + ``timedelta`` arrays can be used as input arrays but ``timedelta`` is not + supported as ``dtype`` parameter. + * When a ``dtype`` is given, it determines the type of the internal + accumulator. When it is not, the selection is made automatically based on + the input array's ``dtype``, mostly following the same rules as NumPy. + However, on 64-bit Windows, Numba uses a 64-bit accumulator for integer + inputs (``int64`` for ``int32`` inputs and ``uint64`` for ``uint32`` + inputs), while NumPy would use a 32-bit accumulator in those cases. * :meth:`~numpy.ndarray.transpose` From 6bc30ed67b2a283f08eebe61c9277b746ffc2c23 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Thu, 26 Sep 2019 22:00:00 +0200 Subject: [PATCH 24/26] removing redundant tests --- numba/tests/test_array_methods.py | 1251 +++++++++++++++++++++++++++++ 1 file changed, 1251 insertions(+) create mode 100644 numba/tests/test_array_methods.py diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py new file mode 100644 index 00000000000..7ccfe1c4078 --- /dev/null +++ b/numba/tests/test_array_methods.py @@ -0,0 +1,1251 @@ +from __future__ import division + +from itertools import product, cycle, permutations +import sys + +import numpy as np + +from numba import unittest_support as unittest +from numba import jit, typeof, types +from numba.compiler import compile_isolated +from numba.errors import TypingError, LoweringError +from numba.numpy_support import (as_dtype, strict_ufunc_typing, + version as numpy_version) +from .support import TestCase, CompilationCache, MemoryLeak, MemoryLeakMixin, tag +from .matmul_usecase import needs_blas + + +def np_around_array(arr, decimals, out): + np.around(arr, decimals, out) + +def np_around_binary(val, decimals): + return np.around(val, decimals) + +def np_around_unary(val): + return np.around(val) + +def np_round_array(arr, decimals, out): + np.round(arr, decimals, out) + +def np_round_binary(val, decimals): + return np.round(val, decimals) + +def np_round_unary(val): + return np.round(val) + +def _fixed_np_round(arr, decimals=0, out=None): + """ + A slightly bugfixed version of np.round(). + """ + if out is not None and arr.dtype.kind == 'c': + # workaround for https://github.com/numpy/numpy/issues/5779 + _fixed_np_round(arr.real, decimals, out.real) + _fixed_np_round(arr.imag, decimals, out.imag) + return out + else: + res = np.round(arr, decimals, out) + if out is None: + # workaround for https://github.com/numpy/numpy/issues/5780 + def fixup_signed_zero(arg, res): + if res == 0.0 and arg < 0: + return -np.abs(res) + else: + return res + if isinstance(arr, (complex, np.complexfloating)): + res = complex(fixup_signed_zero(arr.real, res.real), + fixup_signed_zero(arr.imag, res.imag)) + else: + res = fixup_signed_zero(arr, res) + return res + + +def array_T(arr): + return arr.T + +def array_transpose(arr): + return arr.transpose() + +def array_copy(arr): + return arr.copy() + +def np_copy(arr): + return np.copy(arr) + +def np_asfortranarray(arr): + return np.asfortranarray(arr) + +def np_ascontiguousarray(arr): + return np.ascontiguousarray(arr) + +def array_view(arr, newtype): + return arr.view(newtype) + +def array_take(arr, indices): + return arr.take(indices) + +def array_take_kws(arr, indices, axis): + return arr.take(indices, axis=axis) + +def array_fill(arr, val): + return arr.fill(val) + +# XXX Can't pass a dtype as a Dispatcher argument for now +def make_array_view(newtype): + def array_view(arr): + return arr.view(newtype) + return array_view + +def array_sliced_view(arr, ): + return arr[0:4].view(np.float32)[0] + +def make_array_astype(newtype): + def array_astype(arr): + return arr.astype(newtype) + return array_astype + + +def np_frombuffer(b): + """ + np.frombuffer() on a Python-allocated buffer. + """ + return np.frombuffer(b) + +def np_frombuffer_dtype(b): + return np.frombuffer(b, dtype=np.complex64) + +def np_frombuffer_allocated(shape): + """ + np.frombuffer() on a Numba-allocated buffer. + """ + arr = np.ones(shape, dtype=np.int32) + return np.frombuffer(arr) + +def np_frombuffer_allocated_dtype(shape): + arr = np.ones(shape, dtype=np.int32) + return np.frombuffer(arr, dtype=np.complex64) + +def identity_usecase(a, b): + return (a is b), (a is not b) + +def array_nonzero(a): + return a.nonzero() + +def np_nonzero(a): + return np.nonzero(a) + +def np_where_1(c): + return np.where(c) + +def np_where_3(c, x, y): + return np.where(c, x, y) + +def array_item(a): + return a.item() + +def array_itemset(a, v): + a.itemset(v) + +def array_sum(a, *args): + return a.sum(*args) + +def array_sum_axis_kws(a, axis): + return a.sum(axis=axis) + +def array_sum_dtype_kws(a, dtype): + return a.sum(dtype=dtype) + +def array_sum_axis_dtype_kws(a, dtype, axis): + return a.sum(axis=axis, dtype=dtype) + +def array_sum_axis_dtype_pos(a, a1, a2): + return a.sum(a1, a2) + +def array_sum_const_multi(arr, axis): + # use np.sum with different constant args multiple times to check + # for internal compile cache to see if constant-specialization is + # applied properly. + a = np.sum(arr, axis=4) + b = np.sum(arr, 3) + # the last invocation uses runtime-variable + c = np.sum(arr, axis) + # as method + d = arr.sum(axis=5) + # negative const axis + e = np.sum(arr, axis=-1) + return a, b, c, d, e + +def array_sum_const_axis_neg_one(a, axis): + # use .sum with -1 axis, this is for use with 1D arrays where the above + # "const_multi" variant would raise errors + return a.sum(axis=-1) + +def array_mean(a): + return a.mean() + +def array_mean_axis(a, axis): + return a.mean(axis) + +def array_cumsum(a, *args): + return a.cumsum(*args) + +def array_cumsum_kws(a, axis): + return a.cumsum(axis=axis) + +def array_real(a): + return np.real(a) + +def array_imag(a): + return np.imag(a) + +def array_conj(a): + return a.conj() + +def array_conjugate(a): + return a.conjugate() + +def np_unique(a): + return np.unique(a) + + +def array_dot(a, b): + return a.dot(b) + +def array_dot_chain(a, b): + return a.dot(b).dot(b) + +def array_ctor(n, dtype): + return np.ones(n, dtype=dtype) + + +class TestArrayMethods(MemoryLeakMixin, TestCase): + """ + Test various array methods and array-related functions. + """ + + def setUp(self): + super(TestArrayMethods, self).setUp() + self.ccache = CompilationCache() + + def check_round_scalar(self, unary_pyfunc, binary_pyfunc): + base_values = [-3.0, -2.5, -2.25, -1.5, 1.5, 2.25, 2.5, 2.75] + complex_values = [x * (1 - 1j) for x in base_values] + int_values = [int(x) for x in base_values] + argtypes = (types.float64, types.float32, types.int32, + types.complex64, types.complex128) + argvalues = [base_values, base_values, int_values, + complex_values, complex_values] + + pyfunc = binary_pyfunc + for ty, values in zip(argtypes, argvalues): + cres = compile_isolated(pyfunc, (ty, types.int32)) + cfunc = cres.entry_point + for decimals in (1, 0, -1): + for v in values: + if decimals > 0: + v *= 10 + expected = _fixed_np_round(v, decimals) + got = cfunc(v, decimals) + self.assertPreciseEqual(got, expected) + + pyfunc = unary_pyfunc + for ty, values in zip(argtypes, argvalues): + cres = compile_isolated(pyfunc, (ty,)) + cfunc = cres.entry_point + for v in values: + expected = _fixed_np_round(v) + got = cfunc(v) + self.assertPreciseEqual(got, expected) + + def test_round_scalar(self): + self.check_round_scalar(np_round_unary, np_round_binary) + + def test_around_scalar(self): + self.check_round_scalar(np_around_unary, np_around_binary) + + def check_round_array(self, pyfunc): + def check_round(cfunc, values, inty, outty, decimals): + # Create input and output arrays of the right type + arr = values.astype(as_dtype(inty)) + out = np.zeros_like(arr).astype(as_dtype(outty)) + pyout = out.copy() + _fixed_np_round(arr, decimals, pyout) + self.memory_leak_setup() + cfunc(arr, decimals, out) + self.memory_leak_teardown() + np.testing.assert_allclose(out, pyout) + # Output shape mismatch + with self.assertRaises(ValueError) as raises: + cfunc(arr, decimals, out[1:]) + self.assertEqual(str(raises.exception), + "invalid output shape") + + def check_types(argtypes, outtypes, values): + for inty, outty in product(argtypes, outtypes): + cres = compile_isolated(pyfunc, + (types.Array(inty, 1, 'A'), + types.int32, + types.Array(outty, 1, 'A'))) + cfunc = cres.entry_point + check_round(cres.entry_point, values, inty, outty, 0) + check_round(cres.entry_point, values, inty, outty, 1) + if not isinstance(outty, types.Integer): + check_round(cres.entry_point, values * 10, inty, outty, -1) + else: + # Avoid Numpy bug when output is an int: + # https://github.com/numpy/numpy/issues/5777 + pass + + values = np.array([-3.0, -2.5, -2.25, -1.5, 1.5, 2.25, 2.5, 2.75]) + + if strict_ufunc_typing: + argtypes = (types.float64, types.float32) + else: + argtypes = (types.float64, types.float32, types.int32) + check_types(argtypes, argtypes, values) + + argtypes = (types.complex64, types.complex128) + check_types(argtypes, argtypes, values * (1 - 1j)) + + # Exceptions leak references + self.disable_leak_check() + + def test_round_array(self): + self.check_round_array(np_round_array) + + def test_around_array(self): + self.check_round_array(np_around_array) + + def test_array_view(self): + + def run(arr, dtype): + pyfunc = make_array_view(dtype) + cres = self.ccache.compile(pyfunc, (typeof(arr),)) + return cres.entry_point(arr) + def check(arr, dtype): + expected = arr.view(dtype) + self.memory_leak_setup() + got = run(arr, dtype) + self.assertPreciseEqual(got, expected) + del got + self.memory_leak_teardown() + def check_err(arr, dtype): + with self.assertRaises(ValueError) as raises: + run(arr, dtype) + self.assertEqual(str(raises.exception), + "new type not compatible with array") + + dt1 = np.dtype([('a', np.int8), ('b', np.int8)]) + dt2 = np.dtype([('u', np.int16), ('v', np.int8)]) + dt3 = np.dtype([('x', np.int16), ('y', np.int16)]) + + # C-contiguous + arr = np.arange(24, dtype=np.int8) + check(arr, np.dtype('int16')) + check(arr, np.int16) + check(arr, np.int8) + check(arr, np.float32) + check(arr, np.complex64) + check(arr, dt1) + check(arr, dt2) + check_err(arr, np.complex128) + + # Last dimension must have a compatible size + arr = arr.reshape((3, 8)) + check(arr, np.int8) + check(arr, np.float32) + check(arr, np.complex64) + check(arr, dt1) + check_err(arr, dt2) + check_err(arr, np.complex128) + + # F-contiguous + arr = np.arange(24, dtype=np.int8).reshape((3, 8)).T + check(arr, np.int8) + check(arr, np.float32) + check(arr, np.complex64) + check(arr, dt1) + check_err(arr, dt2) + check_err(arr, np.complex128) + + # Non-contiguous: only a type with the same itemsize can be used + arr = np.arange(16, dtype=np.int32)[::2] + check(arr, np.uint32) + check(arr, np.float32) + check(arr, dt3) + check_err(arr, np.int8) + check_err(arr, np.int16) + check_err(arr, np.int64) + check_err(arr, dt1) + check_err(arr, dt2) + + # Zero-dim array: only a type with the same itemsize can be used + arr = np.array([42], dtype=np.int32).reshape(()) + check(arr, np.uint32) + check(arr, np.float32) + check(arr, dt3) + check_err(arr, np.int8) + check_err(arr, np.int16) + check_err(arr, np.int64) + check_err(arr, dt1) + check_err(arr, dt2) + + # Exceptions leak references + self.disable_leak_check() + + def test_array_sliced_view(self): + """ + Test .view() on A layout array but has contiguous innermost dimension. + """ + pyfunc = array_sliced_view + cres = self.ccache.compile(pyfunc, (types.uint8[:],)) + cfunc = cres.entry_point + + orig = np.array([1.5, 2], dtype=np.float32) + byteary = orig.view(np.uint8) + + expect = pyfunc(byteary) + got = cfunc(byteary) + + self.assertEqual(expect, got) + + def test_array_astype(self): + + def run(arr, dtype): + pyfunc = make_array_astype(dtype) + cres = self.ccache.compile(pyfunc, (typeof(arr),)) + return cres.entry_point(arr) + def check(arr, dtype): + expected = arr.astype(dtype).copy(order='A') + got = run(arr, dtype) + self.assertPreciseEqual(got, expected) + + # C-contiguous + arr = np.arange(24, dtype=np.int8) + check(arr, np.dtype('int16')) + check(arr, np.int32) + check(arr, np.float32) + check(arr, np.complex128) + + # F-contiguous + arr = np.arange(24, dtype=np.int8).reshape((3, 8)).T + check(arr, np.float32) + + # Non-contiguous + arr = np.arange(16, dtype=np.int32)[::2] + check(arr, np.uint64) + + # check read only attr does not get copied + arr = np.arange(16, dtype=np.int32) + arr.flags.writeable = False + check(arr, np.int32) + + # Invalid conversion + dt = np.dtype([('x', np.int8)]) + with self.assertTypingError() as raises: + check(arr, dt) + self.assertIn('cannot convert from int32 to Record', + str(raises.exception)) + + def check_np_frombuffer(self, pyfunc): + def run(buf): + cres = self.ccache.compile(pyfunc, (typeof(buf),)) + return cres.entry_point(buf) + def check(buf): + old_refcnt = sys.getrefcount(buf) + expected = pyfunc(buf) + self.memory_leak_setup() + got = run(buf) + self.assertPreciseEqual(got, expected) + del expected + self.assertEqual(sys.getrefcount(buf), old_refcnt + 1) + del got + self.assertEqual(sys.getrefcount(buf), old_refcnt) + self.memory_leak_teardown() + + b = bytearray(range(16)) + check(b) + if sys.version_info >= (3,): + check(bytes(b)) + check(memoryview(b)) + check(np.arange(12)) + b = np.arange(12).reshape((3, 4)) + check(b) + + # Exceptions leak references + self.disable_leak_check() + + with self.assertRaises(ValueError) as raises: + run(bytearray(b"xxx")) + self.assertEqual("buffer size must be a multiple of element size", + str(raises.exception)) + + def test_np_frombuffer(self): + self.check_np_frombuffer(np_frombuffer) + + def test_np_frombuffer_dtype(self): + self.check_np_frombuffer(np_frombuffer_dtype) + + def check_layout_dependent_func(self, pyfunc, fac=np.arange, + check_sameness=True): + def is_same(a, b): + return a.ctypes.data == b.ctypes.data + def check_arr(arr): + cres = compile_isolated(pyfunc, (typeof(arr),)) + expected = pyfunc(arr) + got = cres.entry_point(arr) + self.assertPreciseEqual(expected, got) + if check_sameness: + self.assertEqual(is_same(expected, arr), is_same(got, arr)) + arr = fac(24) + check_arr(arr) + check_arr(arr.reshape((3, 8))) + check_arr(arr.reshape((3, 8)).T) + check_arr(arr.reshape((3, 8))[::2]) + check_arr(arr.reshape((2, 3, 4))) + check_arr(arr.reshape((2, 3, 4)).T) + check_arr(arr.reshape((2, 3, 4))[::2]) + arr = np.array([0]).reshape(()) + check_arr(arr) + + def test_array_transpose(self): + self.check_layout_dependent_func(array_transpose) + + @tag('important') + def test_array_T(self): + self.check_layout_dependent_func(array_T) + + @tag('important') + def test_array_copy(self): + self.check_layout_dependent_func(array_copy) + + def test_np_copy(self): + self.check_layout_dependent_func(np_copy) + + def test_np_asfortranarray(self): + self.check_layout_dependent_func(np_asfortranarray, + check_sameness=numpy_version >= (1, 8)) + + def test_np_ascontiguousarray(self): + self.check_layout_dependent_func(np_ascontiguousarray, + check_sameness=numpy_version > (1, 11)) + + def check_np_frombuffer_allocated(self, pyfunc): + def run(shape): + cres = self.ccache.compile(pyfunc, (typeof(shape),)) + return cres.entry_point(shape) + def check(shape): + expected = pyfunc(shape) + got = run(shape) + self.assertPreciseEqual(got, expected) + + check((16,)) + check((4, 4)) + check((1, 0, 1)) + + def test_np_frombuffer_allocated(self): + self.check_np_frombuffer_allocated(np_frombuffer_allocated) + + def test_np_frombuffer_allocated2(self): + self.check_np_frombuffer_allocated(np_frombuffer_allocated_dtype) + + def check_nonzero(self, pyfunc): + def fac(N): + np.random.seed(42) + arr = np.random.random(N) + arr[arr < 0.3] = 0.0 + arr[arr > 0.7] = float('nan') + return arr + + def check_arr(arr): + cres = compile_isolated(pyfunc, (typeof(arr),)) + expected = pyfunc(arr) + # NOTE: Numpy 1.9 returns readonly arrays for multidimensional + # arrays. Workaround this by copying the results. + expected = [a.copy() for a in expected] + self.assertPreciseEqual(cres.entry_point(arr), expected) + + arr = np.int16([1, 0, -1, 0]) + check_arr(arr) + arr = np.bool_([1, 0, 1]) + check_arr(arr) + + arr = fac(24) + check_arr(arr) + check_arr(arr.reshape((3, 8))) + check_arr(arr.reshape((3, 8)).T) + check_arr(arr.reshape((3, 8))[::2]) + check_arr(arr.reshape((2, 3, 4))) + check_arr(arr.reshape((2, 3, 4)).T) + check_arr(arr.reshape((2, 3, 4))[::2]) + for v in (0.0, 1.5, float('nan')): + arr = np.array([v]).reshape(()) + check_arr(arr) + + def test_array_nonzero(self): + self.check_nonzero(array_nonzero) + + def test_np_nonzero(self): + self.check_nonzero(np_nonzero) + + def test_np_where_1(self): + self.check_nonzero(np_where_1) + + def test_np_where_3(self): + pyfunc = np_where_3 + def fac(N): + np.random.seed(42) + arr = np.random.random(N) + arr[arr < 0.3] = 0.0 + arr[arr > 0.7] = float('nan') + return arr + + layouts = cycle(['C', 'F', 'A']) + _types = [np.int32, np.int64, np.float32, np.float64, np.complex64, + np.complex128] + + np.random.seed(42) + + def check_arr(arr, layout=False): + np.random.shuffle(_types) + if layout != False: + x = np.zeros_like(arr, dtype=_types[0], order=layout) + y = np.zeros_like(arr, dtype=_types[1], order=layout) + arr = arr.copy(order=layout) + else: + x = np.zeros_like(arr, dtype=_types[0], order=next(layouts)) + y = np.zeros_like(arr, dtype=_types[1], order=next(layouts)) + x.fill(4) + y.fill(9) + cres = compile_isolated(pyfunc, (typeof(arr), typeof(x), typeof(y))) + expected = pyfunc(arr, x, y) + got = cres.entry_point(arr, x, y) + # Contiguity of result varies accross Numpy versions, only + # check contents. NumPy 1.11+ seems to stabilize. + if numpy_version < (1, 11): + self.assertEqual(got.dtype, expected.dtype) + np.testing.assert_array_equal(got, expected) + else: + self.assertPreciseEqual(got, expected) + + def check_scal(scal): + x = 4 + y = 5 + np.random.shuffle(_types) + x = _types[0](4) + y = _types[1](5) + cres = compile_isolated(pyfunc, (typeof(scal), typeof(x), typeof(y))) + expected = pyfunc(scal, x, y) + got = cres.entry_point(scal, x, y) + self.assertPreciseEqual(got, expected) + + arr = np.int16([1, 0, -1, 0]) + check_arr(arr) + arr = np.bool_([1, 0, 1]) + check_arr(arr) + + arr = fac(24) + check_arr(arr) + check_arr(arr.reshape((3, 8))) + check_arr(arr.reshape((3, 8)).T) + check_arr(arr.reshape((3, 8))[::2]) + check_arr(arr.reshape((2, 3, 4))) + check_arr(arr.reshape((2, 3, 4)).T) + check_arr(arr.reshape((2, 3, 4))[::2]) + + check_arr(arr.reshape((2, 3, 4)), layout='F') + check_arr(arr.reshape((2, 3, 4)).T, layout='F') + check_arr(arr.reshape((2, 3, 4))[::2], layout='F') + + for v in (0.0, 1.5, float('nan')): + arr = np.array([v]).reshape(()) + check_arr(arr) + + for x in (0, 1, True, False, 2.5, 0j): + check_scal(x) + + def test_np_where_3_broadcast_x_y_scalar(self): + pyfunc = np_where_3 + cfunc = jit(nopython=True)(pyfunc) + + def check_ok(args): + expected = pyfunc(*args) + got = cfunc(*args) + self.assertPreciseEqual(got, expected) + + def a_variations(): + a = np.linspace(-2, 4, 20) + self.random.shuffle(a) + yield a + yield a.reshape(2, 5, 2) + yield a.reshape(4, 5, order='F') + yield a.reshape(2, 5, 2)[::-1] + + for a in a_variations(): + params = (a > 0, 0, 1) + check_ok(params) + + params = (a < 0, np.nan, 1 + 4j) + check_ok(params) + + params = (a > 1, True, False) + check_ok(params) + + def test_np_where_3_broadcast_x_or_y_scalar(self): + pyfunc = np_where_3 + cfunc = jit(nopython=True)(pyfunc) + + def check_ok(args): + condition, x, y = args + + expected = pyfunc(condition, x, y) + got = cfunc(condition, x, y) + self.assertPreciseEqual(got, expected) + + # swap x and y + expected = pyfunc(condition, y, x) + got = cfunc(condition, y, x) + self.assertPreciseEqual(got, expected) + + def array_permutations(): + x = np.arange(9).reshape(3, 3) + yield x + yield x * 1.1 + yield np.asfortranarray(x) + yield x[::-1] + yield np.linspace(-10, 10, 60).reshape(3, 4, 5) * 1j + + def scalar_permutations(): + yield 0 + yield 4.3 + yield np.nan + yield True + yield 8 + 4j + + for x in array_permutations(): + for y in scalar_permutations(): + x_mean = np.mean(x) + condition = x > x_mean + params = (condition, x, y) + check_ok(params) + + def test_item(self): + pyfunc = array_item + cfunc = jit(nopython=True)(pyfunc) + + def check_ok(arg): + expected = pyfunc(arg) + got = cfunc(arg) + self.assertPreciseEqual(got, expected) + + def check_err(arg): + with self.assertRaises(ValueError) as raises: + cfunc(arg) + self.assertIn("item(): can only convert an array of size 1 to a Python scalar", + str(raises.exception)) + + # Exceptions leak references + self.disable_leak_check() + + # Test on different kinds of scalars and 1-item arrays + check_ok(np.float32([1.5])) + check_ok(np.complex128([[1.5j]])) + check_ok(np.array(1.5)) + check_ok(np.bool_(True)) + check_ok(np.float32(1.5)) + + check_err(np.array([1, 2])) + check_err(np.array([])) + + def test_itemset(self): + pyfunc = array_itemset + cfunc = jit(nopython=True)(pyfunc) + + def check_ok(a, v): + expected = a.copy() + got = a.copy() + pyfunc(expected, v) + cfunc(got, v) + self.assertPreciseEqual(got, expected) + + def check_err(a): + with self.assertRaises(ValueError) as raises: + cfunc(a, 42) + self.assertIn("itemset(): can only write to an array of size 1", + str(raises.exception)) + + # Exceptions leak references + self.disable_leak_check() + + # Test on different kinds of 1-item arrays + check_ok(np.float32([1.5]), 42) + check_ok(np.complex128([[1.5j]]), 42) + check_ok(np.array(1.5), 42) + + check_err(np.array([1, 2])) + check_err(np.array([])) + + def test_sum(self): + """ test sum over a whole range of dtypes, no axis or dtype parameter + """ + pyfunc = array_sum + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, + np.complex64, np.complex128, np.uint32, np.uint64, np.timedelta64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + for arr_list in all_test_arrays: + for arr in arr_list: + with self.subTest("Test np.sum with {} input ".format(arr.dtype)): + self.assertPreciseEqual(pyfunc(arr), cfunc(arr)) + + def test_sum_axis_kws1(self): + """ test sum with axis parameter over a whole range of dtypes """ + pyfunc = array_sum_axis_kws + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.uint64, np.complex64, + np.complex128] + # timedelta test cannot be enabled until issue #4540 is fixed + # all_dtypes += [np.timedelta64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + for arr_list in all_test_arrays: + for arr in arr_list: + for axis in (0, 1, 2): + if axis > len(arr.shape)-1: + continue + with self.subTest("Testing np.sum(axis) with {} " + "input ".format(arr.dtype)): + self.assertPreciseEqual(pyfunc(arr, axis=axis), + cfunc(arr, axis=axis)) + + def test_sum_axis_kws2(self): + """ testing uint32 and int32 separately + + uint32 and int32 must be tested separately because Numpy's current + behaviour is different in 64bits Windows (accumulates as int32) + and 64bits Linux (accumulates as int64), while Numba has decided to always + accumulate as int64, when the OS is 64bits. No testing has been done + for behaviours in 32 bits platforms. + """ + pyfunc = array_sum_axis_kws + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.int32, np.uint32] + # expected return dtypes in Numba + out_dtypes = {np.dtype('int32'): np.int64, np.dtype('uint32'): np.uint64, + np.dtype('int64'): np.int64} + # timedelta test cannot be enabled until issue #4540 is fixed + # all_dtypes += [np.timedelta64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + for arr_list in all_test_arrays: + for arr in arr_list: + for axis in (0, 1, 2): + if axis > len(arr.shape)-1: + continue + with self.subTest("Testing np.sum(axis) with {} " + "input ".format(arr.dtype)): + npy_res = pyfunc(arr, axis=axis) + numba_res = cfunc(arr, axis=axis) + if isinstance(numba_res, np.ndarray): + self.assertPreciseEqual( + npy_res.astype(out_dtypes[arr.dtype]), + numba_res.astype(out_dtypes[arr.dtype])) + else: + # the results are scalars + self.assertEqual(npy_res, numba_res) + + def test_sum_dtype_kws(self): + """ test sum with dtype parameter over a whole range of dtypes """ + pyfunc = array_sum_dtype_kws + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, + np.uint64, np.complex64, np.complex128] + # timedelta test cannot be enabled until issue #4540 is fixed + # all_dtypes += [np.timedelta64] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + out_dtypes = {np.dtype('float64'): [np.float64], + np.dtype('float32'): [np.float64, np.float32], + np.dtype('int64'): [np.float64, np.int64, np.float32], + np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], + np.dtype('uint32'): [np.float64, np.int64, np.float32], + np.dtype('uint64'): [np.float64, np.int64], + np.dtype('complex64'): [np.complex64, np.complex128], + np.dtype('complex128'): [np.complex128], + np.dtype('timedelta64'): [np.timedelta64]} + + for arr_list in all_test_arrays: + for arr in arr_list: + for out_dtype in out_dtypes[arr.dtype]: + subtest_str = ("Testing np.sum with {} input and {} output" + .format(arr.dtype, out_dtype)) + with self.subTest(subtest_str): + self.assertPreciseEqual(pyfunc(arr, dtype=out_dtype), + cfunc(arr, dtype=out_dtype)) + + def test_sum_axis_dtype_kws(self): + """ test sum with axis and dtype parameters over a whole range of dtypes """ + pyfunc = array_sum_axis_dtype_kws + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, + np.uint64, np.complex64, np.complex128] + all_test_arrays = [ + [np.ones((7, 6, 5, 4, 3), arr_dtype), + np.ones(1, arr_dtype), + np.ones((7, 3), arr_dtype) * -5] + for arr_dtype in all_dtypes] + + out_dtypes = {np.dtype('float64'): [np.float64], + np.dtype('float32'): [np.float64, np.float32], + np.dtype('int64'): [np.float64, np.int64, np.float32], + np.dtype('int32'): [np.float64, np.int64, np.float32, np.int32], + np.dtype('uint32'): [np.float64, np.int64, np.float32], + np.dtype('uint64'): [np.float64, np.uint64], + np.dtype('complex64'): [np.complex64, np.complex128], + np.dtype('complex128'): [np.complex128], + np.dtype('timedelta64'): [np.timedelta64]} + + for arr_list in all_test_arrays: + for arr in arr_list: + for out_dtype in out_dtypes[arr.dtype]: + for axis in (0, 1, 2): + if axis > len(arr.shape) - 1: + continue + subtest_str = ("Testing np.sum with {} input and {} output " + .format(arr.dtype, out_dtype)) + with self.subTest(subtest_str): + py_res = pyfunc(arr, axis=axis, dtype=out_dtype) + nb_res = cfunc(arr, axis=axis, dtype=out_dtype) + self.assertPreciseEqual(py_res, nb_res) + + def test_sum_axis_dtype_pos_arg(self): + """ testing that axis and dtype inputs work when passed as positional """ + pyfunc = array_sum_axis_dtype_pos + cfunc = jit(nopython=True)(pyfunc) + dtype = np.float64 + # OK + a = np.ones((7, 6, 5, 4, 3)) + self.assertPreciseEqual(pyfunc(a, 1, dtype), + cfunc(a, 1, dtype)) + + self.assertPreciseEqual(pyfunc(a, 2, dtype), + cfunc(a, 2, dtype)) + + def test_sum_1d_kws(self): + # check 1d reduces to scalar + pyfunc = array_sum_axis_kws + cfunc = jit(nopython=True)(pyfunc) + a = np.arange(10.) + self.assertPreciseEqual(pyfunc(a, axis=0), cfunc(a, axis=0)) + pyfunc = array_sum_const_axis_neg_one + cfunc = jit(nopython=True)(pyfunc) + a = np.arange(10.) + self.assertPreciseEqual(pyfunc(a, axis=-1), cfunc(a, axis=-1)) + + def test_sum_const(self): + pyfunc = array_sum_const_multi + cfunc = jit(nopython=True)(pyfunc) + + arr = np.ones((3, 4, 5, 6, 7, 8)) + axis = 1 + self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) + axis = 2 + self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) + + def test_sum_exceptions(self): + # Exceptions leak references + self.disable_leak_check() + pyfunc = array_sum + cfunc = jit(nopython=True)(pyfunc) + + a = np.ones((7, 6, 5, 4, 3)) + b = np.ones((4, 3)) + # BAD: axis > dimensions + with self.assertRaises(ValueError): + cfunc(b, 2) + # BAD: negative axis + with self.assertRaises(ValueError): + cfunc(a, -1) + # BAD: axis greater than 3 + with self.assertRaises(ValueError): + cfunc(a, 4) + + def test_sum_const_negative(self): + # Exceptions leak references + self.disable_leak_check() + + @jit(nopython=True) + def foo(arr): + return arr.sum(axis=-3) + + # ndim == 4, axis == -3, OK + a = np.ones((1, 2, 3, 4)) + self.assertPreciseEqual(foo(a), foo.py_func(a)) + # ndim == 3, axis == -3, OK + a = np.ones((1, 2, 3)) + self.assertPreciseEqual(foo(a), foo.py_func(a)) + # ndim == 2, axis == -3, BAD + a = np.ones((1, 2)) + with self.assertRaises(LoweringError) as raises: + foo(a) + errmsg = "'axis' entry is out of bounds" + self.assertIn(errmsg, str(raises.exception)) + with self.assertRaises(ValueError) as raises: + foo.py_func(a) + # Numpy 1.13 has a different error message than prior numpy + # Just check for the "out of bounds" phrase in it. + self.assertIn("out of bounds", str(raises.exception)) + + def test_mean(self): + pyfunc = array_mean + cfunc = jit(nopython=True)(pyfunc) + # OK + a = np.ones((7, 6, 5, 4, 3)) + self.assertPreciseEqual(pyfunc(a), cfunc(a)) + + def test_mean_axis(self): + pyfunc = array_mean_axis + cfunc = jit(nopython=True)(pyfunc) + all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, + np.uint64, np.complex64, np.complex128] + all_test_arrays = [np.ones((7, 6, 5, 4, 3), arr_dtype) for arr_dtype in all_dtypes] + a = np.ones((7, 6, 5, 4, 3)) + + for arr in all_test_arrays: + with self.subTest(): + self.assertPreciseEqual(pyfunc(arr, 0), cfunc(arr, 0)) + with self.subTest(): + axis = 1 + self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) + with self.subTest(): + axis = 2 + self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) + with self.subTest(): + # axis -1 is only supported for IntegerLiterals + pyfunc2 = lambda x: x.mean(axis=-1) + cfunc2 = jit(nopython=True)(pyfunc2) + self.assertPreciseEqual(pyfunc2(arr), cfunc2(arr)) + + def test_cumsum(self): + pyfunc = array_cumsum + cfunc = jit(nopython=True)(pyfunc) + # OK + a = np.ones((2, 3)) + self.assertPreciseEqual(pyfunc(a), cfunc(a)) + # BAD: with axis + with self.assertRaises(TypingError): + cfunc(a, 1) + # BAD: with kw axis + pyfunc = array_cumsum_kws + cfunc = jit(nopython=True)(pyfunc) + with self.assertRaises(TypingError): + cfunc(a, axis=1) + + def test_take(self): + pyfunc = array_take + cfunc = jit(nopython=True)(pyfunc) + + def check(arr, ind): + expected = pyfunc(arr, ind) + got = cfunc(arr, ind) + self.assertPreciseEqual(expected, got) + if hasattr(expected, 'order'): + self.assertEqual(expected.order == got.order) + + # need to check: + # 1. scalar index + # 2. 1d array index + # 3. nd array index, >2d and F order + # 4. reflected list + # 5. tuples + + test_indices = [] + test_indices.append(1) + test_indices.append(5) + test_indices.append(11) + test_indices.append(-2) + test_indices.append(np.array([1, 5, 1, 11, 3])) + test_indices.append(np.array([[1, 5, 1], [11, 3, 0]], order='F')) + test_indices.append(np.array([[[1, 5, 1], [11, 3, 0]]])) + test_indices.append(np.array([[[[1, 5]], [[11, 0]], [[1, 2]]]])) + test_indices.append([1, 5, 1, 11, 3]) + test_indices.append((1, 5, 1)) + test_indices.append(((1, 5, 1), (11, 3, 2))) + test_indices.append((((1,), (5,), (1,)), ((11,), (3,), (2,)))) + + layouts = cycle(['C', 'F', 'A']) + + for dt in [np.float64, np.int64, np.complex128]: + A = np.arange(12, dtype=dt).reshape((4, 3), order=next(layouts)) + for ind in test_indices: + check(A, ind) + + #check illegal access raises + A = np.arange(12, dtype=dt).reshape((4, 3), order=next(layouts)) + szA = A.size + illegal_indices = [szA, -szA - 1, np.array(szA), np.array(-szA - 1), + [szA], [-szA - 1]] + for x in illegal_indices: + with self.assertRaises(IndexError): + cfunc(A, x) # oob raises + + # check float indexing raises + with self.assertRaises(TypingError): + cfunc(A, [1.7]) + + # check unsupported arg raises + with self.assertRaises(TypingError): + take_kws = jit(nopython=True)(array_take_kws) + take_kws(A, 1, 1) + + # check kwarg unsupported raises + with self.assertRaises(TypingError): + take_kws = jit(nopython=True)(array_take_kws) + take_kws(A, 1, axis=1) + + #exceptions leak refs + self.disable_leak_check() + + def test_fill(self): + pyfunc = array_fill + cfunc = jit(nopython=True)(pyfunc) + def check(arr, val): + expected = np.copy(arr) + erv = pyfunc(expected, val) + self.assertTrue(erv is None) + got = np.copy(arr) + grv = cfunc(got, val) + self.assertTrue(grv is None) + # check mutation is the same + self.assertPreciseEqual(expected, got) + + # scalar + A = np.arange(1) + for x in [np.float64, np.bool_]: + check(A, x(10)) + + # 2d + A = np.arange(12).reshape(3, 4) + for x in [np.float64, np.bool_]: + check(A, x(10)) + + # 4d + A = np.arange(48, dtype=np.complex64).reshape(2, 3, 4, 2) + for x in [np.float64, np.complex128, np.bool_]: + check(A, x(10)) + + def test_real(self): + pyfunc = array_real + cfunc = jit(nopython=True)(pyfunc) + + x = np.linspace(-10, 10) + np.testing.assert_equal(pyfunc(x), cfunc(x)) + + x, y = np.meshgrid(x, x) + z = x + 1j*y + np.testing.assert_equal(pyfunc(z), cfunc(z)) + + def test_imag(self): + pyfunc = array_imag + cfunc = jit(nopython=True)(pyfunc) + + x = np.linspace(-10, 10) + np.testing.assert_equal(pyfunc(x), cfunc(x)) + + x, y = np.meshgrid(x, x) + z = x + 1j*y + np.testing.assert_equal(pyfunc(z), cfunc(z)) + + def test_conj(self): + for pyfunc in [array_conj, array_conjugate]: + cfunc = jit(nopython=True)(pyfunc) + + x = np.linspace(-10, 10) + np.testing.assert_equal(pyfunc(x), cfunc(x)) + + x, y = np.meshgrid(x, x) + z = x + 1j*y + np.testing.assert_equal(pyfunc(z), cfunc(z)) + + def test_unique(self): + pyfunc = np_unique + cfunc = jit(nopython=True)(pyfunc) + + def check(a): + np.testing.assert_equal(pyfunc(a), cfunc(a)) + + check(np.array([[1, 1, 3], [3, 4, 5]])) + check(np.array(np.zeros(5))) + check(np.array([[3.1, 3.1], [1.7, 2.29], [3.3, 1.7]])) + check(np.array([])) + + @needs_blas + def test_array_dot(self): + # just ensure that the dot impl dispatches correctly, do + # not test dot itself, this is done in test_linalg. + pyfunc = array_dot + cfunc = jit(nopython=True)(pyfunc) + a = np.arange(20.).reshape(4, 5) + b = np.arange(5.) + np.testing.assert_equal(pyfunc(a, b), cfunc(a, b)) + + # check that chaining works + pyfunc = array_dot_chain + cfunc = jit(nopython=True)(pyfunc) + a = np.arange(16.).reshape(4, 4) + np.testing.assert_equal(pyfunc(a, a), cfunc(a, a)) + + def test_array_ctor_with_dtype_arg(self): + # Test using np.dtype and np.generic (i.e. np.dtype.type) has args + pyfunc = array_ctor + cfunc = jit(nopython=True)(pyfunc) + n = 2 + args = n, np.int32 + np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) + args = n, np.dtype('int32') + np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) + args = n, np.float32 + np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) + args = n, np.dtype('f4') + np.testing.assert_array_equal(pyfunc(*args), cfunc(*args)) + + +class TestArrayComparisons(TestCase): + + def test_identity(self): + def check(a, b, expected): + cres = compile_isolated(pyfunc, (typeof(a), typeof(b))) + self.assertPreciseEqual(cres.entry_point(a, b), + (expected, not expected)) + + pyfunc = identity_usecase + + arr = np.zeros(10, dtype=np.int32).reshape((2, 5)) + check(arr, arr, True) + check(arr, arr[:], True) + check(arr, arr.copy(), False) + check(arr, arr.view('uint32'), False) + check(arr, arr.T, False) + check(arr, arr[:-1], False) + + # Other comparison operators ('==', etc.) are tested in test_ufuncs + + +if __name__ == '__main__': + unittest.main() From 80feaa6986cb00945bb1d289df53362420683de4 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Thu, 26 Sep 2019 22:08:32 +0200 Subject: [PATCH 25/26] reverting addition of mean tests --- numba/tests/test_array_methods.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 7ccfe1c4078..1286df6c7c7 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -1012,36 +1012,6 @@ def foo(arr): # Just check for the "out of bounds" phrase in it. self.assertIn("out of bounds", str(raises.exception)) - def test_mean(self): - pyfunc = array_mean - cfunc = jit(nopython=True)(pyfunc) - # OK - a = np.ones((7, 6, 5, 4, 3)) - self.assertPreciseEqual(pyfunc(a), cfunc(a)) - - def test_mean_axis(self): - pyfunc = array_mean_axis - cfunc = jit(nopython=True)(pyfunc) - all_dtypes = [np.float64, np.float32, np.int64, np.int32, np.uint32, - np.uint64, np.complex64, np.complex128] - all_test_arrays = [np.ones((7, 6, 5, 4, 3), arr_dtype) for arr_dtype in all_dtypes] - a = np.ones((7, 6, 5, 4, 3)) - - for arr in all_test_arrays: - with self.subTest(): - self.assertPreciseEqual(pyfunc(arr, 0), cfunc(arr, 0)) - with self.subTest(): - axis = 1 - self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) - with self.subTest(): - axis = 2 - self.assertPreciseEqual(pyfunc(arr, axis), cfunc(arr, axis)) - with self.subTest(): - # axis -1 is only supported for IntegerLiterals - pyfunc2 = lambda x: x.mean(axis=-1) - cfunc2 = jit(nopython=True)(pyfunc2) - self.assertPreciseEqual(pyfunc2(arr), cfunc2(arr)) - def test_cumsum(self): pyfunc = array_cumsum cfunc = jit(nopython=True)(pyfunc) From a3074b42c03e4cbee3309225b704b867a1f2d7b8 Mon Sep 17 00:00:00 2001 From: luk-f-a Date: Thu, 26 Sep 2019 22:10:11 +0200 Subject: [PATCH 26/26] reverting addition of mean tests2 --- numba/tests/test_array_methods.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py index 1286df6c7c7..faae1893635 100644 --- a/numba/tests/test_array_methods.py +++ b/numba/tests/test_array_methods.py @@ -179,12 +179,6 @@ def array_sum_const_axis_neg_one(a, axis): # "const_multi" variant would raise errors return a.sum(axis=-1) -def array_mean(a): - return a.mean() - -def array_mean_axis(a, axis): - return a.mean(axis) - def array_cumsum(a, *args): return a.cumsum(*args)