Skip to content

Commit

Permalink
Add test for 1d, 2d, 3d and C+F order arrays in parfors.
Browse files Browse the repository at this point in the history 
Also fixes some test to use the more descriptive assertEqual instead of assertTrue
  • Loading branch information
@sklam
sklam committed Jan 27, 2021
1 parent ba51ec6 commit 2628ec4
Showing 1 changed file with 141 additions and 36 deletions.
177 changes: 141 additions & 36 deletions numba/tests/test_parfors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@
import operator
from collections import defaultdict, namedtuple
import copy
from itertools import cycle, chain

import numba.parfors.parfor
from numba import njit, prange, set_num_threads, get_num_threads
from numba import njit, prange, set_num_threads, get_num_threads, typeof
from numba.core import (types, utils, typing, errors, ir, rewrites,
typed_passes, inline_closurecall, config, compiler, cpu)
from numba.extending import (overload_method, register_model,
Expand Down Expand Up @@ -529,16 +530,59 @@ def check(self, pyfunc, *args, **kwargs):
cfunc, cpfunc = self.compile_all(pyfunc, *args)
self.check_parfors_vs_others(pyfunc, cfunc, cpfunc, *args, **kwargs)

def gen_linspace(self, n, ct):
"""Make *ct* sample 1D arrays of length *n* using np.linspace().
"""
def gen():
yield np.linspace(0, 1, n)
yield np.linspace(2, 1, n)
yield np.linspace(1, 2, n)

src = cycle(gen())
return [next(src) for i in range(ct)]

def gen_linespace_variants(self, ct):

This comment has been minimized.

Sign in to view

Copy link
@DrTodd13

DrTodd13 Jan 27, 2021

Contributor

Did you mean linspace here instead of linespace?
"""Make 1D, 2D, 3D variants of the data in C and F orders
"""
# 1D
yield self.gen_linspace(10, ct=ct)

# 2D
arr2ds = [x.reshape((2, 3))
for x in self.gen_linspace(n=2 * 3, ct=ct)]
yield arr2ds
# Fortran order
yield [np.asfortranarray(x) for x in arr2ds]

# 3D
arr3ds = [x.reshape((2, 3, 4))
for x in self.gen_linspace(n=2 * 3 * 4, ct=ct)]
yield arr3ds
# Fortran order
yield [np.asfortranarray(x) for x in arr3ds]

def check_variants(self, impl, arg_gen, **kwargs):
"""Run self.check(impl, ...) on array data generated from arg_gen.
"""
for args in arg_gen():
with self.subTest(list(map(typeof, args))):
self.check(impl, *args, **kwargs)

def count_parfors_variants(self, impl, arg_gen, **kwargs):
"""Run self.countParfors(impl, ...) on array types generated from
arg_gen.
"""
for args in arg_gen():
with self.subTest(list(map(typeof, args))):
argtys = tuple(map(typeof, args))
self.assertEqual(countParfors(impl, argtys), 1)

@skip_parfors_unsupported
def test_arraymap(self):
def test_impl(a, x, y):
return a * x + y

A = np.linspace(0, 1, 10)
X = np.linspace(2, 1, 10)
Y = np.linspace(1, 2, 10)

self.check(test_impl, A, X, Y)
self.check_variants(test_impl, lambda: self.gen_linespace_variants(3))

@skip_parfors_unsupported
@needs_blas
Expand All @@ -558,7 +602,7 @@ def test_impl():
Y = np.ones((10, 12))
return np.sum(X + Y)
self.check(test_impl)
self.assertTrue(countParfors(test_impl, ()) == 1)
self.assertEqual(countParfors(test_impl, ()), 1)

@skip_parfors_unsupported
def test_2d_parfor(self):
Expand All @@ -567,7 +611,22 @@ def test_impl():
Y = np.zeros((10, 12))
return np.sum(X + Y)
self.check(test_impl)
self.assertTrue(countParfors(test_impl, ()) == 1)
self.assertEqual(countParfors(test_impl, ()), 1)

@skip_parfors_unsupported
def test_nd_parfor(self):
def case1():
X = np.ones((10, 12))
Y = np.zeros((10, 12))
yield (X, Y)

data_gen = lambda: chain(case1(), self.gen_linespace_variants(2))

def test_impl(X, Y):
return np.sum(X + Y)

self.check_variants(test_impl, data_gen)
self.count_parfors_variants(test_impl, data_gen)

@skip_parfors_unsupported
def test_pi(self):
Expand All @@ -577,8 +636,8 @@ def test_impl(n):
return 4 * np.sum(x**2 + y**2 < 1) / n

self.check(test_impl, 100000, decimal=1)
self.assertTrue(countParfors(test_impl, (types.int64, )) == 1)
self.assertTrue(countArrays(test_impl, (types.intp,)) == 0)
self.assertEqual(countParfors(test_impl, (types.int64, )), 1)
self.assertEqual(countArrays(test_impl, (types.intp,)), 0)

@skip_parfors_unsupported
def test_fuse_argmin_argmax_max_min(self):
Expand All @@ -589,22 +648,22 @@ def test_impl(n):
B = A.sum()
return B + C
self.check(test_impl, 256)
self.assertTrue(countParfors(test_impl, (types.int64, )) == 1)
self.assertTrue(countArrays(test_impl, (types.intp,)) == 0)
self.assertEqual(countParfors(test_impl, (types.int64, )), 1)
self.assertEqual(countArrays(test_impl, (types.intp,)), 0)

@skip_parfors_unsupported
def test_blackscholes(self):
# blackscholes takes 5 1D float array args
args = (numba.float64[:], ) * 5
self.assertTrue(countParfors(blackscholes_impl, args) == 1)
self.assertEqual(countParfors(blackscholes_impl, args), 1)

@skip_parfors_unsupported
@needs_blas
def test_logistic_regression(self):
args = (numba.float64[:], numba.float64[:,:], numba.float64[:],
numba.int64)
self.assertTrue(countParfors(lr_impl, args) == 2)
self.assertTrue(countArrayAllocs(lr_impl, args) == 1)
self.assertEqual(countParfors(lr_impl, args), 2)
self.assertEqual(countArrayAllocs(lr_impl, args), 1)

@skip_parfors_unsupported
def test_kmeans(self):
Expand All @@ -620,8 +679,8 @@ def test_kmeans(self):
# requires recursive parfor counting
arg_typs = (types.Array(types.float64, 2, 'C'), types.intp, types.intp,
types.Array(types.float64, 2, 'C'))
self.assertTrue(
countNonParforArrayAccesses(example_kmeans_test, arg_typs) == 0)
self.assertEqual(
countNonParforArrayAccesses(example_kmeans_test, arg_typs), 0)

@unittest.skipIf(not _32bit, "Only impacts 32 bit hardware")
@needs_blas
Expand Down Expand Up @@ -827,7 +886,7 @@ def test_np_random_func_direct_import(self):
def test_impl(n):
A = randn(n)
return A[0]
self.assertTrue(countParfors(test_impl, (types.int64, )) == 1)
self.assertEqual(countParfors(test_impl, (types.int64, )), 1)

@skip_parfors_unsupported
def test_arange(self):
Expand Down Expand Up @@ -882,8 +941,13 @@ def test_impl(A):
B = np.random.randint(10, size=(N, 3))
self.check(test_impl, A)
self.check(test_impl, B)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 1, 'C'), )) == 1)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 2, 'C'), )) == 1)
self.assertEqual(countParfors(test_impl, (types.Array(types.float64, 1, 'C'), )), 1)
self.assertEqual(countParfors(test_impl, (types.Array(types.float64, 2, 'C'), )), 1)

# Test variants
data_gen = lambda: self.gen_linespace_variants(1)
self.check_variants(test_impl, data_gen)
self.count_parfors_variants(test_impl, data_gen)

@skip_parfors_unsupported
def test_var(self):
Expand All @@ -896,8 +960,13 @@ def test_impl(A):
self.check(test_impl, A)
self.check(test_impl, B)
self.check(test_impl, C)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 1, 'C'), )) == 2)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 2, 'C'), )) == 2)
self.assertEqual(countParfors(test_impl, (types.Array(types.float64, 1, 'C'), )), 2)
self.assertEqual(countParfors(test_impl, (types.Array(types.float64, 2, 'C'), )), 2)

# Test variants
data_gen = lambda: self.gen_linespace_variants(1)
self.check_variants(test_impl, data_gen)
self.count_parfors_variants(test_impl, data_gen)

@skip_parfors_unsupported
def test_std(self):
Expand All @@ -910,8 +979,14 @@ def test_impl(A):
self.check(test_impl, A)
self.check(test_impl, B)
self.check(test_impl, C)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 1, 'C'), )) == 2)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 2, 'C'), )) == 2)
argty = (types.Array(types.float64, 1, 'C'),)
self.assertEqual(countParfors(test_impl, argty), 2)
self.assertEqual(countParfors(test_impl, argty), 2)

# Test variants
data_gen = lambda: self.gen_linespace_variants(1)
self.check_variants(test_impl, data_gen)
self.count_parfors_variants(test_impl, data_gen)

@skip_parfors_unsupported
def test_issue4963_globals(self):
Expand All @@ -938,7 +1013,7 @@ def test_random_parfor(self):
def test_impl(n):
A = np.random.ranf((n, n))
return A
self.assertTrue(countParfors(test_impl, (types.int64, )) == 1)
self.assertEqual(countParfors(test_impl, (types.int64, )), 1)

@skip_parfors_unsupported
def test_randoms(self):
Expand All @@ -957,7 +1032,7 @@ def test_impl(n):
py_output = test_impl(n)
# check results within 5% since random numbers generated in parallel
np.testing.assert_allclose(parfor_output, py_output, rtol=0.05)
self.assertTrue(countParfors(test_impl, (types.int64, )) == 1)
self.assertEqual(countParfors(test_impl, (types.int64, )), 1)

@skip_parfors_unsupported
def test_dead_randoms(self):
Expand All @@ -975,7 +1050,7 @@ def test_impl(n):
parfor_output = cpfunc.entry_point(n)
py_output = test_impl(n)
self.assertEqual(parfor_output, py_output)
self.assertTrue(countParfors(test_impl, (types.int64, )) == 0)
self.assertEqual(countParfors(test_impl, (types.int64, )), 0)

@skip_parfors_unsupported
def test_cfg(self):
Expand Down Expand Up @@ -1034,7 +1109,7 @@ def test_impl(A):
return np.sum(A[B>=3,1:2])
self.check(test_impl, A.reshape((16,10)))
# this doesn't fuse due to mixed indices
self.assertTrue(countParfors(test_impl, (numba.float64[:,:],)) == 2)
self.assertEqual(countParfors(test_impl, (numba.float64[:,:],)), 2)

@skip_parfors_unsupported
def test_min(self):
Expand Down Expand Up @@ -1067,6 +1142,13 @@ def test_impl2(A):
pcfunc.entry_point(np.array([], dtype=np.int64))
self.assertIn(msg, str(e.exception))

# Test variants
data_gen = lambda: self.gen_linespace_variants(1)
self.check_variants(test_impl1, data_gen)
self.count_parfors_variants(test_impl1, data_gen)
self.check_variants(test_impl2, data_gen)
self.count_parfors_variants(test_impl2, data_gen)

@skip_parfors_unsupported
def test_max(self):
def test_impl1(A):
Expand Down Expand Up @@ -1098,6 +1180,13 @@ def test_impl2(A):
pcfunc.entry_point(np.array([], dtype=np.int64))
self.assertIn(msg, str(e.exception))

# Test variants
data_gen = lambda: self.gen_linespace_variants(1)
self.check_variants(test_impl1, data_gen)
self.count_parfors_variants(test_impl1, data_gen)
self.check_variants(test_impl2, data_gen)
self.count_parfors_variants(test_impl2, data_gen)

@skip_parfors_unsupported
def test_use_of_reduction_var1(self):
def test_impl():
Expand Down Expand Up @@ -1139,6 +1228,13 @@ def test_impl2(A):
pcfunc.entry_point(np.array([], dtype=np.int64))
self.assertIn(msg, str(e.exception))

# Test variants
data_gen = lambda: self.gen_linespace_variants(1)
self.check_variants(test_impl1, data_gen)
self.count_parfors_variants(test_impl1, data_gen)
self.check_variants(test_impl2, data_gen)
self.count_parfors_variants(test_impl2, data_gen)

@skip_parfors_unsupported
def test_argmax(self):
def test_impl1(A):
Expand Down Expand Up @@ -1166,6 +1262,13 @@ def test_impl2(A):
pcfunc.entry_point(np.array([], dtype=np.int64))
self.assertIn(msg, str(e.exception))

# Test variants
data_gen = lambda: self.gen_linespace_variants(1)
self.check_variants(test_impl1, data_gen)
self.count_parfors_variants(test_impl1, data_gen)
self.check_variants(test_impl2, data_gen)
self.count_parfors_variants(test_impl2, data_gen)

@skip_parfors_unsupported
def test_parfor_array_access1(self):
# signed index of the prange generated by sum() should be replaced
Expand Down Expand Up @@ -1552,7 +1655,8 @@ def test_impl(x):
return x
x = np.zeros(10)
self.check(test_impl, x)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 1, 'C'),)) == 1)
argty = (types.Array(types.float64, 1, 'C'),)
self.assertEqual(countParfors(test_impl, argty), 1)

@skip_parfors_unsupported
def test_ndarray_fill2d(self):
Expand All @@ -1561,7 +1665,8 @@ def test_impl(x):
return x
x = np.zeros((2,2))
self.check(test_impl, x)
self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 2, 'C'),)) == 1)
argty = (types.Array(types.float64, 2, 'C'),)
self.assertEqual(countParfors(test_impl, argty), 1)

@skip_parfors_unsupported
def test_0d_array(self):
Expand Down Expand Up @@ -1597,9 +1702,9 @@ def test_impl(a, b):
y = np.arange(10 ** 2, dtype=float)

self.check(test_impl, x, y)
self.assertTrue(countParfors(test_impl,
self.assertEqual(countParfors(test_impl,
(types.Array(types.float64, 1, 'C'),
types.Array(types.float64, 1, 'C'))) == 1)
types.Array(types.float64, 1, 'C'))), 1)

@skip_parfors_unsupported
def test_tuple1(self):
Expand Down Expand Up @@ -1686,9 +1791,9 @@ def test_impl(a, b):
X = np.arange(10) + 10
Y = np.arange(10) + 100
self.check(test_impl, X, Y)
self.assertTrue(countParfors(test_impl,
self.assertEqual(countParfors(test_impl,
(types.Array(types.float64, 1, 'C'),
types.Array(types.float64, 1, 'C'))) == 1)
types.Array(types.float64, 1, 'C'))), 1)

@skip_parfors_unsupported
def test_tuple_concat(self):
Expand Down Expand Up @@ -2727,7 +2832,7 @@ def get_gufunc_asm(self, func, schedule_type, *args, **kwargs):
matches = schedty.findall(cres.library.get_llvm_str())
self.assertGreaterEqual(len(matches), 1) # at least 1 parfor call
self.assertEqual(matches[0], schedule_type)
self.assertTrue(asm != {})
self.assertNotEqual(asm, {})

return asm

Expand Down Expand Up @@ -3445,7 +3550,7 @@ def test_impl():
res += arr[i]
return res + dummy[2]

self.assertTrue(get_init_block_size(test_impl, ()) == 0)
self.assertEqual(get_init_block_size(test_impl, ()), 0)

@skip_parfors_unsupported
def test_alias_analysis_for_parfor1(self):
Expand Down

0 comments on commit 2628ec4

Please sign in to comment.