From b069bb39224edd14bd64a0263b69a80f81aaa816 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 7 Feb 2020 12:00:11 +0300 Subject: [PATCH 01/10] Argmin/argmax + nanargmin/nanargmax --- sdc/functions/numpy_like.py | 207 +++++++++++++++++++++++- sdc/tests/test_sdc_numpy.py | 62 ++++++- sdc/tests/tests_perf/test_perf_numpy.py | 18 +++ sdc/utilities/utils.py | 21 +++ 4 files changed, 306 insertions(+), 2 deletions(-) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 07684325c..daa9c24c2 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -36,16 +36,31 @@ from numba import types, jit, prange, numpy_support from numba.errors import TypingError +from numba.targets.arraymath import get_isnan import sdc from sdc.utilities.sdc_typing_utils import TypeChecker from sdc.str_arr_ext import (StringArrayType, pre_alloc_string_array, get_utf8_size) -from sdc.utilities.utils import sdc_overload, sdc_register_jitable +from sdc.utilities.utils import (sdc_overload, sdc_register_jitable, max_, min_, + min_dtype_int_val, max_dtype_int_val, min_dtype_float_val, + max_dtype_float_val) def astype(self, dtype): pass +def argmin(self): + pass + +def argmax(self): + pass + +def nanargmin(self): + pass + +def nanargmax(self): + pass + @sdc_overload(astype) def sdc_astype_overload(self, dtype): @@ -97,3 +112,193 @@ def sdc_astype_number_impl(self, dtype): return arr return sdc_astype_number_impl + + +@sdc_overload(nanargmin) +def sdc_invert_overload(self): + """ + Intel Scalable Dataframe Compiler Developer Guide + ************************************************* + Parallel replacement of numpy.nanargmin. + + .. only:: developer + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k nanargmin + + """ + + ty_checker = TypeChecker("numpy-like 'nanargmin'") + dtype = self.dtype + isnan = get_isnan(dtype) + if isinstance(dtype, types.Integer): + max_ref = max_dtype_int_val(dtype) + + if isinstance(dtype, types.Float): + max_ref = max_dtype_float_val(dtype) + + if not isinstance(self, types.Array): + return None + + if isinstance(dtype, types.Number): + def sdc_nanargmin_impl(self): + res = max_ref + position = max_ + length = len(self) + for i in prange(length): + if min(res, self[i]) == self[i]: + if not isnan(self[i]): + if res == self[i]: + position = min(position, i) + else: + position = i + res = self[i] + return position + + return sdc_nanargmin_impl + + ty_checker.raise_exc(dtype, 'number', 'self.dtype') + + +@sdc_overload(nanargmax) +def sdc_invert_overload(self): + """ + Intel Scalable Dataframe Compiler Developer Guide + ************************************************* + Parallel replacement of numpy.nanargmax. + + .. only:: developer + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k nanargmax + + """ + + ty_checker = TypeChecker("numpy-like 'nanargmax'") + dtype = self.dtype + isnan = get_isnan(dtype) + if isinstance(dtype, types.Integer): + min_ref = min_dtype_int_val(dtype) + + if isinstance(dtype, types.Float): + min_ref = min_dtype_float_val(dtype) + + if not isinstance(self, types.Array): + return None + + if isinstance(dtype, types.Number): + def sdc_nanargmax_impl(self): + res = min_ref + position = max_ + length = len(self) + for i in prange(length): + if max(res, self[i]) == self[i]: + if not isnan(self[i]): + if res == self[i]: + position = min(position, i) + else: + position = i + res = self[i] + return position + + return sdc_nanargmax_impl + + ty_checker.raise_exc(dtype, 'number', 'self.dtype') + + +@sdc_overload(argmin) +def sdc_invert_overload(self): + """ + Intel Scalable Dataframe Compiler Developer Guide + ************************************************* + Parallel replacement of numpy.argmin. + + .. only:: developer + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k argmin + + """ + + ty_checker = TypeChecker("numpy-like 'argmin'") + dtype = self.dtype + isnan = get_isnan(dtype) + if isinstance(dtype, types.Integer): + max_ref = max_dtype_int_val(dtype) + + if isinstance(dtype, types.Float): + max_ref = max_dtype_float_val(dtype) + + if not isinstance(self, types.Array): + return None + + if isinstance(dtype, types.Number): + def sdc_argmin_impl(self): + res = max_ref + position = max_ + length = len(self) + for i in prange(length): + if not isnan(self[i]): + if min(res, self[i]) == self[i]: + if res == self[i]: + position = min(position, i) + else: + position = i + res = self[i] + else: + if numpy.isnan(res): + position = min(position, i) + else: + res = self[i] + position = i + + return position + + return sdc_argmin_impl + + ty_checker.raise_exc(dtype, 'number', 'self.dtype') + + +@sdc_overload(argmax) +def sdc_invert_overload(self): + """ + Intel Scalable Dataframe Compiler Developer Guide + ************************************************* + Parallel replacement of numpy.argmax. + + .. only:: developer + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k argmax + + """ + + ty_checker = TypeChecker("numpy-like 'argmax'") + dtype = self.dtype + isnan = get_isnan(dtype) + if isinstance(dtype, types.Integer): + min_ref = min_dtype_int_val(dtype) + + if isinstance(dtype, types.Float): + min_ref = min_dtype_float_val(dtype) + + if not isinstance(self, types.Array): + return None + + if isinstance(dtype, types.Number): + def sdc_argmax_impl(self): + res = min_ref + position = max_ + length = len(self) + for i in prange(length): + if not isnan(self[i]): + if max(res, self[i]) == self[i]: + if res == self[i]: + position = min(position, i) + else: + position = i + res = self[i] + else: + if numpy.isnan(res): + position = min(position, i) + else: + res = self[i] + position = i + + return position + + return sdc_argmax_impl + + ty_checker.raise_exc(dtype, 'number', 'self.dtype') diff --git a/sdc/tests/test_sdc_numpy.py b/sdc/tests/test_sdc_numpy.py index cb55f9120..eda3754e4 100644 --- a/sdc/tests/test_sdc_numpy.py +++ b/sdc/tests/test_sdc_numpy.py @@ -35,7 +35,7 @@ from sdc.str_ext import std_str_to_unicode, unicode_to_std_str from sdc.tests.test_base import TestCase from sdc.tests.test_utils import skip_numba_jit -from sdc.functions.numpy_like import astype +from sdc.functions import numpy_like class TestArrays(TestCase): @@ -160,5 +160,65 @@ def sdc_impl(a, t): with self.subTest(data=case, type=type_): np.testing.assert_array_equal(sdc_func(a, type_), ref_impl(a, type_)) + def test_argmin(self): + def ref_impl(a): + return np.argmin(a) + + def sdc_impl(a): + return numpy_like.argmin(a) + + sdc_func = self.jit(sdc_impl) + + cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]] + for case in cases: + a = np.array(case) + with self.subTest(data=case): + np.testing.assert_array_equal(sdc_func(a), ref_impl(a)) + + def test_argmax(self): + def ref_impl(a): + return np.argmax(a) + + def sdc_impl(a): + return numpy_like.argmax(a) + + sdc_func = self.jit(sdc_impl) + + cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]] + for case in cases: + a = np.array(case) + with self.subTest(data=case): + np.testing.assert_array_equal(sdc_func(a), ref_impl(a)) + + def test_nanargmin(self): + def ref_impl(a): + return np.nanargmin(a) + + def sdc_impl(a): + return numpy_like.nanargmin(a) + + sdc_func = self.jit(sdc_impl) + + cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]] + for case in cases: + a = np.array(case) + with self.subTest(data=case): + np.testing.assert_array_equal(sdc_func(a), ref_impl(a)) + + def test_nanargmax(self): + def ref_impl(a): + return np.nanargmax(a) + + def sdc_impl(a): + return numpy_like.nanargmax(a) + + sdc_func = self.jit(sdc_impl) + + cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]] + for case in cases: + a = np.array(case) + with self.subTest(data=case): + np.testing.assert_array_equal(sdc_func(a), ref_impl(a)) + if __name__ == "__main__": unittest.main() diff --git a/sdc/tests/tests_perf/test_perf_numpy.py b/sdc/tests/tests_perf/test_perf_numpy.py index 8df13f315..e8619f8a2 100644 --- a/sdc/tests/tests_perf/test_perf_numpy.py +++ b/sdc/tests/tests_perf/test_perf_numpy.py @@ -99,6 +99,24 @@ def _test_case(self, cases, name, total_data_length, data_num=1, input_data=test CE(type_='Numba', code='data.astype(np.int64)', jitted=True), CE(type_='SDC', code='sdc.functions.numpy_like.astype(data, np.int64)', jitted=True), ], usecase_params='data'), + TC(name='nanargmin', size=[10 ** 7], call_expr=[ + CE(type_='Python', code='np.nanargmin(data)', jitted=False), + CE(type_='SDC', code='sdc.functions.numpy_like.nanargmin(data)', jitted=True), + ], usecase_params='data'), + TC(name='nanargmax', size=[10 ** 7], call_expr=[ + CE(type_='Python', code='np.nanargmax(data)', jitted=False), + CE(type_='SDC', code='sdc.functions.numpy_like.nanargmax(data)', jitted=True), + ], usecase_params='data'), + TC(name='argmax', size=[10 ** 7], call_expr=[ + CE(type_='Python', code='np.argmax(data)', jitted=False), + CE(type_='Numba', code='np.argmax(data)', jitted=True), + CE(type_='SDC', code='sdc.functions.numpy_like.argmax(data)', jitted=True), + ], usecase_params='data'), + TC(name='argmin', size=[10 ** 7], call_expr=[ + CE(type_='Python', code='np.argmin(data)', jitted=False), + CE(type_='Numba', code='np.argmin(data)', jitted=True), + CE(type_='SDC', code='sdc.functions.numpy_like.argmin(data)', jitted=True), + ], usecase_params='data'), ] generate_test_cases(cases, TestFunctions, 'function') diff --git a/sdc/utilities/utils.py b/sdc/utilities/utils.py index bd7f0a02d..00b756fe3 100644 --- a/sdc/utilities/utils.py +++ b/sdc/utilities/utils.py @@ -83,6 +83,27 @@ class CTypeEnum(Enum): } +# Maximum and minimum int values +max_ = np.iinfo(np.int64).max +min_ = np.iinfo(np.int64).min + + +def min_dtype_int_val(dtype): + return np.iinfo(str(dtype)).min + + +def max_dtype_int_val(dtype): + return np.iinfo(str(dtype)).max + + +def min_dtype_float_val(dtype): + return np.finfo(str(dtype)).min + + +def max_dtype_float_val(dtype): + return np.finfo(str(dtype)).max + + # silence Numba error messages for now # TODO: customize through @sdc.jit numba.errors.error_extras = { From 0a773a79a73a1dc2b421189bf3ea16b24941963a Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 7 Feb 2020 15:53:53 +0300 Subject: [PATCH 02/10] Remove max and min int --- sdc/functions/numpy_like.py | 18 +++++++++++++----- sdc/tests/test_sdc_numpy.py | 4 ++-- sdc/utilities/utils.py | 18 +++++++++--------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index daa9c24c2..790317dc5 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -41,7 +41,7 @@ import sdc from sdc.utilities.sdc_typing_utils import TypeChecker from sdc.str_arr_ext import (StringArrayType, pre_alloc_string_array, get_utf8_size) -from sdc.utilities.utils import (sdc_overload, sdc_register_jitable, max_, min_, +from sdc.utilities.utils import (sdc_overload, sdc_register_jitable, min_dtype_int_val, max_dtype_int_val, min_dtype_float_val, max_dtype_float_val) @@ -49,15 +49,19 @@ def astype(self, dtype): pass + def argmin(self): pass + def argmax(self): pass + def nanargmin(self): pass + def nanargmax(self): pass @@ -129,6 +133,7 @@ def sdc_invert_overload(self): ty_checker = TypeChecker("numpy-like 'nanargmin'") dtype = self.dtype isnan = get_isnan(dtype) + max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) if isinstance(dtype, types.Integer): max_ref = max_dtype_int_val(dtype) @@ -141,7 +146,7 @@ def sdc_invert_overload(self): if isinstance(dtype, types.Number): def sdc_nanargmin_impl(self): res = max_ref - position = max_ + position = max_int64 length = len(self) for i in prange(length): if min(res, self[i]) == self[i]: @@ -173,6 +178,7 @@ def sdc_invert_overload(self): ty_checker = TypeChecker("numpy-like 'nanargmax'") dtype = self.dtype isnan = get_isnan(dtype) + max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) if isinstance(dtype, types.Integer): min_ref = min_dtype_int_val(dtype) @@ -185,7 +191,7 @@ def sdc_invert_overload(self): if isinstance(dtype, types.Number): def sdc_nanargmax_impl(self): res = min_ref - position = max_ + position = max_int64 length = len(self) for i in prange(length): if max(res, self[i]) == self[i]: @@ -217,6 +223,7 @@ def sdc_invert_overload(self): ty_checker = TypeChecker("numpy-like 'argmin'") dtype = self.dtype isnan = get_isnan(dtype) + max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) if isinstance(dtype, types.Integer): max_ref = max_dtype_int_val(dtype) @@ -229,7 +236,7 @@ def sdc_invert_overload(self): if isinstance(dtype, types.Number): def sdc_argmin_impl(self): res = max_ref - position = max_ + position = max_int64 length = len(self) for i in prange(length): if not isnan(self[i]): @@ -268,6 +275,7 @@ def sdc_invert_overload(self): ty_checker = TypeChecker("numpy-like 'argmax'") dtype = self.dtype isnan = get_isnan(dtype) + max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) if isinstance(dtype, types.Integer): min_ref = min_dtype_int_val(dtype) @@ -280,7 +288,7 @@ def sdc_invert_overload(self): if isinstance(dtype, types.Number): def sdc_argmax_impl(self): res = min_ref - position = max_ + position = max_int64 length = len(self) for i in prange(length): if not isnan(self[i]): diff --git a/sdc/tests/test_sdc_numpy.py b/sdc/tests/test_sdc_numpy.py index eda3754e4..8bba03519 100644 --- a/sdc/tests/test_sdc_numpy.py +++ b/sdc/tests/test_sdc_numpy.py @@ -184,7 +184,7 @@ def sdc_impl(a): sdc_func = self.jit(sdc_impl) - cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]] + cases = [[np.nan, np.nan, np.inf, np.nan], [5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]] for case in cases: a = np.array(case) with self.subTest(data=case): @@ -214,7 +214,7 @@ def sdc_impl(a): sdc_func = self.jit(sdc_impl) - cases = [[5, 2, 0, 333, -4], [3.3, 5.4, np.nan, 7.9, np.nan]] + cases = [[np.nan, np.nan, np.inf, np.nan], [5, 2, -9, 333, -4], [3.3, 5.4, np.nan, 7.9]] for case in cases: a = np.array(case) with self.subTest(data=case): diff --git a/sdc/utilities/utils.py b/sdc/utilities/utils.py index 00b756fe3..cdd0cd9bc 100644 --- a/sdc/utilities/utils.py +++ b/sdc/utilities/utils.py @@ -38,6 +38,7 @@ from numba.typing.templates import infer_global, AbstractTemplate from numba.targets.imputils import lower_builtin from numba.extending import overload, intrinsic, lower_cast +from numba import numpy_support import numpy as np import sdc from sdc.str_ext import string_type, list_string_array_type @@ -83,25 +84,24 @@ class CTypeEnum(Enum): } -# Maximum and minimum int values -max_ = np.iinfo(np.int64).max -min_ = np.iinfo(np.int64).min - - def min_dtype_int_val(dtype): - return np.iinfo(str(dtype)).min + numpy_dtype = numpy_support.as_dtype(dtype) + return np.iinfo(numpy_dtype).min def max_dtype_int_val(dtype): - return np.iinfo(str(dtype)).max + numpy_dtype = numpy_support.as_dtype(dtype) + return np.iinfo(numpy_dtype).max def min_dtype_float_val(dtype): - return np.finfo(str(dtype)).min + numpy_dtype = numpy_support.as_dtype(dtype) + return np.finfo(numpy_dtype).min def max_dtype_float_val(dtype): - return np.finfo(str(dtype)).max + numpy_dtype = numpy_support.as_dtype(dtype) + return np.finfo(numpy_dtype).max # silence Numba error messages for now From 75e91860282fce5074a58a7426633fa7e908ab00 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 7 Feb 2020 15:56:11 +0300 Subject: [PATCH 03/10] astype tests fix --- sdc/tests/test_sdc_numpy.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sdc/tests/test_sdc_numpy.py b/sdc/tests/test_sdc_numpy.py index 8bba03519..0492ab8ff 100644 --- a/sdc/tests/test_sdc_numpy.py +++ b/sdc/tests/test_sdc_numpy.py @@ -45,7 +45,7 @@ def ref_impl(a, t): return a.astype(t) def sdc_impl(a, t): - return astype(a, t) + return numpy_like.astype(a, t) sdc_func = self.jit(sdc_impl) @@ -63,7 +63,7 @@ def ref_impl(a, t): return a.astype(t) def sdc_impl(a, t): - return astype(a, t) + return numpy_like.astype(a, t) sdc_func = self.jit(sdc_impl) @@ -80,7 +80,7 @@ def ref_impl(a): return a.astype('float64') def sdc_impl(a): - return astype(a, 'float64') + return numpy_like.astype(a, 'float64') sdc_func = self.jit(sdc_impl) @@ -95,7 +95,7 @@ def ref_impl(a): return a.astype(np.int64) def sdc_impl(a): - return astype(a, np.int64) + return numpy_like.astype(a, np.int64) sdc_func = self.jit(sdc_impl) @@ -110,7 +110,7 @@ def ref_impl(a): return a.astype(str) def sdc_impl(a): - return astype(a, str) + return numpy_like.astype(a, str) sdc_func = self.jit(sdc_impl) @@ -123,7 +123,7 @@ def ref_impl(a): return a.astype(str) def sdc_impl(a): - return astype(a, str) + return numpy_like.astype(a, str) sdc_func = self.jit(sdc_impl) @@ -135,7 +135,7 @@ def ref_impl(a): return a.astype('str') def sdc_impl(a): - return astype(a, 'str') + return numpy_like.astype(a, 'str') sdc_func = self.jit(sdc_impl) @@ -148,7 +148,7 @@ def ref_impl(a, t): return a.astype(t) def sdc_impl(a, t): - return astype(a, t) + return numpy_like.astype(a, t) sdc_func = self.jit(sdc_impl) From cd4c2ac515568e966cda09ebabbfe6c429b08c3a Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 7 Feb 2020 16:43:00 +0300 Subject: [PATCH 04/10] fix overload names --- sdc/functions/numpy_like.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 790317dc5..69551756a 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -119,7 +119,7 @@ def sdc_astype_number_impl(self, dtype): @sdc_overload(nanargmin) -def sdc_invert_overload(self): +def sdc_nanargmin_overload(self): """ Intel Scalable Dataframe Compiler Developer Guide ************************************************* @@ -164,7 +164,7 @@ def sdc_nanargmin_impl(self): @sdc_overload(nanargmax) -def sdc_invert_overload(self): +def sdc_nanargmax_overload(self): """ Intel Scalable Dataframe Compiler Developer Guide ************************************************* @@ -209,7 +209,7 @@ def sdc_nanargmax_impl(self): @sdc_overload(argmin) -def sdc_invert_overload(self): +def sdc_argmin_overload(self): """ Intel Scalable Dataframe Compiler Developer Guide ************************************************* @@ -261,7 +261,7 @@ def sdc_argmin_impl(self): @sdc_overload(argmax) -def sdc_invert_overload(self): +def sdc_argmax_overload(self): """ Intel Scalable Dataframe Compiler Developer Guide ************************************************* From 3d335df1730199468b34251dcc1c21f5a0c690b9 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Mon, 17 Feb 2020 17:28:45 +0300 Subject: [PATCH 05/10] rewrite with prange --- sdc/functions/numpy_like.py | 191 ++++++++++++++++++++++++---------- sdc/utilities/prange_utils.py | 97 +++++++++++++++++ 2 files changed, 233 insertions(+), 55 deletions(-) create mode 100644 sdc/utilities/prange_utils.py diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index e954ab51a..0c6dc215b 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -33,6 +33,7 @@ import numba import numpy +import sys from numba import types, jit, prange, numpy_support, literally from numba.errors import TypingError @@ -44,6 +45,7 @@ min_dtype_int_val, max_dtype_int_val, min_dtype_float_val, max_dtype_float_val) from sdc.str_arr_ext import (StringArrayType, pre_alloc_string_array, get_utf8_size, str_arr_is_na) +from sdc.utilities.prange_utils import get_chunks def astype(self, dtype): @@ -176,18 +178,35 @@ def sdc_nanargmin_overload(self): if isinstance(dtype, types.Number): def sdc_nanargmin_impl(self): - res = max_ref - position = max_int64 - length = len(self) - for i in prange(length): - if min(res, self[i]) == self[i]: - if not isnan(self[i]): - if res == self[i]: - position = min(position, i) - else: - position = i - res = self[i] - return position + chunks = get_chunks(len(self)) + arr_res = numpy.empty(shape=len(chunks), dtype=dtype) + arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) + for i in prange(len(chunks)): + chunk = chunks[i] + res = max_ref + pos = max_int64 + for j in range(chunk.start, chunk.stop): + if min(res, self[j]) == self[j]: + if not isnan(self[j]): + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] + arr_res[i] = res + arr_pos[i] = pos + + general_res = max_ref + general_pos = max_int64 + for i in range(len(chunks)): + if min(general_res, arr_res[i]) == arr_res[i]: + if general_res == arr_res[i]: + general_pos = min(general_pos, arr_pos[i]) + else: + general_pos = arr_pos[i] + general_res = arr_res[i] + + return general_pos return sdc_nanargmin_impl @@ -221,18 +240,36 @@ def sdc_nanargmax_overload(self): if isinstance(dtype, types.Number): def sdc_nanargmax_impl(self): - res = min_ref - position = max_int64 - length = len(self) - for i in prange(length): - if max(res, self[i]) == self[i]: - if not isnan(self[i]): - if res == self[i]: - position = min(position, i) + chunks = get_chunks(len(self)) + arr_res = numpy.empty(shape=len(chunks), dtype=dtype) + arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) + for i in prange(len(chunks)): + chunk = chunks[i] + res = min_ref + pos = max_int64 + for j in range(chunk.start, chunk.stop): + if max(res, self[j]) == self[j]: + if not isnan(self[j]): + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] + arr_res[i] = res + arr_pos[i] = pos + + general_res = min_ref + general_pos = max_int64 + for i in range(len(chunks)): + if max(general_res, arr_res[i]) == arr_res[i]: + if not isnan(arr_res[i]): + if general_res == arr_res[i]: + general_pos = min(general_pos, arr_pos[i]) else: - position = i - res = self[i] - return position + general_pos = arr_pos[i] + general_res = arr_res[i] + + return general_pos return sdc_nanargmax_impl @@ -266,25 +303,47 @@ def sdc_argmin_overload(self): if isinstance(dtype, types.Number): def sdc_argmin_impl(self): - res = max_ref - position = max_int64 - length = len(self) - for i in prange(length): - if not isnan(self[i]): - if min(res, self[i]) == self[i]: - if res == self[i]: - position = min(position, i) + chunks = get_chunks(len(self)) + arr_res = numpy.empty(shape=len(chunks), dtype=dtype) + arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) + for i in prange(len(chunks)): + chunk = chunks[i] + res = max_ref + pos = max_int64 + for j in range(chunk.start, chunk.stop): + if not isnan(self[j]): + if min(res, self[j]) == self[j]: + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] + else: + if numpy.isnan(res): + pos = min(pos, j) + else: + pos = j + res = self[j] + + arr_res[i] = res + arr_pos[i] = pos + general_res = max_ref + general_pos = max_int64 + for i in range(len(chunks)): + if not isnan(arr_res[i]): + if min(general_res, arr_res[i]) == arr_res[i]: + if general_res == arr_res[i]: + general_pos = min(general_pos, arr_pos[i]) else: - position = i - res = self[i] + general_pos = arr_pos[i] + general_res = arr_res[i] else: - if numpy.isnan(res): - position = min(position, i) + if numpy.isnan(general_res): + general_pos = min(general_pos, arr_pos[i]) else: - res = self[i] - position = i - - return position + general_pos = arr_pos[i] + general_res = arr_res[i] + return general_pos return sdc_argmin_impl @@ -318,25 +377,47 @@ def sdc_argmax_overload(self): if isinstance(dtype, types.Number): def sdc_argmax_impl(self): - res = min_ref - position = max_int64 - length = len(self) - for i in prange(length): - if not isnan(self[i]): - if max(res, self[i]) == self[i]: - if res == self[i]: - position = min(position, i) + chunks = get_chunks(len(self)) + arr_res = numpy.empty(shape=len(chunks), dtype=dtype) + arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) + for i in prange(len(chunks)): + chunk = chunks[i] + res = min_ref + pos = max_int64 + for j in range(chunk.start, chunk.stop): + if not isnan(self[j]): + if max(res, self[j]) == self[j]: + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] + else: + if numpy.isnan(res): + pos = min(pos, j) else: - position = i - res = self[i] + pos = j + res = self[j] + + arr_res[i] = res + arr_pos[i] = pos + general_res = min_ref + general_pos = max_int64 + for i in range(len(chunks)): + if not isnan(arr_res[i]): + if max(general_res, arr_res[i]) == arr_res[i]: + if general_res == arr_res[i]: + general_pos = min(general_pos, arr_pos[i]) + else: + general_pos = arr_pos[i] + general_res = arr_res[i] else: - if numpy.isnan(res): - position = min(position, i) + if numpy.isnan(general_res): + general_pos = min(general_pos, arr_pos[i]) else: - res = self[i] - position = i - - return position + general_pos = arr_pos[i] + general_res = arr_res[i] + return general_pos return sdc_argmax_impl diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py new file mode 100644 index 000000000..63780a74a --- /dev/null +++ b/sdc/utilities/prange_utils.py @@ -0,0 +1,97 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + + +import numba +import sdc + +from typing import NamedTuple +from sdc.utilities.utils import sdc_overload + + +class Chunk(NamedTuple): + start: int + stop: int + + +def get_pool_size(): + if sdc.config.config_use_parallel_overloads: + return numba.config.NUMBA_NUM_THREADS + else: + return 1 + + +@sdc_overload(get_pool_size) +def get_pool_size_overload(): + pool_size = get_pool_size() + + def get_pool_size_impl(): + return pool_size + + return get_pool_size_impl + + +def get_chunks(size, pool_size=0): + if pool_size == 0: + pool_size = get_pool_size() + + chunk_size = (size - 1)//pool_size + 1 + + Chunk = NamedTuple('start', 'stop') + + chunks = [] + + for i in range(pool_size): + start = min(i*chunk_size, size) + stop = min((i + 1)*chunk_size, size) + chunks.append(Chunk(start, stop)) + + return chunks + + +@sdc_overload(get_chunks) +def get_chunks_overload(size, pool_size=0): + def get_chunks_impl(size, pool_size=0): + if pool_size == 0: + pool_size = get_pool_size() + + chunk_size = size // pool_size + + chunks = [] + + for i in range(pool_size): + start = min(i*chunk_size, size) + stop = min((i + 1)*chunk_size, size) + if i == pool_size - 1: + tmp = size - size // pool_size + start = min(i*chunk_size, size) + stop = min((i + 1)*chunk_size+tmp, size) + chunk = Chunk(start, stop) + chunks.append(chunk) + + return chunks + + return get_chunks_impl From 48313c4886ea946bc5e2bf4fd24a8842d31da600 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Tue, 18 Feb 2020 12:42:38 +0300 Subject: [PATCH 06/10] final fix --- sdc/utilities/prange_utils.py | 43 ++++++++--------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index 63780a74a..ee2a4851d 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -29,7 +29,7 @@ import sdc from typing import NamedTuple -from sdc.utilities.utils import sdc_overload +from sdc.utilities.utils import sdc_overload, sdc_register_jitable class Chunk(NamedTuple): @@ -40,8 +40,8 @@ class Chunk(NamedTuple): def get_pool_size(): if sdc.config.config_use_parallel_overloads: return numba.config.NUMBA_NUM_THREADS - else: - return 1 + + return 1 @sdc_overload(get_pool_size) @@ -54,44 +54,21 @@ def get_pool_size_impl(): return get_pool_size_impl +@sdc_register_jitable def get_chunks(size, pool_size=0): if pool_size == 0: pool_size = get_pool_size() + pool_size = min(pool_size, size) chunk_size = (size - 1)//pool_size + 1 - Chunk = NamedTuple('start', 'stop') - chunks = [] - for i in range(pool_size): - start = min(i*chunk_size, size) - stop = min((i + 1)*chunk_size, size) + start = i*chunk_size + stop = min((i+1)*chunk_size, size) + if start >= size: + break + chunks.append(Chunk(start, stop)) return chunks - - -@sdc_overload(get_chunks) -def get_chunks_overload(size, pool_size=0): - def get_chunks_impl(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - - chunk_size = size // pool_size - - chunks = [] - - for i in range(pool_size): - start = min(i*chunk_size, size) - stop = min((i + 1)*chunk_size, size) - if i == pool_size - 1: - tmp = size - size // pool_size - start = min(i*chunk_size, size) - stop = min((i + 1)*chunk_size+tmp, size) - chunk = Chunk(start, stop) - chunks.append(chunk) - - return chunks - - return get_chunks_impl From e957c9cd6c732cbd8997c9b64d0961ccd3b2cf31 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Tue, 18 Feb 2020 13:55:18 +0300 Subject: [PATCH 07/10] reduce op --- sdc/functions/numpy_like.py | 369 ++++++++++++---------------------- sdc/tests/test_sdc_numpy.py | 1 - sdc/utilities/prange_utils.py | 2 +- 3 files changed, 129 insertions(+), 243 deletions(-) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 0c6dc215b..68b4eb24f 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -151,277 +151,164 @@ def sdc_astype_number_impl(self, dtype): return sdc_astype_number_impl -@sdc_overload(nanargmin) -def sdc_nanargmin_overload(self): - """ - Intel Scalable Dataframe Compiler Developer Guide - ************************************************* - Parallel replacement of numpy.nanargmin. - - .. only:: developer - Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k nanargmin - - """ - - ty_checker = TypeChecker("numpy-like 'nanargmin'") - dtype = self.dtype - isnan = get_isnan(dtype) - max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) - if isinstance(dtype, types.Integer): - max_ref = max_dtype_int_val(dtype) - - if isinstance(dtype, types.Float): - max_ref = max_dtype_float_val(dtype) - - if not isinstance(self, types.Array): - return None - - if isinstance(dtype, types.Number): - def sdc_nanargmin_impl(self): - chunks = get_chunks(len(self)) - arr_res = numpy.empty(shape=len(chunks), dtype=dtype) - arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) - for i in prange(len(chunks)): - chunk = chunks[i] - res = max_ref - pos = max_int64 - for j in range(chunk.start, chunk.stop): - if min(res, self[j]) == self[j]: - if not isnan(self[j]): - if res == self[j]: - pos = min(pos, j) - else: - pos = j - res = self[j] - arr_res[i] = res - arr_pos[i] = pos - - general_res = max_ref - general_pos = max_int64 - for i in range(len(chunks)): - if min(general_res, arr_res[i]) == arr_res[i]: - if general_res == arr_res[i]: - general_pos = min(general_pos, arr_pos[i]) - else: - general_pos = arr_pos[i] - general_res = arr_res[i] - - return general_pos - - return sdc_nanargmin_impl - - ty_checker.raise_exc(dtype, 'number', 'self.dtype') - - -@sdc_overload(nanargmax) -def sdc_nanargmax_overload(self): - """ - Intel Scalable Dataframe Compiler Developer Guide - ************************************************* - Parallel replacement of numpy.nanargmax. - - .. only:: developer - Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k nanargmax - - """ - - ty_checker = TypeChecker("numpy-like 'nanargmax'") - dtype = self.dtype - isnan = get_isnan(dtype) - max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) - if isinstance(dtype, types.Integer): - min_ref = min_dtype_int_val(dtype) +def sdc_nanarg_overload(reduce_op): + def nanarg_impl(self): + """ + Intel Scalable Dataframe Compiler Developer Guide + ************************************************* + Parallel replacement of numpy.nanargmin/numpy.nanargmax. + + .. only:: developer + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k nanargmin + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k nanargmax + + """ + + ty_checker = TypeChecker("numpy-like 'nanargmin'/'nanargmax'") + dtype = self.dtype + isnan = get_isnan(dtype) + max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) + if isinstance(dtype, types.Integer): + initial_result = { + min: max_dtype_int_val(dtype), + max: min_dtype_int_val(dtype), + }[reduce_op] - if isinstance(dtype, types.Float): - min_ref = min_dtype_float_val(dtype) + if isinstance(dtype, types.Float): + initial_result = { + min: max_dtype_float_val(dtype), + max: min_dtype_float_val(dtype), + }[reduce_op] - if not isinstance(self, types.Array): - return None + if not isinstance(self, types.Array): + return None - if isinstance(dtype, types.Number): - def sdc_nanargmax_impl(self): - chunks = get_chunks(len(self)) - arr_res = numpy.empty(shape=len(chunks), dtype=dtype) - arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) - for i in prange(len(chunks)): - chunk = chunks[i] - res = min_ref - pos = max_int64 - for j in range(chunk.start, chunk.stop): - if max(res, self[j]) == self[j]: - if not isnan(self[j]): - if res == self[j]: - pos = min(pos, j) - else: - pos = j - res = self[j] - arr_res[i] = res - arr_pos[i] = pos - - general_res = min_ref - general_pos = max_int64 - for i in range(len(chunks)): - if max(general_res, arr_res[i]) == arr_res[i]: - if not isnan(arr_res[i]): + if isinstance(dtype, types.Number): + def sdc_nanargmin_impl(self): + chunks = get_chunks(len(self)) + arr_res = numpy.empty(shape=len(chunks), dtype=dtype) + arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) + for i in prange(len(chunks)): + chunk = chunks[i] + res = initial_result + pos = max_int64 + for j in range(chunk.start, chunk.stop): + if reduce_op(res, self[j]) == self[j]: + if not isnan(self[j]): + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] + arr_res[i] = res + arr_pos[i] = pos + + general_res = initial_result + general_pos = max_int64 + for i in range(len(chunks)): + if reduce_op(general_res, arr_res[i]) == arr_res[i]: if general_res == arr_res[i]: general_pos = min(general_pos, arr_pos[i]) else: general_pos = arr_pos[i] general_res = arr_res[i] - return general_pos - - return sdc_nanargmax_impl - - ty_checker.raise_exc(dtype, 'number', 'self.dtype') - - -@sdc_overload(argmin) -def sdc_argmin_overload(self): - """ - Intel Scalable Dataframe Compiler Developer Guide - ************************************************* - Parallel replacement of numpy.argmin. - - .. only:: developer - Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k argmin - - """ + return general_pos - ty_checker = TypeChecker("numpy-like 'argmin'") - dtype = self.dtype - isnan = get_isnan(dtype) - max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) - if isinstance(dtype, types.Integer): - max_ref = max_dtype_int_val(dtype) - - if isinstance(dtype, types.Float): - max_ref = max_dtype_float_val(dtype) - - if not isinstance(self, types.Array): - return None + return sdc_nanargmin_impl - if isinstance(dtype, types.Number): - def sdc_argmin_impl(self): - chunks = get_chunks(len(self)) - arr_res = numpy.empty(shape=len(chunks), dtype=dtype) - arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) - for i in prange(len(chunks)): - chunk = chunks[i] - res = max_ref - pos = max_int64 - for j in range(chunk.start, chunk.stop): - if not isnan(self[j]): - if min(res, self[j]) == self[j]: - if res == self[j]: - pos = min(pos, j) - else: - pos = j - res = self[j] - else: - if numpy.isnan(res): - pos = min(pos, j) - else: - pos = j - res = self[j] - - arr_res[i] = res - arr_pos[i] = pos - general_res = max_ref - general_pos = max_int64 - for i in range(len(chunks)): - if not isnan(arr_res[i]): - if min(general_res, arr_res[i]) == arr_res[i]: - if general_res == arr_res[i]: - general_pos = min(general_pos, arr_pos[i]) - else: - general_pos = arr_pos[i] - general_res = arr_res[i] - else: - if numpy.isnan(general_res): - general_pos = min(general_pos, arr_pos[i]) - else: - general_pos = arr_pos[i] - general_res = arr_res[i] - return general_pos + ty_checker.raise_exc(dtype, 'number', 'self.dtype') + return nanarg_impl - return sdc_argmin_impl - ty_checker.raise_exc(dtype, 'number', 'self.dtype') +sdc_overload(nanargmin)(sdc_nanarg_overload(min)) +sdc_overload(nanargmax)(sdc_nanarg_overload(max)) -@sdc_overload(argmax) -def sdc_argmax_overload(self): - """ - Intel Scalable Dataframe Compiler Developer Guide - ************************************************* - Parallel replacement of numpy.argmax. +def sdc_arg_overload(reduce_op): + def arg_impl(self): + """ + Intel Scalable Dataframe Compiler Developer Guide + ************************************************* + Parallel replacement of numpy.argmin/numpy.argmax. - .. only:: developer - Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k argmax + .. only:: developer + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k argmin + Test: python -m sdc.runtests sdc.tests.test_sdc_numpy -k argmax - """ + """ - ty_checker = TypeChecker("numpy-like 'argmax'") - dtype = self.dtype - isnan = get_isnan(dtype) - max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) - if isinstance(dtype, types.Integer): - min_ref = min_dtype_int_val(dtype) + ty_checker = TypeChecker("numpy-like 'argmin'/'argmax'") + dtype = self.dtype + isnan = get_isnan(dtype) + max_int64 = max_dtype_int_val(numpy_support.from_dtype(numpy.int64)) + if isinstance(dtype, types.Integer): + initial_result = { + min: max_dtype_int_val(dtype), + max: min_dtype_int_val(dtype), + }[reduce_op] - if isinstance(dtype, types.Float): - min_ref = min_dtype_float_val(dtype) + if isinstance(dtype, types.Float): + initial_result = { + min: max_dtype_float_val(dtype), + max: min_dtype_float_val(dtype), + }[reduce_op] - if not isinstance(self, types.Array): - return None + if not isinstance(self, types.Array): + return None - if isinstance(dtype, types.Number): - def sdc_argmax_impl(self): - chunks = get_chunks(len(self)) - arr_res = numpy.empty(shape=len(chunks), dtype=dtype) - arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) - for i in prange(len(chunks)): - chunk = chunks[i] - res = min_ref - pos = max_int64 - for j in range(chunk.start, chunk.stop): - if not isnan(self[j]): - if max(res, self[j]) == self[j]: - if res == self[j]: + if isinstance(dtype, types.Number): + def sdc_argmin_impl(self): + chunks = get_chunks(len(self)) + arr_res = numpy.empty(shape=len(chunks), dtype=dtype) + arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) + for i in prange(len(chunks)): + chunk = chunks[i] + res = initial_result + pos = max_int64 + for j in range(chunk.start, chunk.stop): + if not isnan(self[j]): + if reduce_op(res, self[j]) == self[j]: + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] + else: + if numpy.isnan(res): pos = min(pos, j) else: pos = j - res = self[j] + res = self[j] + + arr_res[i] = res + arr_pos[i] = pos + general_res = initial_result + general_pos = max_int64 + for i in range(len(chunks)): + if not isnan(arr_res[i]): + if reduce_op(general_res, arr_res[i]) == arr_res[i]: + if general_res == arr_res[i]: + general_pos = min(general_pos, arr_pos[i]) + else: + general_pos = arr_pos[i] + general_res = arr_res[i] else: - if numpy.isnan(res): - pos = min(pos, j) - else: - pos = j - res = self[j] - - arr_res[i] = res - arr_pos[i] = pos - general_res = min_ref - general_pos = max_int64 - for i in range(len(chunks)): - if not isnan(arr_res[i]): - if max(general_res, arr_res[i]) == arr_res[i]: - if general_res == arr_res[i]: + if numpy.isnan(general_res): general_pos = min(general_pos, arr_pos[i]) else: general_pos = arr_pos[i] - general_res = arr_res[i] - else: - if numpy.isnan(general_res): - general_pos = min(general_pos, arr_pos[i]) - else: - general_pos = arr_pos[i] - general_res = arr_res[i] - return general_pos + general_res = arr_res[i] + return general_pos + + return sdc_argmin_impl + + ty_checker.raise_exc(dtype, 'number', 'self.dtype') + return arg_impl - return sdc_argmax_impl - ty_checker.raise_exc(dtype, 'number', 'self.dtype') +sdc_overload(argmin)(sdc_arg_overload(min)) +sdc_overload(argmax)(sdc_arg_overload(max)) @sdc_overload(copy) diff --git a/sdc/tests/test_sdc_numpy.py b/sdc/tests/test_sdc_numpy.py index 27c297010..c23423947 100644 --- a/sdc/tests/test_sdc_numpy.py +++ b/sdc/tests/test_sdc_numpy.py @@ -240,7 +240,6 @@ def sdc_impl(): sdc_func = self.jit(sdc_impl) np.testing.assert_array_equal(sdc_func(), ref_impl()) - def test_argmin(self): def ref_impl(a): return np.argmin(a) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index ee2a4851d..05b1f4919 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -68,7 +68,7 @@ def get_chunks(size, pool_size=0): stop = min((i+1)*chunk_size, size) if start >= size: break - + chunks.append(Chunk(start, stop)) return chunks From d7863895acc0dce8cbe94b0e3a8c0b11992e0fae Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Wed, 19 Feb 2020 09:34:06 +0300 Subject: [PATCH 08/10] new prange utils --- sdc/utilities/prange_utils.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index 05b1f4919..0f24d34ea 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -37,6 +37,7 @@ class Chunk(NamedTuple): stop: int +@sdc_register_jitable def get_pool_size(): if sdc.config.config_use_parallel_overloads: return numba.config.NUMBA_NUM_THREADS @@ -44,31 +45,21 @@ def get_pool_size(): return 1 -@sdc_overload(get_pool_size) -def get_pool_size_overload(): - pool_size = get_pool_size() - - def get_pool_size_impl(): - return pool_size - - return get_pool_size_impl - - @sdc_register_jitable -def get_chunks(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - +def get_chunks(size, pool_size): pool_size = min(pool_size, size) - chunk_size = (size - 1)//pool_size + 1 + chunk_size = size // pool_size + overload_size = size % pool_size chunks = [] for i in range(pool_size): - start = i*chunk_size - stop = min((i+1)*chunk_size, size) - if start >= size: - break - + start = i * chunk_size + min(i, overload_size) + stop = (i + 1) * chunk_size + min(i + 1, overload_size) chunks.append(Chunk(start, stop)) return chunks + + +@sdc_register_jitable +def parallel_chunks(size): + return get_chunks(size, get_pool_size()) From 20d707398945bcdbb5b45bd59e6bcade493dbfd4 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Wed, 19 Feb 2020 10:15:56 +0300 Subject: [PATCH 09/10] Call numpy-like arg --- sdc/datatypes/hpat_pandas_series_functions.py | 110 +++++++++++++++--- sdc/functions/numpy_like.py | 6 +- sdc/tests/tests_perf/test_perf_series.py | 6 +- 3 files changed, 103 insertions(+), 19 deletions(-) diff --git a/sdc/datatypes/hpat_pandas_series_functions.py b/sdc/datatypes/hpat_pandas_series_functions.py index 658d6460c..f971d2d31 100644 --- a/sdc/datatypes/hpat_pandas_series_functions.py +++ b/sdc/datatypes/hpat_pandas_series_functions.py @@ -2926,7 +2926,7 @@ def hpat_pandas_series_take_impl(self, indices, axis=0, is_copy=False): @sdc_overload_method(SeriesType, 'idxmax') -def hpat_pandas_series_idxmax(self, axis=None, skipna=True): +def hpat_pandas_series_idxmax(self, axis=None, skipna=None): """ Intel Scalable Dataframe Compiler User Guide ******************************************** @@ -2975,22 +2975,63 @@ def hpat_pandas_series_idxmax(self, axis=None, skipna=True): if not isinstance(self.data.dtype, types.Number): ty_checker.raise_exc(self.data.dtype, 'int, float', 'self.data.dtype') - if not (isinstance(skipna, (types.Omitted, types.Boolean, bool)) or skipna is True): + if not (isinstance(skipna, (types.Omitted, types.Boolean, bool)) or skipna is None): ty_checker.raise_exc(skipna, 'bool', 'skipna') if not (isinstance(axis, types.Omitted) or axis is None): ty_checker.raise_exc(axis, 'None', 'axis') if isinstance(self.index, types.NoneType) or self.index is None: - def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=True): - return numpy.argmax(self._data) + if isinstance(self.data, StringArrayType): + def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=None): + if skipna is None: + _skipna = True + else: + raise ValueError("Method idxmax(). Unsupported parameter 'skipna'=False with str data") + + return numpy.argmax(self._data) + + return hpat_pandas_series_idxmax_impl + + def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=None): + # return numpy.argmax(self._data) + if skipna is None: + _skipna = True + else: + _skipna = skipna + + if _skipna: + return numpy_like.nanargmax(self._data) + + return numpy_like.argmax(self._data) return hpat_pandas_series_idxmax_impl else: - def hpat_pandas_series_idxmax_index_impl(self, axis=None, skipna=True): - # no numpy.nanargmax is supported by Numba at this time - result = numpy.argmax(self._data) + if isinstance(self.data, StringArrayType) or isinstance(self.index, StringArrayType): + def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=None): + # no numpy.nanargmax is supported by Numba at this time + if skipna is None: + _skipna = True + else: + raise ValueError("Method idxmax(). Unsupported parameter 'skipna'=False with str data") + + result = numpy.argmax(self._data) + return self._index[int(result)] + + return hpat_pandas_series_idxmax_impl + + def hpat_pandas_series_idxmax_index_impl(self, axis=None, skipna=None): + if skipna is None: + _skipna = True + else: + _skipna = skipna + + if _skipna: + result = numpy_like.nanargmax(self._data) + else: + result = numpy_like.argmax(self._data) + return self._index[int(result)] return hpat_pandas_series_idxmax_index_impl @@ -3987,7 +4028,7 @@ def hpat_pandas_series_ge_impl(self, other, level=None, fill_value=None, axis=0) @sdc_overload_method(SeriesType, 'idxmin') -def hpat_pandas_series_idxmin(self, axis=None, skipna=True): +def hpat_pandas_series_idxmin(self, axis=None, skipna=None): """ Intel Scalable Dataframe Compiler User Guide ******************************************** @@ -4036,22 +4077,63 @@ def hpat_pandas_series_idxmin(self, axis=None, skipna=True): if not isinstance(self.data.dtype, types.Number): ty_checker.raise_exc(self.data.dtype, 'int, float', 'self.data.dtype') - if not (isinstance(skipna, (types.Omitted, types.Boolean, bool)) or skipna is True): + if not (isinstance(skipna, (types.Omitted, types.Boolean, bool)) or skipna is None): ty_checker.raise_exc(skipna, 'bool', 'skipna') if not (isinstance(axis, types.Omitted) or axis is None): ty_checker.raise_exc(axis, 'None', 'axis') if isinstance(self.index, types.NoneType) or self.index is None: - def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=True): - return numpy.argmin(self._data) + if isinstance(self.data, StringArrayType): + def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=None): + if skipna is None: + _skipna = True + else: + raise ValueError("Method idxmin(). Unsupported parameter 'skipna'=False with str data") + + return numpy.argmin(self._data) + + return hpat_pandas_series_idxmin_impl + + def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=None): + # return numpy.argmin(self._data) + if skipna is None: + _skipna = True + else: + _skipna = skipna + + if _skipna: + return numpy_like.nanargmin(self._data) + + return numpy_like.argmin(self._data) return hpat_pandas_series_idxmin_impl else: - def hpat_pandas_series_idxmin_index_impl(self, axis=None, skipna=True): - # no numpy.nanargmin is supported by Numba at this time - result = numpy.argmin(self._data) + if isinstance(self.data, StringArrayType) or isinstance(self.index, StringArrayType): + def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=None): + # no numpy.nanargmin is supported by Numba at this time + if skipna is None: + _skipna = True + else: + raise ValueError("Method idxmin(). Unsupported parameter 'skipna'=False with str data") + + result = numpy.argmin(self._data) + return self._index[int(result)] + + return hpat_pandas_series_idxmin_impl + + def hpat_pandas_series_idxmin_index_impl(self, axis=None, skipna=None): + if skipna is None: + _skipna = True + else: + _skipna = skipna + + if _skipna: + result = numpy_like.nanargmin(self._data) + else: + result = numpy_like.argmin(self._data) + return self._index[int(result)] return hpat_pandas_series_idxmin_index_impl diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index e670437c2..c1c9c0083 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -46,7 +46,7 @@ min_dtype_int_val, max_dtype_int_val, min_dtype_float_val, max_dtype_float_val) from sdc.str_arr_ext import (StringArrayType, pre_alloc_string_array, get_utf8_size, str_arr_is_na) -from sdc.utilities.prange_utils import get_chunks +from sdc.utilities.prange_utils import parallel_chunks def astype(self, dtype): @@ -186,7 +186,7 @@ def nanarg_impl(self): if isinstance(dtype, types.Number): def sdc_nanargmin_impl(self): - chunks = get_chunks(len(self)) + chunks = parallel_chunks(len(self)) arr_res = numpy.empty(shape=len(chunks), dtype=dtype) arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) for i in prange(len(chunks)): @@ -260,7 +260,7 @@ def arg_impl(self): if isinstance(dtype, types.Number): def sdc_argmin_impl(self): - chunks = get_chunks(len(self)) + chunks = parallel_chunks(len(self)) arr_res = numpy.empty(shape=len(chunks), dtype=dtype) arr_pos = numpy.empty(shape=len(chunks), dtype=numpy.int64) for i in prange(len(chunks)): diff --git a/sdc/tests/tests_perf/test_perf_series.py b/sdc/tests/tests_perf/test_perf_series.py index 272d211bd..a17c2d56a 100644 --- a/sdc/tests/tests_perf/test_perf_series.py +++ b/sdc/tests/tests_perf/test_perf_series.py @@ -89,8 +89,10 @@ def _test_case(self, pyfunc, name, total_data_length, data_num=1, input_data=tes TC(name='gt', size=[10 ** 7],params='other', data_num=2), TC(name='head', size=[10 ** 8]), TC(name='iat', size=[10 ** 7], call_expr='data.iat[100000]', usecase_params='data'), - TC(name='idxmax', size=[10 ** 8]), - TC(name='idxmin', size=[10 ** 8]), + TC(name='idxmax', size=[10 ** 8], params='skipna=True'), + TC(name='idxmax', size=[10 ** 8], params='skipna=False'), + TC(name='idxmin', size=[10 ** 8], params='skipna=True'), + TC(name='idxmin', size=[10 ** 8], params='skipna=False'), TC(name='iloc', size=[10 ** 7], call_expr='data.iloc[100000]', usecase_params='data'), TC(name='index', size=[10 ** 7], call_expr='data.index', usecase_params='data'), TC(name='isin', size=[10 ** 7], call_expr='data.isin([0])', usecase_params='data'), From fc27331511af9b9a17c5ae6919738a151014143e Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Wed, 19 Feb 2020 15:47:05 +0300 Subject: [PATCH 10/10] optimize algo --- sdc/datatypes/hpat_pandas_series_functions.py | 136 +++++++----------- sdc/functions/numpy_like.py | 55 +++---- 2 files changed, 80 insertions(+), 111 deletions(-) diff --git a/sdc/datatypes/hpat_pandas_series_functions.py b/sdc/datatypes/hpat_pandas_series_functions.py index f971d2d31..ff9a0ab52 100644 --- a/sdc/datatypes/hpat_pandas_series_functions.py +++ b/sdc/datatypes/hpat_pandas_series_functions.py @@ -2981,60 +2981,42 @@ def hpat_pandas_series_idxmax(self, axis=None, skipna=None): if not (isinstance(axis, types.Omitted) or axis is None): ty_checker.raise_exc(axis, 'None', 'axis') - if isinstance(self.index, types.NoneType) or self.index is None: - if isinstance(self.data, StringArrayType): - def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=None): - if skipna is None: - _skipna = True - else: - raise ValueError("Method idxmax(). Unsupported parameter 'skipna'=False with str data") - - return numpy.argmax(self._data) - - return hpat_pandas_series_idxmax_impl - - def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=None): - # return numpy.argmax(self._data) + none_index = isinstance(self.index, types.NoneType) or self.index is None + if isinstance(self.data, StringArrayType): + def hpat_pandas_series_idxmax_str_impl(self, axis=None, skipna=None): if skipna is None: _skipna = True else: - _skipna = skipna - - if _skipna: - return numpy_like.nanargmax(self._data) + raise ValueError("Method idxmax(). Unsupported parameter 'skipna'=False with str data") - return numpy_like.argmax(self._data) - - return hpat_pandas_series_idxmax_impl - - else: - if isinstance(self.data, StringArrayType) or isinstance(self.index, StringArrayType): - def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=None): - # no numpy.nanargmax is supported by Numba at this time - if skipna is None: - _skipna = True - else: - raise ValueError("Method idxmax(). Unsupported parameter 'skipna'=False with str data") - - result = numpy.argmax(self._data) + result = numpy.argmax(self._data) + if none_index == True: # noqa + return result + else: return self._index[int(result)] - return hpat_pandas_series_idxmax_impl + return hpat_pandas_series_idxmax_str_impl - def hpat_pandas_series_idxmax_index_impl(self, axis=None, skipna=None): - if skipna is None: - _skipna = True - else: - _skipna = skipna + def hpat_pandas_series_idxmax_impl(self, axis=None, skipna=None): + # return numpy.argmax(self._data) + if skipna is None: + _skipna = True + else: + _skipna = skipna - if _skipna: - result = numpy_like.nanargmax(self._data) - else: - result = numpy_like.argmax(self._data) + if _skipna: + result = numpy_like.nanargmax(self._data) + else: + result = numpy_like.argmax(self._data) + if none_index == True: # noqa + return result + else: return self._index[int(result)] - return hpat_pandas_series_idxmax_index_impl + return numpy_like.argmax(self._data) + + return hpat_pandas_series_idxmax_impl @sdc_overload_method(SeriesType, 'mul') @@ -4083,60 +4065,42 @@ def hpat_pandas_series_idxmin(self, axis=None, skipna=None): if not (isinstance(axis, types.Omitted) or axis is None): ty_checker.raise_exc(axis, 'None', 'axis') - if isinstance(self.index, types.NoneType) or self.index is None: - if isinstance(self.data, StringArrayType): - def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=None): - if skipna is None: - _skipna = True - else: - raise ValueError("Method idxmin(). Unsupported parameter 'skipna'=False with str data") - - return numpy.argmin(self._data) - - return hpat_pandas_series_idxmin_impl - - def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=None): - # return numpy.argmin(self._data) + none_index = isinstance(self.index, types.NoneType) or self.index is None + if isinstance(self.data, StringArrayType): + def hpat_pandas_series_idxmin_str_impl(self, axis=None, skipna=None): if skipna is None: _skipna = True else: - _skipna = skipna - - if _skipna: - return numpy_like.nanargmin(self._data) + raise ValueError("Method idxmin(). Unsupported parameter 'skipna'=False with str data") - return numpy_like.argmin(self._data) - - return hpat_pandas_series_idxmin_impl - - else: - if isinstance(self.data, StringArrayType) or isinstance(self.index, StringArrayType): - def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=None): - # no numpy.nanargmin is supported by Numba at this time - if skipna is None: - _skipna = True - else: - raise ValueError("Method idxmin(). Unsupported parameter 'skipna'=False with str data") - - result = numpy.argmin(self._data) + result = numpy.argmin(self._data) + if none_index == True: # noqa + return result + else: return self._index[int(result)] - return hpat_pandas_series_idxmin_impl + return hpat_pandas_series_idxmin_str_impl - def hpat_pandas_series_idxmin_index_impl(self, axis=None, skipna=None): - if skipna is None: - _skipna = True - else: - _skipna = skipna + def hpat_pandas_series_idxmin_impl(self, axis=None, skipna=None): + # return numpy.argmin(self._data) + if skipna is None: + _skipna = True + else: + _skipna = skipna - if _skipna: - result = numpy_like.nanargmin(self._data) - else: - result = numpy_like.argmin(self._data) + if _skipna: + result = numpy_like.nanargmin(self._data) + else: + result = numpy_like.argmin(self._data) + if none_index == True: # noqa + return result + else: return self._index[int(result)] - return hpat_pandas_series_idxmin_index_impl + return numpy_like.argmin(self._data) + + return hpat_pandas_series_idxmin_impl @sdc_overload_method(SeriesType, 'lt') diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index c1c9c0083..f2c9a898f 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -194,25 +194,28 @@ def sdc_nanargmin_impl(self): res = initial_result pos = max_int64 for j in range(chunk.start, chunk.stop): - if reduce_op(res, self[j]) == self[j]: - if not isnan(self[j]): - if res == self[j]: - pos = min(pos, j) - else: - pos = j - res = self[j] + if reduce_op(res, self[j]) != self[j]: + continue + if isnan(self[j]): + continue + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] arr_res[i] = res arr_pos[i] = pos general_res = initial_result general_pos = max_int64 for i in range(len(chunks)): - if reduce_op(general_res, arr_res[i]) == arr_res[i]: - if general_res == arr_res[i]: - general_pos = min(general_pos, arr_pos[i]) - else: - general_pos = arr_pos[i] - general_res = arr_res[i] + if reduce_op(general_res, arr_res[i]) != arr_res[i]: + continue + if general_res == arr_res[i]: + general_pos = min(general_pos, arr_pos[i]) + else: + general_pos = arr_pos[i] + general_res = arr_res[i] return general_pos @@ -269,12 +272,13 @@ def sdc_argmin_impl(self): pos = max_int64 for j in range(chunk.start, chunk.stop): if not isnan(self[j]): - if reduce_op(res, self[j]) == self[j]: - if res == self[j]: - pos = min(pos, j) - else: - pos = j - res = self[j] + if reduce_op(res, self[j]) != self[j]: + continue + if res == self[j]: + pos = min(pos, j) + else: + pos = j + res = self[j] else: if numpy.isnan(res): pos = min(pos, j) @@ -288,12 +292,13 @@ def sdc_argmin_impl(self): general_pos = max_int64 for i in range(len(chunks)): if not isnan(arr_res[i]): - if reduce_op(general_res, arr_res[i]) == arr_res[i]: - if general_res == arr_res[i]: - general_pos = min(general_pos, arr_pos[i]) - else: - general_pos = arr_pos[i] - general_res = arr_res[i] + if reduce_op(general_res, arr_res[i]) != arr_res[i]: + continue + if general_res == arr_res[i]: + general_pos = min(general_pos, arr_pos[i]) + else: + general_pos = arr_pos[i] + general_res = arr_res[i] else: if numpy.isnan(general_res): general_pos = min(general_pos, arr_pos[i])