From 51cbb746c62ff6559104e4f2df39651bdf811120 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 14 Feb 2020 12:10:03 +0300 Subject: [PATCH 1/9] wip --- sdc/functions/numpy_like.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 9e74a76bd..d2b511148 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -393,3 +393,87 @@ def nanprod_impl(a): return c return nanprod_impl + + +def get_pool_size(): + return numba.config.NUMBA_NUM_THREADS + +@sdc_overload(get_pool_size) +def get_pool_size_overload(): + pool_size = get_pool_size() + def get_pool_size_impl(): + return pool_size + + return get_pool_size_impl + +def get_chunks(size, pool_size=0): + if pool_size == 0: + pool_size = get_pool_size() + + chunk_size = size//pool_size + 1 + + Chunk = namedtuple('start', 'stop') + + chunks = [] + + for i in range(pool_size): + start = min(i*chunk_size, size) + stop = min((i + 1)*chunk_size, size) + chunks.append(Chunk(start, stop)) + + return chunks + +@sdc_overload(get_chunks) +def get_chunks_overload(size, pool_size=0): + Chunk = namedtuple('Chunk', ['start', 'stop']) + + def get_chunks_impl(size, pool_size=0): + if pool_size == 0: + pool_size = get_pool_size() + + chunk_size = size//pool_size + 1 + + chunks = [] + + for i in range(pool_size): + start = min(i*chunk_size, size) + stop = min((i + 1)*chunk_size, size) + chunk = Chunk(start, stop) + chunks.append(chunk) + + return chunks + + return get_chunks_impl + +def dropna(arr): + pass + +@sdc_overload(dropna) +def dropna_overload(arr): + dtype = arr.dtype + isnan = get_isnan(dtype) + def dropna_impl(arr): + chunks = get_chunks(len(arr)) + # partial_sum = numpy.zeros(len(chunks), dtype=arr.dtype) + # result = numpy.empty_like(arr) + + for i in prange(len(chunks)): + chunk = chunks[i] + length = chunk.stop - chunk.start + # res = numpy.empty(shape=length, dtype=dtype) + # partial = 0 + for j in range(chunk.start, chunk.stop): + partial += arr[j] + result[j] = partial + + partial_sum[i] = partial + + for i in prange(len(chunks)): + prefix = sum(partial_sum[0:i]) + chunk = chunks[i] + for j in range(chunk.start, chunk.stop): + result[j] += prefix + + return result + + return cumsum_impl From 398497b0bdabfb5119877525105ddb238bc256f9 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 14 Feb 2020 16:42:03 +0300 Subject: [PATCH 2/9] Series.dropna draft --- sdc/datatypes/hpat_pandas_series_functions.py | 22 +++++--- sdc/functions/numpy_like.py | 54 ++++++++++++------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/sdc/datatypes/hpat_pandas_series_functions.py b/sdc/datatypes/hpat_pandas_series_functions.py index 36f629a20..bbfab3cfe 100644 --- a/sdc/datatypes/hpat_pandas_series_functions.py +++ b/sdc/datatypes/hpat_pandas_series_functions.py @@ -4932,14 +4932,22 @@ def hpat_pandas_series_dropna(self, axis=0, inplace=False): if not (inplace is False or isinstance(inplace, types.Omitted)): ty_checker.raise_exc(inplace, 'bool', 'inplace') - def hpat_pandas_series_dropna_impl(self, axis=0, inplace=False): - # generate Series index if needed by using SeriesType.index (i.e. not self._index) - na_data_arr = sdc.hiframes.api.get_nan_mask(self._data) - data = self._data[~na_data_arr] - index = self.index[~na_data_arr] - return pandas.Series(data, index, self._name) + if isinstance(self.data.dtype, types.Number) and isinstance(self.index, (types.Number, types.NoneType)): + def hpat_pandas_series_dropna_impl(self, axis=0, inplace=False): + index = self.index + return numpy_like.dropna(self._data, index, self._name) + + return hpat_pandas_series_dropna_impl + + else: + def hpat_pandas_series_dropna_str_impl(self, axis=0, inplace=False): + # generate Series index if needed by using SeriesType.index (i.e. not self._index) + na_data_arr = sdc.hiframes.api.get_nan_mask(self._data) + data = self._data[~na_data_arr] + index = self.index[~na_data_arr] + return pandas.Series(data, index, self._name) - return hpat_pandas_series_dropna_impl + return hpat_pandas_series_dropna_str_impl @sdc_overload_method(SeriesType, 'fillna') diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index d2b511148..2865be466 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -33,10 +33,12 @@ import numba import numpy +import pandas from numba import types, jit, prange, numpy_support, literally from numba.errors import TypingError from numba.targets.arraymath import get_isnan +from typing import NamedTuple import sdc from sdc.utilities.sdc_typing_utils import TypeChecker @@ -412,7 +414,7 @@ def get_chunks(size, pool_size=0): chunk_size = size//pool_size + 1 - Chunk = namedtuple('start', 'stop') + Chunk = NamedTuple('start', 'stop') chunks = [] @@ -423,9 +425,13 @@ def get_chunks(size, pool_size=0): return chunks +class Chunk(NamedTuple): + start: int + stop: int + @sdc_overload(get_chunks) def get_chunks_overload(size, pool_size=0): - Chunk = namedtuple('Chunk', ['start', 'stop']) + # Chunk = NamedTuple('Chunk', ['start', 'stop']) def get_chunks_impl(size, pool_size=0): if pool_size == 0: @@ -445,35 +451,47 @@ def get_chunks_impl(size, pool_size=0): return get_chunks_impl -def dropna(arr): +def dropna(arr, idx, name): pass @sdc_overload(dropna) -def dropna_overload(arr): +def dropna_overload(arr, idx, name): dtype = arr.dtype + dtype_idx = idx.dtype isnan = get_isnan(dtype) - def dropna_impl(arr): + def dropna_impl(arr, idx, name): chunks = get_chunks(len(arr)) - # partial_sum = numpy.zeros(len(chunks), dtype=arr.dtype) - # result = numpy.empty_like(arr) + arr_len = numpy.empty(len(chunks), dtype=numpy.int64) + length = 0 for i in prange(len(chunks)): chunk = chunks[i] - length = chunk.stop - chunk.start - # res = numpy.empty(shape=length, dtype=dtype) - # partial = 0 + res = 0 for j in range(chunk.start, chunk.stop): - partial += arr[j] - result[j] = partial - - partial_sum[i] = partial + if not isnan(arr[j]): + res += 1 + length += res + arr_len[i] = res + result_data = numpy.empty(shape=length, dtype=dtype) + result_index = numpy.empty(shape=length, dtype=dtype_idx) for i in prange(len(chunks)): - prefix = sum(partial_sum[0:i]) chunk = chunks[i] + if i == 0: + new_start = 0 + new_stop = arr_len[0] + else: + for x in range(i): + new_start += arr_len[x] + new_stop = new_start + arr_len[i] + for j in range(chunk.start, chunk.stop): - result[j] += prefix + if new_start < new_stop: + if not isnan(arr[j]): + result_data[new_start] = arr[j] + result_index[new_start] = idx[j] + new_start += 1 - return result + return pandas.Series(result_data, result_index, name) - return cumsum_impl + return dropna_impl From 6b50702e09b06d36ffc9c4b88db0bd9cd1d44bdb Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 14 Feb 2020 18:23:06 +0300 Subject: [PATCH 3/9] add check parallel --- sdc/functions/numpy_like.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 2865be466..9dbb6715f 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -398,7 +398,10 @@ def nanprod_impl(a): def get_pool_size(): - return numba.config.NUMBA_NUM_THREADS + if sdc.config.config_use_parallel_overloads: + return numba.config.NUMBA_NUM_THREADS + else: + return 1 @sdc_overload(get_pool_size) def get_pool_size_overload(): From 2b3e7938f5e70eff27dfc81dc806e052f29a578f Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Mon, 17 Feb 2020 10:58:31 +0300 Subject: [PATCH 4/9] Remove prange utils to other file --- sdc/functions/numpy_like.py | 60 +---------------------- sdc/utilities/prange_utils.py | 92 +++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 58 deletions(-) create mode 100644 sdc/utilities/prange_utils.py diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 9dbb6715f..f5060453b 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -38,12 +38,12 @@ from numba import types, jit, prange, numpy_support, literally from numba.errors import TypingError from numba.targets.arraymath import get_isnan -from typing import NamedTuple import sdc from sdc.utilities.sdc_typing_utils import TypeChecker from sdc.str_arr_ext import (StringArrayType, pre_alloc_string_array, get_utf8_size, str_arr_is_na) from sdc.utilities.utils import sdc_overload, sdc_register_jitable +from sdc.utilities.prange_utils import get_chunks def astype(self, dtype): @@ -397,66 +397,10 @@ def nanprod_impl(a): return nanprod_impl -def get_pool_size(): - if sdc.config.config_use_parallel_overloads: - return numba.config.NUMBA_NUM_THREADS - else: - return 1 - -@sdc_overload(get_pool_size) -def get_pool_size_overload(): - pool_size = get_pool_size() - def get_pool_size_impl(): - return pool_size - - return get_pool_size_impl - -def get_chunks(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - - chunk_size = size//pool_size + 1 - - Chunk = NamedTuple('start', 'stop') - - chunks = [] - - for i in range(pool_size): - start = min(i*chunk_size, size) - stop = min((i + 1)*chunk_size, size) - chunks.append(Chunk(start, stop)) - - return chunks - -class Chunk(NamedTuple): - start: int - stop: int - -@sdc_overload(get_chunks) -def get_chunks_overload(size, pool_size=0): - # Chunk = NamedTuple('Chunk', ['start', 'stop']) - - def get_chunks_impl(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - - chunk_size = size//pool_size + 1 - - chunks = [] - - for i in range(pool_size): - start = min(i*chunk_size, size) - stop = min((i + 1)*chunk_size, size) - chunk = Chunk(start, stop) - chunks.append(chunk) - - return chunks - - return get_chunks_impl - def dropna(arr, idx, name): pass + @sdc_overload(dropna) def dropna_overload(arr, idx, name): dtype = arr.dtype diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py new file mode 100644 index 000000000..c916ec1e8 --- /dev/null +++ b/sdc/utilities/prange_utils.py @@ -0,0 +1,92 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + + +import numba +import sdc + +from typing import NamedTuple +from sdc.utilities.utils import sdc_overload + + +class Chunk(NamedTuple): + start: int + stop: int + + +def get_pool_size(): + if sdc.config.config_use_parallel_overloads: + return numba.config.NUMBA_NUM_THREADS + else: + return 1 + + +@sdc_overload(get_pool_size) +def get_pool_size_overload(): + pool_size = get_pool_size() + def get_pool_size_impl(): + return pool_size + + return get_pool_size_impl + + +def get_chunks(size, pool_size=0): + if pool_size == 0: + pool_size = get_pool_size() + + chunk_size = (size - 1)//pool_size + 1 + + Chunk = NamedTuple('start', 'stop') + + chunks = [] + + for i in range(pool_size): + start = min(i*chunk_size, size) + stop = min((i + 1)*chunk_size, size) + chunks.append(Chunk(start, stop)) + + return chunks + + +@sdc_overload(get_chunks) +def get_chunks_overload(size, pool_size=0): + def get_chunks_impl(size, pool_size=0): + if pool_size == 0: + pool_size = get_pool_size() + + chunk_size = size//pool_size + 1 + + chunks = [] + + for i in range(pool_size): + start = min(i*chunk_size, size) + stop = min((i + 1)*chunk_size, size) + chunk = Chunk(start, stop) + chunks.append(chunk) + + return chunks + + return get_chunks_impl From 3806bc50176c83aef19c53b12782442e58821288 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Mon, 17 Feb 2020 11:00:16 +0300 Subject: [PATCH 5/9] pep --- sdc/functions/numpy_like.py | 1 + sdc/utilities/prange_utils.py | 1 + 2 files changed, 2 insertions(+) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index f5060453b..1d52b46b1 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -406,6 +406,7 @@ def dropna_overload(arr, idx, name): dtype = arr.dtype dtype_idx = idx.dtype isnan = get_isnan(dtype) + def dropna_impl(arr, idx, name): chunks = get_chunks(len(arr)) arr_len = numpy.empty(len(chunks), dtype=numpy.int64) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index c916ec1e8..56ab5b640 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -47,6 +47,7 @@ def get_pool_size(): @sdc_overload(get_pool_size) def get_pool_size_overload(): pool_size = get_pool_size() + def get_pool_size_impl(): return pool_size From cf92c1fc73cb27d92f4b04195238f09dd5818ba7 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Mon, 17 Feb 2020 11:24:51 +0300 Subject: [PATCH 6/9] chunk size fix --- sdc/utilities/prange_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index 56ab5b640..961eba266 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -78,7 +78,7 @@ def get_chunks_impl(size, pool_size=0): if pool_size == 0: pool_size = get_pool_size() - chunk_size = size//pool_size + 1 + chunk_size = (size - 1)//pool_size + 1 chunks = [] From 25e0f9fd2e5429cedfebb8738afa9124bf557bb0 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Tue, 18 Feb 2020 17:55:49 +0300 Subject: [PATCH 7/9] small fixes --- sdc/functions/numpy_like.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 1d52b46b1..1913a5748 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -425,20 +425,15 @@ def dropna_impl(arr, idx, name): result_index = numpy.empty(shape=length, dtype=dtype_idx) for i in prange(len(chunks)): chunk = chunks[i] - if i == 0: - new_start = 0 - new_stop = arr_len[0] - else: - for x in range(i): - new_start += arr_len[x] - new_stop = new_start + arr_len[i] + new_start = int(sum(arr_len[0:i])) + new_stop = new_start + arr_len[i] + current_pos = new_start for j in range(chunk.start, chunk.stop): - if new_start < new_stop: - if not isnan(arr[j]): - result_data[new_start] = arr[j] - result_index[new_start] = idx[j] - new_start += 1 + if not isnan(arr[j]): + result_data[current_pos] = arr[j] + result_index[current_pos] = idx[j] + current_pos += 1 return pandas.Series(result_data, result_index, name) From 2806c48a32af2b35386d049971db52b0e43cb31b Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Wed, 19 Feb 2020 09:28:15 +0300 Subject: [PATCH 8/9] new prange utils --- sdc/utilities/prange_utils.py | 54 +++++++++-------------------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index 961eba266..0f24d34ea 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -29,7 +29,7 @@ import sdc from typing import NamedTuple -from sdc.utilities.utils import sdc_overload +from sdc.utilities.utils import sdc_overload, sdc_register_jitable class Chunk(NamedTuple): @@ -37,57 +37,29 @@ class Chunk(NamedTuple): stop: int +@sdc_register_jitable def get_pool_size(): if sdc.config.config_use_parallel_overloads: return numba.config.NUMBA_NUM_THREADS - else: - return 1 + return 1 -@sdc_overload(get_pool_size) -def get_pool_size_overload(): - pool_size = get_pool_size() - def get_pool_size_impl(): - return pool_size - - return get_pool_size_impl - - -def get_chunks(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - - chunk_size = (size - 1)//pool_size + 1 - - Chunk = NamedTuple('start', 'stop') +@sdc_register_jitable +def get_chunks(size, pool_size): + pool_size = min(pool_size, size) + chunk_size = size // pool_size + overload_size = size % pool_size chunks = [] - for i in range(pool_size): - start = min(i*chunk_size, size) - stop = min((i + 1)*chunk_size, size) + start = i * chunk_size + min(i, overload_size) + stop = (i + 1) * chunk_size + min(i + 1, overload_size) chunks.append(Chunk(start, stop)) return chunks -@sdc_overload(get_chunks) -def get_chunks_overload(size, pool_size=0): - def get_chunks_impl(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - - chunk_size = (size - 1)//pool_size + 1 - - chunks = [] - - for i in range(pool_size): - start = min(i*chunk_size, size) - stop = min((i + 1)*chunk_size, size) - chunk = Chunk(start, stop) - chunks.append(chunk) - - return chunks - - return get_chunks_impl +@sdc_register_jitable +def parallel_chunks(size): + return get_chunks(size, get_pool_size()) From ed7e4a5428e6f437b4e8ae16363eaa31e76ed30c Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Wed, 19 Feb 2020 10:24:53 +0300 Subject: [PATCH 9/9] fix prange utils call --- sdc/functions/numpy_like.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdc/functions/numpy_like.py b/sdc/functions/numpy_like.py index 6f4687ee3..266520c13 100644 --- a/sdc/functions/numpy_like.py +++ b/sdc/functions/numpy_like.py @@ -44,7 +44,7 @@ from sdc.utilities.sdc_typing_utils import TypeChecker from sdc.str_arr_ext import (StringArrayType, pre_alloc_string_array, get_utf8_size, str_arr_is_na) from sdc.utilities.utils import sdc_overload, sdc_register_jitable -from sdc.utilities.prange_utils import get_chunks +from sdc.utilities.prange_utils import parallel_chunks def astype(self, dtype): @@ -488,7 +488,7 @@ def dropna_overload(arr, idx, name): isnan = get_isnan(dtype) def dropna_impl(arr, idx, name): - chunks = get_chunks(len(arr)) + chunks = parallel_chunks(len(arr)) arr_len = numpy.empty(len(chunks), dtype=numpy.int64) length = 0