From 25182ca9fde5a7f531b517a5b0c32bdeaad0ef11 Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Tue, 18 Feb 2020 15:26:40 +0300 Subject: [PATCH 1/7] Add tests for get_chunks and change formula for get_chunks --- sdc/tests/__init__.py | 1 + sdc/tests/test_prange_utils.py | 82 ++++++++++++++++++++++++++++++++++ sdc/utilities/prange_utils.py | 32 +++---------- 3 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 sdc/tests/test_prange_utils.py diff --git a/sdc/tests/__init__.py b/sdc/tests/__init__.py index def2a3a4a..41236156a 100644 --- a/sdc/tests/__init__.py +++ b/sdc/tests/__init__.py @@ -45,6 +45,7 @@ from sdc.tests.test_hpat_jit import * from sdc.tests.test_sdc_numpy import * +from sdc.tests.test_prange_utils import * # performance tests import sdc.tests.tests_perf diff --git a/sdc/tests/test_prange_utils.py b/sdc/tests/test_prange_utils.py new file mode 100644 index 000000000..162b70180 --- /dev/null +++ b/sdc/tests/test_prange_utils.py @@ -0,0 +1,82 @@ +# ***************************************************************************** +# Copyright (c) 2019, Intel Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +from sdc.tests.test_base import TestCase + +from sdc.utilities.prange_utils import get_chunks, Chunk + + +class ChunkTest(TestCase): + + def _get_chunks_date(self): + yield (5, 5), [ + Chunk(start=0, stop=1), + Chunk(start=1, stop=2), + Chunk(start=2, stop=3), + Chunk(start=3, stop=4), + Chunk(start=4, stop=5), + ] + yield (5, 6), [ + Chunk(start=0, stop=1), + Chunk(start=1, stop=2), + Chunk(start=2, stop=3), + Chunk(start=3, stop=4), + Chunk(start=4, stop=5), + ] + yield (9, 5), [ + Chunk(start=0, stop=2), + Chunk(start=2, stop=4), + Chunk(start=4, stop=6), + Chunk(start=6, stop=8), + Chunk(start=8, stop=9), + ] + yield (9, 4), [ + Chunk(start=0, stop=3), + Chunk(start=3, stop=5), + Chunk(start=5, stop=7), + Chunk(start=7, stop=9), + ] + yield (9, 2), [ + Chunk(start=0, stop=5), + Chunk(start=5, stop=9), + ] + yield (9, 3), [ + Chunk(start=0, stop=3), + Chunk(start=3, stop=6), + Chunk(start=6, stop=9), + ] + + def _check_get_chunks(self, args, expected_chunks): + pyfunc = get_chunks + cfunc = self.jit(pyfunc) + + self.assertEqual(pyfunc(*args), expected_chunks) + self.assertEqual(cfunc(*args), expected_chunks) + + def test_get_chunks(self): + for args, expected_chunks in self._get_chunks_date(): + with self.subTest(args=args): + self.assertEqual(get_chunks(*args), expected_chunks) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index 2380bb513..866b55b91 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -29,7 +29,7 @@ import sdc from typing import NamedTuple -from sdc.utilities.utils import sdc_overload +from sdc.utilities.utils import sdc_overload, sdc_register_jitable class Chunk(NamedTuple): @@ -53,37 +53,19 @@ def get_pool_size_impl(): return get_pool_size_impl - +@sdc_register_jitable def get_chunks(size, pool_size=0): if pool_size == 0: pool_size = get_pool_size() - chunk_size = (size - 1) // pool_size + 1 + pool_size = min(pool_size, size) + chunk_size = size // pool_size + overload_size = size % pool_size chunks = [] for i in range(pool_size): - start = min(i * chunk_size, size) - stop = min((i + 1) * chunk_size, size) + start = i * chunk_size + min(i, overload_size) + stop = (i + 1) * chunk_size + min(i + 1, overload_size) chunks.append(Chunk(start, stop)) return chunks - - -@sdc_overload(get_chunks) -def get_chunks_overload(size, pool_size=0): - def get_chunks_impl(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - - chunk_size = (size - 1) // pool_size + 1 - - chunks = [] - for i in range(pool_size): - start = min(i * chunk_size, size) - stop = min((i + 1) * chunk_size, size) - chunk = Chunk(start, stop) - chunks.append(chunk) - - return chunks - - return get_chunks_impl From 3814371a78c99d6aef9ee98ab9c9a298c329bdac Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Tue, 18 Feb 2020 15:50:39 +0300 Subject: [PATCH 2/7] fix pep8 --- sdc/utilities/prange_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index 866b55b91..d43ff49ca 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -53,6 +53,7 @@ def get_pool_size_impl(): return get_pool_size_impl + @sdc_register_jitable def get_chunks(size, pool_size=0): if pool_size == 0: From 6aa288a743a3dcc1ea21c09a55af140c793c55dd Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Tue, 18 Feb 2020 18:16:24 +0300 Subject: [PATCH 3/7] Pass pool_size explicitly --- sdc/utilities/prange_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index d43ff49ca..17c8acbad 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -55,10 +55,7 @@ def get_pool_size_impl(): @sdc_register_jitable -def get_chunks(size, pool_size=0): - if pool_size == 0: - pool_size = get_pool_size() - +def get_chunks(size, pool_size): pool_size = min(pool_size, size) chunk_size = size // pool_size overload_size = size % pool_size From 48c0f92e2f6f067313a3de14f363922675078aca Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Tue, 18 Feb 2020 18:35:44 +0300 Subject: [PATCH 4/7] Fix year --- sdc/tests/test_prange_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdc/tests/test_prange_utils.py b/sdc/tests/test_prange_utils.py index 162b70180..1f1938946 100644 --- a/sdc/tests/test_prange_utils.py +++ b/sdc/tests/test_prange_utils.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2019, Intel Corporation All rights reserved. +# Copyright (c) 2020, Intel Corporation All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: From 987ccf1f0742e0a4d21af94b64972080cb089830 Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Tue, 18 Feb 2020 18:37:53 +0300 Subject: [PATCH 5/7] Use _check_get_chunks() --- sdc/tests/test_prange_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdc/tests/test_prange_utils.py b/sdc/tests/test_prange_utils.py index 1f1938946..231e985cc 100644 --- a/sdc/tests/test_prange_utils.py +++ b/sdc/tests/test_prange_utils.py @@ -79,4 +79,4 @@ def _check_get_chunks(self, args, expected_chunks): def test_get_chunks(self): for args, expected_chunks in self._get_chunks_date(): with self.subTest(args=args): - self.assertEqual(get_chunks(*args), expected_chunks) + self._check_get_chunks(args, expected_chunks) From b8bacc4c0ba3aca90813d769c9da9c0539a2510b Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Tue, 18 Feb 2020 18:38:55 +0300 Subject: [PATCH 6/7] Rename date with data --- sdc/tests/test_prange_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdc/tests/test_prange_utils.py b/sdc/tests/test_prange_utils.py index 231e985cc..5ae0f629b 100644 --- a/sdc/tests/test_prange_utils.py +++ b/sdc/tests/test_prange_utils.py @@ -31,7 +31,7 @@ class ChunkTest(TestCase): - def _get_chunks_date(self): + def _get_chunks_data(self): yield (5, 5), [ Chunk(start=0, stop=1), Chunk(start=1, stop=2), @@ -77,6 +77,6 @@ def _check_get_chunks(self, args, expected_chunks): self.assertEqual(cfunc(*args), expected_chunks) def test_get_chunks(self): - for args, expected_chunks in self._get_chunks_date(): + for args, expected_chunks in self._get_chunks_data(): with self.subTest(args=args): self._check_get_chunks(args, expected_chunks) From fd48d168b52ac4d6cb349b14550ee1036248a359 Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Tue, 18 Feb 2020 19:00:04 +0300 Subject: [PATCH 7/7] Add parallel_chunks() --- sdc/utilities/prange_utils.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/sdc/utilities/prange_utils.py b/sdc/utilities/prange_utils.py index 17c8acbad..0f24d34ea 100644 --- a/sdc/utilities/prange_utils.py +++ b/sdc/utilities/prange_utils.py @@ -37,6 +37,7 @@ class Chunk(NamedTuple): stop: int +@sdc_register_jitable def get_pool_size(): if sdc.config.config_use_parallel_overloads: return numba.config.NUMBA_NUM_THREADS @@ -44,16 +45,6 @@ def get_pool_size(): return 1 -@sdc_overload(get_pool_size) -def get_pool_size_overload(): - pool_size = get_pool_size() - - def get_pool_size_impl(): - return pool_size - - return get_pool_size_impl - - @sdc_register_jitable def get_chunks(size, pool_size): pool_size = min(pool_size, size) @@ -67,3 +58,8 @@ def get_chunks(size, pool_size): chunks.append(Chunk(start, stop)) return chunks + + +@sdc_register_jitable +def parallel_chunks(size): + return get_chunks(size, get_pool_size())