diff --git a/README.rst b/README.rst index b831059..3a3734d 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,7 @@ on top of `OpenTURNS `_, with its users as the target audience. Documentation is available `here `_. The module provides : +- An integrated progress bar for unit evaluation of a sample without parallelization. - A `Parallelizer` class that converts any `ot.Function `_ into a parallel wrapper using either diff --git a/otwrapy/_otwrapy.py b/otwrapy/_otwrapy.py index ad67579..816756c 100644 --- a/otwrapy/_otwrapy.py +++ b/otwrapy/_otwrapy.py @@ -12,6 +12,7 @@ import logging import openturns as ot import numpy as np +from tqdm import tqdm __author__ = "Felipe Aguirre Martinez" __copyright__ = "Copyright 2015-2019 Phimeca" @@ -338,6 +339,36 @@ def __exit__(self, type, value, traceback): shutil.rmtree(self.dirname) +def _exec_sample_serial(func, verbosity): + """Return a function that evaluate the sample and provide a progress bar. + + Parameters + ---------- + func : Function or callable + A callable python object, usually a function. The function should take + an input vector as argument and return an output vector. + verbosity : int + If value greater than 0, the progress bar is displayed. + + Returns + ------- + _exec_sample : Function or callable + The function with the progress bar. + """ + + def _exec_sample(X): + X = ot.Sample(X) + if verbosity > 0 and X.getSize() > 1: + Y = ot.Sample(0, func.getOutputDimension()) + for x in tqdm(X): + Y.add(func(x)) + else: + Y = func(X) + return ot.Sample(Y) + + return _exec_sample + + def _exec_sample_joblib(func, n_cpus, verbosity): """Return a function that executes a sample in parallel using joblib. @@ -508,15 +539,17 @@ class Parallelizer(ot.OpenTURNSPythonFunction): openturns wrapper to be distributed backend : string (Optional) - Whether to parallelize using 'ipyparallel', 'joblib', pathos, or - 'multiprocessing'. + Whether to parallelize using 'ipyparallel', 'joblib', 'pathos', 'multiprocessing' or + 'serial'. Serial backend means unit evaluation, with a progress bar if verbosity > 0. n_cpus : int (Optional) Number of CPUs on which the simulations will be distributed. Needed Only if using 'joblib', pathos or 'multiprocessing' as backend. + If n_cpus = 1, the behavior is the same as 'serial'. verbosity : int (Optional) - verbose parameter when using 'joblib' or 'dask'. Default is 10. + verbose parameter when using 'joblib', 'dask' or without parallelization. Default is 10. + For 'serial', if verbosity > 0, a progress bar is displayed using tqdm module. When 'dask' is used, 0 means no progress bar, whereas other value activate the progress bar. dask_args : dict (Optional) @@ -566,13 +599,13 @@ def __init__(self, wrapper, backend='multiprocessing', n_cpus=-1, verbosity=10, self.setInputDescription(self.wrapper.getInputDescription()) self.setOutputDescription(self.wrapper.getOutputDescription()) - assert backend in ['ipython', 'ipyparallel', + assert backend in ['serial', 'ipython', 'ipyparallel', 'multiprocessing', 'pathos', 'joblib', 'dask'], "Unknown backend" # This configures how to run samples on the model : - if self.n_cpus == 1: - self._exec_sample = self.wrapper + if backend == 'serial' or self.n_cpus == 1: + self._exec_sample = _exec_sample_serial(self.wrapper, verbosity) elif (backend == 'ipython') or (backend == 'ipyparallel'): # Check that ipyparallel is installed diff --git a/setup.py b/setup.py index a956d44..98a8d9f 100644 --- a/setup.py +++ b/setup.py @@ -25,11 +25,12 @@ name='otwrapy', version=version, packages=find_packages(), - extras_require = { + extras_require={ 'joblib': ["joblib>=0.9.3"], 'ipyparallel': ["ipyparallel>=5.0.1"], 'pathos': ["pathos>=0.2.0"], - 'dask': ["dask>=2021.01.0", "asyncssh"] + 'dask': ["dask>=2021.01.0", "asyncssh"], + 'tqdm': ["tqdm>=4.0.0"] }, author="Felipe Aguirre Martinez", author_email="aguirre@phimeca.com", @@ -37,8 +38,8 @@ long_description=long_description, setup_requires=['pytest-runner'], tests_require=['pytest'], - include_package_data = True, - package_data = {'otwrapy': ['examples/beam/*']}, + include_package_data=True, + package_data={'otwrapy': ['examples/beam/*']}, scripts=['otwrapy/examples/beam/beam_wrapper'], zip_safe=False ) diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index f2f15f6..c278b5f 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -17,13 +17,16 @@ def backendtest(backend): dask_args = {'scheduler': 'localhost', 'workers': {'localhost': n_cpu}} else: dask_args = None - model_parallel = otw.Parallelizer(model, backend=backend, dask_args=dask_args) + model_parallel = otw.Parallelizer(model, backend=backend, n_cpus=n_cpu, dask_args=dask_args) for size in sizes: X_sample = X_distribution.getSample(size) Y_ref = model(X_sample) Y_sample = ot.Sample(model_parallel(X_sample)) assert Y_ref == Y_sample, 'samples do not match' +def test_serial(): + backendtest('serial') + def test_joblib(): backendtest('joblib')