From 965c94bc9d6cb725b691f7220bb09b52979a5751 Mon Sep 17 00:00:00 2001 From: Eric Giguere Date: Tue, 31 Jan 2023 15:08:18 -0500 Subject: [PATCH 1/2] Remove example using parfor --- doc/QuTiP_tree_plot/qutip-structure.py | 2 +- doc/apidoc/functions.rst | 2 +- doc/guide/guide-parfor.rst | 66 +++++++------------------- qutip/ipynbtools.py | 4 +- qutip/solve/parallel.py | 2 +- 5 files changed, 22 insertions(+), 54 deletions(-) diff --git a/doc/QuTiP_tree_plot/qutip-structure.py b/doc/QuTiP_tree_plot/qutip-structure.py index 6ba99fcc79..cf18a9f171 100755 --- a/doc/QuTiP_tree_plot/qutip-structure.py +++ b/doc/QuTiP_tree_plot/qutip-structure.py @@ -62,7 +62,7 @@ ("#bf5730", { "fileio", "utilities", "ipynbtools", "sparse", "graph", "simdiag", "permute", "demos", "about", "parallel", "version", "testing", - "parfor", "hardware_info", "ui", "cite", "lattice", + "hardware_info", "ui", "cite", }), ] diff --git a/doc/apidoc/functions.rst b/doc/apidoc/functions.rst index acacc97c80..78a06941b5 100644 --- a/doc/apidoc/functions.rst +++ b/doc/apidoc/functions.rst @@ -316,7 +316,7 @@ IPython Notebook Tools ---------------------- .. automodule:: qutip.ipynbtools - :members: parfor, parallel_map, version_table + :members: parallel_map, version_table .. _functions-misc: diff --git a/doc/guide/guide-parfor.rst b/doc/guide/guide-parfor.rst index abdeeba867..7cb6412b71 100644 --- a/doc/guide/guide-parfor.rst +++ b/doc/guide/guide-parfor.rst @@ -7,30 +7,14 @@ Parallel computation Parallel map and parallel for-loop ---------------------------------- -Often one is interested in the output of a given function as a single-parameter is varied. For instance, we can calculate the steady-state response of our system as the driving frequency is varied. In cases such as this, where each iteration is independent of the others, we can speedup the calculation by performing the iterations in parallel. In QuTiP, parallel computations may be performed using the :func:`qutip.parallel.parallel_map` function or the :func:`qutip.parallel.parfor` (parallel-for-loop) function. +Often one is interested in the output of a given function as a single-parameter is varied. +For instance, we can calculate the steady-state response of our system as the driving frequency is varied. +In cases such as this, where each iteration is independent of the others, we can speedup the calculation by performing the iterations in parallel. +In QuTiP, parallel computations may be performed using the :func:`qutip.solver.parallel.parallel_map` function. -To use the these functions we need to define a function of one or more variables, and the range over which one of these variables are to be evaluated. For example: +To use the this function we need to define a function of one or more variables, and the range over which one of these variables are to be evaluated. For example: -.. doctest:: - :skipif: not os_nt - :options: +NORMALIZE_WHITESPACE - - >>> def func1(x): return x, x**2, x**3 - - >>> a, b, c = parfor(func1, range(10)) - - >>> print(a) - [0 1 2 3 4 5 6 7 8 9] - - >>> print(b) - [ 0 1 4 9 16 25 36 49 64 81] - - >>> print(c) - [ 0 1 8 27 64 125 216 343 512 729] - -or - .. doctest:: :skipif: not os_nt :options: +NORMALIZE_WHITESPACE @@ -48,25 +32,11 @@ or >>> print(result_array[:, 2]) # == c [ 0 1 8 27 64 125 216 343 512 729] - -Note that the return values are arranged differently for the :func:`qutip.parallel.parallel_map` and the :func:`qutip.parallel.parfor` functions, as illustrated below. In particular, the return value of :func:`qutip.parallel.parallel_map` is not enforced to be NumPy arrays, which can avoid unnecessary copying if all that is needed is to iterate over the resulting list: - - -.. doctest:: - :skipif: not os_nt - :options: +NORMALIZE_WHITESPACE - - >>> result = parfor(func1, range(5)) - - >>> print(result) - [array([0, 1, 2, 3, 4]), array([ 0, 1, 4, 9, 16]), array([ 0, 1, 8, 27, 64])] - - >>> result = parallel_map(func1, range(5)) - >>> print(result) [(0, 0, 0), (1, 1, 1), (2, 4, 8), (3, 9, 27), (4, 16, 64)] -The :func:`qutip.parallel.parallel_map` and :func:`qutip.parallel.parfor` functions are not limited to just numbers, but also works for a variety of outputs: + +The :func:`qutip.solver.parallel.parallel_map` function is not limited to just numbers, but also works for a variety of outputs: .. doctest:: :skipif: not os_nt @@ -74,12 +44,12 @@ The :func:`qutip.parallel.parallel_map` and :func:`qutip.parallel.parfor` functi >>> def func2(x): return x, Qobj(x), 'a' * x - >>> a, b, c = parfor(func2, range(5)) + >>> results = parallel_map(func2, range(5)) - >>> print(a) + >>> print([result[0] for result in results]) [0 1 2 3 4] - >>> print(b) + >>> print([result[1] for result in results]) [Quantum object: dims = [[1], [1]], shape = (1, 1), type = bra Qobj data = [[0.]] @@ -96,12 +66,11 @@ The :func:`qutip.parallel.parallel_map` and :func:`qutip.parallel.parfor` functi Qobj data = [[4.]]] - >>>print(c) + >>>print([result[2] for result in results]) ['' 'a' 'aa' 'aaa' 'aaaa'] -One can also define functions with **multiple** input arguments and even keyword arguments. Here the :func:`qutip.parallel.parallel_map` and :func:`qutip.parallel.parfor` functions behaves differently: -While :func:`qutip.parallel.parallel_map` only iterate over the values `arguments`, the :func:`qutip.parallel.parfor` function simultaneously iterates over all arguments: +One can also define functions with **multiple** input arguments and keyword arguments. .. doctest:: :skipif: not os_nt @@ -109,17 +78,14 @@ While :func:`qutip.parallel.parallel_map` only iterate over the values `argument >>> def sum_diff(x, y, z=0): return x + y, x - y, z - >>> parfor(sum_diff, [1, 2, 3], [4, 5, 6], z=5.0) - [array([5, 7, 9]), array([-3, -3, -3]), array([5., 5., 5.])] - >>> parallel_map(sum_diff, [1, 2, 3], task_args=(np.array([4, 5, 6]),), task_kwargs=dict(z=5.0)) [(array([5, 6, 7]), array([-3, -4, -5]), 5.0), (array([6, 7, 8]), array([-2, -3, -4]), 5.0), (array([7, 8, 9]), array([-1, -2, -3]), 5.0)] -Note that the keyword arguments can be anything you like, but the keyword values are **not** iterated over. The keyword argument *num_cpus* is reserved as it sets the number of CPU's used by parfor. By default, this value is set to the total number of physical processors on your system. You can change this number to a lower value, however setting it higher than the number of CPU's will cause a drop in performance. In :func:`qutip.parallel.parallel_map`, keyword arguments to the task function are specified using `task_kwargs` argument, so there is no special reserved keyword arguments. -The :func:`qutip.parallel.parallel_map` function also supports progressbar, using the keyword argument `progress_bar` which can be set to `True` or to an instance of :class:`qutip.ui.progressbar.BaseProgressBar`. There is a function called :func:`qutip.parallel.serial_map` that works as a non-parallel drop-in replacement for :func:`qutip.parallel.parallel_map`, which allows easy switching between serial and parallel computation. +The :func:`qutip.solver.parallel.parallel_map` function supports progressbar by setting the keyword argument `progress_bar` to `True`. +The number of cpu used can also be controlled using the `map_kw` keyword, per default, all available cpus are used. .. doctest:: :options: +SKIP @@ -128,7 +94,7 @@ The :func:`qutip.parallel.parallel_map` function also supports progressbar, usin >>> def func(x): time.sleep(1) - >>> result = parallel_map(func, range(50), progress_bar=True) + >>> result = parallel_map(func, range(50), progress_bar=True, map_kw={"num_cpus": 2}) 10.0%. Run time: 3.10s. Est. time left: 00:00:00:27 20.0%. Run time: 5.11s. Est. time left: 00:00:00:20 @@ -142,6 +108,8 @@ The :func:`qutip.parallel.parallel_map` function also supports progressbar, usin 100.0%. Run time: 25.15s. Est. time left: 00:00:00:00 Total run time: 28.91s +There is a function called :func:`qutip.solver.parallel.serial_map` that works as a non-parallel drop-in replacement for :func:`qutip.solver.parallel.parallel_map`, which allows easy switching between serial and parallel computation. +Qutip as also the function :func:`qutip.solver.parallel.loky_map` as another drop-in replacement. It use the `loky` module instead of `multiprocessing` to run in parallel. Parallel processing is useful for repeated tasks such as generating plots corresponding to the dynamical evolution of your system, or simultaneously simulating different parameter configurations. diff --git a/qutip/ipynbtools.py b/qutip/ipynbtools.py index afe269dc3b..e2d906b637 100644 --- a/qutip/ipynbtools.py +++ b/qutip/ipynbtools.py @@ -10,14 +10,14 @@ if IPython.version_info[0] >= 4: try: from ipyparallel import Client - __all__ = ['version_table', 'parfor', 'plot_animation', + __all__ = ['version_table', 'plot_animation', 'parallel_map', 'HTMLProgressBar'] except: __all__ = ['version_table', 'plot_animation', 'HTMLProgressBar'] else: try: from IPython.parallel import Client - __all__ = ['version_table', 'parfor', 'plot_animation', + __all__ = ['version_table', 'plot_animation', 'parallel_map', 'HTMLProgressBar'] except: __all__ = ['version_table', 'plot_animation', 'HTMLProgressBar'] diff --git a/qutip/solve/parallel.py b/qutip/solve/parallel.py index 9edd77c7f1..36e35b8531 100644 --- a/qutip/solve/parallel.py +++ b/qutip/solve/parallel.py @@ -2,7 +2,7 @@ This function provides functions for parallel execution of loops and function mappings, using the builtin Python module multiprocessing. """ -__all__ = ['parfor', 'parallel_map', 'serial_map'] +__all__ = ['parallel_map', 'serial_map'] from scipy import array import multiprocessing From 4ff75f2fd338dd137cd6aaa26196ac1434f5ef72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Gigu=C3=A8re?= Date: Wed, 1 Feb 2023 09:59:25 -0500 Subject: [PATCH 2/2] Update doc/guide/guide-parfor.rst Co-authored-by: Simon Cross --- doc/guide/guide-parfor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/guide/guide-parfor.rst b/doc/guide/guide-parfor.rst index 7cb6412b71..4491cf30af 100644 --- a/doc/guide/guide-parfor.rst +++ b/doc/guide/guide-parfor.rst @@ -109,7 +109,7 @@ The number of cpu used can also be controlled using the `map_kw` keyword, per de Total run time: 28.91s There is a function called :func:`qutip.solver.parallel.serial_map` that works as a non-parallel drop-in replacement for :func:`qutip.solver.parallel.parallel_map`, which allows easy switching between serial and parallel computation. -Qutip as also the function :func:`qutip.solver.parallel.loky_map` as another drop-in replacement. It use the `loky` module instead of `multiprocessing` to run in parallel. +Qutip also has the function :func:`qutip.solver.parallel.loky_map` as another drop-in replacement. It use the `loky` module instead of `multiprocessing` to run in parallel. Parallel processing is useful for repeated tasks such as generating plots corresponding to the dynamical evolution of your system, or simultaneously simulating different parameter configurations.