From 13e7f7e2b98685daed5f9e1ac09671e383df2a49 Mon Sep 17 00:00:00 2001 From: Eric Prestat Date: Sat, 2 Sep 2023 12:30:21 +0100 Subject: [PATCH] Rename `max_workers` to `num_workers` to be consistent with dask --- doc/user_guide/signal/generic_tools.rst | 26 ------------- doc/user_guide/signal2d.rst | 4 +- hyperspy/_signals/complex_signal.py | 8 ++-- hyperspy/_signals/eels.py | 11 ++---- hyperspy/_signals/signal1d.py | 50 ++++++++++++------------- hyperspy/_signals/signal2d.py | 14 +++---- hyperspy/docstrings/signal.py | 14 ++----- hyperspy/signal.py | 14 +++---- hyperspy/tests/signals/test_cuda.py | 6 +-- upcoming_changes/3198.api.rst | 1 + 10 files changed, 57 insertions(+), 91 deletions(-) diff --git a/doc/user_guide/signal/generic_tools.rst b/doc/user_guide/signal/generic_tools.rst index d1b403a659..21350fc24b 100644 --- a/doc/user_guide/signal/generic_tools.rst +++ b/doc/user_guide/signal/generic_tools.rst @@ -345,32 +345,6 @@ data (default, ``True``) or storing it to a new signal (``False``). (512, 512) (724, 724) -.. _parallel-map-label: - -The execution can be sped up by passing ``parallel`` keyword to the -:py:meth:`~.api.signals.BaseSignal.map` method: - -.. code-block:: python - - >>> import time - >>> def slow_func(data): - ... time.sleep(1.) - ... return data + 1 - >>> s = hs.signals.Signal1D(np.arange(40).reshape((20, 2))) - >>> s - - >>> s.map(slow_func, parallel=False) - 100%|██████████████████████████████████████| 20/20 [00:20<00:00, 1.00s/it] - >>> # some operations will be done in parallel: - >>> s.map(slow_func, parallel=True) - 100%|██████████████████████████████████████| 20/20 [00:02<00:00, 6.73it/s] - -.. note:: - - HyperSpy implements *thread-based* parallelism for the :py:meth:`~.api.signals.BaseSignal.map` - method. You can control the number of threads that are created by passing an integer value - to the ``max_workers`` keyword argument. By default, it will use ``min(32, os.cpu_count())``. - .. versionadded:: 1.4 Iterating over signal using a parameter with no navigation dimension. diff --git a/doc/user_guide/signal2d.rst b/doc/user_guide/signal2d.rst index 47f69b947f..0e7a22ec0e 100644 --- a/doc/user_guide/signal2d.rst +++ b/doc/user_guide/signal2d.rst @@ -53,7 +53,7 @@ Sub-pixel accuracy can be achieved in two ways: If you have a large stack of images, the image alignment is automatically done in parallel. -You can control the number of threads used with the ``max_workers`` argument. Or by adjusting +You can control the number of threads used with the ``num_workers`` argument. Or by adjusting the scheduler of the :ref:`dask ` backend. .. code-block:: python @@ -62,7 +62,7 @@ the scheduler of the :ref:`dask ` backend. >>> shifts = s.estimate_shift2D() # Align images in parallel using 4 threads - >>> s.align2D(shifts=shifts, max_workers=4) + >>> s.align2D(shifts=shifts, num_workers=4) .. _signal2D.crop: diff --git a/hyperspy/_signals/complex_signal.py b/hyperspy/_signals/complex_signal.py index 02b35d48d1..971d9e1c32 100644 --- a/hyperspy/_signals/complex_signal.py +++ b/hyperspy/_signals/complex_signal.py @@ -31,7 +31,7 @@ ) from hyperspy.docstrings.signal import ( SHOW_PROGRESSBAR_ARG, - MAX_WORKERS_ARG, + NUM_WORKERS_ARG, LAZYSIGNAL_DOC, ) from hyperspy.misc.utils import parse_quantity @@ -167,7 +167,7 @@ def change_dtype(self, dtype): 'Complex data can only be converted into other complex dtypes!') def unwrapped_phase(self, wrap_around=False, seed=None, - show_progressbar=None, max_workers=None): + show_progressbar=None, num_workers=None): """Return the unwrapped phase as an appropriate HyperSpy signal. Parameters @@ -202,11 +202,11 @@ def unwrapped_phase(self, wrap_around=False, seed=None, phase = self.phase phase.map(unwrap_phase, wrap_around=wrap_around, seed=seed, show_progressbar=show_progressbar, ragged=False, - max_workers=max_workers) + num_workers=num_workers) phase.metadata.General.title = f'unwrapped {phase.metadata.General.title}' return phase # Now unwrapped! - unwrapped_phase.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG) + unwrapped_phase.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG) def __call__(self, axes_manager=None, power_spectrum=False, fft_shift=False, as_numpy=None): diff --git a/hyperspy/_signals/eels.py b/hyperspy/_signals/eels.py index 9e9105ee86..594734e41b 100644 --- a/hyperspy/_signals/eels.py +++ b/hyperspy/_signals/eels.py @@ -46,8 +46,7 @@ ) from hyperspy.docstrings.signal import ( SHOW_PROGRESSBAR_ARG, - PARALLEL_ARG, - MAX_WORKERS_ARG, + NUM_WORKERS_ARG, SIGNAL_MASK_ARG, NAVIGATION_MASK_ARG, LAZYSIGNAL_DOC, @@ -1032,7 +1031,7 @@ def fourier_ratio_deconvolution(self, ll, def richardson_lucy_deconvolution(self, psf, iterations=15, show_progressbar=None, - parallel=None, max_workers=None): + num_workers=None): """1D Richardson-Lucy Poissonian deconvolution of the spectrum by the given kernel. @@ -1047,7 +1046,6 @@ def richardson_lucy_deconvolution(self, psf, iterations=15, increasing the value will increase the noise amplification. %s %s - %s Raises ------ @@ -1085,8 +1083,7 @@ def deconv_function(signal, kernel=None, ds = self.map(deconv_function, kernel=psf, iterations=iterations, psf_size=psf_size, show_progressbar=show_progressbar, - parallel=parallel, max_workers=max_workers, - ragged=False, inplace=False) + num_workers=num_workers, ragged=False, inplace=False) ds.metadata.General.title += ( ' after Richardson-Lucy deconvolution %i iterations' % @@ -1096,7 +1093,7 @@ def deconv_function(signal, kernel=None, '_after_R-L_deconvolution_%iiter' % iterations) return ds - richardson_lucy_deconvolution.__doc__ %= (SHOW_PROGRESSBAR_ARG, PARALLEL_ARG, MAX_WORKERS_ARG) + richardson_lucy_deconvolution.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG) def _are_microscope_parameters_missing(self, ignore_parameters=[]): """ diff --git a/hyperspy/_signals/signal1d.py b/hyperspy/_signals/signal1d.py index 5a6775cc2f..367c8ee1f7 100644 --- a/hyperspy/_signals/signal1d.py +++ b/hyperspy/_signals/signal1d.py @@ -50,7 +50,7 @@ from hyperspy.docstrings.signal1d import CROP_PARAMETER_DOC, SPIKES_REMOVAL_TOOL_DOCSTRING from hyperspy.docstrings.signal import ( SHOW_PROGRESSBAR_ARG, - MAX_WORKERS_ARG, + NUM_WORKERS_ARG, SIGNAL_MASK_ARG, NAVIGATION_MASK_ARG, LAZYSIGNAL_DOC, @@ -393,7 +393,7 @@ def shift1D( expand=False, fill_value=np.nan, show_progressbar=None, - max_workers=None, + num_workers=None, ): """Shift the data in place over the signal axis by the amount specified by an array. @@ -498,7 +498,7 @@ def shift1D( fill_value=fill_value, kind=interpolation_method, show_progressbar=show_progressbar, - max_workers=max_workers, + num_workers=num_workers, ragged=False) if crop and not expand: @@ -509,7 +509,7 @@ def shift1D( ihigh) self.events.data_changed.trigger(obj=self) - shift1D.__doc__ %= (CROP_PARAMETER_DOC, SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG) + shift1D.__doc__ %= (CROP_PARAMETER_DOC, SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG) def interpolate_in_between( self, @@ -517,7 +517,7 @@ def interpolate_in_between( end, delta=3, show_progressbar=None, - max_workers=None, + num_workers=None, **kwargs, ): """Replace the data in a given range by interpolation. @@ -572,11 +572,11 @@ def interpolating_function(dat): interpolating_function, ragged=False, show_progressbar=show_progressbar, - max_workers=max_workers, + num_workers=num_workers, ) self.events.data_changed.trigger(obj=self) - interpolate_in_between.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG) + interpolate_in_between.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG) def estimate_shift1D( self, @@ -588,7 +588,7 @@ def estimate_shift1D( number_of_interpolation_points=5, mask=None, show_progressbar=None, - max_workers=None, + num_workers=None, ): """Estimate the shifts in the current signal axis using cross-correlation. @@ -669,7 +669,7 @@ def estimate_shift1D( ragged=False, inplace=False, show_progressbar=show_progressbar, - max_workers=max_workers, + num_workers=num_workers, ) shift_array = shift_signal.data if max_shift is not None: @@ -686,7 +686,7 @@ def estimate_shift1D( shift_array = shift_array.compute() return shift_array - estimate_shift1D.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG) + estimate_shift1D.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG) def align1D(self, start=None, @@ -832,7 +832,7 @@ def smooth_savitzky_golay( polynomial_order=None, window_length=None, differential_order=0, - max_workers=None, + num_workers=None, display=True, toolkit=None, ): @@ -876,7 +876,7 @@ def smooth_savitzky_golay( axis = self.axes_manager.signal_axes[0] self.map(savgol_filter, window_length=window_length, polyorder=polynomial_order, deriv=differential_order, - delta=axis.scale, ragged=False, max_workers=max_workers) + delta=axis.scale, ragged=False, num_workers=num_workers) else: # Interactive mode smoother = SmoothingSavitzkyGolay(self) @@ -887,14 +887,14 @@ def smooth_savitzky_golay( smoother.window_length = window_length return smoother.gui(display=display, toolkit=toolkit) - smooth_savitzky_golay.__doc__ %= (MAX_WORKERS_ARG, DISPLAY_DT, TOOLKIT_DT) + smooth_savitzky_golay.__doc__ %= (NUM_WORKERS_ARG, DISPLAY_DT, TOOLKIT_DT) def smooth_lowess( self, smoothing_parameter=None, number_of_iterations=None, show_progressbar=None, - max_workers=None, + num_workers=None, display=True, toolkit=None, ): @@ -937,14 +937,14 @@ def smooth_lowess( n_iter=number_of_iterations, show_progressbar=show_progressbar, ragged=False, - max_workers=max_workers) - smooth_lowess.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG, DISPLAY_DT, TOOLKIT_DT) + num_workers=num_workers) + smooth_lowess.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG, DISPLAY_DT, TOOLKIT_DT) def smooth_tv( self, smoothing_parameter=None, show_progressbar=None, - max_workers=None, + num_workers=None, display=True, toolkit=None, ): @@ -980,9 +980,9 @@ def smooth_tv( self.map(_tv_denoise_1d, weight=smoothing_parameter, ragged=False, show_progressbar=show_progressbar, - max_workers=max_workers) + num_workers=num_workers) - smooth_tv.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG, DISPLAY_DT, TOOLKIT_DT) + smooth_tv.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG, DISPLAY_DT, TOOLKIT_DT) def filter_butterworth(self, cutoff_frequency_ratio=None, @@ -1349,7 +1349,7 @@ def find_peaks1D_ohaver(self, xdim=None, medfilt_radius=5, maxpeakn=30000, peakgroup=10, - max_workers=None): + num_workers=None): """Find positive peaks along a 1D Signal. It detects peaks by looking for downward zero-crossings in the first derivative that exceed 'slope_thresh'. @@ -1412,12 +1412,12 @@ def find_peaks1D_ohaver(self, xdim=None, peakgroup=peakgroup, subchannel=subchannel, ragged=True, - max_workers=max_workers, + num_workers=num_workers, inplace=False, lazy_output=False) return peaks.data - find_peaks1D_ohaver.__doc__ %= ( MAX_WORKERS_ARG) + find_peaks1D_ohaver.__doc__ %= ( NUM_WORKERS_ARG) def estimate_peak_width( self, @@ -1425,7 +1425,7 @@ def estimate_peak_width( window=None, return_interval=False, show_progressbar=None, - max_workers=None, + num_workers=None, ): """Estimate the width of the highest intensity of peak of the spectra at a given fraction of its maximum. @@ -1499,7 +1499,7 @@ def estimating_function(spectrum, ragged=False, inplace=False, show_progressbar=show_progressbar, - max_workers=max_workers, + num_workers=num_workers, ) left, right = both.T.split() width = right - left @@ -1530,7 +1530,7 @@ def estimating_function(spectrum, else: return width - estimate_peak_width.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG) + estimate_peak_width.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG) def plot(self, navigator="auto", diff --git a/hyperspy/_signals/signal2d.py b/hyperspy/_signals/signal2d.py index 89db9716ec..25a63a6c01 100644 --- a/hyperspy/_signals/signal2d.py +++ b/hyperspy/_signals/signal2d.py @@ -46,7 +46,7 @@ PLOT2D_KWARGS_DOCSTRING) from hyperspy.docstrings.signal import ( SHOW_PROGRESSBAR_ARG, - MAX_WORKERS_ARG, + NUM_WORKERS_ARG, LAZYSIGNAL_DOC, ) from hyperspy.ui_registry import DISPLAY_DT, TOOLKIT_DT @@ -611,7 +611,7 @@ def align2D( expand=False, interpolation_order=1, show_progressbar=None, - max_workers=None, + num_workers=None, **kwargs, ): """Align the images in-place using :py:func:`scipy.ndimage.shift`. @@ -738,7 +738,7 @@ def align2D( shift_image, shift=signal_shifts, show_progressbar=show_progressbar, - max_workers=max_workers, + num_workers=num_workers, ragged=False, inplace=True, fill_value=fill_value, @@ -772,7 +772,7 @@ def align2D( if return_shifts: return shifts - align2D.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG) + align2D.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG) def calibrate( self, @@ -923,7 +923,7 @@ def add_ramp(self, ramp_x, ramp_y, offset=0): def find_peaks(self, method='local_max', interactive=True, current_index=False, show_progressbar=None, - max_workers=None, display=True, toolkit=None, + num_workers=None, display=True, toolkit=None, get_intensity=False, **kwargs): """Find peaks in a 2D signal. @@ -1042,12 +1042,12 @@ def find_peaks(self, method='local_max', interactive=True, else: peaks = self.map(method_func, show_progressbar=show_progressbar, inplace=False, ragged=True, - max_workers=max_workers, **kwargs) + num_workers=num_workers, **kwargs) peaks.metadata.add_node("Peaks") # add information about the signal Axes peaks.metadata.Peaks.signal_axes = deepcopy(self.axes_manager.signal_axes) return peaks - find_peaks.__doc__ %= (SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG, + find_peaks.__doc__ %= (SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG, DISPLAY_DT, TOOLKIT_DT) diff --git a/hyperspy/docstrings/signal.py b/hyperspy/docstrings/signal.py index e6308c9507..61b9db57d6 100644 --- a/hyperspy/docstrings/signal.py +++ b/hyperspy/docstrings/signal.py @@ -67,12 +67,6 @@ If ``True``, display a progress bar. If ``None``, the default from the preferences settings is used.""" -PARALLEL_ARG = \ - """parallel : None or bool - If ``True``, perform computation in parallel using multithreading. If - ``None``, the default from the preferences settings is used. The number - of threads is controlled by the ``max_workers`` argument.""" - LAZY_OUTPUT_ARG = \ """lazy_output : None or bool If ``True``, the output will be returned as a lazy signal. This means @@ -83,10 +77,10 @@ If ``None`` the output will be lazy if the input signal is lazy, and non-lazy if the input signal is non-lazy.""" -MAX_WORKERS_ARG = \ - """max_workers : None or int - Maximum number of threads used when ``parallel=True``. If None, defaults - to ``min(32, os.cpu_count())``.""" +NUM_WORKERS_ARG = \ + """num_workers : None or int + Number of worker used by dask. If None, default + to dask default value.""" CLUSTER_SIGNALS_ARG = \ """signal : {"mean", "sum", "centroid"}, optional diff --git a/hyperspy/signal.py b/hyperspy/signal.py index 87819bd7c6..a6411788cd 100644 --- a/hyperspy/signal.py +++ b/hyperspy/signal.py @@ -69,7 +69,7 @@ from hyperspy.misc.utils import _get_block_pattern from hyperspy.docstrings.signal import ( ONE_AXIS_PARAMETER, MANY_AXIS_PARAMETER, OUT_ARG, NAN_FUNC, OPTIMIZE_ARG, - RECHUNK_ARG, SHOW_PROGRESSBAR_ARG, MAX_WORKERS_ARG, + RECHUNK_ARG, SHOW_PROGRESSBAR_ARG, NUM_WORKERS_ARG, CLUSTER_SIGNALS_ARG, HISTOGRAM_BIN_ARGS, HISTOGRAM_MAX_BIN_ARGS, LAZY_OUTPUT_ARG) from hyperspy.docstrings.plot import (BASE_PLOT_DOCSTRING, PLOT1D_DOCSTRING, BASE_PLOT_DOCSTRING_PARAMETERS, @@ -4732,7 +4732,7 @@ def map( self, function, show_progressbar=None, - max_workers=None, + num_workers=None, inplace=True, ragged=None, navigation_chunks=None, @@ -4960,7 +4960,7 @@ def map( ragged=ragged, inplace=inplace, lazy_output=lazy_output, - max_workers=max_workers, + num_workers=num_workers, output_dtype=output_dtype, output_signal_size=output_signal_size, navigation_chunks=navigation_chunks, @@ -4971,7 +4971,7 @@ def map( else: self.events.data_changed.trigger(obj=self) - map.__doc__ %= (SHOW_PROGRESSBAR_ARG, LAZY_OUTPUT_ARG, MAX_WORKERS_ARG) + map.__doc__ %= (SHOW_PROGRESSBAR_ARG, LAZY_OUTPUT_ARG, NUM_WORKERS_ARG) def _map_all(self, function, inplace=True, **kwargs): """ @@ -5006,7 +5006,7 @@ def _map_iterate( output_signal_size=None, output_dtype=None, lazy_output=None, - max_workers=None, + num_workers=None, navigation_chunks="auto", **kwargs, ): @@ -5113,7 +5113,7 @@ def _map_iterate( self.data, dtype=mapped.dtype, compute=True, - num_workers=max_workers, + num_workers=num_workers, ) data_stored = True else: @@ -5142,7 +5142,7 @@ def _map_iterate( sig._assign_subclass() if not lazy_output and not data_stored: - sig.data = sig.data.compute(num_workers=max_workers) + sig.data = sig.data.compute(num_workers=num_workers) if show_progressbar: pbar.unregister() diff --git a/hyperspy/tests/signals/test_cuda.py b/hyperspy/tests/signals/test_cuda.py index 907149ea2f..8679cfb2b1 100644 --- a/hyperspy/tests/signals/test_cuda.py +++ b/hyperspy/tests/signals/test_cuda.py @@ -61,15 +61,15 @@ def test_as_signal(self): s = self.s _ = s.as_signal2D([0, 1]) - @pytest.mark.parametrize('parallel', [True, False, None]) - def test_map(self, parallel): + @pytest.mark.parametrize('num_workers', [1, 2, None]) + def test_map(self, num_workers): s = self.s data_ref = s.data.copy() def dummy_function(data): return data * 10 - s.map(dummy_function, parallel, inplace=True, + s.map(dummy_function, inplace=True, num_workers=num_workers, output_signal_size=s.axes_manager.signal_shape, output_dtype=s.data.dtype) diff --git a/upcoming_changes/3198.api.rst b/upcoming_changes/3198.api.rst index 5dc7b760ff..47ba19d69b 100644 --- a/upcoming_changes/3198.api.rst +++ b/upcoming_changes/3198.api.rst @@ -1,6 +1,7 @@ Changes and improvement of the map function: - Removes the ``parallel`` argument +- Replace the ``max_workers`` with the ``num_workers`` argument to be consistent with ``dask`` - Adds more documentation on setting the dask backend and how to use multiple core - Adds ``navigation_chunk`` argument for setting the chunks with a non-lazy signal - Fix axes handling when the function to be mapped can be applied to the whole dataset - typically when it has the ``axis`` or ``axes`` keyword argument.