From a97f4be7611d035ff05e19a65391fc54093e4c55 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 26 Sep 2022 12:38:13 -0400 Subject: [PATCH 01/13] update for versioned GPU extensions --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a46c7582e5..0714800c1c 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,10 @@ def read(fname): # for example gpu packages which may not install for all users, # or developer tools that are handy but not required for users. extras_require={ - "gpu": ["pycuda", "cupy", "cufinufft==1.2"], + "gpu_102": ["pycuda", "cupy-cuda102", "cufinufft==1.2"], + "gpu_110": ["pycuda", "cupy-cuda110", "cufinufft==1.2"], + "gpu_111": ["pycuda", "cupy-cuda111", "cufinufft==1.2"], + "gpu_11x": ["pycuda", "cupy-cuda11x", "cufinufft==1.2"], "dev": [ "black", "bumpversion", From da1c58e6efb282f02f358374edfc8d8b14aea6b3 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 3 Oct 2022 08:34:21 -0400 Subject: [PATCH 02/13] add basic gpu ci workflow, Co-authored-by: Chris Langfield --- .github/workflows/workflow.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 96bfc69195..b77f4d9ccb 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -74,3 +74,13 @@ jobs: run: "docs/buildsite.sh" shell: bash + decaf_ampere: + runs-on: self-hosted + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev,gpu_102]" + - name: Run + run: pytest From cf35060581372e7284cb647a0eaaf743440f35d4 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 28 Sep 2022 09:28:40 -0400 Subject: [PATCH 03/13] GPU first in config --- src/aspire/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/config.ini b/src/aspire/config.ini index 251d13dabd..d43a80c952 100644 --- a/src/aspire/config.ini +++ b/src/aspire/config.ini @@ -24,7 +24,7 @@ cg_tol = 1e-5 regularizer = 0. [nfft] -backends = finufft, cufinufft, pynfft +backends = cufinufft, finufft, pynfft [ray] # Ray will default to a OS specific tmp dir. From a7311c392aff886e2b5374ea1887c1e1810125c9 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 3 Oct 2022 14:25:25 -0400 Subject: [PATCH 04/13] Force doubles for cufinufft --- src/aspire/nufft/cufinufft.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/aspire/nufft/cufinufft.py b/src/aspire/nufft/cufinufft.py index c2e83dd9a8..15a41ae5af 100644 --- a/src/aspire/nufft/cufinufft.py +++ b/src/aspire/nufft/cufinufft.py @@ -28,7 +28,8 @@ def __init__(self, sz, fourier_pts, epsilon=1e-15, ntransforms=1, **kwargs): self.ntransforms = ntransforms # Basic dtype passthough. - dtype = fourier_pts.dtype + self._dtype = fourier_pts.dtype + dtype = np.float64 if dtype == np.float64 or dtype == np.complex128: self.dtype = np.float64 self.complex_dtype = np.complex128 @@ -132,7 +133,7 @@ def transform(self, signal): result = result_gpu.get() - return result + return result.astype(self._dtype, copy=False def adjoint(self, signal): """ @@ -172,4 +173,4 @@ def adjoint(self, signal): result = result_gpu.get() - return result + return result.astype(self._dtype, copy=False) From 58bb38e2a5f55b01fc9864b0f3611e8a4913835d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 3 Oct 2022 14:26:32 -0400 Subject: [PATCH 05/13] some dtype regression cleanup --- src/aspire/nufft/cufinufft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/nufft/cufinufft.py b/src/aspire/nufft/cufinufft.py index 15a41ae5af..9098fc9c19 100644 --- a/src/aspire/nufft/cufinufft.py +++ b/src/aspire/nufft/cufinufft.py @@ -133,7 +133,7 @@ def transform(self, signal): result = result_gpu.get() - return result.astype(self._dtype, copy=False + return result.astype(self._dtype, copy=False) def adjoint(self, signal): """ From 4018b308b792b7fdde6961cf1e202bb4cc4e902f Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 3 Oct 2022 15:34:11 -0400 Subject: [PATCH 06/13] doubles doesn't fix --- src/aspire/nufft/cufinufft.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/aspire/nufft/cufinufft.py b/src/aspire/nufft/cufinufft.py index 9098fc9c19..c2e83dd9a8 100644 --- a/src/aspire/nufft/cufinufft.py +++ b/src/aspire/nufft/cufinufft.py @@ -28,8 +28,7 @@ def __init__(self, sz, fourier_pts, epsilon=1e-15, ntransforms=1, **kwargs): self.ntransforms = ntransforms # Basic dtype passthough. - self._dtype = fourier_pts.dtype - dtype = np.float64 + dtype = fourier_pts.dtype if dtype == np.float64 or dtype == np.complex128: self.dtype = np.float64 self.complex_dtype = np.complex128 @@ -133,7 +132,7 @@ def transform(self, signal): result = result_gpu.get() - return result.astype(self._dtype, copy=False) + return result def adjoint(self, signal): """ @@ -173,4 +172,4 @@ def adjoint(self, signal): result = result_gpu.get() - return result.astype(self._dtype, copy=False) + return result From f6f5fc876bb8f55d092646eeceffbc7d62ce51fe Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 24 Oct 2022 13:49:04 -0400 Subject: [PATCH 07/13] Attempt workaround cufinufft singles bug cast to doubles \(and back\) as needed --- src/aspire/nufft/cufinufft.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/aspire/nufft/cufinufft.py b/src/aspire/nufft/cufinufft.py index c2e83dd9a8..82ffc51bbb 100644 --- a/src/aspire/nufft/cufinufft.py +++ b/src/aspire/nufft/cufinufft.py @@ -7,6 +7,7 @@ from cufinufft import cufinufft from aspire.nufft import Plan +from aspire.utils import complex_type logger = logging.getLogger(__name__) @@ -27,6 +28,12 @@ def __init__(self, sz, fourier_pts, epsilon=1e-15, ntransforms=1, **kwargs): # Passing "ntransforms" > 1 expects one large higher dimensional array later. self.ntransforms = ntransforms + # Workaround cufinufft A100 singles issue + # ASPIRE-Python/703 + # Cast to doubles. + self._original_dtype = fourier_pts.dtype + fourier_pts = fourier_pts.astype(np.float64, copy=False) + # Basic dtype passthough. dtype = fourier_pts.dtype if dtype == np.float64 or dtype == np.complex128: @@ -95,7 +102,9 @@ def transform(self, signal): `(ntransforms, num_pts)`. """ - if not (signal.dtype == self.dtype or signal.dtype == self.complex_dtype): + if (self._original_dtype == self.dtype) and not ( + signal.dtype == self.dtype or signal.dtype == self.complex_dtype + ): logger.warning( "Incorrect dtypes passed to (a)nufft." " In the future this will be an error." @@ -131,6 +140,8 @@ def transform(self, signal): self._transform_plan.execute(result_gpu, signal_gpu) result = result_gpu.get() + # ASPIRE-Python/703 + result = result.astype(complex_type(self._original_dtype), copy=False) return result @@ -145,7 +156,9 @@ def adjoint(self, signal): :returns: Transformed signal `(sz)` or `(sz, ntransforms)`. """ - if not (signal.dtype == self.complex_dtype or signal.dtype == self.dtype): + if (self._original_dtype == self.dtype) and not ( + signal.dtype == self.complex_dtype or signal.dtype == self.dtype + ): logger.warning( "Incorrect dtypes passed to (a)nufft." " In the future this will be an error." @@ -171,5 +184,7 @@ def adjoint(self, signal): self._adjoint_plan.execute(signal_gpu, result_gpu) result = result_gpu.get() + # ASPIRE-Python/703 + result = result.astype(complex_type(self._original_dtype), copy=False) return result From 94ccf971fb30791672877facc2562f7f09af61a8 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 24 Oct 2022 13:49:26 -0400 Subject: [PATCH 08/13] set cufinufft tolerance similar to finufft --- src/aspire/nufft/cufinufft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/nufft/cufinufft.py b/src/aspire/nufft/cufinufft.py index 82ffc51bbb..79d5279f6e 100644 --- a/src/aspire/nufft/cufinufft.py +++ b/src/aspire/nufft/cufinufft.py @@ -13,7 +13,7 @@ class CufinufftPlan(Plan): - def __init__(self, sz, fourier_pts, epsilon=1e-15, ntransforms=1, **kwargs): + def __init__(self, sz, fourier_pts, epsilon=1e-8, ntransforms=1, **kwargs): """ A plan for non-uniform FFT in 2D or 3D. From 584f2890c4d11d9be4d9e19677f0f01d532b0acb Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 31 Oct 2022 14:38:14 -0400 Subject: [PATCH 09/13] Update docs with Installing GPU Extension section --- README.md | 7 ------- docs/source/installation.rst | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1b46cad8b4..e6cd86f486 100644 --- a/README.md +++ b/README.md @@ -64,13 +64,6 @@ pip install -e ".[dev]" If you prefer not to use Anaconda, or want to manage environments yourself, you should be able to use `pip` with Python >= 3.7. Please see the full documentation for details. -You may optionally install additional packages for GPU extensions: - -``` -# Additional GPU packages (requires CUDA) -pip install -e ".[gpu]" -``` - ### Make sure everything works Once ASPIRE is installed, make sure the unit tests run correctly on your platform by doing: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index f81abe078d..ac3b7f4dea 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -99,6 +99,42 @@ Make sure all unit tests run correctly by doing: Tests currently take around 5 minutes to run, but this depends on your specific machine's resources. +Installing GPU Extensions +************************* + +GPU extensions can be installed using pip. +Extensions are grouped based on CUDA versions. +To find the CUDA driver version, run ``nvidia-smi``. + +.. list-table:: CUDA GPU Extension Versions + :widths: 25 25 + :header-rows: 1 + + * - CUDA Version + - ASPIRE Extension + * - 10.2 + - gpu_102 + * - 11.0 + - gpu_110 + * - 11.2 + - gpu_111 + * - >=11.2 + - gpu_11x + +For example, if you have CUDA 11.7 installed on your system, +the command below would install GPU packages required for ASPIRE. + +:: + + # From PyPI + pip install -e "aspire[gpu_112]" + + # From a local git repo + pip install -e ".[gpu_112]" + +By default if GPU extensions are correctly installed, +ASPIRE should automatically begin using the GPU for select components +(such as those using ``nufft``). Generating Documentation ************************ From 724e59c5937f1819fb9eef7403e77026603553ce Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 31 Oct 2022 14:44:59 -0400 Subject: [PATCH 10/13] version typo --- docs/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index ac3b7f4dea..1a0eef8c54 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -116,7 +116,7 @@ To find the CUDA driver version, run ``nvidia-smi``. - gpu_102 * - 11.0 - gpu_110 - * - 11.2 + * - 11.1 - gpu_111 * - >=11.2 - gpu_11x From 6c373173baa255990937e844d8d783629216bef0 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 31 Oct 2022 14:45:19 -0400 Subject: [PATCH 11/13] roll the decaf cuda version forward --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index b77f4d9ccb..cf263d0f72 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -81,6 +81,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[dev,gpu_102]" + pip install -e ".[dev,gpu_11x]" - name: Run run: pytest From 5cd1bd43d21f1196246db40986e6821d0476540c Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 1 Nov 2022 18:30:34 -0400 Subject: [PATCH 12/13] Fix gpu_11x version typo in install rst --- docs/source/installation.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 1a0eef8c54..46092615f3 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -126,12 +126,12 @@ the command below would install GPU packages required for ASPIRE. :: - # From PyPI - pip install -e "aspire[gpu_112]" - - # From a local git repo - pip install -e ".[gpu_112]" + # From PyPI + pip install -e "aspire[gpu_11x]" + # From a local git repo + pip install -e ".[gpu_11x]" + By default if GPU extensions are correctly installed, ASPIRE should automatically begin using the GPU for select components (such as those using ``nufft``). From 1983cbd3f19fed71e991d88688bfb23ac211072f Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 1 Nov 2022 18:38:05 -0400 Subject: [PATCH 13/13] Add clarifying comment for warning branch short circuit --- src/aspire/nufft/cufinufft.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/aspire/nufft/cufinufft.py b/src/aspire/nufft/cufinufft.py index 79d5279f6e..dba4cdf4a5 100644 --- a/src/aspire/nufft/cufinufft.py +++ b/src/aspire/nufft/cufinufft.py @@ -102,6 +102,9 @@ def transform(self, signal): `(ntransforms, num_pts)`. """ + # Check we're not forcing a dtype workaround for ASPIRE-Python/703, + # then check if we have a dtype mismatch. + # This avoids false positive complaint for the workaround. if (self._original_dtype == self.dtype) and not ( signal.dtype == self.dtype or signal.dtype == self.complex_dtype ): @@ -156,6 +159,9 @@ def adjoint(self, signal): :returns: Transformed signal `(sz)` or `(sz, ntransforms)`. """ + # Check we're not forcing a dtype workaround for ASPIRE-Python/703, + # then check if we have a dtype mismatch. + # This avoids false positive complaint for the workaround. if (self._original_dtype == self.dtype) and not ( signal.dtype == self.complex_dtype or signal.dtype == self.dtype ):