Develop ~~> Main v0.13.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.12.3
+current_version = 0.13.0
 commit = True
 tag = True
 

.github/workflows/long_workflow.yml

@@ -1,14 +1,14 @@
 name: ASPIRE Python Long Running Test Suite
 
 on:
-  push:
-    branches:
-      - 'main'
-      - 'develop'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
 
 jobs:
   expensive_tests:
     runs-on: self-hosted
+    # Only run on review ready pull_requests
+    if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == false }}
     timeout-minutes: 360
     steps:
     - uses: actions/checkout@v4
@@ -33,8 +33,9 @@ jobs:
         cat ${WORK_DIR}/config.yaml
     - name: Run
       run: |
+        export OMP_NUM_THREADS=1
         ASPIREDIR=${{ env.WORK_DIR }} python -c \
         "import aspire; print(aspire.config['ray']['temp_dir'])"
-        ASPIREDIR=${{ env.WORK_DIR }} python -m pytest -m "expensive" --durations=0
+        ASPIREDIR=${{ env.WORK_DIR }} python -m pytest -n8 -m "expensive" --durations=0
     - name: Cleanup
       run: rm -rf ${{ env.WORK_DIR }}

.github/workflows/workflow.yml

@@ -55,6 +55,8 @@ jobs:
       run: tox --skip-missing-interpreters false -e py${{ matrix.python-version }}-${{ matrix.pyenv }}
     - name: Upload Coverage to CodeCov
       uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
 
   conda-build:
     needs: check
@@ -147,14 +149,17 @@ jobs:
         echo "Stash the WORK_DIR to GitHub env so we can clean it up later."
         echo "WORK_DIR=${WORK_DIR}" >> $GITHUB_ENV
         echo -e "ray:\n    temp_dir: ${WORK_DIR}\n" > ${WORK_DIR}/config.yaml
-        echo -e "common:\n    cache_dir: ${CI_CACHE_DIR}\n" >> ${WORK_DIR}/config.yaml
+        echo -e "common:" >> ${WORK_DIR}/config.yaml
+        echo -e "    cache_dir: ${CI_CACHE_DIR}" >> ${WORK_DIR}/config.yaml
+        echo -e "    numeric: cupy" >> ${WORK_DIR}/config.yaml
+        echo -e "    fft: cupy\n" >> ${WORK_DIR}/config.yaml
         echo "Log the config: ${WORK_DIR}/config.yaml"
         cat ${WORK_DIR}/config.yaml
     - name: Run
       run: |
         ASPIREDIR=${{ env.WORK_DIR }} python -c \
         "import aspire; print(aspire.config['ray']['temp_dir'])"
-        ASPIREDIR=${{ env.WORK_DIR }} python -m pytest --durations=50
+        ASPIREDIR=${{ env.WORK_DIR }} PYTHONWARNINGS=error python -m pytest --durations=50
     - name: Cache Data
       run: |
         ASPIREDIR=${{ env.WORK_DIR }} python -c \
@@ -219,29 +224,18 @@ jobs:
           retention-days: 7
 
   osx_arm:
-    defaults:
-      run:
-        shell: bash -l {0}
     needs: check
     runs-on: macos-14
     # Run on every code push, but only on review ready PRs
     if: ${{ github.event_name == 'push' || github.event.pull_request.draft == false }}
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Conda
-      uses: conda-incubator/setup-miniconda@v2.3.0
+    - uses: actions/setup-python@v5
       with:
-        miniconda-version: "latest"
-        auto-update-conda: true
-        python-version: '3.8'
-        activate-environment: aspire
-        environment-file: environment-accelerate.yml
-        auto-activate-base: false
-    - name: Complete Install and Log Environment ${{ matrix.os }} Python ${{ matrix.python-version }}
+        python-version: '3.11'
+    - name: Complete Install and Log Environment
       run: |
-        conda info
-        conda list
-        conda install pyshtools  # debug depends issues
+        python --version
         pip install -e ".[dev]"  # install aspire
         pip freeze
     - name: Test

MANIFEST.in

@@ -17,6 +17,7 @@ recursive-include docs *.rst
 recursive-include docs Makefile
 recursive-include docs *.sh
 recursive-include src *.conf
+recursive-include src *.cu
 recursive-include src *.yaml
 prune docs/build
 prune docs/source

README.md

@@ -5,7 +5,7 @@
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5657281.svg)](https://doi.org/10.5281/zenodo.5657281)
 [![Downloads](https://static.pepy.tech/badge/aspire/month)](https://pepy.tech/project/aspire)
 
-# ASPIRE - Algorithms for Single Particle Reconstruction - v0.12.3
+# ASPIRE - Algorithms for Single Particle Reconstruction - v0.13.0
 
 The ASPIRE-Python project supersedes [Matlab ASPIRE](https://github.com/PrincetonUniversity/aspire).
 
@@ -20,7 +20,7 @@ For more information about the project, algorithms, and related publications ple
 Please cite using the following DOI. This DOI represents all versions, and will always resolve to the latest one.
 
 ```
-ComputationalCryoEM/ASPIRE-Python: v0.12.3 https://doi.org/10.5281/zenodo.5657281
+ComputationalCryoEM/ASPIRE-Python: v0.13.0 https://doi.org/10.5281/zenodo.5657281
 
 ```
 
@@ -37,11 +37,7 @@ install `aspire` safely in that environment.
 
 If you are unfamiliar with `conda`, the
 [Miniconda](https://docs.conda.io/en/latest/miniconda.html)
-distribution for `x86_64` is recommended.  For Apple silicon to use
-the osx-arm platform, patching and building some dependencies from
-source is currently required.  The Intel `osx-64` install is still
-preferred even for Apple silicon users, otherwise [notes are
-provided.](https://github.com/ComputationalCryoEM/ASPIRE-Python/discussions/969)
+distribution for `x86_64` is recommended.
 
 Assuming you have `conda` and a compatible system, the following steps
 will checkout current code release, create an environment, and install

docs/source/conf.py

@@ -86,7 +86,7 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = version = "0.12.3"
+release = version = "0.13.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/index.rst

@@ -1,4 +1,4 @@
-Aspire v0.12.3
+Aspire v0.13.0
 ==============
 
 Algorithms for Single Particle Reconstruction

docs/source/installation.rst

@@ -34,12 +34,6 @@ to view Conda's installation instructions.
 .. note:: If you're not sure which distribution is right for you, go
    with `Miniconda <https://conda.io/miniconda.html>`__
 
-.. note:: For Apple silicon to use the osx-arm platform, patching and
-   building some dependencies from source is currently required.  The
-   Intel ``osx-64`` install is still preferred even for Apple silicon
-   users, otherwise `notes are
-   provided. <https://github.com/ComputationalCryoEM/ASPIRE-Python/discussions/969>`_
-
 Getting Started - Installation
 ************************************
 
@@ -129,25 +123,17 @@ an M1 laptop:
 Installing GPU Extensions
 *************************
 
-ASPIRE does support GPUs, depending on several external packages.  The
-collection of GPU extensions can be installed using ``pip``.
-Extensions are grouped based on CUDA versions.  To find the CUDA
-driver version, run ``nvidia-smi`` on the intended system.
+ASPIRE does support using a GPU, depending on several external
+packages.  The collection of GPU extensions can be installed using
+``pip``.  Extensions are grouped based on CUDA versions.  To find the
+CUDA driver version, run ``nvidia-smi`` on the intended system.
 
 .. list-table:: CUDA GPU Extension Versions
    :widths: 25 25
    :header-rows: 1
 
    * - CUDA Version
      - ASPIRE Extension
-   * - 10.2
-     - gpu-102
-   * - 11.0
-     - gpu-110
-   * - 11.1
-     - gpu-111
-   * - >=11.2
-     - gpu-11x
    * - >=12
      - gpu-12x
 
@@ -164,12 +150,15 @@ the command below would install GPU packages required for ASPIRE.
 
 
 By default if the required GPU extensions are correctly installed,
-ASPIRE should automatically begin using the GPU for select components
-(such as those using ``nufft``).
-
-Because GPU extensions depend on several third party packages and
-libraries, we can only offer limited support if one of the packages
-has a problem on your system.
+ASPIRE should automatically begin using the GPU calls to our ``nufft`` module.
+
+Using GPU in other areas of the code is still an experimental feature
+and requires a minor configuration setting to enable ``cupy``. See the
+:ref:`sphx_glr_auto_tutorials_configuration.py` for details. Because
+GPU extensions depend on several third party softwares and machines
+vary wildly, we can only offer limited support if one of the packages
+has a problem on your system.  We are currently expanding GPU code
+coverage.
 
 Generating Documentation
 ************************

environment-accelerate.yml

@@ -7,7 +7,6 @@ channels:
 dependencies:
   - pip
   - python=3.8
-  - pyshtools
   - numpy=1.24.1
   - scipy=1.10.1
   - scikit-learn

gallery/experiments/experimental_abinitio_pipeline_10081.py

@@ -59,7 +59,11 @@
 
 # Create a source object for the experimental images
 src = RelionSource(
-    starfile_in, pixel_size=pixel_size, max_rows=n_imgs, data_folder=data_folder
+    starfile_in,
+    pixel_size=pixel_size,
+    max_rows=n_imgs,
+    data_folder=data_folder,
+    symmetry_group="C4",
 )
 
 # Downsample the images
@@ -115,12 +119,13 @@
 # Volume Reconstruction
 # ----------------------
 #
-# Using the oriented source, attempt to reconstruct a volume.
-# Since this is a Cn symmetric molecule, as indicated by
-# ``symmetry="C4"`` above, the ``avgs`` images set will be repeated
-# for each of the 3 additional rotations during the back-projection
-# step.  This boosts the effective number of images used in the
-# reconstruction from ``n_classes`` to ``4*n_classes``.
+# Using the oriented source, attempt to reconstruct a volume.  Since
+# this is a Cn symmetric molecule, as specified by ``RelionSource(...,
+# symmetry_group="C4", ...)``, the ``symmetry_group`` source attribute
+# will flow through the pipeline to ``avgs``. Then each image will be
+# repeated for each of the 3 additional rotations during
+# back-projection.  This boosts the effective number of images used in
+# the reconstruction from ``n_classes`` to ``4*n_classes``.
 
 logger.info("Begin Volume reconstruction")
 

gallery/experiments/simulated_abinitio_pipeline.py

@@ -54,13 +54,9 @@
 # ---------------
 # Start with the hi-res volume map EMDB-2660 sourced from EMDB,
 # https://www.ebi.ac.uk/emdb/EMD-2660, and dowloaded via ASPIRE's downloader utility.
-og_v = emdb_2660()
+og_v = emdb_2660().astype(np.float64)
 logger.info("Original volume map data" f" shape: {og_v.shape} dtype:{og_v.dtype}")
 
-logger.info(f"Downsampling to {(img_size,)*3}")
-v = og_v.downsample(img_size)
-L = v.resolution
-
 
 # Then create a filter based on that variance
 # This is an example of a custom noise profile
@@ -70,15 +66,15 @@ def noise_function(x, y):
     # White
     f1 = noise_variance
     # Violet-ish
-    f2 = noise_variance * (x * x + y * y) / L * L
+    f2 = noise_variance * (x * x + y * y) / img_size * img_size
     return (alpha * f1 + beta * f2) / 2.0
 
 
 custom_noise = CustomNoiseAdder(noise_filter=FunctionFilter(noise_function))
 
 logger.info("Initialize CTF filters.")
 # Create some CTF effects
-pixel_size = 5 * 65 / img_size  # Pixel size of the images (in angstroms)
+pixel_size = og_v.pixel_size  # Pixel size (in angstroms)
 voltage = 200  # Voltage (in KV)
 defocus_min = 1.5e4  # Minimum defocus value (in angstroms)
 defocus_max = 2.5e4  # Maximum defocus value (in angstroms)
@@ -94,13 +90,16 @@ def noise_function(x, y):
 
 # Finally create the Simulation
 src = Simulation(
-    L=v.resolution,
     n=num_imgs,
-    vols=v,
+    vols=og_v,
     noise_adder=custom_noise,
     unique_filters=ctf_filters,
-    dtype=v.dtype,
+    dtype=np.float64,
 )
+
+# Downsample
+src = src.downsample(img_size).cache()
+
 # Peek
 if interactive:
     src.images[:10].show()
@@ -115,7 +114,7 @@ def noise_function(x, y):
 
 # Plot the noise profile for inspection
 if interactive:
-    plt.imshow(aiso_noise_estimator.filter.evaluate_grid(L))
+    plt.imshow(aiso_noise_estimator.filter.evaluate_grid(img_size))
     plt.show()
 
 # Peek, what do the whitened images look like...

gallery/tutorials/aspire_introduction.py

@@ -571,7 +571,7 @@ def noise_function(x, y):
 
 # Generate several CTFs.
 ctf_filters = [
-    RadialCTFFilter(pixel_size=5, defocus=d)
+    RadialCTFFilter(pixel_size=vol_ds.pixel_size, defocus=d)
     for d in np.linspace(defocus_min, defocus_max, defocus_ct)
 ]
 

gallery/tutorials/configuration.py

@@ -102,6 +102,36 @@
     time.sleep(1)
 print("Done Loop 2\n")
 
+# %%
+# Enabling GPU Acceleration
+# -------------------------
+# Enabling GPU acceleration requires installing supporting software
+# packages and small config changes. Installing the supporting
+# software is most easily accomplished by installing ASPIRE with one
+# of the published GPU extensions, for example ``pip install
+# "aspire[dev,gpu_12x]"``.  Once the packages are installed users
+# should find that the NUFFT calls are automatically running on the
+# GPU.  Additional acceleration is achieved by enabling `cupy` for
+# `numeric` and `fft` components.
+#
+#    .. code-block:: yaml
+#
+#      common:
+#          # numeric module to use - one of numpy/cupy
+#          numeric: cupy
+#          # fft backend to use - one of pyfftw/scipy/cupy/mkl
+#          fft: cupy
+#
+# Alternatively, like other config options, this can be changed
+# dynamically with code.
+#
+#    .. code-block:: python
+#
+#      from aspire import config
+#
+#      config["common"]["numeric"] = "cupy"
+#      config["common"]["fft"] = "cupy"
+#
 
 # %%
 # Resolution

gallery/tutorials/pipeline_demo.py

@@ -63,7 +63,7 @@
 defocus_ct = 7
 
 ctf_filters = [
-    RadialCTFFilter(pixel_size=5, defocus=d)
+    RadialCTFFilter(pixel_size=vol.pixel_size, defocus=d)
     for d in np.linspace(defocus_min, defocus_max, defocus_ct)
 ]
 

gallery/tutorials/tutorials/cov3d_simulation.py

@@ -47,7 +47,9 @@
     L=img_size,
     n=num_imgs,
     vols=vols,
-    unique_filters=[RadialCTFFilter(defocus=d) for d in np.linspace(1.5e4, 2.5e4, 7)],
+    unique_filters=[
+        RadialCTFFilter(pixel_size=10, defocus=d) for d in np.linspace(1.5e4, 2.5e4, 7)
+    ],
     dtype=dtype,
 )