From 20e686fd34d8ac1b96df2276d21ad9b6af9295be Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 08:51:19 -0400 Subject: [PATCH 001/294] initial migration of basis filter mat expansion. Requires DiagMatrix. --- src/aspire/basis/fb_2d.py | 51 +++++++++++++++++++++++++++++++++++ src/aspire/basis/fle_2d.py | 37 +++++++++++++++++++++++++ src/aspire/basis/steerable.py | 15 +++++++++++ 3 files changed, 103 insertions(+) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index b2bcc68f69..de3789265f 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -5,6 +5,7 @@ from aspire.basis import FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import unique_coords_nd +from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type, real_type, roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape @@ -401,3 +402,53 @@ def calculate_bispectrum( filter_nonzero_freqs=filter_nonzero_freqs, freq_cutoff=freq_cutoff, ) + + def filter_to_basis_mat(self, f, matrix_type=None): + """ + See SteerableBasis2D.filter_to_basis_mat. + """ + + # These form a circular dependence, import locally until time to clean up. + from aspire.basis.basis_utils import lgwt + + # Get the filter's evaluate function. + h_fun = f.evaluate + + # Set same dimensions as basis object + n_k = self.n_r + n_theta = self.n_theta + radial = self.get_radial() + + # get 2D grid in polar coordinate + k_vals, wts = lgwt(n_k, 0, 0.5, dtype=self.dtype) + k, theta = np.meshgrid( + k_vals, np.arange(n_theta) * 2 * np.pi / (2 * n_theta), indexing="ij" + ) + + # Get function values in polar 2D grid and average out angle contribution + omegax = k * np.cos(theta) + omegay = k * np.sin(theta) + omega = 2 * np.pi * np.vstack((omegax.flatten("C"), omegay.flatten("C"))) + h_vals2d = h_fun(omega).reshape(n_k, n_theta).astype(self.dtype) + h_vals = np.sum(h_vals2d, axis=1) / n_theta + + # Represent 1D function values in basis + h_basis = BlkDiagMatrix.empty(2 * self.ell_max + 1, dtype=self.dtype) + ind_ell = 0 + for ell in range(0, self.ell_max + 1): + k_max = self.k_max[ell] + rmat = 2 * k_vals.reshape(n_k, 1) * self.r0[ell][0:k_max].T + basis_vals = np.zeros_like(rmat) + ind_radial = np.sum(self.k_max[0:ell]) + basis_vals[:, 0:k_max] = radial[ind_radial : ind_radial + k_max].T + h_basis_vals = basis_vals * h_vals.reshape(n_k, 1) + h_basis_ell = basis_vals.T @ ( + h_basis_vals * k_vals.reshape(n_k, 1) * wts.reshape(n_k, 1) + ) + h_basis[ind_ell] = h_basis_ell + ind_ell += 1 + if ell > 0: + h_basis[ind_ell] = h_basis[ind_ell - 1] + ind_ell += 1 + + return h_basis diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index e9a595d727..aad3d6f25e 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -14,6 +14,7 @@ ) from aspire.nufft import anufft, nufft from aspire.numeric import fft +from aspire.operators import DiagMatrix from aspire.utils import complex_type, grid_2d logger = logging.getLogger(__name__) @@ -712,3 +713,39 @@ def _radial_convolve_weights(self, b): a[self.idx_list[i]] = y[i] return a.flatten() + + def filter_to_basis_mat(self, f, matrix_type=None): + """ + See SteerableBasis2D.filter_to_basis_mat. + """ + # These form a circular dependence, import locally until time to clean up. + from aspire.basis.basis_utils import lgwt + + # Get the filter's evaluate function. + h_fun = f.evaluate + + # Set same dimensions as basis object + n_k = 2 * self.num_radial_nodes # self.n_r + n_theta = self.num_angular_nodes # self.n_theta + # radial = self.get_radial() + + # get 2D grid in polar coordinate + k_vals, wts = lgwt(n_k, 0, 0.5, dtype=self.dtype) + k, theta = np.meshgrid( + k_vals, np.arange(n_theta) * 2 * np.pi / (2 * n_theta), indexing="ij" + ) + + # Get function values in polar 2D grid and average out angle contribution + # NOTE: should probably just let the ctf objects handle this... + omegax = k * np.cos(theta) + omegay = k * np.sin(theta) + omega = 2 * np.pi * np.vstack((omegax.flatten("C"), omegay.flatten("C"))) + h_vals2d = h_fun(omega).reshape(n_k, n_theta).astype(self.dtype) + h_vals = np.sum(h_vals2d, axis=1) / n_theta + + h_basis = np.zeros(self.count, dtype=self.dtype) + # For now we just need handle 1D (stack of one ctf) + for j in range(self.ell_p_max + 1): + h_basis[self.idx_list[j]] = self.A3[j] @ h_vals + + return DiagMatrix(h_basis) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 955c92dc4e..e0d1d4ca92 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -1,3 +1,4 @@ +import abc import logging from collections.abc import Iterable @@ -274,3 +275,17 @@ def shift(self, coef, shifts): ) return self.evaluate_t(self.evaluate(coef).shift(shifts)) + + @abc.abstractmethod + def filter_to_basis_mat(self, f, matrix_type=None): + """ + Convert a filter into a basis representation. + + :param f: `Filter` object, usually a `CTFFilter`. + :param matrix_type: Optional override, Example, `BlkDiagMatrix` or + `DiagMatrix`. Default `None` returns the default for + this basis. + + :return: `BlkDiagMatrix` or `DiagMatrix` instance + representation of filter in `basis`. + """ From d87744ac5793f66f9a173ed631b41d6a7ac62df2 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 08:51:24 -0400 Subject: [PATCH 002/294] migrate covar2d test away from parametrized --- tests/test_covar2d.py | 499 ++++++++++++++++++++++++------------------ 1 file changed, 281 insertions(+), 218 deletions(-) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index b8e223d92f..daa325152f 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -1,9 +1,8 @@ import os import os.path -from unittest import TestCase import numpy as np -from parameterized import parameterized +import pytest from pytest import raises from aspire.basis import FFBBasis2D @@ -11,249 +10,313 @@ from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter from aspire.source.simulation import Simulation -from aspire.utils import utest_tolerance +from aspire.utils import randi, utest_tolerance from aspire.volume import Volume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") -class Cov2DTestCase(TestCase): +IMG_SIZES = [8] +DTYPES = [np.float32] +# Basis used in FSPCA for class averaging. +BASIS = [ + FFBBasis2D, +] + +# Hard coded to match legacy files. +NOISE_VAR = 1.3957e-4 + +# These variables support parameterized arg checking in `test_shrinkage` +SHRINKERS = [None, "frobenius_norm", "operator_norm", "soft_threshold"] + +CTF_ENABLED = [True, False] + + +@pytest.fixture(params=CTF_ENABLED, ids=lambda x: f"ctf={x}") +def ctf_enabled(request): + return request.param + + +@pytest.fixture(params=SHRINKERS, ids=lambda x: f"shrinker={x}") +def shrinker(request): + return request.param + + +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +def dtype(request): + return request.param + + +@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}") +def img_size(request): + return request.param + + +@pytest.fixture +def volume(dtype, img_size): + # Get a volume + v = Volume( + np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype(dtype) + ) + # 1e3 is hardcoded to match legacy test files. + return v.downsample(img_size) * 1.0e3 + + +@pytest.fixture(params=BASIS, ids=lambda x: f"basis={x}") +def basis(request, img_size, dtype): + cls = request.param + # Setup a Basis + basis = cls(img_size, dtype=dtype) + return basis + + +@pytest.fixture +def cov2d_fixture(volume, basis, ctf_enabled): """ - Cov2D Test without CTFFilters populated. + Cov2D Test Fixture. """ + n = 32 + # Default CTF params unique_filters = None h_idx = None h_ctf_fb = None + # Popluate CTF + if ctf_enabled: + unique_filters = [ + RadialCTFFilter( + 5.0 * 65 / volume.resolution, 200, defocus=d, Cs=2.0, alpha=0.1 + ) + for d in np.linspace(1.5e4, 2.5e4, 7) + ] + + # Copied from simulation defaults to match legacy test files. + h_idx = randi(len(unique_filters), n, seed=0) - 1 + + h_ctf_fb = [basis.filter_to_basis_mat(f) for f in unique_filters] + + noise_adder = WhiteNoiseAdder(var=NOISE_VAR) + + sim = Simulation( + n=n, + vols=volume, + unique_filters=unique_filters, + filter_indices=h_idx, + offsets=0.0, + amplitudes=1.0, + dtype=volume.dtype, + noise_adder=noise_adder, + ) + sim.cache() + + cov2d = RotCov2D(basis) + coeff_clean = basis.evaluate_t(sim.projections[:]) + coeff = basis.evaluate_t(sim.images[:]) + + return sim, cov2d, coeff_clean, coeff, h_ctf_fb, h_idx + + +def test_get_mean(cov2d_fixture): + results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_mean.npy")) + cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + + mean_coeff = cov2d._get_mean(coeff_clean) + assert np.allclose(results, mean_coeff, atol=utest_tolerance(cov2d.dtype)) + + +def test_get_covar(cov2d_fixture): + results = np.load( + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covar.npy"), + allow_pickle=True, + ) + + cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + covar_coeff = cov2d._get_covar(coeff_clean) + + for im, mat in enumerate(results.tolist()): + assert np.allclose(mat, covar_coeff[im]) + + +def test_get_mean_ctf(cov2d_fixture, ctf_enabled): + """ + Compare `get_mean` (no CTF args) with `_get_mean` (no CTF model). + """ + sim, cov2d, coeff_clean, coeff, h_ctf_fb, h_idx = cov2d_fixture + + mean_coeff_ctf = cov2d.get_mean(coeff, h_ctf_fb, h_idx) + + if ctf_enabled: + result = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_meanctf.npy")) + else: + result = cov2d._get_mean(coeff_clean) + + assert np.allclose(mean_coeff_ctf, result, atol=0.002) + + +def test_get_cwf_coeffs_clean(cov2d_fixture): + results = np.load( + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff_clean.npy") + ) + + cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + + coeff_cwf_clean = cov2d.get_cwf_coeffs(coeff_clean, noise_var=0) + assert np.allclose(results, coeff_cwf_clean, atol=utest_tolerance(cov2d.dtype)) + + +def test_get_cwf_coeffs_clean_ctf(cov2d_fixture): + """ + Test case of clean images (coeff_clean and noise_var=0) + while using a non Identity CTF. + + This case may come up when a developer switches between + clean and dirty images. + """ + sim, cov2d, coeff_clean, _, h_ctf_fb, h_idx = cov2d_fixture + + coeff_cwf = cov2d.get_cwf_coeffs(coeff_clean, h_ctf_fb, h_idx, noise_var=0) + + # Reconstruct images from output of get_cwf_coeffs + img_est = cov2d.basis.evaluate(coeff_cwf) + # Compare with clean images + delta = np.mean(np.square((sim.projections[:] - img_est).asnumpy())) + assert delta < 0.02 + + +def test_shrinker_inputs(cov2d_fixture): + """ + Check we raise with specific message for unsupporting shrinker arg. + """ + cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] - # These class variables support parameterized arg checking in `testShrinkers` - shrinkers = [(None,), "frobenius_norm", "operator_norm", "soft_threshold"] bad_shrinker_inputs = ["None", "notashrinker", ""] - def setUp(self): - self.dtype = np.float32 + for shrinker in bad_shrinker_inputs: + with raises(AssertionError, match="Unsupported shrink method"): + _ = cov2d.get_covar(coeff_clean, covar_est_opt={"shrinker": shrinker}) + - self.L = L = 8 - n = 32 +def test_shrinkage(cov2d_fixture, shrinker): + """ + Test all the shrinkers we know about run without crashing, + """ + cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] - self.noise_var = 1.3957e-4 - noise_adder = WhiteNoiseAdder(var=self.noise_var) + results = np.load( + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covar.npy"), + allow_pickle=True, + ) - vols = Volume( - np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype( - self.dtype - ) - ) # RCOPT - vols = vols.downsample(L) * 1.0e3 - # Since FFBBasis2D doesn't yet implement dtype, we'll set this to double to match its built in types. - self.sim = Simulation( - n=n, - L=L, - vols=vols, - unique_filters=self.unique_filters, - offsets=0.0, - amplitudes=1.0, - dtype=self.dtype, - noise_adder=noise_adder, - ) - - self.basis = FFBBasis2D((L, L), dtype=self.dtype) - - self.imgs_clean = self.sim.projections[:] - self.imgs_ctf_clean = self.sim.clean_images[:] - self.imgs_ctf_noise = self.sim.images[:n] - - self.cov2d = RotCov2D(self.basis) - self.coeff_clean = self.basis.evaluate_t(self.imgs_clean) - self.coeff = self.basis.evaluate_t(self.imgs_ctf_noise) - - def tearDown(self): - pass - - def testGetMean(self): - results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_mean.npy")) - mean_coeff = self.cov2d._get_mean(self.coeff_clean) - self.assertTrue(np.allclose(results, mean_coeff)) - - def testGetCovar(self): - results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covar.npy"), - allow_pickle=True, - ) - covar_coeff = self.cov2d._get_covar(self.coeff_clean) - - for im, mat in enumerate(results.tolist()): - self.assertTrue(np.allclose(mat, covar_coeff[im])) - - def testGetMeanCTF(self): - """ - Compare `get_mean` (no CTF args) with `_get_mean` (no CTF model). - """ - mean_coeff_ctf = self.cov2d.get_mean(self.coeff, self.h_ctf_fb, self.h_idx) - mean_coeff = self.cov2d._get_mean(self.coeff_clean) - self.assertTrue(np.allclose(mean_coeff_ctf, mean_coeff, atol=0.002)) - - def testGetCWFCoeffsClean(self): - results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff_clean.npy") - ) - coeff_cwf_clean = self.cov2d.get_cwf_coeffs(self.coeff_clean, noise_var=0) - self.assertTrue( - np.allclose(results, coeff_cwf_clean, atol=utest_tolerance(self.dtype)) - ) - - def testGetCWFCoeffsCleanCTF(self): - """ - Test case of clean images (coeff_clean and noise_var=0) - while using a non Identity CTF. - - This case may come up when a developer switches between - clean and dirty images. - """ - - coeff_cwf = self.cov2d.get_cwf_coeffs( - self.coeff_clean, self.h_ctf_fb, self.h_idx, noise_var=0 - ) - - # Reconstruct images from output of get_cwf_coeffs - img_est = self.basis.evaluate(coeff_cwf) - # Compare with clean images - delta = np.mean(np.square((self.imgs_clean - img_est).asnumpy())) - self.assertTrue(delta < 0.02) - - # Note, parameterized module can be removed at a later date - # and replaced with pytest if ASPIRE-Python moves away from - # the TestCase class style tests. - # Paramaterize over known shrinkers and some bad values - @parameterized.expand(shrinkers + bad_shrinker_inputs) - def testShrinkers(self, shrinker): - """Test all the shrinkers we know about run without crashing, - and check we raise with specific message for unsupporting shrinker arg.""" - - if shrinker in self.bad_shrinker_inputs: - with raises(AssertionError, match="Unsupported shrink method"): - _ = self.cov2d.get_covar( - self.coeff_clean, covar_est_opt={"shrinker": shrinker} - ) - return - - results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covar.npy"), - allow_pickle=True, - ) - - covar_coeff = self.cov2d.get_covar( - self.coeff_clean, covar_est_opt={"shrinker": shrinker} - ) - - for im, mat in enumerate(results.tolist()): - self.assertTrue( - np.allclose(mat, covar_coeff[im], atol=utest_tolerance(self.dtype)) - ) + covar_coeff = cov2d.get_covar(coeff_clean, covar_est_opt={"shrinker": shrinker}) + + for im, mat in enumerate(results.tolist()): + assert np.allclose(mat, covar_coeff[im], atol=utest_tolerance(cov2d.dtype)) -class Cov2DTestCaseCTF(Cov2DTestCase): +def test_get_cwf_coeffs_ctf_args(cov2d_fixture): """ - Cov2D Test with CTFFilters populated. + Test we raise when user supplies incorrect CTF arguments, + and that the error message matches. """ + sim, cov2d, _, coeff, h_ctf_fb, _ = cov2d_fixture - @property - def unique_filters(self): - return [ - RadialCTFFilter(5.0 * 65 / self.L, 200, defocus=d, Cs=2.0, alpha=0.1) - for d in np.linspace(1.5e4, 2.5e4, 7) - ] + # When half the ctf info (h_ctf_fb) is populated, + # set the other ctf param (h_idx) to none. + h_idx = None + if h_ctf_fb is None: + # And when h_ctf_fb is None, we'll populate the other half (h_idx) + h_idx = sim.filter_indices - @property - def h_idx(self): - return self.sim.filter_indices + # Both the above situations should be an error. + with raises(RuntimeError, match=r".*Given ctf_.*"): + _ = cov2d.get_cwf_coeffs(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) - @property - def h_ctf_fb(self): - return [filt.fb_mat(self.basis) for filt in self.unique_filters] - def testGetCWFCoeffsCTFargs(self): - """ - Test we raise when user supplies incorrect CTF arguments, - and that the error message matches. - """ +def test_get_cwf_coeffs(cov2d_fixture, ctf_enabled): + """ + Tests `get_cwf_coeffs` with poulated CTF. + """ + _, cov2d, coeff_clean, coeff, h_ctf_fb, h_idx = cov2d_fixture - with raises(RuntimeError, match=r".*Given ctf_fb.*"): - _ = self.cov2d.get_cwf_coeffs( - self.coeff, self.h_ctf_fb, None, noise_var=self.noise_var - ) + # Hard coded file expects sim with ctf. + if not ctf_enabled: + pytest.skip(reason="Reference file n/a.") - def testGetMeanCTF(self): - """ - Compare `get_mean` with saved legacy cov2d results. - """ - results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_meanctf.npy")) - mean_coeff_ctf = self.cov2d.get_mean(self.coeff, self.h_ctf_fb, self.h_idx) - self.assertTrue(np.allclose(results, mean_coeff_ctf)) - - def testGetCWFCoeffs(self): - """ - Tests `get_cwf_coeffs` with poulated CTF. - """ - results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff.npy") - ) - coeff_cwf = self.cov2d.get_cwf_coeffs( - self.coeff, self.h_ctf_fb, self.h_idx, noise_var=self.noise_var - ) - self.assertTrue( - np.allclose(results, coeff_cwf, atol=utest_tolerance(self.dtype)) - ) + results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff.npy")) + + coeff_cwf = cov2d.get_cwf_coeffs(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) + + assert np.allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) + + +def test_get_cwf_coeffs_without_ctf_args(cov2d_fixture, ctf_enabled): + """ + Tests `get_cwf_coeffs` with poulated CTF. + """ + + _, cov2d, _, coeff, _, _ = cov2d_fixture + + # Hard coded file expects sim with ctf. + if not ctf_enabled: + pytest.skip(reason="Reference file n/a.") # Note, I think this file is incorrectly named... # It appears to have come from operations on images with ctf applied. - def testGetCWFCoeffsNoCTF(self): - """ - Tests `get_cwf_coeffs` without providing CTF. (Internally uses IdentityCTF). - """ - results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff_noCTF.npy") - ) - coeff_cwf_noCTF = self.cov2d.get_cwf_coeffs( - self.coeff, noise_var=self.noise_var - ) - - self.assertTrue( - np.allclose(results, coeff_cwf_noCTF, atol=utest_tolerance(self.dtype)) - ) - - def testGetCovarCTF(self): - results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covarctf.npy"), - allow_pickle=True, - ) - covar_coeff_ctf = self.cov2d.get_covar( - self.coeff, self.h_ctf_fb, self.h_idx, noise_var=self.noise_var - ) - for im, mat in enumerate(results.tolist()): - self.assertTrue(np.allclose(mat, covar_coeff_ctf[im])) - - def testGetCovarCTFShrink(self): - results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covarctf_shrink.npy"), - allow_pickle=True, - ) - covar_opt = { - "shrinker": "frobenius_norm", - "verbose": 0, - "max_iter": 250, - "iter_callback": [], - "store_iterates": False, - "rel_tolerance": 1e-12, - "precision": self.dtype, - } - covar_coeff_ctf_shrink = self.cov2d.get_covar( - self.coeff, - self.h_ctf_fb, - self.h_idx, - noise_var=self.noise_var, - covar_est_opt=covar_opt, - ) - - for im, mat in enumerate(results.tolist()): - self.assertTrue(np.allclose(mat, covar_coeff_ctf_shrink[im])) + results = np.load( + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff_noCTF.npy") + ) + + coeff_cwf = cov2d.get_cwf_coeffs(coeff, noise_var=NOISE_VAR) + + assert np.allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) + + +def test_get_covar_ctf(cov2d_fixture, ctf_enabled): + # Hard coded file expects sim with ctf. + if not ctf_enabled: + pytest.skip(reason="Reference file n/a.") + + sim, cov2d, _, coeff, h_ctf_fb, h_idx = cov2d_fixture + + results = np.load( + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covarctf.npy"), + allow_pickle=True, + ) + + covar_coeff_ctf = cov2d.get_covar(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) + for im, mat in enumerate(results.tolist()): + assert np.allclose(mat, covar_coeff_ctf[im]) + + +def test_get_covar_ctf_shrink(cov2d_fixture, ctf_enabled): + sim, cov2d, _, coeff, h_ctf_fb, h_idx = cov2d_fixture + + # Hard coded file expects sim with ctf. + if not ctf_enabled: + pytest.skip(reason="Reference file n/a.") + + results = np.load( + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covarctf_shrink.npy"), + allow_pickle=True, + ) + + covar_opt = { + "shrinker": "frobenius_norm", + "verbose": 0, + "max_iter": 250, + "iter_callback": [], + "store_iterates": False, + "rel_tolerance": 1e-12, + "precision": cov2d.dtype, + } + + covar_coeff_ctf_shrink = cov2d.get_covar( + coeff, + h_ctf_fb, + h_idx, + noise_var=NOISE_VAR, + covar_est_opt=covar_opt, + ) + + for im, mat in enumerate(results.tolist()): + assert np.allclose(mat, covar_coeff_ctf_shrink[im]) From a1c84072b6059734d753a334e2fa9441c034887b Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 08:58:32 -0400 Subject: [PATCH 003/294] update tutorial with new basis mat method --- gallery/tutorials/tutorials/cov2d_simulation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/tutorials/tutorials/cov2d_simulation.py b/gallery/tutorials/tutorials/cov2d_simulation.py index 162eda4e80..b75450e138 100644 --- a/gallery/tutorials/tutorials/cov2d_simulation.py +++ b/gallery/tutorials/tutorials/cov2d_simulation.py @@ -116,7 +116,7 @@ h_idx = sim.filter_indices # Evaluate CTF in the 8X8 FB basis -h_ctf_fb = [filt.fb_mat(ffbbasis) for filt in ctf_filters] +h_ctf_fb = [ffbbasis.filter_to_basis_mat(filt) for filt in ctf_filters] # Get clean images from projections of 3D map. logger.info("Apply CTF filters to clean images.") From ffbb399dd99b37791b2d13143b285b923c71c8d0 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 09:00:34 -0400 Subject: [PATCH 004/294] update batched covar with new basis mat method --- tests/test_batched_covar2d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_batched_covar2d.py b/tests/test_batched_covar2d.py index 71c3366870..c8af4b0bf2 100644 --- a/tests/test_batched_covar2d.py +++ b/tests/test_batched_covar2d.py @@ -260,4 +260,4 @@ def ctf_idx(self): @property def ctf_fb(self): - return [f.fb_mat(self.basis) for f in self.src.unique_filters] + return [self.basis.filter_to_basis_mat(f) for f in self.src.unique_filters] From aae78db289d82ed4aa4f868fc9e51029f7602668 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 09:13:21 -0400 Subject: [PATCH 005/294] migrate fb_mat to basis_mat --- gallery/tutorials/aspire_introduction.py | 2 +- src/aspire/operators/filters.py | 11 +++-- src/aspire/utils/filter_to_fb_mat.py | 60 ------------------------ 3 files changed, 8 insertions(+), 65 deletions(-) delete mode 100644 src/aspire/utils/filter_to_fb_mat.py diff --git a/gallery/tutorials/aspire_introduction.py b/gallery/tutorials/aspire_introduction.py index e12b3651be..f87afaee2f 100644 --- a/gallery/tutorials/aspire_introduction.py +++ b/gallery/tutorials/aspire_introduction.py @@ -245,7 +245,7 @@ # classDiagram # class Filter{ # +evaluate() -# +fb_mat() +# +basis_mat() # +scale() # +evaluate_grid() # +dual() diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index 839a6dffe5..e74bf7b02e 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -5,7 +5,6 @@ from scipy.interpolate import RegularGridInterpolator from aspire.utils import grid_2d, voltage_to_wavelength -from aspire.utils.filter_to_fb_mat import filter_to_fb_mat logger = logging.getLogger(__name__) @@ -89,11 +88,15 @@ def evaluate(self, omega): def _evaluate(self, omega): raise NotImplementedError("Subclasses should implement this method") - def fb_mat(self, fbasis): + def basis_mat(self, basis): """ - Represent the filter in FB basis matrix + Represent the filter in `basis`. + + :param basis: 2D Basis. + :return: `basis` representation of this filter. + Return type will depend on `basis`. """ - return filter_to_fb_mat(self.evaluate, fbasis) + return basis.filter_to_basis_mat(self, basis) def scale(self, c=1): """ diff --git a/src/aspire/utils/filter_to_fb_mat.py b/src/aspire/utils/filter_to_fb_mat.py deleted file mode 100644 index 9933ffe3ba..0000000000 --- a/src/aspire/utils/filter_to_fb_mat.py +++ /dev/null @@ -1,60 +0,0 @@ -import numpy as np - -from aspire.operators import BlkDiagMatrix - - -def filter_to_fb_mat(h_fun, fbasis): - """ - Convert a nonradial function in k space into a basis representation. - - :param h_fun: The function form in k space. - :param fbasis: The basis object for expanding. - - :return: a BlkDiagMatrix instance representation using the - `fbasis` expansion. - """ - - # These form a circular dependence, import locally until time to clean up. - from aspire.basis import FFBBasis2D - from aspire.basis.basis_utils import lgwt - - if not isinstance(fbasis, FFBBasis2D): - raise NotImplementedError("Currently only fast FB method is supported") - # Set same dimensions as basis object - n_k = fbasis.n_r - n_theta = fbasis.n_theta - radial = fbasis.get_radial() - - # get 2D grid in polar coordinate - k_vals, wts = lgwt(n_k, 0, 0.5, dtype=fbasis.dtype) - k, theta = np.meshgrid( - k_vals, np.arange(n_theta) * 2 * np.pi / (2 * n_theta), indexing="ij" - ) - - # Get function values in polar 2D grid and average out angle contribution - omegax = k * np.cos(theta) - omegay = k * np.sin(theta) - omega = 2 * np.pi * np.vstack((omegax.flatten("C"), omegay.flatten("C"))) - h_vals2d = h_fun(omega).reshape(n_k, n_theta).astype(fbasis.dtype) - h_vals = np.sum(h_vals2d, axis=1) / n_theta - - # Represent 1D function values in fbasis - h_fb = BlkDiagMatrix.empty(2 * fbasis.ell_max + 1, dtype=fbasis.dtype) - ind_ell = 0 - for ell in range(0, fbasis.ell_max + 1): - k_max = fbasis.k_max[ell] - rmat = 2 * k_vals.reshape(n_k, 1) * fbasis.r0[ell][0:k_max].T - fb_vals = np.zeros_like(rmat) - ind_radial = np.sum(fbasis.k_max[0:ell]) - fb_vals[:, 0:k_max] = radial[ind_radial : ind_radial + k_max].T - h_fb_vals = fb_vals * h_vals.reshape(n_k, 1) - h_fb_ell = fb_vals.T @ ( - h_fb_vals * k_vals.reshape(n_k, 1) * wts.reshape(n_k, 1) - ) - h_fb[ind_ell] = h_fb_ell - ind_ell += 1 - if ell > 0: - h_fb[ind_ell] = h_fb[ind_ell - 1] - ind_ell += 1 - - return h_fb From 134a5aea1217e7d32a98b3fafb17d2fd89c69acf Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 12:56:58 -0400 Subject: [PATCH 006/294] migrate tests to basis_mat from fb_mat implementation --- src/aspire/basis/basis.py | 6 + src/aspire/basis/fle_2d.py | 3 + src/aspire/basis/steerable.py | 25 ++ src/aspire/covariance/covar2d.py | 357 ++++++++++++++----------- src/aspire/denoising/denoiser_cov2d.py | 2 +- tests/test_batched_covar2d.py | 33 +-- 6 files changed, 255 insertions(+), 171 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 9a78b55b9d..372e13f241 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -233,3 +233,9 @@ def expand(self, x): # return v coefficients with the last dimension of self.count v = v.reshape((*sz_roll, self.count)) return v + + @property + def blk_diag_cov_shape(self): + raise NotImplementedError( + "This method should be implemented for any steerable 2D basis." + ) diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index aad3d6f25e..db134de719 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -31,6 +31,9 @@ class FLEBasis2D(SteerableBasis2D, FBBasisMixin): https://arxiv.org/pdf/2207.13674.pdf """ + # Default matrix type for basis representation. + matrix_type = DiagMatrix + def __init__( self, size, bandlimit=None, epsilon=1e-10, dtype=np.float32, match_fb=True ): diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index e0d1d4ca92..745660c590 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -5,6 +5,7 @@ import numpy as np from aspire.basis import Basis +from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type logger = logging.getLogger(__name__) @@ -16,6 +17,9 @@ class SteerableBasis2D(Basis): `rotation` (steerable) and `calculate_bispectrum` methods. """ + # Default matrix type for basis representation. + matrix_type = BlkDiagMatrix + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -26,6 +30,9 @@ def __init__(self, *args, **kwargs): self._pos_angular_inds = (self.signs_indices == 1) & (self.angular_indices != 0) self._neg_angular_inds = self.signs_indices == -1 + # Cache the blk_diag shape once known. + self._blk_diag_cov_shape = None + def calculate_bispectrum( self, complex_coef, flatten=False, filter_nonzero_freqs=False, freq_cutoff=None ): @@ -289,3 +296,21 @@ def filter_to_basis_mat(self, f, matrix_type=None): :return: `BlkDiagMatrix` or `DiagMatrix` instance representation of filter in `basis`. """ + + @property + def blk_diag_cov_shape(self): + # Compute the _blk_diag_cov_shape as needed. + if self._blk_diag_cov_shape is None: + blks = [] + for ell in range(self.ell_max + 1): + sgns = (1,) if ell == 0 else (1, -1) + for _ in sgns: + blks.append( + [ + self.k_max[ell], + ] + * 2 + ) + self._blk_diag_cov_shape = np.array(blks) + + return self._blk_diag_cov_shape diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index d0b1507009..2aace9d86b 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -4,7 +4,8 @@ from numpy.linalg import eig, inv from scipy.linalg import solve, sqrtm -from aspire.operators import BlkDiagMatrix, RadialCTFFilter +from aspire.basis import FFBBasis2D +from aspire.operators import BlkDiagMatrix, DiagMatrix, RadialCTFFilter from aspire.optimization import conj_grad, fill_struct from aspire.utils import make_symmat from aspire.utils.matlab_compat import m_reshape @@ -100,6 +101,23 @@ def __init__(self, basis): self.dtype = self.basis.dtype assert basis.ndim == 2, "Only two-dimensional basis functions are needed." + # Abstract the basis matrix type (BlkDiagMatrix/DiagMatrix) + self.ctf_matrix_type = self.basis.matrix_type + + def _ctf_identity_mat(self): + """ + Returns CTF identity corresponding to the `matrix_type` of `self.basis`. + + :return: Identity BlkDiagMatrix or DiagMatrix + """ + if self.basis.matrix_type == DiagMatrix: + # TODO: compute this without computing filter/ctf + return DiagMatrix.ones( + len(self.basis.filter_to_basis_mat(RadialCTFFilter())), dtype=self.dtype + ) + else: + return BlkDiagMatrix.eye(self.basis.blk_diag_cov_shape, dtype=self.dtype) + def _get_mean(self, coeffs): """ Calculate the mean vector from the expansion coefficients of 2D images without CTF information. @@ -107,104 +125,49 @@ def _get_mean(self, coeffs): :param coeffs: A coefficient vector (or an array of coefficient vectors) to be averaged. :return: The mean value vector for all images. """ + if coeffs.size == 0: raise RuntimeError("The coefficients need to be calculated first!") + mask = self.basis._indices["ells"] == 0 mean_coeff = np.zeros(self.basis.count, dtype=coeffs.dtype) + # Use array for manually masking, since Coef.__getitem__ tries to return a Coef. mean_coeff[mask] = np.mean(coeffs[..., mask], axis=0) return mean_coeff - def _get_covar(self, coeffs, mean_coeff=None, do_refl=True): - """ - Calculate the covariance matrix from the expansion coefficients without CTF information. - - :param coeffs: A coefficient vector (or an array of coefficient vectors) calculated from 2D images. - :param mean_coeff: The mean vector calculated from the `coeffs`. - :param do_refl: If true, enforce invariance to reflection (default false). - :return: The covariance matrix of coefficients for all images. - """ - if coeffs.size == 0: - raise RuntimeError("The coefficients need to be calculated first!") - if mean_coeff is None: - mean_coeff = self._get_mean(coeffs) - - # Initialize a totally empty BlkDiagMatrix, build incrementally. - covar_coeff = BlkDiagMatrix.empty(0, dtype=coeffs.dtype) - ell = 0 - mask = self.basis._indices["ells"] == ell - coeff_ell = coeffs[..., mask] - mean_coeff[mask] - covar_ell = np.array(coeff_ell.T @ coeff_ell / coeffs.shape[0]) - covar_coeff.append(covar_ell) - - for ell in range(1, self.basis.ell_max + 1): - mask_ell = self.basis._indices["ells"] == ell - mask_pos = mask_ell & (self.basis._indices["sgns"] == +1) - mask_neg = mask_ell & (self.basis._indices["sgns"] == -1) - - covar_ell_diag = np.array( - coeffs[:, mask_pos].T @ coeffs[:, mask_pos] - + coeffs[:, mask_neg].T @ coeffs[:, mask_neg] - ) / (2 * coeffs.shape[0]) - - if do_refl: - covar_coeff.append(covar_ell_diag) - covar_coeff.append(covar_ell_diag) - else: - covar_ell_off = np.array( - ( - coeffs[:, mask_pos] @ coeffs[:, mask_neg].T / coeffs.shape[0] - - coeffs[:, mask_pos].T @ coeffs[:, mask_neg] - ) - / (2 * coeffs.shape[0]) - ) - - hsize = covar_ell_diag.shape[0] - covar_coeff_blk = np.zeros((2, hsize, 2, hsize)) - - covar_coeff_blk[0:2, :, 0:2, :] = covar_ell_diag[:hsize, :hsize] - covar_coeff_blk[0, :, 1, :] = covar_ell_off[:hsize, :hsize] - covar_coeff_blk[1, :, 0, :] = covar_ell_off.T[:hsize, :hsize] - - covar_coeff.append(covar_coeff_blk.reshape(2 * hsize, 2 * hsize)) - - return covar_coeff - - def get_mean(self, coeffs, ctf_fb=None, ctf_idx=None): + def get_mean(self, coeffs, ctf_basis=None, ctf_idx=None): """ Calculate the mean vector from the expansion coefficients with CTF information. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be averaged. - :param ctf_fb: The CFT functions in the FB expansion. + :param ctf_basis: The CFT functions in the BASIS expansion. :param ctf_idx: An array of the CFT function indices for all 2D images. - If ctf_fb or ctf_idx is None, the identity filter will be applied. + If ctf_basis or ctf_idx is None, the identity filter will be applied. :return: The mean value vector for all images. """ + # TODO: Redundant, remove? if coeffs.size == 0: raise RuntimeError("The coefficients need to be calculated!") # should assert we require none or both... - if (ctf_fb is None) or (ctf_idx is None): + if (ctf_basis is None) or (ctf_idx is None): ctf_idx = np.zeros(coeffs.shape[0], dtype=int) - ctf_fb = [ - BlkDiagMatrix.eye_like( - RadialCTFFilter().fb_mat(self.basis), dtype=self.dtype - ) - ] + ctf_basis = [self._ctf_identity_mat()] b = np.zeros(self.basis.count, dtype=coeffs.dtype) - A = BlkDiagMatrix.zeros_like(ctf_fb[0]) + A = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) for k in np.unique(ctf_idx[:]).T: coeff_k = coeffs[ctf_idx == k] weight = coeff_k.shape[0] / coeffs.shape[0] mean_coeff_k = self._get_mean(coeff_k) - ctf_fb_k = ctf_fb[k] - ctf_fb_k_t = ctf_fb_k.T - b += weight * ctf_fb_k_t.apply(mean_coeff_k) - A += weight * (ctf_fb_k_t @ ctf_fb_k) + ctf_basis_k = ctf_basis[k] + ctf_basis_k_t = ctf_basis_k.T + b += weight * ctf_basis_k_t.apply(mean_coeff_k) + A += weight * (ctf_basis_k_t @ ctf_basis_k) mean_coeff = A.solve(b) return mean_coeff @@ -212,7 +175,7 @@ def get_mean(self, coeffs, ctf_fb=None, ctf_idx=None): def get_covar( self, coeffs, - ctf_fb=None, + ctf_basis=None, ctf_idx=None, mean_coeff=None, do_refl=True, @@ -224,9 +187,9 @@ def get_covar( Calculate the covariance matrix from the expansion coefficients and CTF information. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. - :param ctf_fb: The CFT functions in the FB expansion. + :param ctf_basis: The CFT functions in the BASIS expansion. :param ctf_idx: An array of the CFT function indices for all 2D images. - If ctf_fb or ctf_idx is None, the identity filter will be applied. + If ctf_basis or ctf_idx is None, the identity filter will be applied. :param mean_coeff: The mean value vector from all images. :param noise_var: The estimated variance of noise. The value should be zero for `coeffs` from clean images of simulation data. @@ -237,20 +200,16 @@ def get_covar( block diagonal matrices are implemented as BlkDiagMatrix instances. The covariance is calculated from the images represented by the coeffs array, along with all possible rotations and reflections. As a result, the computed covariance - matrix is invariant to both reflection and rotation. The effect of the filters in ctf_fb + matrix is invariant to both reflection and rotation. The effect of the filters in ctf_basis are accounted for and inverted to yield a covariance estimate of the unfiltered images. """ if coeffs.size == 0: raise RuntimeError("The coefficients need to be calculated!") - if (ctf_fb is None) or (ctf_idx is None): + if (ctf_basis is None) or (ctf_idx is None): ctf_idx = np.zeros(coeffs.shape[0], dtype=int) - ctf_fb = [ - BlkDiagMatrix.eye_like( - RadialCTFFilter().fb_mat(self.basis), dtype=self.dtype - ) - ] + ctf_basis = [self._ctf_identity_mat()] def identity(x): return x @@ -269,31 +228,35 @@ def identity(x): covar_est_opt = fill_struct(covar_est_opt, default_est_opt) if mean_coeff is None: - mean_coeff = self.get_mean(coeffs, ctf_fb, ctf_idx) + mean_coeff = self.get_mean(coeffs, ctf_basis, ctf_idx) - b_coeff = BlkDiagMatrix.zeros_like(ctf_fb[0]) - b_noise = BlkDiagMatrix.zeros_like(ctf_fb[0]) + b_coeff = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) + b_noise = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) A = [] - for _ in range(len(ctf_fb)): - A.append(BlkDiagMatrix.zeros_like(ctf_fb[0])) + for _ in range(len(ctf_basis)): + A.append(BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape)) - M = BlkDiagMatrix.zeros_like(ctf_fb[0]) + M = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) for k in np.unique(ctf_idx[:]): coeff_k = coeffs[ctf_idx == k].astype(self.dtype) weight = coeff_k.shape[0] / coeffs.shape[0] - ctf_fb_k = ctf_fb[k] - ctf_fb_k_t = ctf_fb_k.T - mean_coeff_k = ctf_fb_k.apply(mean_coeff) + ctf_basis_k = ctf_basis[k] + ctf_basis_k_t = ctf_basis_k.T + mean_coeff_k = ctf_basis_k.apply(mean_coeff) covar_coeff_k = self._get_covar(coeff_k, mean_coeff_k) - b_coeff += weight * (ctf_fb_k_t @ covar_coeff_k @ ctf_fb_k) + b_coeff += weight * (ctf_basis_k_t @ covar_coeff_k @ ctf_basis_k) - ctf_fb_k_sq = ctf_fb_k_t @ ctf_fb_k - b_noise += weight * ctf_fb_k_sq + ctf_basis_k_sq = ctf_basis_k_t @ ctf_basis_k + b_noise += weight * ctf_basis_k_sq - A[k] = np.sqrt(weight) * ctf_fb_k_sq + A_k = np.sqrt(weight) * ctf_basis_k_sq + if not isinstance(A_k, BlkDiagMatrix): + A_k = DiagMatrix(A_k).as_blk_diag(self.basis.blk_diag_cov_shape) + + A[k] = A_k M += A[k] if not b_coeff.check_psd(): @@ -319,7 +282,7 @@ def identity(x): cg_opt = covar_est_opt - covar_coeff = BlkDiagMatrix.zeros_like(ctf_fb[0]) + covar_coeff = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) def precond_fun(S, x): p = np.size(S, 0) @@ -359,6 +322,63 @@ def apply(A, x): return covar_coeff + def _get_covar(self, coeffs, mean_coeff=None, do_refl=True): + """ + Calculate the covariance matrix from the expansion coefficients without CTF information. + + :param coeffs: A coefficient vector (or an array of coefficient vectors) calculated from 2D images. + :param mean_coeff: The mean vector calculated from the `coeffs`. + :param do_refl: If true, enforce invariance to reflection (default false). + :return: The covariance matrix of coefficients for all images. + """ + if coeffs.size == 0: + raise RuntimeError("The coefficients need to be calculated first!") + if mean_coeff is None: + mean_coeff = self._get_mean(coeffs) + + # Initialize a totally empty BlkDiagMatrix, build incrementally. + covar_coeff = BlkDiagMatrix.empty(0, dtype=coeffs.dtype) + ell = 0 + + mask = self.basis._indices["ells"] == ell + + coeff_ell = coeffs[..., mask] - mean_coeff[mask] + covar_ell = np.array(coeff_ell.T @ coeff_ell / coeffs.shape[0]) + covar_coeff.append(covar_ell) + + for ell in range(1, self.basis.ell_max + 1): + mask_ell = self.basis._indices["ells"] == ell + mask_pos = mask_ell & (self.basis._indices["sgns"] == +1) + mask_neg = mask_ell & (self.basis._indices["sgns"] == -1) + + covar_ell_diag = np.array( + coeffs[:, mask_pos].T @ coeffs[:, mask_pos] + + coeffs[:, mask_neg].T @ coeffs[:, mask_neg] + ) / (2 * coeffs.shape[0]) + + if do_refl: + covar_coeff.append(covar_ell_diag) + covar_coeff.append(covar_ell_diag) + else: + covar_ell_off = np.array( + ( + coeffs[:, mask_pos] @ coeffs[:, mask_neg].T / coeffs.shape[0] + - coeffs[:, mask_pos].T @ coeffs[:, mask_neg] + ) + / (2 * coeffs.shape[0]) + ) + + hsize = covar_ell_diag.shape[0] + covar_coeff_blk = np.zeros((2, hsize, 2, hsize)) + + covar_coeff_blk[0:2, :, 0:2, :] = covar_ell_diag[:hsize, :hsize] + covar_coeff_blk[0, :, 1, :] = covar_ell_off[:hsize, :hsize] + covar_coeff_blk[1, :, 0, :] = covar_ell_off.T[:hsize, :hsize] + + covar_coeff.append(covar_coeff_blk.reshape(2 * hsize, 2 * hsize)) + + return covar_coeff + def shrink_covar_backward(self, b, b_noise, n, noise_var, shrinker): """ Apply the shrinking method to the 2D covariance of coefficients. @@ -386,7 +406,7 @@ def shrink_covar_backward(self, b, b_noise, n, noise_var, shrinker): def get_cwf_coeffs( self, coeffs, - ctf_fb=None, + ctf_basis=None, ctf_idx=None, mean_coeff=None, covar_coeff=None, @@ -396,9 +416,9 @@ def get_cwf_coeffs( Estimate the expansion coefficients using the Covariance Wiener Filtering (CWF) method. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. - :param ctf_fb: The CFT functions in the FB expansion. + :param ctf_basis: The CFT functions in the BASIS expansion. :param ctf_idx: An array of the CFT function indices for all 2D images. - If ctf_fb or ctf_idx is None, the identity filter will be applied. + If ctf_basis or ctf_idx is None, the identity filter will be applied. :param mean_coeff: The mean value vector from all images. :param covar_coeff: The block diagonal covariance matrix of the clean coefficients represented by a cell array. :param noise_var: The estimated variance of noise. The value should be zero for `coeffs` @@ -409,24 +429,24 @@ def get_cwf_coeffs( """ if mean_coeff is None: - mean_coeff = self.get_mean(coeffs, ctf_fb, ctf_idx) + mean_coeff = self.get_mean(coeffs, ctf_basis, ctf_idx) if covar_coeff is None: covar_coeff = self.get_covar( - coeffs, ctf_fb, ctf_idx, mean_coeff, noise_var=noise_var + coeffs, ctf_basis, ctf_idx, mean_coeff, noise_var=noise_var ) # Handle CTF arguments. - if (ctf_fb is None) ^ (ctf_idx is None): + if (ctf_basis is None) ^ (ctf_idx is None): raise RuntimeError( - "Both `ctf_fb` and `ctf_idx` should be provided," + "Both `ctf_basis` and `ctf_idx` should be provided," " or both should be `None`." - f' Given {"ctf_fb" if ctf_idx is None else "ctf_idx"}' + f' Given {"ctf_basis" if ctf_idx is None else "ctf_idx"}' ) - elif ctf_fb is None: + elif ctf_basis is None: # Setup defaults for CTF ctf_idx = np.zeros(coeffs.shape[0], dtype=int) - ctf_fb = [BlkDiagMatrix.eye_like(covar_coeff)] + ctf_basis = [BlkDiagMatrix.eye_like(covar_coeff)] noise_covar_coeff = noise_var * BlkDiagMatrix.eye_like(covar_coeff) @@ -434,20 +454,20 @@ def get_cwf_coeffs( for k in np.unique(ctf_idx[:]): coeff_k = coeffs[ctf_idx == k] - ctf_fb_k = ctf_fb[k] - ctf_fb_k_t = ctf_fb_k.T + ctf_basis_k = ctf_basis[k] + ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_fb_k.apply(mean_coeff) + mean_coeff_k = ctf_basis_k.apply(mean_coeff) coeff_est_k = coeff_k - mean_coeff_k if noise_var == 0: - coeff_est_k = ctf_fb_k.solve(coeff_est_k.T).T + coeff_est_k = ctf_basis_k.solve(coeff_est_k.T).T else: - sig_covar_coeff = ctf_fb_k @ covar_coeff @ ctf_fb_k_t + sig_covar_coeff = ctf_basis_k @ covar_coeff @ ctf_basis_k_t sig_noise_covar_coeff = sig_covar_coeff + noise_covar_coeff coeff_est_k = sig_noise_covar_coeff.solve(coeff_est_k.T).T - coeff_est_k = (covar_coeff @ ctf_fb_k_t).apply(coeff_est_k.T).T + coeff_est_k = (covar_coeff @ ctf_basis_k_t).apply(coeff_est_k.T).T coeff_est_k = coeff_est_k + mean_coeff coeffs_est[ctf_idx == k] = coeff_est_k @@ -492,33 +512,32 @@ def _build(self): src = self.src if self.basis is None: - from aspire.basis import FFBBasis2D - self.basis = FFBBasis2D((src.L, src.L), dtype=self.dtype) if not src.unique_filters: logger.info("CTF filters are not included in Cov2D denoising") # set all CTF filters to an identity filter self.ctf_idx = np.zeros(src.n, dtype=int) - self.ctf_fb = [BlkDiagMatrix.eye_like(RadialCTFFilter().fb_mat(self.basis))] + self.ctf_basis = [self._ctf_identity_mat()] + else: - logger.info("Represent CTF filters in FB basis") + logger.info("Represent CTF filters in basis") unique_filters = src.unique_filters self.ctf_idx = src.filter_indices - self.ctf_fb = [f.fb_mat(self.basis) for f in unique_filters] + self.ctf_basis = [self.basis.filter_to_basis_mat(f) for f in unique_filters] def _calc_rhs(self): src = self.src basis = self.basis - ctf_fb = self.ctf_fb + ctf_basis = self.ctf_basis ctf_idx = self.ctf_idx zero_coeff = np.zeros((basis.count,), dtype=self.dtype) - b_mean = [np.zeros(basis.count, dtype=self.dtype) for _ in ctf_fb] + b_mean = [np.zeros(basis.count, dtype=self.dtype) for _ in ctf_basis] - b_covar = BlkDiagMatrix.zeros_like(ctf_fb[0]) + b_covar = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape, dtype=self.dtype) for start in range(0, src.n, self.batch_size): batch = np.arange(start, min(start + self.batch_size, src.n)) @@ -532,17 +551,18 @@ def _calc_rhs(self): mean_coeff_k = self._get_mean(coeff_k) - ctf_fb_k = ctf_fb[k] - ctf_fb_k_t = ctf_fb_k.T + ctf_basis_k = ctf_basis[k] + ctf_basis_k_t = ctf_basis_k.T - b_mean_k = weight * ctf_fb_k_t.apply(mean_coeff_k) + b_mean_k = weight * ctf_basis_k_t.apply(mean_coeff_k) b_mean[k] += b_mean_k covar_coeff_k = self._get_covar(coeff_k, zero_coeff) - b_covar_k = ctf_fb_k_t @ covar_coeff_k - b_covar_k = b_covar_k @ ctf_fb_k + b_covar_k = ctf_basis_k_t @ covar_coeff_k + + b_covar_k = b_covar_k @ ctf_basis_k b_covar_k *= weight b_covar += b_covar_k @@ -553,24 +573,26 @@ def _calc_rhs(self): def _calc_op(self): src = self.src - ctf_fb = self.ctf_fb + ctf_basis = self.ctf_basis ctf_idx = self.ctf_idx - A_mean = BlkDiagMatrix.zeros_like(ctf_fb[0]) - A_covar = [None for _ in ctf_fb] - M_covar = BlkDiagMatrix.zeros_like(ctf_fb[0]) + A_mean = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape, self.dtype) + A_covar = [None for _ in ctf_basis] + M_covar = BlkDiagMatrix.zeros_like(A_mean) for k in np.unique(ctf_idx): weight = np.count_nonzero(ctf_idx == k) / src.n - ctf_fb_k = ctf_fb[k] - ctf_fb_k_t = ctf_fb_k.T + ctf_basis_k = ctf_basis[k] + ctf_basis_k_t = ctf_basis_k.T - ctf_fb_k_sq = ctf_fb_k_t @ ctf_fb_k - A_mean_k = weight * ctf_fb_k_sq + ctf_basis_k_sq = ctf_basis_k_t @ ctf_basis_k + A_mean_k = weight * ctf_basis_k_sq + # blk + diag A_mean += A_mean_k - - A_covar_k = np.sqrt(weight) * ctf_fb_k_sq + # why is rmul returning an array?! + # A_covar_k = np.sqrt(weight) * ctf_basis_k_sq + A_covar_k = ctf_basis_k_sq * np.sqrt(weight) A_covar[k] = A_covar_k M_covar += A_covar_k @@ -582,10 +604,10 @@ def _calc_op(self): def _mean_correct_covar_rhs(self, b_covar, b_mean, mean_coeff): src = self.src - ctf_fb = self.ctf_fb + ctf_basis = self.ctf_basis ctf_idx = self.ctf_idx - partition = ctf_fb[0].partition + partition = self.basis.blk_diag_cov_shape # Note: If we don't do this, we'll be modifying the stored `b_covar` # since the operations below are in-place. @@ -594,11 +616,11 @@ def _mean_correct_covar_rhs(self, b_covar, b_mean, mean_coeff): for k in np.unique(ctf_idx): weight = np.count_nonzero(ctf_idx == k) / src.n - ctf_fb_k = ctf_fb[k] - ctf_fb_k_t = ctf_fb_k.T + ctf_basis_k = ctf_basis[k] + ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_fb_k.apply(mean_coeff) - mean_coeff_k = ctf_fb_k_t.apply(mean_coeff_k) + mean_coeff_k = ctf_basis_k.apply(mean_coeff) + mean_coeff_k = ctf_basis_k_t.apply(mean_coeff_k) mean_coeff_k = mean_coeff_k[: partition[0][0]] b_mean_k = b_mean[k][: partition[0][0]] @@ -625,8 +647,31 @@ def _noise_correct_covar_rhs(self, b_covar, b_noise, noise_var, shrinker): return b_covar def _solve_covar(self, A_covar, b_covar, M, covar_est_opt): - ctf_fb = self.ctf_fb + method = self._solve_covar_cg + if self.basis.matrix_type == DiagMatrix: + method = self._solve_covar_direct + return method(A_covar, b_covar, M, covar_est_opt) + + def _solve_covar_direct(self, A_covar, b_covar, M, covar_est_opt): + # A_covar is a list of Diags, ctf in basis + # b_covar is a blk diag + # M is a blk diag + + # # A_covar is (CTF.T * CTF) (weighted a**2 from chalkboard) + # most similar to wr_k2 from Yunpeng's code + + # maybe need to unweight A + A_covar = DiagMatrix(np.concatenate([x.asnumpy() for x in A_covar])) + A2i = A_covar * A_covar + + L = A2i.lr_scale(b_covar.partition) + + return b_covar / L + + # M is sum of weighted A squared, only for cg, ignore + + def _solve_covar_cg(self, A_covar, b_covar, M, covar_est_opt): def precond_fun(S, x): p = np.size(S, 0) assert np.size(x) == p * p, "The sizes of S and x are not consistent." @@ -645,7 +690,9 @@ def apply(A, x): return y cg_opt = covar_est_opt - covar_coeff = BlkDiagMatrix.zeros_like(ctf_fb[0]) + covar_coeff = BlkDiagMatrix.zeros( + self.basis.blk_diag_cov_shape, dtype=self.dtype + ) for ell in range(0, len(b_covar)): A_ell = [] @@ -770,15 +817,15 @@ def identity(x): return covar_coeff def get_cwf_coeffs( - self, coeffs, ctf_fb, ctf_idx, mean_coeff, covar_coeff, noise_var=0 + self, coeffs, ctf_basis, ctf_idx, mean_coeff, covar_coeff, noise_var=0 ): """ Estimate the expansion coefficients using the Covariance Wiener Filtering (CWF) method. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. - :param ctf_fb: The CFT functions in the FB expansion. + :param ctf_basis: The CFT functions in the BASIS expansion. :param ctf_idx: An array of the CFT function indices for all 2D images. - If ctf_fb or ctf_idx is None, the identity filter will be applied. + If ctf_basis or ctf_idx is None, the identity filter will be applied. :param mean_coeff: The mean value vector from all images. :param covar_coeff: The block diagonal covariance matrix of the clean coefficients represented by a cell array. :param noise_var: The estimated variance of noise. The value should be zero for `coeffs` @@ -795,16 +842,16 @@ def get_cwf_coeffs( covar_coeff = self.get_covar(noise_var=noise_var, mean_coeff=mean_coeff) # Handle CTF arguments. - if (ctf_fb is None) ^ (ctf_idx is None): + if (ctf_basis is None) ^ (ctf_idx is None): raise RuntimeError( - "Both `ctf_fb` and `ctf_idx` should be provided," + "Both `ctf_basis` and `ctf_idx` should be provided," " or both should be `None`." - f' Given {"ctf_fb" if ctf_idx is None else "ctf_idx"}' + f' Given {"ctf_basis" if ctf_idx is None else "ctf_idx"}' ) - elif ctf_fb is None: + elif ctf_basis is None: # Setup defaults for CTF ctf_idx = np.zeros(coeffs.shape[0], dtype=int) - ctf_fb = [BlkDiagMatrix.eye_like(covar_coeff)] + ctf_basis = [BlkDiagMatrix.eye_like(covar_coeff)] noise_covar_coeff = noise_var * BlkDiagMatrix.eye_like(covar_coeff) @@ -812,20 +859,20 @@ def get_cwf_coeffs( for k in np.unique(ctf_idx[:]): coeff_k = coeffs[ctf_idx == k] - ctf_fb_k = ctf_fb[k] - ctf_fb_k_t = ctf_fb_k.T + ctf_basis_k = ctf_basis[k] + ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_fb_k.apply(mean_coeff) + mean_coeff_k = ctf_basis_k.apply(mean_coeff) coeff_est_k = coeff_k - mean_coeff_k if noise_var == 0: - coeff_est_k = ctf_fb_k.solve(coeff_est_k.T).T + coeff_est_k = ctf_basis_k.solve(coeff_est_k.T).T else: - sig_covar_coeff = ctf_fb_k @ covar_coeff @ ctf_fb_k_t + sig_covar_coeff = ctf_basis_k @ covar_coeff @ ctf_basis_k_t sig_noise_covar_coeff = sig_covar_coeff + noise_covar_coeff coeff_est_k = sig_noise_covar_coeff.solve(coeff_est_k.T).T - coeff_est_k = (covar_coeff @ ctf_fb_k_t).apply(coeff_est_k.T).T + coeff_est_k = (covar_coeff @ ctf_basis_k_t).apply(coeff_est_k.T).T coeff_est_k = coeff_est_k + mean_coeff coeffs_est[ctf_idx == k] = coeff_est_k diff --git a/src/aspire/denoising/denoiser_cov2d.py b/src/aspire/denoising/denoiser_cov2d.py index b5581013e4..a00dcc3078 100644 --- a/src/aspire/denoising/denoiser_cov2d.py +++ b/src/aspire/denoising/denoiser_cov2d.py @@ -187,7 +187,7 @@ def images(self, istart=0, batch_size=512): ) coeffs_estim = self.cov2d.get_cwf_coeffs( coeffs_noise, - self.cov2d.ctf_fb, + self.cov2d.ctf_basis, self.cov2d.ctf_idx[img_start:img_end], mean_coeff=self.mean_est, covar_coeff=self.covar_est, diff --git a/tests/test_batched_covar2d.py b/tests/test_batched_covar2d.py index c8af4b0bf2..801ff8b7c6 100644 --- a/tests/test_batched_covar2d.py +++ b/tests/test_batched_covar2d.py @@ -17,7 +17,7 @@ class BatchedRotCov2DTestCase(TestCase): filters = None ctf_idx = None - ctf_fb = None + ctf_basis = None def setUp(self): n = 32 @@ -60,12 +60,12 @@ def testMeanCovar(self): # Test basic functionality against RotCov2D. mean_cov2d = self.cov2d.get_mean( - self.coeff, ctf_fb=self.ctf_fb, ctf_idx=self.ctf_idx + self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_cov2d = self.cov2d.get_covar( self.coeff, mean_coeff=mean_cov2d, - ctf_fb=self.ctf_fb, + ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, noise_var=self.noise_var, ) @@ -88,7 +88,10 @@ def testZeroMean(self): zero_coeff = np.zeros((self.basis.count,), dtype=self.dtype) covar_cov2d = self.cov2d.get_covar( - self.coeff, mean_coeff=zero_coeff, ctf_fb=self.ctf_fb, ctf_idx=self.ctf_idx + self.coeff, + mean_coeff=zero_coeff, + ctf_basis=self.ctf_basis, + ctf_idx=self.ctf_idx, ) covar_bcov2d = self.bcov2d.get_covar(mean_coeff=zero_coeff) @@ -102,7 +105,7 @@ def testZeroMean(self): def testAutoMean(self): # Make sure it automatically calls get_mean if needed. covar_cov2d = self.cov2d.get_covar( - self.coeff, ctf_fb=self.ctf_fb, ctf_idx=self.ctf_idx + self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_bcov2d = self.bcov2d.get_covar() @@ -127,7 +130,7 @@ def testShrink(self): covar_cov2d = self.cov2d.get_covar( self.coeff, - ctf_fb=self.ctf_fb, + ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, covar_est_opt=covar_est_opt, ) @@ -152,11 +155,11 @@ def testAutoBasis(self): def testCWFCoeff(self): # Calculate CWF coefficients using Cov2D base class mean_cov2d = self.cov2d.get_mean( - self.coeff, ctf_fb=self.ctf_fb, ctf_idx=self.ctf_idx + self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_cov2d = self.cov2d.get_covar( self.coeff, - ctf_fb=self.ctf_fb, + ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, noise_var=self.noise_var, make_psd=True, @@ -164,7 +167,7 @@ def testCWFCoeff(self): coeff_cov2d = self.cov2d.get_cwf_coeffs( self.coeff, - self.ctf_fb, + self.ctf_basis, self.ctf_idx, mean_coeff=mean_cov2d, covar_coeff=covar_cov2d, @@ -177,7 +180,7 @@ def testCWFCoeff(self): coeff_bcov2d = self.bcov2d.get_cwf_coeffs( self.coeff, - self.ctf_fb, + self.ctf_basis, self.ctf_idx, mean_bcov2d, covar_bcov2d, @@ -202,11 +205,11 @@ def testCWFCoeffCleanCTF(self): # Calculate CWF coefficients using Cov2D base class mean_cov2d = self.cov2d.get_mean( - self.coeff, ctf_fb=self.ctf_fb, ctf_idx=self.ctf_idx + self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_cov2d = self.cov2d.get_covar( self.coeff, - ctf_fb=self.ctf_fb, + ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, noise_var=self.noise_var, make_psd=True, @@ -214,7 +217,7 @@ def testCWFCoeffCleanCTF(self): coeff_cov2d = self.cov2d.get_cwf_coeffs( self.coeff, - self.ctf_fb, + self.ctf_basis, self.ctf_idx, mean_coeff=mean_cov2d, covar_coeff=covar_cov2d, @@ -227,7 +230,7 @@ def testCWFCoeffCleanCTF(self): coeff_bcov2d = self.bcov2d.get_cwf_coeffs( self.coeff, - self.ctf_fb, + self.ctf_basis, self.ctf_idx, mean_bcov2d, covar_bcov2d, @@ -259,5 +262,5 @@ def ctf_idx(self): return self.src.filter_indices @property - def ctf_fb(self): + def ctf_basis(self): return [self.basis.filter_to_basis_mat(f) for f in self.src.unique_filters] From cf320a2f1b4350b806c09c01ace53c73553b74bf Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 13:35:44 -0400 Subject: [PATCH 007/294] update class_src test towards params --- tests/test_class_src.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_class_src.py b/tests/test_class_src.py index 6e1b9ca903..88be36a013 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from aspire.basis import FFBBasis2D +from aspire.basis import FBBasis2D, FFBBasis2D from aspire.classification import ( BandedSNRImageQualityFunction, BFRAverager2D, @@ -49,6 +49,20 @@ NUM_PROCS = 1 +BASIS = [ + FFBBasis2D, + pytest.param(FBBasis2D, marks=pytest.mark.expensive), +] + + +@pytest.fixture(params=BASIS, ids=lambda x: f"basis={x}") +def basis(request, img_size, dtype): + cls = request.param + # Setup a Basis + basis = cls(img_size, dtype=dtype) + return basis + + def sim_fixture_id(params): res = params[0] dtype = params[1] @@ -234,9 +248,9 @@ def test_online_selector(cls_fixture, selector): "quality_function", QUALITY_FUNCTIONS, ids=lambda param: f"Quality Function={param}" ) @pytest.mark.expensive -def test_global_selector(class_sim_fixture, cls_fixture, selector, quality_function): - basis = FFBBasis2D(class_sim_fixture.L, dtype=class_sim_fixture.dtype) - +def test_global_selector( + class_sim_fixture, cls_fixture, selector, quality_function, basis +): averager = BFRAverager2D(basis, class_sim_fixture, num_procs=NUM_PROCS) fun = quality_function() From 1bde1071796e22efb433e844151948fd938a8179 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 3 Aug 2023 13:45:21 -0400 Subject: [PATCH 008/294] update class2D test towards params --- tests/test_class2D.py | 651 ++++++++++++++++++++++-------------------- 1 file changed, 340 insertions(+), 311 deletions(-) diff --git a/tests/test_class2D.py b/tests/test_class2D.py index bec7348442..1e2422a484 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -1,12 +1,11 @@ import logging import os -from unittest import TestCase import numpy as np import pytest from sklearn import datasets -from aspire.basis import FFBBasis2D, FSPCABasis +from aspire.basis import FBBasis2D, FFBBasis2D, FSPCABasis from aspire.classification import RIRClass2D from aspire.classification.legacy_implementations import bispec_2drot_large, pca_y from aspire.noise import WhiteNoiseAdder @@ -23,369 +22,399 @@ SEED = 42 -class FSPCATestCase(TestCase): - def setUp(self): - self.resolution = 16 - self.dtype = np.float32 +IMG_SIZES = [16] +DTYPES = [np.float32] +# Basis used in FSPCA for class averaging. +BASIS = [ + FFBBasis2D, + pytest.param(FBBasis2D, marks=pytest.mark.expensive), +] - # Get a volume - v = Volume( - np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype( - self.dtype - ) - ) - v = v.downsample(self.resolution) - # Create a src from the volume - self.src = Simulation( - L=self.resolution, n=321, vols=v, dtype=self.dtype, seed=SEED - ) - self.src = self.src.cache() # Precompute image stack +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +def dtype(request): + return request.param - # Calculate some projection images - self.imgs = self.src.images[:] - # Configure an FSPCA basis - self.fspca_basis = FSPCABasis(self.src, noise_var=0) +@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}") +def img_size(request): + return request.param - def testExpandEval(self): - coef = self.fspca_basis.expand_from_image_basis(self.imgs) - recon = self.fspca_basis.evaluate_to_image_basis(coef) - # Check recon is close to imgs - rmse = np.sqrt(np.mean(np.square(self.imgs.asnumpy() - recon.asnumpy()))) - logger.info(f"FSPCA Expand Eval Image Round True RMSE: {rmse}") - self.assertTrue(rmse < utest_tolerance(self.dtype)) +@pytest.fixture +def volume(dtype, img_size): + # Get a volume + v = Volume( + np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype(dtype) + ) - def testComplexConversionErrors(self): - """ - Test we raise when passed incorrect dtypes. + return v.downsample(img_size) - Also checks we can handle 0d vector in `to_real`. - Most other cases covered by classification unit tests. - """ +@pytest.fixture +def sim_fixture(volume, img_size, dtype): + # Create a src from the volume + src = Simulation(L=img_size, n=321, vols=volume, dtype=dtype, seed=SEED) + src = src.cache() # Precompute image stack - with pytest.raises( - TypeError, match="coef provided to to_complex should be real." - ): - _ = self.fspca_basis.to_complex( - np.arange(self.fspca_basis.count, dtype=np.complex64) - ) + # Calculate some projection images + imgs = src.images[:] - with pytest.raises( - TypeError, match="coef provided to to_real should be complex." - ): - _ = self.fspca_basis.to_real( - np.arange(self.fspca_basis.count, dtype=np.float32).flatten() - ) + # Configure an FSPCA basis + fspca_basis = FSPCABasis(src, noise_var=0) - def testRotate(self): - """ - Trivial test of rotation in FSPCA Basis. - - Also covers to_real and to_complex conversions in FSPCA Basis. - """ - coef = self.fspca_basis.expand_from_image_basis(self.imgs) - # rotate by pi - rot_coef = self.fspca_basis.rotate(coef, radians=np.pi) - rot_imgs = self.fspca_basis.evaluate_to_image_basis(rot_coef) - - for i, img in enumerate(self.imgs): - rmse = np.sqrt(np.mean(np.square(np.flip(img) - rot_imgs[i]))) - self.assertTrue(rmse < 10 * utest_tolerance(self.dtype)) - - def testBasisTooSmall(self): - """ - When number of components is more than basis functions raise with descriptive error. - """ - fb_basis = FFBBasis2D((self.resolution, self.resolution), dtype=self.dtype) - - with pytest.raises(ValueError, match=r".*Reduce components.*"): - # Configure an FSPCA basis - _ = FSPCABasis( - self.src, basis=fb_basis, components=fb_basis.count * 2, noise_var=0 - ) + return imgs, src, fspca_basis -class RIRClass2DTestCase(TestCase): - def setUp(self): - self.n_classes = 5 - self.resolution = 16 - self.dtype = np.float64 - self.n_img = 150 +@pytest.fixture(params=BASIS, ids=lambda x: f"basis={x}") +def basis(request, img_size, dtype): + cls = request.param + # Setup a Basis + basis = cls(img_size, dtype=dtype) + return basis - # Create some projections - v = Volume( - np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype( - self.dtype - ) - ) - v = v.downsample(self.resolution) - # Clean - self.clean_src = Simulation( - L=self.resolution, n=self.n_img, vols=v, dtype=self.dtype, seed=SEED +def test_expand_eval(sim_fixture): + imgs, _, fspca_basis = sim_fixture + coef = fspca_basis.expand_from_image_basis(imgs) + recon = fspca_basis.evaluate_to_image_basis(coef) + + # Check recon is close to imgs + rmse = np.sqrt(np.mean(np.square(imgs.asnumpy() - recon.asnumpy()))) + logger.info(f"FSPCA Expand Eval Image Round True RMSE: {rmse}") + assert rmse < utest_tolerance(fspca_basis.dtype) + + +def test_complex_conversions_errors(sim_fixture): + """ + Test we raise when passed incorrect dtypes. + + Also checks we can handle 0d vector in `to_real`. + + Most other cases covered by classification unit tests. + """ + imgs, _, fspca_basis = sim_fixture + + with pytest.raises(TypeError, match="coef provided to to_complex should be real."): + _ = fspca_basis.to_complex( + np.arange(fspca_basis.count), ) - # With Noise - noise_var = 0.01 * np.var(np.sum(v[0], axis=0)) - noise_adder = WhiteNoiseAdder(var=noise_var) - self.noisy_src = Simulation( - L=self.resolution, - n=self.n_img, - vols=v, - dtype=self.dtype, - noise_adder=noise_adder, - seed=SEED, + with pytest.raises(TypeError, match="coef provided to to_real should be complex."): + _ = fspca_basis.to_real( + np.arange(fspca_basis.count, dtype=np.float32).flatten() ) - # Set up FFB - # Setup a Basis - self.basis = FFBBasis2D((self.resolution, self.resolution), dtype=self.dtype) - - # Create Basis, use precomputed Basis - self.clean_fspca_basis = FSPCABasis( - self.clean_src, self.basis, noise_var=0 - ) # Note noise_var assigned zero, skips eigval filtering. - - self.clean_fspca_basis_compressed = FSPCABasis( - self.clean_src, self.basis, components=101, noise_var=0 - ) # Note noise_var assigned zero, skips eigval filtering. - - # Ceate another fspca_basis, use autogeneration FFB2D Basis - self.noisy_fspca_basis = FSPCABasis(self.noisy_src) - - def testSourceTooSmall(self): - """ - When number of images in source is less than requested bispectrum components, - raise with descriptive error. - """ - - with pytest.raises( - RuntimeError, match=r".*Increase number of images or reduce components.*" - ): - _ = RIRClass2D( - self.clean_src, - fspca_components=self.clean_src.n * 4, - bispectrum_components=self.clean_src.n * 2, - ) - def testIncorrectComponents(self): - """ - Check we raise with inconsistent configuration of FSPCA components. - """ - - with pytest.raises( - RuntimeError, match=r"`pca_basis` components.*provided by user." - ): - _ = RIRClass2D( - self.clean_src, - self.clean_fspca_basis, # 400 components - fspca_components=100, - large_pca_implementation="legacy", - nn_implementation="legacy", - bispectrum_implementation="legacy", - ) +def test_rotate(sim_fixture): + """ + Trivial test of rotation in FSPCA Basis. + + Also covers to_real and to_complex conversions in FSPCA Basis. + """ + imgs, _, fspca_basis = sim_fixture + + coef = fspca_basis.expand_from_image_basis(imgs) + # rotate by pi + rot_coef = fspca_basis.rotate(coef, radians=np.pi) + rot_imgs = fspca_basis.evaluate_to_image_basis(rot_coef) + + for i, img in enumerate(imgs): + rmse = np.sqrt(np.mean(np.square(np.flip(img) - rot_imgs[i]))) + assert rmse < 10 * utest_tolerance(fspca_basis.dtype) + - # Explicitly providing the same number should be okay. +def test_basis_too_small(sim_fixture, basis): + """ + When number of components is more than basis functions raise with descriptive error. + """ + src = sim_fixture[1] + + with pytest.raises(ValueError, match=r".*Reduce components.*"): + # Configure an FSPCA basis + _ = FSPCABasis(src, basis=basis, components=basis.count * 2, noise_var=0) + + +@pytest.fixture +def sim_fixture2(volume, basis, img_size, dtype): + n_img = 150 + + # Clean + clean_src = Simulation(L=img_size, n=n_img, vols=volume, dtype=dtype, seed=SEED) + + # With Noise + noise_var = 0.01 * np.var(np.sum(volume[0], axis=0)) + noise_adder = WhiteNoiseAdder(var=noise_var) + noisy_src = Simulation( + L=img_size, + n=n_img, + vols=volume, + dtype=dtype, + noise_adder=noise_adder, + seed=SEED, + ) + + # Create Basis, use precomputed Basis + clean_fspca_basis = FSPCABasis( + clean_src, basis, noise_var=0 + ) # Note noise_var assigned zero, skips eigval filtering. + + clean_fspca_basis_compressed = FSPCABasis( + clean_src, basis, components=101, noise_var=0 + ) # Note noise_var assigned zero, skips eigval filtering. + + # Ceate another fspca_basis, use autogeneration Basis + noisy_fspca_basis = FSPCABasis(noisy_src) + + return ( + clean_src, + noisy_src, + clean_fspca_basis, + clean_fspca_basis_compressed, + noisy_fspca_basis, + ) + + +def test_source_too_small(sim_fixture2): + """ + When number of images in source is less than requested bispectrum components, + raise with descriptive error. + """ + clean_src = sim_fixture2[0] + + with pytest.raises( + RuntimeError, match=r".*Increase number of images or reduce components.*" + ): _ = RIRClass2D( - self.clean_src, - self.clean_fspca_basis, # 400 components - fspca_components=self.clean_fspca_basis.components, - bispectrum_components=100, - large_pca_implementation="legacy", - nn_implementation="legacy", - bispectrum_implementation="legacy", - seed=SEED, + clean_src, + fspca_components=clean_src.n * 4, + bispectrum_components=clean_src.n * 2, ) - def testRIRLegacy(self): - """ - Currently just tests for runtime errors. - """ - clean_fspca_basis = FSPCABasis( - self.clean_src, self.basis, noise_var=0, components=100 - ) # Note noise_var assigned zero, skips eigval filtering. +def test_incorrect_components(sim_fixture2): + """ + Check we raise with inconsistent configuration of FSPCA components. + """ + clean_src, clean_fspca_basis = sim_fixture2[0], sim_fixture2[2] - rir = RIRClass2D( - self.clean_src, - clean_fspca_basis, - bispectrum_components=42, + with pytest.raises( + RuntimeError, match=r"`pca_basis` components.*provided by user." + ): + _ = RIRClass2D( + clean_src, + clean_fspca_basis, # 400 components + fspca_components=100, large_pca_implementation="legacy", nn_implementation="legacy", bispectrum_implementation="legacy", - seed=SEED, ) - _ = rir.classify() + # Explicitly providing the same number should be okay. + _ = RIRClass2D( + clean_src, + clean_fspca_basis, # 400 components + fspca_components=clean_fspca_basis.components, + bispectrum_components=100, + large_pca_implementation="legacy", + nn_implementation="legacy", + bispectrum_implementation="legacy", + seed=SEED, + ) - def testRIRDevelBisp(self): - """ - Currently just tests for runtime errors. - """ - # Use the basis class setup, only requires a Source. - rir = RIRClass2D( - self.clean_src, - fspca_components=self.clean_fspca_basis.components, - bispectrum_components=self.clean_fspca_basis.components - 1, - large_pca_implementation="legacy", - nn_implementation="legacy", - bispectrum_implementation="devel", - ) +def test_RIR_legacy(basis, sim_fixture2): + """ + Currently just tests for runtime errors. + """ + clean_src = sim_fixture2[0] - _ = rir.classify() + clean_fspca_basis = FSPCABasis( + clean_src, basis, noise_var=0, components=100 + ) # Note noise_var assigned zero, skips eigval filtering. - def testRIRsk(self): - """ - Excercises the eigenvalue based filtering, - along with other swappable components. + rir = RIRClass2D( + clean_src, + clean_fspca_basis, + bispectrum_components=42, + large_pca_implementation="legacy", + nn_implementation="legacy", + bispectrum_implementation="legacy", + seed=SEED, + ) - Currently just tests for runtime errors. - """ - rir = RIRClass2D( - self.noisy_src, - self.noisy_fspca_basis, - bispectrum_components=100, - sample_n=42, - large_pca_implementation="sklearn", - nn_implementation="sklearn", - bispectrum_implementation="devel", - seed=SEED, + _ = rir.classify() + + +def test_RIR_devel_disp(sim_fixture2): + """ + Currently just tests for runtime errors. + """ + clean_src, fspca_basis = sim_fixture2[0], sim_fixture2[3] + + # Use the basis class setup, only requires a Source. + rir = RIRClass2D( + clean_src, + fspca_components=fspca_basis.components, + bispectrum_components=fspca_basis.components - 1, + large_pca_implementation="legacy", + nn_implementation="legacy", + bispectrum_implementation="devel", + ) + + _ = rir.classify() + + +def test_RIR_sk(sim_fixture2): + """ + Excercises the eigenvalue based filtering, + along with other swappable components. + + Currently just tests for runtime errors. + """ + noisy_src, noisy_fspca_basis = sim_fixture2[1], sim_fixture2[4] + + rir = RIRClass2D( + noisy_src, + noisy_fspca_basis, + bispectrum_components=100, + sample_n=42, + large_pca_implementation="sklearn", + nn_implementation="sklearn", + bispectrum_implementation="devel", + seed=SEED, + ) + + _ = rir.classify() + + +def test_eigein_images(sim_fixture2): + """ + Test we can return eigenimages. + """ + clean_fspca_basis, clean_fspca_basis_compressed = sim_fixture2[2], sim_fixture2[3] + + # Get the eigenimages from an FSPCA basis for testing + eigimg_uncompressed = clean_fspca_basis.eigen_images() + + # Get the eigenimages from a compressed FSPCA basis for testing + eigimg_compressed = clean_fspca_basis_compressed.eigen_images() + + # Check they are close. + # Note it is expected the compression reorders the eigvecs, + # and thus the eigimages. + # We sum over all the eigimages to yield an "average" for comparison + assert np.allclose( + np.sum(eigimg_uncompressed.asnumpy(), axis=0), + np.sum(eigimg_compressed.asnumpy(), axis=0), + atol=utest_tolerance(clean_fspca_basis.dtype), + ) + + +def test_component_size(sim_fixture2): + """ + Tests we raise when number of components are too small. + + Also tests dtype mismatch behavior. + """ + clean_src, compressed_fspca_basis = sim_fixture2[0], sim_fixture2[3] + + with pytest.raises(RuntimeError, match=r".*Reduce bispectrum_components.*"): + _ = RIRClass2D( + clean_src, + compressed_fspca_basis, + bispectrum_components=clean_src.n + 1, ) - _ = rir.classify() - def testEigenImages(self): - """ - Test we can return eigenimages. - """ +def test_implementations(basis, sim_fixture2): + """ + Test optional implementations handle bad inputs with a descriptive error. + """ + clean_src, clean_fspca_basis = sim_fixture2[0], sim_fixture2[2] - # Get the eigenimages from an FSPCA basis for testing - eigimg_uncompressed = self.clean_fspca_basis.eigen_images() + # Nearest Neighbhor component + with pytest.raises(ValueError, match=r"Provided nn_implementation.*"): + _ = RIRClass2D( + clean_src, + clean_fspca_basis, + bispectrum_components=int(0.75 * clean_fspca_basis.basis.count), + nn_implementation="badinput", + ) - # Get the eigenimages from a compressed FSPCA basis for testing - eigimg_compressed = self.clean_fspca_basis_compressed.eigen_images() + # Large PCA component + with pytest.raises(ValueError, match=r"Provided large_pca_implementation.*"): + _ = RIRClass2D( + clean_src, + clean_fspca_basis, + large_pca_implementation="badinput", + ) - # Check they are close. - # Note it is expected the compression reorders the eigvecs, - # and thus the eigimages. - # We sum over all the eigimages to yield an "average" for comparison - self.assertTrue( - np.allclose( - np.sum(eigimg_uncompressed.asnumpy(), axis=0), - np.sum(eigimg_compressed.asnumpy(), axis=0), - ) + # Bispectrum component + with pytest.raises(ValueError, match=r"Provided bispectrum_implementation.*"): + _ = RIRClass2D( + clean_src, + clean_fspca_basis, + bispectrum_implementation="badinput", + ) + + # Legacy Bispectrum implies legacy bispectrum (they're integrated). + with pytest.raises( + ValueError, match=r'"legacy" bispectrum_implementation implies.*' + ): + _ = RIRClass2D( + clean_src, + clean_fspca_basis, + bispectrum_implementation="legacy", + large_pca_implementation="sklearn", ) - def testComponentSize(self): - """ - Tests we raise when number of components are too small. + # Currently we only FSPCA Basis in RIRClass2D + with pytest.raises( + RuntimeError, + match="RIRClass2D has currently only been developed for pca_basis as a FSPCABasis.", + ): + _ = RIRClass2D(clean_src, basis) - Also tests dtype mismatch behavior. - """ - with pytest.raises(RuntimeError, match=r".*Reduce bispectrum_components.*"): - _ = RIRClass2D( - self.clean_src, - self.clean_fspca_basis, - bispectrum_components=self.clean_src.n + 1, - dtype=np.float64, - ) +# Cover branches of Legacy code not taken by the classification unit tests. - def testImplementations(self): - """ - Test optional implementations handle bad inputs with a descriptive error. - """ - - # Nearest Neighbhor component - with pytest.raises(ValueError, match=r"Provided nn_implementation.*"): - _ = RIRClass2D( - self.clean_src, - self.clean_fspca_basis, - bispectrum_components=int(0.75 * self.clean_fspca_basis.basis.count), - nn_implementation="badinput", - ) - # Large PCA component - with pytest.raises(ValueError, match=r"Provided large_pca_implementation.*"): - _ = RIRClass2D( - self.clean_src, - self.clean_fspca_basis, - large_pca_implementation="badinput", - ) +def test_pca_y(): + """ + We want to check that real inputs and differing input matrix shapes work. - # Bispectrum component - with pytest.raises(ValueError, match=r"Provided bispectrum_implementation.*"): - _ = RIRClass2D( - self.clean_src, - self.clean_fspca_basis, - bispectrum_implementation="badinput", - ) + Most of pca_y is covered by the classificiation unit tests. + """ - # Legacy Bispectrum implies legacy bispectrum (they're integrated). - with pytest.raises( - ValueError, match=r'"legacy" bispectrum_implementation implies.*' - ): - _ = RIRClass2D( - self.clean_src, - self.clean_fspca_basis, - bispectrum_implementation="legacy", - large_pca_implementation="sklearn", - ) + # The iris dataset is a small 150 sample by 5 feature dataset in float64 + iris = datasets.load_iris() - # Currently we only FSPCA Basis in RIRClass2D - with pytest.raises( - RuntimeError, - match="RIRClass2D has currently only been developed for pca_basis as a FSPCABasis.", - ): - _ = RIRClass2D(self.clean_src, self.basis) + # Extract the data matrix, run once as is (150, 5), + # and once tranposed so shape[0] < shape[1] (5, 150) + for x in (iris.data, iris.data.T): + # Run pca_y and check reconstruction holds + lsvec, svals, rsvec = pca_y(x, 5) + # svd ~~> A = U S V = (U S) V + recon = np.dot(lsvec * svals, rsvec) -class LegacyImplementationTestCase(TestCase): + assert np.allclose(x, recon) + + +def test_bispect_overflow(): """ - Cover branches of Legacy code not taken by the classification unit tests. + A zero value coeff will cause a div0 error in log call. + Check it is raised. """ - def setUp(self): - pass - - def test_pca_y(self): - """ - We want to check that real inputs and differing input matrix shapes work. - - Most of pca_y is covered by the classificiation unit tests. - """ - - # The iris dataset is a small 150 sample by 5 feature dataset in float64 - iris = datasets.load_iris() - - # Extract the data matrix, run once as is (150, 5), - # and once tranposed so shape[0] < shape[1] (5, 150) - for x in (iris.data, iris.data.T): - # Run pca_y and check reconstruction holds - lsvec, svals, rsvec = pca_y(x, 5) - - # svd ~~> A = U S V = (U S) V - recon = np.dot(lsvec * svals, rsvec) - - self.assertTrue(np.allclose(x, recon)) - - def testBispectOverflow(self): - """ - A zero value coeff will cause a div0 error in log call. - Check it is raised. - """ - - with pytest.raises(ValueError, match="coeff_norm should not be -inf"): - # This should emit a warning before raising - with self.assertWarns(RuntimeWarning): - bispec_2drot_large( - coeff=np.arange(10), - freqs=np.arange(1, 11), - eigval=np.arange(10), - alpha=1 / 3, - sample_n=4000, - ) + with pytest.raises(ValueError, match="coeff_norm should not be -inf"): + # This should emit a warning before raising + with pytest.warns(RuntimeWarning): + bispec_2drot_large( + coeff=np.arange(10), + freqs=np.arange(1, 11), + eigval=np.arange(10), + alpha=1 / 3, + sample_n=4000, + ) From 0a1d08750af87afd662766d9f488f4c78ef0de55 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 30 Aug 2023 13:21:53 -0400 Subject: [PATCH 009/294] Remove some misc code from parent branch --- src/aspire/basis/steerable.py | 10 +++++++++- src/aspire/covariance/covar2d.py | 18 +----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 745660c590..7e06de1b70 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -30,7 +30,7 @@ def __init__(self, *args, **kwargs): self._pos_angular_inds = (self.signs_indices == 1) & (self.angular_indices != 0) self._neg_angular_inds = self.signs_indices == -1 - # Cache the blk_diag shape once known. + # Attribute for caching the blk_diag shape once known. self._blk_diag_cov_shape = None def calculate_bispectrum( @@ -299,6 +299,13 @@ def filter_to_basis_mat(self, f, matrix_type=None): @property def blk_diag_cov_shape(self): + """ + Return the `BlkDiagMatrix` partition shapes. + + If the shape has already been cached, + returns cached value. Otherwise, will + compute the shape and cache in this instance. + """ # Compute the _blk_diag_cov_shape as needed. if self._blk_diag_cov_shape is None: blks = [] @@ -313,4 +320,5 @@ def blk_diag_cov_shape(self): ) self._blk_diag_cov_shape = np.array(blks) + # Return the cached shape return self._blk_diag_cov_shape diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 2aace9d86b..7c2e4b5508 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -131,7 +131,6 @@ def _get_mean(self, coeffs): mask = self.basis._indices["ells"] == 0 mean_coeff = np.zeros(self.basis.count, dtype=coeffs.dtype) - # Use array for manually masking, since Coef.__getitem__ tries to return a Coef. mean_coeff[mask] = np.mean(coeffs[..., mask], axis=0) return mean_coeff @@ -654,22 +653,7 @@ def _solve_covar(self, A_covar, b_covar, M, covar_est_opt): return method(A_covar, b_covar, M, covar_est_opt) def _solve_covar_direct(self, A_covar, b_covar, M, covar_est_opt): - # A_covar is a list of Diags, ctf in basis - # b_covar is a blk diag - # M is a blk diag - - # # A_covar is (CTF.T * CTF) (weighted a**2 from chalkboard) - # most similar to wr_k2 from Yunpeng's code - - # maybe need to unweight A - A_covar = DiagMatrix(np.concatenate([x.asnumpy() for x in A_covar])) - A2i = A_covar * A_covar - - L = A2i.lr_scale(b_covar.partition) - - return b_covar / L - - # M is sum of weighted A squared, only for cg, ignore + raise NotImplementedError("To be implemented in future changeset.") def _solve_covar_cg(self, A_covar, b_covar, M, covar_est_opt): def precond_fun(S, x): From 303e1d09327b9be87c8c1d6b6579115b2e7cc607 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 30 Aug 2023 13:48:10 -0400 Subject: [PATCH 010/294] Remove vestigial matrix_type from filter_to_basis_mat conversion --- src/aspire/basis/fb_2d.py | 2 +- src/aspire/basis/fle_2d.py | 2 +- src/aspire/basis/steerable.py | 9 +++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index de3789265f..a05ac12fd9 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -403,7 +403,7 @@ def calculate_bispectrum( freq_cutoff=freq_cutoff, ) - def filter_to_basis_mat(self, f, matrix_type=None): + def filter_to_basis_mat(self, f): """ See SteerableBasis2D.filter_to_basis_mat. """ diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index db134de719..bbfce2717a 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -717,7 +717,7 @@ def _radial_convolve_weights(self, b): return a.flatten() - def filter_to_basis_mat(self, f, matrix_type=None): + def filter_to_basis_mat(self, f): """ See SteerableBasis2D.filter_to_basis_mat. """ diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 7e06de1b70..2dca059a36 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -284,17 +284,14 @@ def shift(self, coef, shifts): return self.evaluate_t(self.evaluate(coef).shift(shifts)) @abc.abstractmethod - def filter_to_basis_mat(self, f, matrix_type=None): + def filter_to_basis_mat(self, f): """ Convert a filter into a basis representation. :param f: `Filter` object, usually a `CTFFilter`. - :param matrix_type: Optional override, Example, `BlkDiagMatrix` or - `DiagMatrix`. Default `None` returns the default for - this basis. - :return: `BlkDiagMatrix` or `DiagMatrix` instance - representation of filter in `basis`. + :return: Representation of filter in `basis`. + Return type will be based on the class's `matrix_type`. """ @property From 17f22ddcaa5cbbd61d4686f3bd22f4ecd5c0b9fc Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 30 Aug 2023 14:05:11 -0400 Subject: [PATCH 011/294] additional cleanup, mostly string replace typos --- src/aspire/covariance/covar2d.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 7c2e4b5508..5b01205cec 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -111,10 +111,7 @@ def _ctf_identity_mat(self): :return: Identity BlkDiagMatrix or DiagMatrix """ if self.basis.matrix_type == DiagMatrix: - # TODO: compute this without computing filter/ctf - return DiagMatrix.ones( - len(self.basis.filter_to_basis_mat(RadialCTFFilter())), dtype=self.dtype - ) + return DiagMatrix.eye(self.basis.count, dtype=self.dtype) else: return BlkDiagMatrix.eye(self.basis.blk_diag_cov_shape, dtype=self.dtype) @@ -140,8 +137,8 @@ def get_mean(self, coeffs, ctf_basis=None, ctf_idx=None): Calculate the mean vector from the expansion coefficients with CTF information. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be averaged. - :param ctf_basis: The CFT functions in the BASIS expansion. - :param ctf_idx: An array of the CFT function indices for all 2D images. + :param ctf_basis: The CTF functions in the Basis expansion. + :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. :return: The mean value vector for all images. """ @@ -186,8 +183,8 @@ def get_covar( Calculate the covariance matrix from the expansion coefficients and CTF information. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. - :param ctf_basis: The CFT functions in the BASIS expansion. - :param ctf_idx: An array of the CFT function indices for all 2D images. + :param ctf_basis: The CTF functions in the Basis expansion. + :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. :param mean_coeff: The mean value vector from all images. :param noise_var: The estimated variance of noise. The value should be zero for `coeffs` @@ -415,8 +412,8 @@ def get_cwf_coeffs( Estimate the expansion coefficients using the Covariance Wiener Filtering (CWF) method. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. - :param ctf_basis: The CFT functions in the BASIS expansion. - :param ctf_idx: An array of the CFT function indices for all 2D images. + :param ctf_basis: The CTF functions in the Basis expansion. + :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. :param mean_coeff: The mean value vector from all images. :param covar_coeff: The block diagonal covariance matrix of the clean coefficients represented by a cell array. @@ -807,8 +804,8 @@ def get_cwf_coeffs( Estimate the expansion coefficients using the Covariance Wiener Filtering (CWF) method. :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. - :param ctf_basis: The CFT functions in the BASIS expansion. - :param ctf_idx: An array of the CFT function indices for all 2D images. + :param ctf_basis: The CTF functions in the Basis expansion. + :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. :param mean_coeff: The mean value vector from all images. :param covar_coeff: The block diagonal covariance matrix of the clean coefficients represented by a cell array. From a22f91789dcaf526841d399e0b18d8775fa39bb0 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 30 Aug 2023 14:08:07 -0400 Subject: [PATCH 012/294] Remove parameterized --- pyproject.toml | 1 - tests/test_covar2d.py | 2 +- tests/test_diag_matrix.py | 1 - tox.ini | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 70cfc85960..a78f893a7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,6 @@ dev = [ "pooch", "pyflakes", "pydocstyle", - "parameterized", "pytest", "pytest-cov", "pytest-random-order", diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index daa325152f..f5fd7fe679 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -26,7 +26,7 @@ # Hard coded to match legacy files. NOISE_VAR = 1.3957e-4 -# These variables support parameterized arg checking in `test_shrinkage` +# Cover `test_shrinkage` SHRINKERS = [None, "frobenius_norm", "operator_norm", "soft_threshold"] CTF_ENABLED = [True, False] diff --git a/tests/test_diag_matrix.py b/tests/test_diag_matrix.py index e883d7f55f..e2269f0575 100644 --- a/tests/test_diag_matrix.py +++ b/tests/test_diag_matrix.py @@ -139,7 +139,6 @@ def test_dtype_mismatch(): _ = d1 + d2 -# Explicit Tests (non parameterized). def test_dtype_passthrough(): """ Test that the datatype is inferred correctly. diff --git a/tox.ini b/tox.ini index cd98bab956..6969a99719 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ minversion = 3.8.0 [testenv] changedir = tests deps = - parameterized pooch pytest pytest-cov From be84411d23354931b560af53cd4ba092538a7c05 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 30 Aug 2023 14:23:41 -0400 Subject: [PATCH 013/294] tox! --- src/aspire/covariance/covar2d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 5b01205cec..2b5f6b52e3 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -5,7 +5,7 @@ from scipy.linalg import solve, sqrtm from aspire.basis import FFBBasis2D -from aspire.operators import BlkDiagMatrix, DiagMatrix, RadialCTFFilter +from aspire.operators import BlkDiagMatrix, DiagMatrix from aspire.optimization import conj_grad, fill_struct from aspire.utils import make_symmat from aspire.utils.matlab_compat import m_reshape From a1f8dcc9e1246339b740a6bc3b85f114ec33a535 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 7 Sep 2023 09:46:14 -0400 Subject: [PATCH 014/294] cleanup for a smaller diff --- src/aspire/covariance/covar2d.py | 117 +++++++++++++++---------------- 1 file changed, 57 insertions(+), 60 deletions(-) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 2b5f6b52e3..0719e409f7 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -101,9 +101,6 @@ def __init__(self, basis): self.dtype = self.basis.dtype assert basis.ndim == 2, "Only two-dimensional basis functions are needed." - # Abstract the basis matrix type (BlkDiagMatrix/DiagMatrix) - self.ctf_matrix_type = self.basis.matrix_type - def _ctf_identity_mat(self): """ Returns CTF identity corresponding to the `matrix_type` of `self.basis`. @@ -132,6 +129,63 @@ def _get_mean(self, coeffs): return mean_coeff + def _get_covar(self, coeffs, mean_coeff=None, do_refl=True): + """ + Calculate the covariance matrix from the expansion coefficients without CTF information. + + :param coeffs: A coefficient vector (or an array of coefficient vectors) calculated from 2D images. + :param mean_coeff: The mean vector calculated from the `coeffs`. + :param do_refl: If true, enforce invariance to reflection (default false). + :return: The covariance matrix of coefficients for all images. + """ + if coeffs.size == 0: + raise RuntimeError("The coefficients need to be calculated first!") + if mean_coeff is None: + mean_coeff = self._get_mean(coeffs) + + # Initialize a totally empty BlkDiagMatrix, build incrementally. + covar_coeff = BlkDiagMatrix.empty(0, dtype=coeffs.dtype) + ell = 0 + + mask = self.basis._indices["ells"] == ell + + coeff_ell = coeffs[..., mask] - mean_coeff[mask] + covar_ell = np.array(coeff_ell.T @ coeff_ell / coeffs.shape[0]) + covar_coeff.append(covar_ell) + + for ell in range(1, self.basis.ell_max + 1): + mask_ell = self.basis._indices["ells"] == ell + mask_pos = mask_ell & (self.basis._indices["sgns"] == +1) + mask_neg = mask_ell & (self.basis._indices["sgns"] == -1) + + covar_ell_diag = np.array( + coeffs[:, mask_pos].T @ coeffs[:, mask_pos] + + coeffs[:, mask_neg].T @ coeffs[:, mask_neg] + ) / (2 * coeffs.shape[0]) + + if do_refl: + covar_coeff.append(covar_ell_diag) + covar_coeff.append(covar_ell_diag) + else: + covar_ell_off = np.array( + ( + coeffs[:, mask_pos] @ coeffs[:, mask_neg].T / coeffs.shape[0] + - coeffs[:, mask_pos].T @ coeffs[:, mask_neg] + ) + / (2 * coeffs.shape[0]) + ) + + hsize = covar_ell_diag.shape[0] + covar_coeff_blk = np.zeros((2, hsize, 2, hsize)) + + covar_coeff_blk[0:2, :, 0:2, :] = covar_ell_diag[:hsize, :hsize] + covar_coeff_blk[0, :, 1, :] = covar_ell_off[:hsize, :hsize] + covar_coeff_blk[1, :, 0, :] = covar_ell_off.T[:hsize, :hsize] + + covar_coeff.append(covar_coeff_blk.reshape(2 * hsize, 2 * hsize)) + + return covar_coeff + def get_mean(self, coeffs, ctf_basis=None, ctf_idx=None): """ Calculate the mean vector from the expansion coefficients with CTF information. @@ -318,63 +372,6 @@ def apply(A, x): return covar_coeff - def _get_covar(self, coeffs, mean_coeff=None, do_refl=True): - """ - Calculate the covariance matrix from the expansion coefficients without CTF information. - - :param coeffs: A coefficient vector (or an array of coefficient vectors) calculated from 2D images. - :param mean_coeff: The mean vector calculated from the `coeffs`. - :param do_refl: If true, enforce invariance to reflection (default false). - :return: The covariance matrix of coefficients for all images. - """ - if coeffs.size == 0: - raise RuntimeError("The coefficients need to be calculated first!") - if mean_coeff is None: - mean_coeff = self._get_mean(coeffs) - - # Initialize a totally empty BlkDiagMatrix, build incrementally. - covar_coeff = BlkDiagMatrix.empty(0, dtype=coeffs.dtype) - ell = 0 - - mask = self.basis._indices["ells"] == ell - - coeff_ell = coeffs[..., mask] - mean_coeff[mask] - covar_ell = np.array(coeff_ell.T @ coeff_ell / coeffs.shape[0]) - covar_coeff.append(covar_ell) - - for ell in range(1, self.basis.ell_max + 1): - mask_ell = self.basis._indices["ells"] == ell - mask_pos = mask_ell & (self.basis._indices["sgns"] == +1) - mask_neg = mask_ell & (self.basis._indices["sgns"] == -1) - - covar_ell_diag = np.array( - coeffs[:, mask_pos].T @ coeffs[:, mask_pos] - + coeffs[:, mask_neg].T @ coeffs[:, mask_neg] - ) / (2 * coeffs.shape[0]) - - if do_refl: - covar_coeff.append(covar_ell_diag) - covar_coeff.append(covar_ell_diag) - else: - covar_ell_off = np.array( - ( - coeffs[:, mask_pos] @ coeffs[:, mask_neg].T / coeffs.shape[0] - - coeffs[:, mask_pos].T @ coeffs[:, mask_neg] - ) - / (2 * coeffs.shape[0]) - ) - - hsize = covar_ell_diag.shape[0] - covar_coeff_blk = np.zeros((2, hsize, 2, hsize)) - - covar_coeff_blk[0:2, :, 0:2, :] = covar_ell_diag[:hsize, :hsize] - covar_coeff_blk[0, :, 1, :] = covar_ell_off[:hsize, :hsize] - covar_coeff_blk[1, :, 0, :] = covar_ell_off.T[:hsize, :hsize] - - covar_coeff.append(covar_coeff_blk.reshape(2 * hsize, 2 * hsize)) - - return covar_coeff - def shrink_covar_backward(self, b, b_noise, n, noise_var, shrinker): """ Apply the shrinking method to the 2D covariance of coefficients. From e2363038c34c102d731fa2470b623cee2d2357e8 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 7 Sep 2023 11:25:31 -0400 Subject: [PATCH 015/294] remove leftover (pre-DiagMatrix) comments --- src/aspire/covariance/covar2d.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 0719e409f7..c6d380afa4 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -581,11 +581,8 @@ def _calc_op(self): ctf_basis_k_sq = ctf_basis_k_t @ ctf_basis_k A_mean_k = weight * ctf_basis_k_sq - # blk + diag A_mean += A_mean_k - # why is rmul returning an array?! - # A_covar_k = np.sqrt(weight) * ctf_basis_k_sq - A_covar_k = ctf_basis_k_sq * np.sqrt(weight) + A_covar_k = np.sqrt(weight) * ctf_basis_k_sq A_covar[k] = A_covar_k M_covar += A_covar_k From f73f5dee9a41087bc527969b38c5dd60b967f743 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 07:40:03 -0400 Subject: [PATCH 016/294] review remarks --- src/aspire/basis/fle_2d.py | 1 - tests/test_covar2d.py | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index bbfce2717a..6f37cddc00 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -730,7 +730,6 @@ def filter_to_basis_mat(self, f): # Set same dimensions as basis object n_k = 2 * self.num_radial_nodes # self.n_r n_theta = self.num_angular_nodes # self.n_theta - # radial = self.get_radial() # get 2D grid in polar coordinate k_vals, wts = lgwt(n_k, 0, 0.5, dtype=self.dtype) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index f5fd7fe679..4e72c579d4 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -145,12 +145,14 @@ def test_get_mean_ctf(cov2d_fixture, ctf_enabled): mean_coeff_ctf = cov2d.get_mean(coeff, h_ctf_fb, h_idx) + tol = utest_tolerance(sim.dtype) if ctf_enabled: result = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_meanctf.npy")) else: result = cov2d._get_mean(coeff_clean) + tol = 0.002 - assert np.allclose(mean_coeff_ctf, result, atol=0.002) + np.testing.assert_allclose(mean_coeff_ctf, result, atol=tol) def test_get_cwf_coeffs_clean(cov2d_fixture): @@ -179,7 +181,7 @@ def test_get_cwf_coeffs_clean_ctf(cov2d_fixture): # Reconstruct images from output of get_cwf_coeffs img_est = cov2d.basis.evaluate(coeff_cwf) # Compare with clean images - delta = np.mean(np.square((sim.projections[:] - img_est).asnumpy())) + delta = np.mean(np.square((sim.clean_images[:] - img_est).asnumpy())) assert delta < 0.02 From 42741819e7dd3cff5593ee96a44196c5348b9fdc Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 07:54:19 -0400 Subject: [PATCH 017/294] waste time --- tests/test_covar2d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index 4e72c579d4..ff64920195 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -182,7 +182,7 @@ def test_get_cwf_coeffs_clean_ctf(cov2d_fixture): img_est = cov2d.basis.evaluate(coeff_cwf) # Compare with clean images delta = np.mean(np.square((sim.clean_images[:] - img_est).asnumpy())) - assert delta < 0.02 + np.testing.assert_array_less(delta, 0.01) def test_shrinker_inputs(cov2d_fixture): From 1c83cb2a05537a6547876a35cf8d4285978afdf8 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 08:25:48 -0400 Subject: [PATCH 018/294] since the tests might fail now --- tests/test_covar2d.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index ff64920195..dd7850a1ed 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -121,7 +121,7 @@ def test_get_mean(cov2d_fixture): cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] mean_coeff = cov2d._get_mean(coeff_clean) - assert np.allclose(results, mean_coeff, atol=utest_tolerance(cov2d.dtype)) + np.testing.assert_allclose(results, mean_coeff, atol=utest_tolerance(cov2d.dtype)) def test_get_covar(cov2d_fixture): @@ -134,7 +134,7 @@ def test_get_covar(cov2d_fixture): covar_coeff = cov2d._get_covar(coeff_clean) for im, mat in enumerate(results.tolist()): - assert np.allclose(mat, covar_coeff[im]) + np.testing.assert_allclose(mat, covar_coeff[im], rtol=1e-05) def test_get_mean_ctf(cov2d_fixture, ctf_enabled): @@ -163,7 +163,9 @@ def test_get_cwf_coeffs_clean(cov2d_fixture): cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] coeff_cwf_clean = cov2d.get_cwf_coeffs(coeff_clean, noise_var=0) - assert np.allclose(results, coeff_cwf_clean, atol=utest_tolerance(cov2d.dtype)) + np.testing.assert_allclose( + results, coeff_cwf_clean, atol=utest_tolerance(cov2d.dtype) + ) def test_get_cwf_coeffs_clean_ctf(cov2d_fixture): @@ -212,7 +214,9 @@ def test_shrinkage(cov2d_fixture, shrinker): covar_coeff = cov2d.get_covar(coeff_clean, covar_est_opt={"shrinker": shrinker}) for im, mat in enumerate(results.tolist()): - assert np.allclose(mat, covar_coeff[im], atol=utest_tolerance(cov2d.dtype)) + np.testing.assert_allclose( + mat, covar_coeff[im], atol=utest_tolerance(cov2d.dtype) + ) def test_get_cwf_coeffs_ctf_args(cov2d_fixture): @@ -248,7 +252,7 @@ def test_get_cwf_coeffs(cov2d_fixture, ctf_enabled): coeff_cwf = cov2d.get_cwf_coeffs(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) - assert np.allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) + np.testing.assert_allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) def test_get_cwf_coeffs_without_ctf_args(cov2d_fixture, ctf_enabled): @@ -270,7 +274,7 @@ def test_get_cwf_coeffs_without_ctf_args(cov2d_fixture, ctf_enabled): coeff_cwf = cov2d.get_cwf_coeffs(coeff, noise_var=NOISE_VAR) - assert np.allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) + np.testing.assert_allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) def test_get_covar_ctf(cov2d_fixture, ctf_enabled): @@ -287,7 +291,7 @@ def test_get_covar_ctf(cov2d_fixture, ctf_enabled): covar_coeff_ctf = cov2d.get_covar(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) for im, mat in enumerate(results.tolist()): - assert np.allclose(mat, covar_coeff_ctf[im]) + np.testing.assert_allclose(mat, covar_coeff_ctf[im], rtol=1e-05, atol=1e-08) def test_get_covar_ctf_shrink(cov2d_fixture, ctf_enabled): @@ -321,4 +325,4 @@ def test_get_covar_ctf_shrink(cov2d_fixture, ctf_enabled): ) for im, mat in enumerate(results.tolist()): - assert np.allclose(mat, covar_coeff_ctf_shrink[im]) + np.testing.assert_allclose(mat, covar_coeff_ctf_shrink[im]) From 7b01cfbc453dffe567c65176510aaf54e436ffae Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 08:51:46 -0400 Subject: [PATCH 019/294] Add docstring for class2d test sim fixtures --- tests/test_class2D.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_class2D.py b/tests/test_class2D.py index 1e2422a484..40074c3c80 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -53,6 +53,10 @@ def volume(dtype, img_size): @pytest.fixture def sim_fixture(volume, img_size, dtype): + """ + Provides a clean simulation parameterized by `img_size` and `dtype`. + """ + # Create a src from the volume src = Simulation(L=img_size, n=321, vols=volume, dtype=dtype, seed=SEED) src = src.cache() # Precompute image stack @@ -137,6 +141,15 @@ def test_basis_too_small(sim_fixture, basis): @pytest.fixture def sim_fixture2(volume, basis, img_size, dtype): + """ + Provides clean/noisy pair of smaller parameterized simulations, + along with corresponding clean/noisy basis and an additional + compressed basis. + + These are slightly smaller than `sim_fixture` and support covering + additional code and corner cases. + """ + n_img = 150 # Clean From 5aa3dfc1d0c9e487d4071c4ec1e637d0a526e2c5 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 10:50:47 -0400 Subject: [PATCH 020/294] typo in upstream wrapper function --- src/aspire/operators/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index e74bf7b02e..d838f6387d 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -96,7 +96,7 @@ def basis_mat(self, basis): :return: `basis` representation of this filter. Return type will depend on `basis`. """ - return basis.filter_to_basis_mat(self, basis) + return basis.filter_to_basis_mat(self) def scale(self, c=1): """ From eac0489363670a771090bd690e50f404205dc048 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 18 Sep 2023 07:06:32 -0400 Subject: [PATCH 021/294] remove '.dontrun' files --- .../experiments/cov2d_experiment.py.dontrun | 58 ------------- gallery/experiments/cov3d_experiment.dontrun | 66 -------------- .../experiments/orient3d_experiment.dontrun | 33 ------- .../preprocess_imgs_exp.py.dontrun | 87 ------------------- 4 files changed, 244 deletions(-) delete mode 100755 gallery/experiments/cov2d_experiment.py.dontrun delete mode 100644 gallery/experiments/cov3d_experiment.dontrun delete mode 100644 gallery/experiments/orient3d_experiment.dontrun delete mode 100644 gallery/experiments/preprocess_imgs_exp.py.dontrun diff --git a/gallery/experiments/cov2d_experiment.py.dontrun b/gallery/experiments/cov2d_experiment.py.dontrun deleted file mode 100755 index 24d2e22799..0000000000 --- a/gallery/experiments/cov2d_experiment.py.dontrun +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -""" -This script illustrates denoising 2D images using batched Cov2D class -from experime dataset and outputing to mrcs file. -""" - -import logging - -from aspire.basis import FFBBasis2D -from aspire.denoising.denoiser_cov2d import DenoiserCov2D -from aspire.noise import AnisotropicNoiseEstimator -from aspire.source.relion import RelionSource - -logger = logging.getLogger(__name__) - - -# Set input path and files and initialize other parameters -DATA_FOLDER = "/path/to/untarred/empiar/dataset/" -STARFILE_IN = "/path/to/untarred/empiar/dataset/input.star" -STARFILE_OUT = "/path/to/output/ouput.star" -PIXEL_SIZE = 1.34 -MAX_ROWS = 1024 -MAX_RESOLUTION = 60 - -# Create a source object for 2D images -logger.info(f"Read in images from {STARFILE_IN} and preprocess the images.") -source = RelionSource( - STARFILE_IN, DATA_FOLDER, pixel_size=PIXEL_SIZE, max_rows=MAX_ROWS -) - -# Downsample the images -logger.info(f"Set the resolution to {MAX_RESOLUTION} X {MAX_RESOLUTION}") -if MAX_RESOLUTION < source.L: - source = source.downsample(MAX_RESOLUTION) -else: - logger.warn(f"Unable to downsample to {max_resolution}, using {source.L}") - - -# Specify the fast FB basis method for expending the 2D images -basis = FFBBasis2D((source.L, source.L)) - -# Estimate the noise of images -logger.info(f"Estimate the noise of images using anisotropic method") -noise_estimator = AnisotropicNoiseEstimator(source) -var_noise = noise_estimator.estimate() -logger.info(f"var_noise before whitening {var_noise}") - -# Whiten the noise of images -logger.info(f"Whiten the noise of images from the noise estimator") -source = source.whiten(noise_estimator) -# Note this changes the noise variance, -# flattening spectrum and converging towards 1. -# Noise variance will be recomputed in DenoiserCov2D by default. - -logger.info(f"Denoise the images using batched cov2D method.") -denoiser = DenoiserCov2D(source, basis) -denoised_src = denoiser.denoise(batch_size=512) -denoised_src.save(STARFILE_OUT, batch_size=512, save_mode="single", overwrite=False) diff --git a/gallery/experiments/cov3d_experiment.dontrun b/gallery/experiments/cov3d_experiment.dontrun deleted file mode 100644 index a907649480..0000000000 --- a/gallery/experiments/cov3d_experiment.dontrun +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -""" -This script illustrates Cov3D analysis using experimental dataset -""" -import numpy as np - -from aspire.basis import FBBasis3D -from aspire.covariance import CovarianceEstimator -from aspire.noise import AnisotropicNoiseEstimator -from aspire.reconstruction import MeanEstimator -from aspire.source.relion import RelionSource -from aspire.utils import eigs -from aspire.volume import Volume - -# Set input path and files and initialize other parameters -DATA_FOLDER = "/path/to/untarred/empiar/dataset/" -STARFILE = "/path/to/untarred/empiar/dataset/input.star" -PIXEL_SIZE = 5.0 -MAX_ROWS = 1024 -MAX_RESOLUTION = 8 -CG_TOL = 1e-5 - -# Set number of eigen-vectors to keep -NUM_EIGS = 16 - -# Create a source object for experimental 2D images with estimated rotation angles -print(f"Read in images from {STARFILE} and preprocess the images.") -source = RelionSource( - STARFILE, data_folder=DATA_FOLDER, pixel_size=PIXEL_SIZE, max_rows=MAX_ROWS -) - -# Downsample the images -print(f"Set the resolution to {MAX_RESOLUTION} X {MAX_RESOLUTION}") -if MAX_RESOLUTION < source.L: - source = source.downsample(MAX_RESOLUTION) - -# Estimate the noise of images -print("Estimate the noise of images using anisotropic method") -noise_estimator = AnisotropicNoiseEstimator(source, batchSize=512) - -# Whiten the noise of images -print("Whiten the noise of images from the noise estimator") -source = source.whiten(noise_estimator) -# Estimate the noise variance. This is needed for the covariance estimation step below. -noise_variance = noise_estimator.estimate() -print(f"Noise Variance = {noise_variance}") - -# Specify the fast FB basis method for expanding the 2D images -basis = FBBasis3D((MAX_RESOLUTION, MAX_RESOLUTION, MAX_RESOLUTION), dtype=source.dtype) - -mean_estimator = MeanEstimator(source, basis, batch_size=512) -mean_est = mean_estimator.estimate() - -# Passing in a mean_kernel argument to the following constructor speeds up some calculations -covar_estimator = CovarianceEstimator(source, basis, mean_kernel=mean_estimator.kernel) -covar_est = covar_estimator.estimate(mean_est, noise_variance, tol=CG_TOL) - -# Extract the top eigenvectors and eigenvalues of the covariance estimate. -eigs_est, lambdas_est = eigs(covar_est, NUM_EIGS) -for i in range(NUM_EIGS): - print(f"Top {i}th eigen value: {lambdas_est[i, i]}") - -# Eigs should probably return a Volume, for now hack it. -# move the last axis to the first -eigs_est_c = np.moveaxis(eigs_est, -1, 0) -eigs_est = Volume(eigs_est_c) diff --git a/gallery/experiments/orient3d_experiment.dontrun b/gallery/experiments/orient3d_experiment.dontrun deleted file mode 100644 index ec7a9eb079..0000000000 --- a/gallery/experiments/orient3d_experiment.dontrun +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -""" -This script illustrates the estimation of orientation angles for experimental dataset -""" - -from aspire.abinitio import CLSyncVoting -from aspire.source.relion import RelionSource - -# Set input path and files and initialize other parameters -DATA_FOLDER = "/path/to/untarred/empiar/dataset/" -STARFILE_IN = "/path/to/untarred/empiar/dataset/input.star" -STARFILE_OUT = "/path/to/output/output.star" -PIXEL_SIZE = 1.34 -MAX_ROWS = 1024 - -# Create a source object for 2D images -print(f"Read in images from {STARFILE_IN}.") -source = RelionSource( - STARFILE_IN, DATA_FOLDER, pixel_size=PIXEL_SIZE, max_rows=MAX_ROWS -) - -# Estimate rotation matrices -print("Estimate rotation matrices.") -orient_est = CLSyncVoting(source) -orient_est.estimate_rotations() - -# Create new source object and save estimate rotation matrices -print("Save estimate rotation matrices.") -orient_est_src = orient_est.save_rotations() - -# Output orientational angles -print("Save orientational angles to STAR file.") -orient_est_src.save_metadata(STARFILE_OUT) diff --git a/gallery/experiments/preprocess_imgs_exp.py.dontrun b/gallery/experiments/preprocess_imgs_exp.py.dontrun deleted file mode 100644 index 9a02e43dd9..0000000000 --- a/gallery/experiments/preprocess_imgs_exp.py.dontrun +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -""" -This script illustrates how to preprocess experimental cryo-EM images -before starting the pipeline of reconstructing 3D map. -""" - -import matplotlib.pyplot as plt - -from aspire.noise import WhiteNoiseEstimator -from aspire.source.relion import RelionSource - -# Set input path and files and initialize other parameters -DATA_FOLDER = '/path/to/untarred/empiar/dataset/' -STARFILE_IN = '/path/to/untarred/empiar/dataset/input.star' -PIXEL_SIZE = 1.34 -NUM_IMGS = 100 - -print('This script illustrates how to preprocess experimental cryo-EM images') -print(f'Read in images from {STARFILE_IN} and preprocess the images') -source = RelionSource( - STARFILE_IN, - DATA_FOLDER, - pixel_size=PIXEL_SIZE, - max_rows=NUM_IMGS -) - -# number of images to extract for plotting -nimgs_ext = 1 - -print('Obtain original images') -imgs_od = source.images[:nimgs_ext] - -print('Perform phase flip to input images') -source = source.phase_flip() -imgs_pf = source.images[:nimgs_ext] - -max_resolution = 60 -print(f'Downsample resolution to {max_resolution} X {max_resolution}') -source = source.downsample(max_resolution) -imgs_ds = source.images[:nimgs_ext] - -print('Normalize images to noise background') -source = source.normalize_background() -imgs_nb = source.images[:nimgs_ext] - -print('Whiten noise of images') -noise_estimator = WhiteNoiseEstimator(source) -source = source.whiten(noise_estimator) -imgs_wt = source.images[:nimgs_ext] - -print('Invert global density contrast') -source = source.invert_contrast() -imgs_rc = source.images[:nimgs_ext] - -# plot the first images -print('plot the first images') -idm = 0 -plt.subplot(2, 3, 1) -plt.imshow(imgs_od[idm], cmap='gray') -plt.colorbar(orientation='horizontal') -plt.title('original image') - -plt.subplot(2, 3, 2) -plt.imshow(imgs_pf[idm], cmap='gray') -plt.colorbar(orientation='horizontal') -plt.title('phase flip') - -plt.subplot(2, 3, 3) -plt.imshow(imgs_ds[idm], cmap='gray') -plt.colorbar(orientation='horizontal') -plt.title('downsample') - -plt.subplot(2, 3, 4) -plt.imshow(imgs_nb[idm], cmap='gray') -plt.colorbar(orientation='horizontal') -plt.title('normalize background') - -plt.subplot(2, 3, 5) -plt.imshow(imgs_wt[idm], cmap='gray') -plt.colorbar(orientation='horizontal') -plt.title('noise whitening') - -plt.subplot(2, 3, 6) -plt.imshow(imgs_rc[idm], cmap='gray') -plt.colorbar(orientation='horizontal') -plt.title('invert contrast') -plt.show() From a3eb1c4f660e97b0025d5c900e139b7ef3686146 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 09:17:36 -0400 Subject: [PATCH 022/294] Add fuzzy_mask to commonline_base --- src/aspire/abinitio/commonline_base.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/aspire/abinitio/commonline_base.py b/src/aspire/abinitio/commonline_base.py index a7a044b60e..c8576b7423 100644 --- a/src/aspire/abinitio/commonline_base.py +++ b/src/aspire/abinitio/commonline_base.py @@ -5,7 +5,7 @@ import scipy.sparse as sparse from aspire.operators import PolarFT -from aspire.utils import common_line_from_rots +from aspire.utils import common_line_from_rots, fuzzy_mask from aspire.utils.random import choice logger = logging.getLogger(__name__) @@ -17,7 +17,14 @@ class CLOrient3D: """ def __init__( - self, src, n_rad=None, n_theta=360, n_check=None, max_shift=0.15, shift_step=1 + self, + src, + n_rad=None, + n_theta=360, + n_check=None, + max_shift=0.15, + shift_step=1, + mask=False, ): """ Initialize an object for estimating 3D orientations using common lines @@ -34,6 +41,8 @@ def __init__( of the resolution. Default is 0.15. :param shift_step: Resolution of shift estimation in pixels. Default is 1 pixel. + :param mask: Option to mask `src.images` with a fuzzy mask (boolean). + Default, `False`, does not apply a mask. """ self.src = src # Note dtype is inferred from self.src @@ -46,6 +55,7 @@ def __init__( self.clmatrix = None self.max_shift = math.ceil(max_shift * self.n_res) self.shift_step = shift_step + self.mask = mask self.rotations = None self._build() @@ -69,6 +79,13 @@ def _build(self): imgs = self.src.images[:] + if self.mask: + # Apply fuzzy mask to images using Matlab default values for risetime and rad. + risetime = np.floor(0.05 * self.n_res) + rad = np.floor(0.45 * self.n_res) + fuzz_mask = fuzzy_mask((self.n_res, self.n_res), rad, risetime, self.dtype) + imgs = imgs * fuzz_mask + # Obtain coefficients of polar Fourier transform for input 2D images self.pft = PolarFT( (self.n_res, self.n_res), self.n_rad, self.n_theta, dtype=self.dtype From 23d736294041870fa12799baf5c2c893faa39681 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 09:19:16 -0400 Subject: [PATCH 023/294] Add fuzzy_mask to commonline_c2 --- src/aspire/abinitio/commonline_c2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/aspire/abinitio/commonline_c2.py b/src/aspire/abinitio/commonline_c2.py index 382078cc52..404acfc887 100644 --- a/src/aspire/abinitio/commonline_c2.py +++ b/src/aspire/abinitio/commonline_c2.py @@ -43,6 +43,7 @@ def __init__( degree_res=1, min_dist_cls=25, seed=None, + mask=False, ): """ Initialize object for estimating 3D orientations for molecules with C2 symmetry. @@ -57,6 +58,8 @@ def __init__( :param degree_res: Degree resolution for estimating in-plane rotations. :param min_dist_cls: Minimum distance between mutual common-lines. Default = 25 degrees. :param seed: Optional seed for RNG. + :param mask: Option to mask `src.images` with a fuzzy mask (boolean). + Default, `False`, does not apply a mask. """ super().__init__( src, @@ -69,6 +72,7 @@ def __init__( max_iters=max_iters, degree_res=degree_res, seed=seed, + mask=mask, ) self.min_dist_cls = min_dist_cls From dfe8b04b970fa414965d6c44816cf89586b87c1b Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 09:20:51 -0400 Subject: [PATCH 024/294] Add fuzzy_mask to commonline_c3_c4 --- src/aspire/abinitio/commonline_c3_c4.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/aspire/abinitio/commonline_c3_c4.py b/src/aspire/abinitio/commonline_c3_c4.py index 7ddfbde7cd..b8c42c4498 100644 --- a/src/aspire/abinitio/commonline_c3_c4.py +++ b/src/aspire/abinitio/commonline_c3_c4.py @@ -51,6 +51,7 @@ def __init__( max_iters=1000, degree_res=1, seed=None, + mask=False, ): """ Initialize object for estimating 3D orientations for molecules with C3 and C4 symmetry. @@ -65,6 +66,8 @@ def __init__( :param max_iter: Maximum iterations for the power method. :param degree_res: Degree resolution for estimating in-plane rotations. :param seed: Optional seed for RNG. + :param mask: Option to mask `src.images` with a fuzzy mask (boolean). + Default, `False`, does not apply a mask. """ super().__init__( @@ -73,6 +76,7 @@ def __init__( n_theta=n_theta, max_shift=max_shift, shift_step=shift_step, + mask=mask, ) self._check_symmetry(symmetry) From e921543da0eb755609a89c4b7d64b42768e4de13 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 09:21:56 -0400 Subject: [PATCH 025/294] Add fuzzy_mask to commonline_cn --- src/aspire/abinitio/commonline_cn.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/aspire/abinitio/commonline_cn.py b/src/aspire/abinitio/commonline_cn.py index d267a367c1..d2efae9650 100644 --- a/src/aspire/abinitio/commonline_cn.py +++ b/src/aspire/abinitio/commonline_cn.py @@ -40,6 +40,7 @@ def __init__( n_points_sphere=500, equator_threshold=10, seed=None, + mask=False, ): """ Initialize object for estimating 3D orientations for molecules with Cn symmetry, n>4. @@ -57,6 +58,8 @@ def __init__( :param equator_threshold: Threshold for removing candidate rotations within `equator_threshold` degrees of being an equator image. Default is 10 degrees. :param seed: Optional seed for RNG. + :param mask: Option to mask `src.images` with a fuzzy mask (boolean). + Default, `False`, does not apply a mask. """ super().__init__( @@ -70,6 +73,7 @@ def __init__( max_iters=max_iters, degree_res=degree_res, seed=seed, + mask=mask, ) self.n_points_sphere = n_points_sphere From 107b899741f9c5a46e9848911e3a1cd54275792a Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 09:23:16 -0400 Subject: [PATCH 026/294] Add fuzzy_mask to commonline_sync --- src/aspire/abinitio/commonline_sync.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/aspire/abinitio/commonline_sync.py b/src/aspire/abinitio/commonline_sync.py index c19640bb35..806aaa0c3d 100644 --- a/src/aspire/abinitio/commonline_sync.py +++ b/src/aspire/abinitio/commonline_sync.py @@ -22,7 +22,9 @@ class CLSyncVoting(CLOrient3D, SyncVotingMixin): Journal of Structural Biology, 169, 312-322 (2010). """ - def __init__(self, src, n_rad=None, n_theta=360, max_shift=0.15, shift_step=1): + def __init__( + self, src, n_rad=None, n_theta=360, max_shift=0.15, shift_step=1, mask=False + ): """ Initialize an object for estimating 3D orientations using synchronization matrix @@ -33,6 +35,8 @@ def __init__(self, src, n_rad=None, n_theta=360, max_shift=0.15, shift_step=1): :param max_shift: Determines maximum range for shifts as a proportion of the resolution. Default is 0.15. :param shift_step: Resolution for shift estimation in pixels. Default is 1 pixel. + :param mask: Option to mask `src.images` with a fuzzy mask (boolean). + Default, `False`, does not apply a mask. """ super().__init__( src, @@ -40,6 +44,7 @@ def __init__(self, src, n_rad=None, n_theta=360, max_shift=0.15, shift_step=1): n_theta=n_theta, max_shift=max_shift, shift_step=shift_step, + mask=mask, ) self.syncmatrix = None From 9fa2bdd66d5e76548842ee5102bd874bf4fb3a74 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 09:25:08 -0400 Subject: [PATCH 027/294] Add dtype to fuzzy_mask function. --- src/aspire/utils/misc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/aspire/utils/misc.py b/src/aspire/utils/misc.py index 7a91f66168..91cbffe5e7 100644 --- a/src/aspire/utils/misc.py +++ b/src/aspire/utils/misc.py @@ -274,7 +274,7 @@ def inverse_r(size, x0=0, y0=0, peak=1, dtype=np.float64): return (peak / vals).astype(dtype) -def fuzzy_mask(L, r0, risetime, origin=None): +def fuzzy_mask(L, r0, risetime, dtype, origin=None): """ Create a centered 1D to 3D fuzzy mask of radius r0 @@ -283,6 +283,7 @@ def fuzzy_mask(L, r0, risetime, origin=None): :param L: The sizes of image in tuple structure :param r0: The specified radius :param risetime: The rise time for `erf` function + :param dtype: dtype for fuzzy mask :param origin: The coordinates of origin :return: The desired fuzzy mask """ @@ -291,7 +292,9 @@ def fuzzy_mask(L, r0, risetime, origin=None): if origin is None: origin = center - grids = [np.arange(1 - org, ell - org + 1) for ell, org in zip(L, origin)] + grids = [ + np.arange(1 - org, ell - org + 1, dtype=dtype) for ell, org in zip(L, origin) + ] XYZ = np.meshgrid(*grids, indexing="ij") XYZ_sq = [X**2 for X in XYZ] R = np.sqrt(np.sum(XYZ_sq, axis=0)) From 6adf9602c87f52f2ed0a1a184a055b96f9f36e2c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 09:59:40 -0400 Subject: [PATCH 028/294] dtype in fuzzy_mask test. --- tests/test_orient_sync_voting.py | 46 +++++++++++++++++++++++++++++--- tests/test_utils.py | 2 +- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index 8b427984ef..c3a17aa3e4 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -8,6 +8,7 @@ from aspire.abinitio import CLOrient3D, CLSyncVoting from aspire.commands.orient3d import orient3d +from aspire.noise import WhiteNoiseAdder from aspire.source import Simulation from aspire.utils import ( Rotation, @@ -99,9 +100,7 @@ def test_estimate_rotations(source_orientation_objs): # Register estimates to ground truth rotations and compute the # angular distance between them (in degrees). - Q_mat, flag = register_rotations(orient_est.rotations, src.rotations) - regrot = get_aligned_rotations(orient_est.rotations, Q_mat, flag) - mean_ang_dist = Rotation.mean_angular_distance(regrot, src.rotations) * 180 / np.pi + mean_ang_dist = check_rotations(orient_est.rotations, src.rotations) # Assert that mean angular distance is less than 1 degree (5 degrees with shifts). degree_tol = 1 @@ -119,6 +118,37 @@ def test_estimate_shifts(source_orientation_objs): assert np.allclose(est_shifts, src.offsets) +def test_estimate_rotations_fuzzy_mask(): + noisy_src = Simulation( + n=30, + vols=AsymmetricVolume(L=40, C=1, K=100, seed=0).generate(), + offsets=0, + amplitudes=1, + noise_adder=WhiteNoiseAdder.from_snr(snr=5), + seed=0, + ) + + # Orientation estimation without fuzzy_mask. + max_shift = 1 / noisy_src.L + shift_step = 1 + orient_est = CLSyncVoting(noisy_src, max_shift=max_shift, shift_step=shift_step) + orient_est.estimate_rotations() + + # Orientation estimation with fuzzy mask. + orient_est_fuzzy = CLSyncVoting( + noisy_src, max_shift=max_shift, shift_step=shift_step, mask=True + ) + orient_est_fuzzy.estimate_rotations() + + # Check that fuzzy_mask improves orientation estimation. + mean_angle_dist = check_rotations(orient_est.rotations, noisy_src.rotations) + mean_angle_dist_fuzzy = check_rotations( + orient_est_fuzzy.rotations, noisy_src.rotations + ) + + assert mean_angle_dist_fuzzy < mean_angle_dist + + def test_theta_error(): """ Test that CLSyncVoting when instantiated with odd value for `n_theta` @@ -161,3 +191,13 @@ def test_command_line(): ) # check that the command completed successfully assert result.exit_code == 0 + + +def check_rotations(rots_est, rots_gt): + # Register estimates to ground truth rotations and compute the + # angular distance between them (in degrees). + Q_mat, flag = register_rotations(rots_est, rots_gt) + regrot = get_aligned_rotations(rots_est, Q_mat, flag) + mean_ang_dist = Rotation.mean_angular_distance(regrot, rots_gt) * 180 / np.pi + + return mean_ang_dist diff --git a/tests/test_utils.py b/tests/test_utils.py index 3b9c26fa44..449d4607f0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -350,7 +350,7 @@ def test_fuzzy_mask(): ], ] ) - fmask = fuzzy_mask((8, 8), 2, 2) + fmask = fuzzy_mask((8, 8), 2, 2, dtype=results.dtype) assert np.allclose(results, fmask, atol=1e-7) From cd235ea02369de9a5aeeef28eadeef28a6b62f97 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Sep 2023 15:35:32 -0400 Subject: [PATCH 029/294] Use fuzzy_mask by default. --- gallery/tutorials/tutorials/orient3d_simulation.py | 2 +- src/aspire/abinitio/commonline_base.py | 4 ++-- src/aspire/abinitio/commonline_c2.py | 4 ++-- src/aspire/abinitio/commonline_c3_c4.py | 4 ++-- src/aspire/abinitio/commonline_cn.py | 4 ++-- src/aspire/abinitio/commonline_sync.py | 4 ++-- tests/test_orient_symmetric.py | 1 + tests/test_orient_sync_voting.py | 10 +++++++--- tests/test_oriented_source.py | 2 +- 9 files changed, 20 insertions(+), 15 deletions(-) diff --git a/gallery/tutorials/tutorials/orient3d_simulation.py b/gallery/tutorials/tutorials/orient3d_simulation.py index 433223e6d2..480469428a 100644 --- a/gallery/tutorials/tutorials/orient3d_simulation.py +++ b/gallery/tutorials/tutorials/orient3d_simulation.py @@ -89,7 +89,7 @@ # Initialize an orientation estimation object and create an ``OrientedSource`` object # to perform viewing angle estimation logger.info("Estimate rotation angles using synchronization matrix and voting method.") -orient_est = CLSyncVoting(sim, n_theta=36) +orient_est = CLSyncVoting(sim, n_theta=36, mask=False) oriented_src = OrientedSource(sim, orient_est) rots_est = oriented_src.rotations diff --git a/src/aspire/abinitio/commonline_base.py b/src/aspire/abinitio/commonline_base.py index c8576b7423..6b019d072c 100644 --- a/src/aspire/abinitio/commonline_base.py +++ b/src/aspire/abinitio/commonline_base.py @@ -24,7 +24,7 @@ def __init__( n_check=None, max_shift=0.15, shift_step=1, - mask=False, + mask=True, ): """ Initialize an object for estimating 3D orientations using common lines @@ -42,7 +42,7 @@ def __init__( :param shift_step: Resolution of shift estimation in pixels. Default is 1 pixel. :param mask: Option to mask `src.images` with a fuzzy mask (boolean). - Default, `False`, does not apply a mask. + Default, `True`, applies a mask. """ self.src = src # Note dtype is inferred from self.src diff --git a/src/aspire/abinitio/commonline_c2.py b/src/aspire/abinitio/commonline_c2.py index 404acfc887..6a762a59c9 100644 --- a/src/aspire/abinitio/commonline_c2.py +++ b/src/aspire/abinitio/commonline_c2.py @@ -43,7 +43,7 @@ def __init__( degree_res=1, min_dist_cls=25, seed=None, - mask=False, + mask=True, ): """ Initialize object for estimating 3D orientations for molecules with C2 symmetry. @@ -59,7 +59,7 @@ def __init__( :param min_dist_cls: Minimum distance between mutual common-lines. Default = 25 degrees. :param seed: Optional seed for RNG. :param mask: Option to mask `src.images` with a fuzzy mask (boolean). - Default, `False`, does not apply a mask. + Default, `True`, applies a mask. """ super().__init__( src, diff --git a/src/aspire/abinitio/commonline_c3_c4.py b/src/aspire/abinitio/commonline_c3_c4.py index b8c42c4498..b3c6e3df9d 100644 --- a/src/aspire/abinitio/commonline_c3_c4.py +++ b/src/aspire/abinitio/commonline_c3_c4.py @@ -51,7 +51,7 @@ def __init__( max_iters=1000, degree_res=1, seed=None, - mask=False, + mask=True, ): """ Initialize object for estimating 3D orientations for molecules with C3 and C4 symmetry. @@ -67,7 +67,7 @@ def __init__( :param degree_res: Degree resolution for estimating in-plane rotations. :param seed: Optional seed for RNG. :param mask: Option to mask `src.images` with a fuzzy mask (boolean). - Default, `False`, does not apply a mask. + Default, `True`, applies a mask. """ super().__init__( diff --git a/src/aspire/abinitio/commonline_cn.py b/src/aspire/abinitio/commonline_cn.py index d2efae9650..94c9fa3ca3 100644 --- a/src/aspire/abinitio/commonline_cn.py +++ b/src/aspire/abinitio/commonline_cn.py @@ -40,7 +40,7 @@ def __init__( n_points_sphere=500, equator_threshold=10, seed=None, - mask=False, + mask=True, ): """ Initialize object for estimating 3D orientations for molecules with Cn symmetry, n>4. @@ -59,7 +59,7 @@ def __init__( degrees of being an equator image. Default is 10 degrees. :param seed: Optional seed for RNG. :param mask: Option to mask `src.images` with a fuzzy mask (boolean). - Default, `False`, does not apply a mask. + Default, `True`, applies a mask. """ super().__init__( diff --git a/src/aspire/abinitio/commonline_sync.py b/src/aspire/abinitio/commonline_sync.py index 806aaa0c3d..6fbec4c8ef 100644 --- a/src/aspire/abinitio/commonline_sync.py +++ b/src/aspire/abinitio/commonline_sync.py @@ -23,7 +23,7 @@ class CLSyncVoting(CLOrient3D, SyncVotingMixin): """ def __init__( - self, src, n_rad=None, n_theta=360, max_shift=0.15, shift_step=1, mask=False + self, src, n_rad=None, n_theta=360, max_shift=0.15, shift_step=1, mask=True ): """ Initialize an object for estimating 3D orientations using synchronization matrix @@ -36,7 +36,7 @@ def __init__( of the resolution. Default is 0.15. :param shift_step: Resolution for shift estimation in pixels. Default is 1 pixel. :param mask: Option to mask `src.images` with a fuzzy mask (boolean). - Default, `False`, does not apply a mask. + Default, `True`, applies a mask. """ super().__init__( src, diff --git a/tests/test_orient_symmetric.py b/tests/test_orient_symmetric.py index 746213913c..9f16fbe2a3 100644 --- a/tests/test_orient_symmetric.py +++ b/tests/test_orient_symmetric.py @@ -84,6 +84,7 @@ def source_orientation_objs(n_img, L, order, dtype): n_theta=360, max_shift=1 / L, seed=seed, + mask=False, ) if order in [3, 4]: diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index c3a17aa3e4..e31c527221 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -69,7 +69,9 @@ def source_orientation_objs(resolution, offsets, dtype): if src.offsets.all() != 0: max_shift = 0.20 shift_step = 0.25 # Reduce shift steps for non-integer offsets of Simulation. - orient_est = CLSyncVoting(src, max_shift=max_shift, shift_step=shift_step) + orient_est = CLSyncVoting( + src, max_shift=max_shift, shift_step=shift_step, mask=False + ) return src, orient_est @@ -131,12 +133,14 @@ def test_estimate_rotations_fuzzy_mask(): # Orientation estimation without fuzzy_mask. max_shift = 1 / noisy_src.L shift_step = 1 - orient_est = CLSyncVoting(noisy_src, max_shift=max_shift, shift_step=shift_step) + orient_est = CLSyncVoting( + noisy_src, max_shift=max_shift, shift_step=shift_step, mask=False + ) orient_est.estimate_rotations() # Orientation estimation with fuzzy mask. orient_est_fuzzy = CLSyncVoting( - noisy_src, max_shift=max_shift, shift_step=shift_step, mask=True + noisy_src, max_shift=max_shift, shift_step=shift_step ) orient_est_fuzzy.estimate_rotations() diff --git a/tests/test_oriented_source.py b/tests/test_oriented_source.py index e91b36c0c7..54c34ea404 100644 --- a/tests/test_oriented_source.py +++ b/tests/test_oriented_source.py @@ -39,7 +39,7 @@ def src_fixture(request): # Generate an origianl source and an oriented source. og_src = Simulation(L=L, n=n, vols=vol, offsets=0) - orient_est = estimator(og_src, max_shift=1 / L, **estimator_kwargs) + orient_est = estimator(og_src, max_shift=1 / L, mask=False, **estimator_kwargs) oriented_src = OrientedSource(og_src, orient_est) return og_src, oriented_src From 0efe8ff5bc8882e7080c304195a876e9fec17171 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 8 Sep 2023 09:02:09 -0400 Subject: [PATCH 030/294] Reword comment. --- src/aspire/abinitio/commonline_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/abinitio/commonline_base.py b/src/aspire/abinitio/commonline_base.py index 6b019d072c..1f21f24424 100644 --- a/src/aspire/abinitio/commonline_base.py +++ b/src/aspire/abinitio/commonline_base.py @@ -80,7 +80,7 @@ def _build(self): imgs = self.src.images[:] if self.mask: - # Apply fuzzy mask to images using Matlab default values for risetime and rad. + # Apply fuzzy mask to images using values for risetime and rad found in the Matlab code. risetime = np.floor(0.05 * self.n_res) rad = np.floor(0.45 * self.n_res) fuzz_mask = fuzzy_mask((self.n_res, self.n_res), rad, risetime, self.dtype) From ed7ae889a5284fe43db339ce450f3e416e59db0f Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 8 Sep 2023 09:32:40 -0400 Subject: [PATCH 031/294] Clarifying comment in tutorial. --- gallery/tutorials/tutorials/orient3d_simulation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gallery/tutorials/tutorials/orient3d_simulation.py b/gallery/tutorials/tutorials/orient3d_simulation.py index 480469428a..0a6513057a 100644 --- a/gallery/tutorials/tutorials/orient3d_simulation.py +++ b/gallery/tutorials/tutorials/orient3d_simulation.py @@ -87,7 +87,11 @@ # ---------------------------------------- # Initialize an orientation estimation object and create an ``OrientedSource`` object -# to perform viewing angle estimation +# to perform viewing angle estimation. Here, because of the small image size of the +# ``Simulation``, we customize the ``CLSyncVoting`` method to use fewer theta values +# when searching for common-lines between pairs of images. Additionally, since we are +# processing images with no noise, we opt not to use a ``fuzzy_mask``, an option that +# improves common-line detection in higher noise regimes. logger.info("Estimate rotation angles using synchronization matrix and voting method.") orient_est = CLSyncVoting(sim, n_theta=36, mask=False) oriented_src = OrientedSource(sim, orient_est) From 2c5caf03904560ef2ade26d453320f062d44d1a2 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 12 Sep 2023 10:54:14 -0400 Subject: [PATCH 032/294] use grid_*d in fuzzy_mask. --- src/aspire/utils/misc.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/aspire/utils/misc.py b/src/aspire/utils/misc.py index 91cbffe5e7..399ebd09b9 100644 --- a/src/aspire/utils/misc.py +++ b/src/aspire/utils/misc.py @@ -274,7 +274,7 @@ def inverse_r(size, x0=0, y0=0, peak=1, dtype=np.float64): return (peak / vals).astype(dtype) -def fuzzy_mask(L, r0, risetime, dtype, origin=None): +def fuzzy_mask(L, r0, risetime, dtype): """ Create a centered 1D to 3D fuzzy mask of radius r0 @@ -284,19 +284,28 @@ def fuzzy_mask(L, r0, risetime, dtype, origin=None): :param r0: The specified radius :param risetime: The rise time for `erf` function :param dtype: dtype for fuzzy mask - :param origin: The coordinates of origin + :return: The desired fuzzy mask """ - center = [sz // 2 + 1 for sz in L] - if origin is None: - origin = center + dim = len(L) + axes = ["x"] + grid_kwargs = {"n": L[0], "shifted": False, "normalized": False, "dtype": dtype} + + if dim == 1: + grid = grid_1d(**grid_kwargs) + elif dim == 2: + grid = grid_2d(**grid_kwargs) + axes.append("y") + elif dim == 3: + grid = grid_3d(**grid_kwargs) + axes.extend(["y", "z"]) + else: + raise RuntimeError( + f"Only 1D, 2D, or 3D fuzzy_mask supported. Found {dim}-dimensional `L`." + ) - grids = [ - np.arange(1 - org, ell - org + 1, dtype=dtype) for ell, org in zip(L, origin) - ] - XYZ = np.meshgrid(*grids, indexing="ij") - XYZ_sq = [X**2 for X in XYZ] + XYZ_sq = [grid[axis] ** 2 for axis in axes] R = np.sqrt(np.sum(XYZ_sq, axis=0)) k = 1.782 / risetime m = 0.5 * (1 - erf(k * (R - r0))) From dc187c6187b5152553268e636644385b57f122e2 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 12 Sep 2023 11:12:00 -0400 Subject: [PATCH 033/294] Add default values to fuzzy_mask. --- src/aspire/abinitio/commonline_base.py | 5 +---- src/aspire/utils/misc.py | 17 +++++++++++------ tests/test_utils.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/aspire/abinitio/commonline_base.py b/src/aspire/abinitio/commonline_base.py index 1f21f24424..0074d4b562 100644 --- a/src/aspire/abinitio/commonline_base.py +++ b/src/aspire/abinitio/commonline_base.py @@ -80,10 +80,7 @@ def _build(self): imgs = self.src.images[:] if self.mask: - # Apply fuzzy mask to images using values for risetime and rad found in the Matlab code. - risetime = np.floor(0.05 * self.n_res) - rad = np.floor(0.45 * self.n_res) - fuzz_mask = fuzzy_mask((self.n_res, self.n_res), rad, risetime, self.dtype) + fuzz_mask = fuzzy_mask((self.n_res, self.n_res), self.dtype) imgs = imgs * fuzz_mask # Obtain coefficients of polar Fourier transform for input 2D images diff --git a/src/aspire/utils/misc.py b/src/aspire/utils/misc.py index 399ebd09b9..7598556b49 100644 --- a/src/aspire/utils/misc.py +++ b/src/aspire/utils/misc.py @@ -274,19 +274,24 @@ def inverse_r(size, x0=0, y0=0, peak=1, dtype=np.float64): return (peak / vals).astype(dtype) -def fuzzy_mask(L, r0, risetime, dtype): +def fuzzy_mask(L, dtype, r0=None, risetime=None): """ - Create a centered 1D to 3D fuzzy mask of radius r0 + Create a centered 1D to 3D fuzzy mask of radius r0. Made with an error function with effective rise time. - :param L: The sizes of image in tuple structure - :param r0: The specified radius - :param risetime: The rise time for `erf` function - :param dtype: dtype for fuzzy mask + :param L: The sizes of image in tuple structure. + :param dtype: dtype for fuzzy mask. + :param r0: The specified radius. Defaults to floor(0.45 * L) + :param risetime: The rise time for `erf` function. Defaults to floor(0.05 * L) :return: The desired fuzzy mask """ + # Note: default values for r0 and risetime are from Matlab common-lines code. + if r0 is None: + r0 = np.floor(0.45 * L[0]) + if risetime is None: + risetime = np.floor(0.05 * L[0]) dim = len(L) axes = ["x"] diff --git a/tests/test_utils.py b/tests/test_utils.py index 449d4607f0..f3d4193bf1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -350,7 +350,7 @@ def test_fuzzy_mask(): ], ] ) - fmask = fuzzy_mask((8, 8), 2, 2, dtype=results.dtype) + fmask = fuzzy_mask((8, 8), dtype=results.dtype, r0=2, risetime=2) assert np.allclose(results, fmask, atol=1e-7) From 8a93955019ee2d2d1e386642ee8a02a2aac7fda2 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 12 Sep 2023 11:29:30 -0400 Subject: [PATCH 034/294] SMoke tests for fuzzy_mask. --- tests/test_utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index f3d4193bf1..8e6a61b28c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -350,9 +350,17 @@ def test_fuzzy_mask(): ], ] ) - fmask = fuzzy_mask((8, 8), dtype=results.dtype, r0=2, risetime=2) + fmask = fuzzy_mask((8, 8), results.dtype, r0=2, risetime=2) assert np.allclose(results, fmask, atol=1e-7) + # Smoke test for 1D, 2D, and 3D fuzzy_mask. + for dim in range(1, 4): + _ = fuzzy_mask((8,) * dim, np.float32) + + # Check that we raise an error for bad dimension. + with pytest.raises(RuntimeError, match=r"Only 1D, 2D, or 3D fuzzy_mask*"): + _ = fuzzy_mask((8,) * 4, np.float32) + def test_multiprocessing_utils(): """ From 40355d71151a7ae711bd9579875dd9cfd6b629fc Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 12 Sep 2023 12:10:21 -0400 Subject: [PATCH 035/294] public check_rotations method and test. --- src/aspire/utils/__init__.py | 1 + src/aspire/utils/coor_trans.py | 19 +++++++++++++++++++ tests/test_coor_trans.py | 13 +++++++++++++ tests/test_orient_sync_voting.py | 17 +---------------- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/aspire/utils/__init__.py b/src/aspire/utils/__init__.py index 6aa71fb8f5..f5f133cf09 100644 --- a/src/aspire/utils/__init__.py +++ b/src/aspire/utils/__init__.py @@ -1,6 +1,7 @@ from .types import complex_type, real_type, utest_tolerance # isort:skip from .coor_trans import ( # isort:skip common_line_from_rots, + check_rotations, crop_pad_2d, crop_pad_3d, get_aligned_rotations, diff --git a/src/aspire/utils/coor_trans.py b/src/aspire/utils/coor_trans.py index a29e041966..17b7bc4490 100644 --- a/src/aspire/utils/coor_trans.py +++ b/src/aspire/utils/coor_trans.py @@ -9,6 +9,7 @@ from scipy.linalg import svd from aspire.utils.random import Random +from aspire.utils.rotation import Rotation def cart2pol(x, y): @@ -283,6 +284,24 @@ def get_rots_mse(rots_reg, rots_ref): return mse +def check_rotations(rots_est, rots_gt): + """ + Register estimates to ground truth rotations and compute the + mean angular distance between them (in degrees). + + :param rots_est: A set of estimated rotations of size nx3x3. + :param rots_gt: A set of ground truth rotations of size nx3x3. + + :return: The mean angular distance between registered estimates + and the ground truth (in degrees). + """ + Q_mat, flag = register_rotations(rots_est, rots_gt) + regrot = get_aligned_rotations(rots_est, Q_mat, flag) + mean_ang_dist = Rotation.mean_angular_distance(regrot, rots_gt) * 180 / np.pi + + return mean_ang_dist + + def common_line_from_rots(r1, r2, ell): """ Compute the common line induced by rotation matrices r1 and r2. diff --git a/tests/test_coor_trans.py b/tests/test_coor_trans.py index 6da99cbe9b..9509cf0225 100644 --- a/tests/test_coor_trans.py +++ b/tests/test_coor_trans.py @@ -5,6 +5,7 @@ from aspire.utils import ( Rotation, + check_rotations, crop_pad_2d, crop_pad_3d, get_aligned_rotations, @@ -335,3 +336,15 @@ def testCrop3DFillValue(self): a = np.ones((4, 4, 3)) b = crop_pad_3d(a, 4, fill_value=-1) self.assertTrue(np.array_equal(b[:, :, 0], -1 * np.ones((4, 4)))) + + +def test_check_rotations(): + n_rots = 10 + dtype = np.float32 + rots_gt = Rotation.generate_random_rotations(n_rots, dtype=dtype).matrices + + # Create a set of rotations that can be exactly globally aligned to rots_gt. + rots_est = rots_gt[0] @ rots_gt + + # Check that the mean angular distance is zero degrees. + np.testing.assert_allclose(check_rotations(rots_est, rots_gt), 0.0) diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index e31c527221..0e4ed9d79a 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -10,12 +10,7 @@ from aspire.commands.orient3d import orient3d from aspire.noise import WhiteNoiseAdder from aspire.source import Simulation -from aspire.utils import ( - Rotation, - get_aligned_rotations, - register_rotations, - rots_to_clmatrix, -) +from aspire.utils import Rotation, check_rotations, rots_to_clmatrix from aspire.volume import AsymmetricVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -195,13 +190,3 @@ def test_command_line(): ) # check that the command completed successfully assert result.exit_code == 0 - - -def check_rotations(rots_est, rots_gt): - # Register estimates to ground truth rotations and compute the - # angular distance between them (in degrees). - Q_mat, flag = register_rotations(rots_est, rots_gt) - regrot = get_aligned_rotations(rots_est, Q_mat, flag) - mean_ang_dist = Rotation.mean_angular_distance(regrot, rots_gt) * 180 / np.pi - - return mean_ang_dist From 20feb8accce485c8b8d3cb93cb563cfffb9f87ba Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 12 Sep 2023 13:43:49 -0400 Subject: [PATCH 036/294] update gallery and tests with check_rotations. --- .../simulated_abinitio_pipeline.py | 15 +++++--------- gallery/tutorials/pipeline_demo.py | 20 +++++++------------ .../tutorials/orient3d_simulation.py | 17 ++++++++-------- tests/test_orient_symmetric.py | 17 +++++++++------- tests/test_orient_sync_voting.py | 2 +- 5 files changed, 31 insertions(+), 40 deletions(-) diff --git a/gallery/experiments/simulated_abinitio_pipeline.py b/gallery/experiments/simulated_abinitio_pipeline.py index ec775fc973..2a861dfbb3 100644 --- a/gallery/experiments/simulated_abinitio_pipeline.py +++ b/gallery/experiments/simulated_abinitio_pipeline.py @@ -28,11 +28,7 @@ from aspire.operators import FunctionFilter, RadialCTFFilter from aspire.reconstruction import MeanEstimator from aspire.source import OrientedSource, Simulation -from aspire.utils.coor_trans import ( - get_aligned_rotations, - get_rots_mse, - register_rotations, -) +from aspire.utils import check_rotations logger = logging.getLogger(__name__) @@ -198,12 +194,11 @@ def noise_function(x, y): oriented_src = OrientedSource(avgs, orient_est) logger.info("Compare with known rotations") -# Compare with known true rotations -Q_mat, flag = register_rotations(oriented_src.rotations, true_rotations) -regrot = get_aligned_rotations(oriented_src.rotations, Q_mat, flag) -mse_reg = get_rots_mse(regrot, true_rotations) +# Compare with known true rotations. ``check_rotations`` globally aligns the estimated +# rotations to the ground truth and finds the mean angular distance between them. +mean_ang_dist = check_rotations(oriented_src.rotations, true_rotations) logger.info( - f"MSE deviation of the estimated rotations using register_rotations : {mse_reg}\n" + f"Mean angular distance between globally aligned estimates and ground truth rotations: {mean_ang_dist}\n" ) # %% diff --git a/gallery/tutorials/pipeline_demo.py b/gallery/tutorials/pipeline_demo.py index 3119b8e1bd..8e571b9fe3 100644 --- a/gallery/tutorials/pipeline_demo.py +++ b/gallery/tutorials/pipeline_demo.py @@ -215,21 +215,15 @@ # %% # Mean Squared Error # ------------------ -# ASPIRE has some built-in utility functions for globally aligning the -# estimated rotations to the true rotations and computing the mean -# squared error. - -from aspire.utils.coor_trans import ( - get_aligned_rotations, - get_rots_mse, - register_rotations, -) +# ASPIRE has the built-in utility function, ``check_rotations``, which globally +# aligns the estimated rotations to the true rotations and computes the mean +# angular distance (in degrees). + +from aspire.utils import check_rotations # Compare with known true rotations -Q_mat, flag = register_rotations(oriented_src.rotations, true_rotations) -regrot = get_aligned_rotations(oriented_src.rotations, Q_mat, flag) -mse_reg = get_rots_mse(regrot, true_rotations) -mse_reg +mean_ang_dist = check_rotations(oriented_src.rotations, true_rotations) +mean_ang_dist # %% diff --git a/gallery/tutorials/tutorials/orient3d_simulation.py b/gallery/tutorials/tutorials/orient3d_simulation.py index 0a6513057a..2058a11f3f 100644 --- a/gallery/tutorials/tutorials/orient3d_simulation.py +++ b/gallery/tutorials/tutorials/orient3d_simulation.py @@ -14,7 +14,7 @@ from aspire.abinitio import CLSyncVoting from aspire.operators import RadialCTFFilter from aspire.source import OrientedSource, Simulation -from aspire.utils import get_aligned_rotations, get_rots_mse, register_rotations +from aspire.utils import check_rotations from aspire.volume import Volume logger = logging.getLogger(__name__) @@ -98,16 +98,15 @@ rots_est = oriented_src.rotations # %% -# Mean Squared Error -# ------------------ +# Mean Angular Distance +# --------------------- -# Get register rotations after performing global alignment -Q_mat, flag = register_rotations(rots_est, rots_true) -regrot = get_aligned_rotations(rots_est, Q_mat, flag) -mse_reg = get_rots_mse(regrot, rots_true) +# ``check_rotations`` will perform global alignment of the estimated rotations +# to the ground truth and find the mean angular distance between them (in degrees). +mean_ang_dist = check_rotations(rots_est, rots_true) logger.info( - f"MSE deviation of the estimated rotations using register_rotations : {mse_reg}" + f"Mean angular distance between estimates and ground truth: {mean_ang_dist} degrees" ) # Basic Check -assert mse_reg < 0.06 +assert mean_ang_dist < 10 diff --git a/tests/test_orient_symmetric.py b/tests/test_orient_symmetric.py index 9f16fbe2a3..d269f36a5e 100644 --- a/tests/test_orient_symmetric.py +++ b/tests/test_orient_symmetric.py @@ -6,10 +6,15 @@ from aspire.abinitio import CLSymmetryC2, CLSymmetryC3C4, CLSymmetryCn from aspire.abinitio.commonline_cn import MeanOuterProductEstimator from aspire.source import Simulation -from aspire.utils import Rotation, utest_tolerance -from aspire.utils.coor_trans import get_aligned_rotations, register_rotations -from aspire.utils.misc import J_conjugate, all_pairs, cyclic_rotations -from aspire.utils.random import randn +from aspire.utils import ( + J_conjugate, + Rotation, + all_pairs, + check_rotations, + cyclic_rotations, + randn, + utest_tolerance, +) from aspire.volume import CnSymmetricVolume # A set of these parameters are marked expensive to reduce testing time. @@ -119,9 +124,7 @@ def test_estimate_rotations(n_img, L, order, dtype): # Register estimates to ground truth rotations and compute the # angular distance between them (in degrees). - Q_mat, flag = register_rotations(rots_est, rots_gt_sync) - regrot = get_aligned_rotations(rots_est, Q_mat, flag) - mean_ang_dist = Rotation.mean_angular_distance(regrot, rots_gt_sync) * 180 / np.pi + mean_ang_dist = check_rotations(rots_est, rots_gt_sync) # Assert mean angular distance is reasonable. assert mean_ang_dist < 3 diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index 0e4ed9d79a..1ea5ed2e6b 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -10,7 +10,7 @@ from aspire.commands.orient3d import orient3d from aspire.noise import WhiteNoiseAdder from aspire.source import Simulation -from aspire.utils import Rotation, check_rotations, rots_to_clmatrix +from aspire.utils import check_rotations, rots_to_clmatrix from aspire.volume import AsymmetricVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") From 2b751047a1f01642a9e4d8baf11d3c9f759b8633 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 12 Sep 2023 14:43:49 -0400 Subject: [PATCH 037/294] improve cl fuzzy mask test. --- tests/test_orient_sync_voting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index 1ea5ed2e6b..a28587e6dd 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -117,11 +117,11 @@ def test_estimate_shifts(source_orientation_objs): def test_estimate_rotations_fuzzy_mask(): noisy_src = Simulation( - n=30, - vols=AsymmetricVolume(L=40, C=1, K=100, seed=0).generate(), + n=35, + vols=AsymmetricVolume(L=128, C=1, K=400, seed=0).generate(), offsets=0, amplitudes=1, - noise_adder=WhiteNoiseAdder.from_snr(snr=5), + noise_adder=WhiteNoiseAdder.from_snr(snr=2), seed=0, ) @@ -145,7 +145,7 @@ def test_estimate_rotations_fuzzy_mask(): orient_est_fuzzy.rotations, noisy_src.rotations ) - assert mean_angle_dist_fuzzy < mean_angle_dist + assert mean_angle_dist_fuzzy < mean_angle_dist < 10 def test_theta_error(): From 6321490cbb868458983690ecbf22e48c23a17c09 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 14 Sep 2023 09:21:19 -0400 Subject: [PATCH 038/294] Add degree_tol option to check_rotations. --- src/aspire/utils/coor_trans.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/aspire/utils/coor_trans.py b/src/aspire/utils/coor_trans.py index 17b7bc4490..930fe1ddf0 100644 --- a/src/aspire/utils/coor_trans.py +++ b/src/aspire/utils/coor_trans.py @@ -284,13 +284,16 @@ def get_rots_mse(rots_reg, rots_ref): return mse -def check_rotations(rots_est, rots_gt): +def check_rotations(rots_est, rots_gt, degree_tol=None): """ Register estimates to ground truth rotations and compute the mean angular distance between them (in degrees). :param rots_est: A set of estimated rotations of size nx3x3. :param rots_gt: A set of ground truth rotations of size nx3x3. + :param degree_tol: Option to assert if the mean angular distance is + less than `degree_tol` degrees. If `None`, returns the mean + aligned angular distance. :return: The mean angular distance between registered estimates and the ground truth (in degrees). @@ -299,6 +302,10 @@ def check_rotations(rots_est, rots_gt): regrot = get_aligned_rotations(rots_est, Q_mat, flag) mean_ang_dist = Rotation.mean_angular_distance(regrot, rots_gt) * 180 / np.pi + if degree_tol is not None: + np.testing.assert_array_less(mean_ang_dist, degree_tol) + return + return mean_ang_dist From 7c34e3562838b642a2061960da18a8052ec5c133 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 14 Sep 2023 09:32:52 -0400 Subject: [PATCH 039/294] check_rotations ~~> mean_aligned_angular_distance. --- gallery/experiments/simulated_abinitio_pipeline.py | 6 +++--- gallery/tutorials/pipeline_demo.py | 6 +++--- gallery/tutorials/tutorials/orient3d_simulation.py | 6 +++--- src/aspire/utils/__init__.py | 2 +- src/aspire/utils/coor_trans.py | 2 +- tests/test_coor_trans.py | 6 +++--- tests/test_orient_symmetric.py | 4 ++-- tests/test_orient_sync_voting.py | 8 ++++---- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/gallery/experiments/simulated_abinitio_pipeline.py b/gallery/experiments/simulated_abinitio_pipeline.py index 2a861dfbb3..e4501daf8f 100644 --- a/gallery/experiments/simulated_abinitio_pipeline.py +++ b/gallery/experiments/simulated_abinitio_pipeline.py @@ -28,7 +28,7 @@ from aspire.operators import FunctionFilter, RadialCTFFilter from aspire.reconstruction import MeanEstimator from aspire.source import OrientedSource, Simulation -from aspire.utils import check_rotations +from aspire.utils import mean_aligned_angular_distance logger = logging.getLogger(__name__) @@ -194,9 +194,9 @@ def noise_function(x, y): oriented_src = OrientedSource(avgs, orient_est) logger.info("Compare with known rotations") -# Compare with known true rotations. ``check_rotations`` globally aligns the estimated +# Compare with known true rotations. ``mean_aligned_angular_distance`` globally aligns the estimated # rotations to the ground truth and finds the mean angular distance between them. -mean_ang_dist = check_rotations(oriented_src.rotations, true_rotations) +mean_ang_dist = mean_aligned_angular_distance(oriented_src.rotations, true_rotations) logger.info( f"Mean angular distance between globally aligned estimates and ground truth rotations: {mean_ang_dist}\n" ) diff --git a/gallery/tutorials/pipeline_demo.py b/gallery/tutorials/pipeline_demo.py index 8e571b9fe3..6c60e6f9ab 100644 --- a/gallery/tutorials/pipeline_demo.py +++ b/gallery/tutorials/pipeline_demo.py @@ -215,14 +215,14 @@ # %% # Mean Squared Error # ------------------ -# ASPIRE has the built-in utility function, ``check_rotations``, which globally +# ASPIRE has the built-in utility function, ``mean_aligned_angular_distance``, which globally # aligns the estimated rotations to the true rotations and computes the mean # angular distance (in degrees). -from aspire.utils import check_rotations +from aspire.utils import mean_aligned_angular_distance # Compare with known true rotations -mean_ang_dist = check_rotations(oriented_src.rotations, true_rotations) +mean_ang_dist = mean_aligned_angular_distance(oriented_src.rotations, true_rotations) mean_ang_dist diff --git a/gallery/tutorials/tutorials/orient3d_simulation.py b/gallery/tutorials/tutorials/orient3d_simulation.py index 2058a11f3f..5934eec030 100644 --- a/gallery/tutorials/tutorials/orient3d_simulation.py +++ b/gallery/tutorials/tutorials/orient3d_simulation.py @@ -14,7 +14,7 @@ from aspire.abinitio import CLSyncVoting from aspire.operators import RadialCTFFilter from aspire.source import OrientedSource, Simulation -from aspire.utils import check_rotations +from aspire.utils import mean_aligned_angular_distance from aspire.volume import Volume logger = logging.getLogger(__name__) @@ -101,9 +101,9 @@ # Mean Angular Distance # --------------------- -# ``check_rotations`` will perform global alignment of the estimated rotations +# ``mean_aligned_angular_distance`` will perform global alignment of the estimated rotations # to the ground truth and find the mean angular distance between them (in degrees). -mean_ang_dist = check_rotations(rots_est, rots_true) +mean_ang_dist = mean_aligned_angular_distance(rots_est, rots_true) logger.info( f"Mean angular distance between estimates and ground truth: {mean_ang_dist} degrees" ) diff --git a/src/aspire/utils/__init__.py b/src/aspire/utils/__init__.py index f5f133cf09..8f07078316 100644 --- a/src/aspire/utils/__init__.py +++ b/src/aspire/utils/__init__.py @@ -1,7 +1,7 @@ from .types import complex_type, real_type, utest_tolerance # isort:skip from .coor_trans import ( # isort:skip common_line_from_rots, - check_rotations, + mean_aligned_angular_distance, crop_pad_2d, crop_pad_3d, get_aligned_rotations, diff --git a/src/aspire/utils/coor_trans.py b/src/aspire/utils/coor_trans.py index 930fe1ddf0..2b7f432111 100644 --- a/src/aspire/utils/coor_trans.py +++ b/src/aspire/utils/coor_trans.py @@ -284,7 +284,7 @@ def get_rots_mse(rots_reg, rots_ref): return mse -def check_rotations(rots_est, rots_gt, degree_tol=None): +def mean_aligned_angular_distance(rots_est, rots_gt, degree_tol=None): """ Register estimates to ground truth rotations and compute the mean angular distance between them (in degrees). diff --git a/tests/test_coor_trans.py b/tests/test_coor_trans.py index 9509cf0225..b2ac28297b 100644 --- a/tests/test_coor_trans.py +++ b/tests/test_coor_trans.py @@ -5,7 +5,7 @@ from aspire.utils import ( Rotation, - check_rotations, + mean_aligned_angular_distance, crop_pad_2d, crop_pad_3d, get_aligned_rotations, @@ -338,7 +338,7 @@ def testCrop3DFillValue(self): self.assertTrue(np.array_equal(b[:, :, 0], -1 * np.ones((4, 4)))) -def test_check_rotations(): +def test_mean_aligned_angular_distance(): n_rots = 10 dtype = np.float32 rots_gt = Rotation.generate_random_rotations(n_rots, dtype=dtype).matrices @@ -347,4 +347,4 @@ def test_check_rotations(): rots_est = rots_gt[0] @ rots_gt # Check that the mean angular distance is zero degrees. - np.testing.assert_allclose(check_rotations(rots_est, rots_gt), 0.0) + np.testing.assert_allclose(mean_aligned_angular_distance(rots_est, rots_gt), 0.0) diff --git a/tests/test_orient_symmetric.py b/tests/test_orient_symmetric.py index d269f36a5e..8e18f408c3 100644 --- a/tests/test_orient_symmetric.py +++ b/tests/test_orient_symmetric.py @@ -10,7 +10,7 @@ J_conjugate, Rotation, all_pairs, - check_rotations, + mean_aligned_angular_distance, cyclic_rotations, randn, utest_tolerance, @@ -124,7 +124,7 @@ def test_estimate_rotations(n_img, L, order, dtype): # Register estimates to ground truth rotations and compute the # angular distance between them (in degrees). - mean_ang_dist = check_rotations(rots_est, rots_gt_sync) + mean_ang_dist = mean_aligned_angular_distance(rots_est, rots_gt_sync) # Assert mean angular distance is reasonable. assert mean_ang_dist < 3 diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index a28587e6dd..176f5ad05c 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -10,7 +10,7 @@ from aspire.commands.orient3d import orient3d from aspire.noise import WhiteNoiseAdder from aspire.source import Simulation -from aspire.utils import check_rotations, rots_to_clmatrix +from aspire.utils import mean_aligned_angular_distance, rots_to_clmatrix from aspire.volume import AsymmetricVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -97,7 +97,7 @@ def test_estimate_rotations(source_orientation_objs): # Register estimates to ground truth rotations and compute the # angular distance between them (in degrees). - mean_ang_dist = check_rotations(orient_est.rotations, src.rotations) + mean_ang_dist = mean_aligned_angular_distance(orient_est.rotations, src.rotations) # Assert that mean angular distance is less than 1 degree (5 degrees with shifts). degree_tol = 1 @@ -140,8 +140,8 @@ def test_estimate_rotations_fuzzy_mask(): orient_est_fuzzy.estimate_rotations() # Check that fuzzy_mask improves orientation estimation. - mean_angle_dist = check_rotations(orient_est.rotations, noisy_src.rotations) - mean_angle_dist_fuzzy = check_rotations( + mean_angle_dist = mean_aligned_angular_distance(orient_est.rotations, noisy_src.rotations) + mean_angle_dist_fuzzy = mean_aligned_angular_distance( orient_est_fuzzy.rotations, noisy_src.rotations ) From 37c2b74a2cc9bbb6b5d52a421bb6175d919f75be Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 14 Sep 2023 10:49:33 -0400 Subject: [PATCH 040/294] adjust tests to use degree_tol --- tests/test_coor_trans.py | 5 ++++- tests/test_orient_symmetric.py | 11 ++++------- tests/test_orient_sync_voting.py | 13 ++++++------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/test_coor_trans.py b/tests/test_coor_trans.py index b2ac28297b..70d4dc37a4 100644 --- a/tests/test_coor_trans.py +++ b/tests/test_coor_trans.py @@ -5,12 +5,12 @@ from aspire.utils import ( Rotation, - mean_aligned_angular_distance, crop_pad_2d, crop_pad_3d, get_aligned_rotations, grid_2d, grid_3d, + mean_aligned_angular_distance, register_rotations, uniform_random_angles, ) @@ -348,3 +348,6 @@ def test_mean_aligned_angular_distance(): # Check that the mean angular distance is zero degrees. np.testing.assert_allclose(mean_aligned_angular_distance(rots_est, rots_gt), 0.0) + + # Test internal assert using the `degree_tol` argument. + mean_aligned_angular_distance(rots_est, rots_gt, degree_tol=0.1) diff --git a/tests/test_orient_symmetric.py b/tests/test_orient_symmetric.py index 8e18f408c3..b008769e72 100644 --- a/tests/test_orient_symmetric.py +++ b/tests/test_orient_symmetric.py @@ -10,8 +10,8 @@ J_conjugate, Rotation, all_pairs, - mean_aligned_angular_distance, cyclic_rotations, + mean_aligned_angular_distance, randn, utest_tolerance, ) @@ -122,12 +122,9 @@ def test_estimate_rotations(n_img, L, order, dtype): # g-synchronize ground truth rotations. rots_gt_sync = cl_symm.g_sync(rots_est, order, rots_gt) - # Register estimates to ground truth rotations and compute the - # angular distance between them (in degrees). - mean_ang_dist = mean_aligned_angular_distance(rots_est, rots_gt_sync) - - # Assert mean angular distance is reasonable. - assert mean_ang_dist < 3 + # Register estimates to ground truth rotations and check that the + # mean angular distance between them is less than 3 degrees. + mean_aligned_angular_distance(rots_est, rots_gt_sync, degree_tol=3) @pytest.mark.parametrize("n_img, L, order, dtype", param_list_c3_c4) diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index 176f5ad05c..b5d97d9d03 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -96,12 +96,9 @@ def test_estimate_rotations(source_orientation_objs): orient_est.estimate_rotations() # Register estimates to ground truth rotations and compute the - # angular distance between them (in degrees). - mean_ang_dist = mean_aligned_angular_distance(orient_est.rotations, src.rotations) - - # Assert that mean angular distance is less than 1 degree (5 degrees with shifts). - degree_tol = 1 - assert mean_ang_dist < degree_tol + # mean angular distance between them (in degrees). + # Assert that mean angular distance is less than 1 degree. + mean_aligned_angular_distance(orient_est.rotations, src.rotations, degree_tol=1) def test_estimate_shifts(source_orientation_objs): @@ -140,7 +137,9 @@ def test_estimate_rotations_fuzzy_mask(): orient_est_fuzzy.estimate_rotations() # Check that fuzzy_mask improves orientation estimation. - mean_angle_dist = mean_aligned_angular_distance(orient_est.rotations, noisy_src.rotations) + mean_angle_dist = mean_aligned_angular_distance( + orient_est.rotations, noisy_src.rotations + ) mean_angle_dist_fuzzy = mean_aligned_angular_distance( orient_est_fuzzy.rotations, noisy_src.rotations ) From b35d37f8924b609eeee5d696aebb43178dc155e6 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 15 Sep 2023 09:49:39 -0400 Subject: [PATCH 041/294] address review. --- gallery/tutorials/pipeline_demo.py | 2 +- src/aspire/utils/coor_trans.py | 1 - tests/test_utils.py | 2 +- tox.ini | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gallery/tutorials/pipeline_demo.py b/gallery/tutorials/pipeline_demo.py index 6c60e6f9ab..77b34af9b9 100644 --- a/gallery/tutorials/pipeline_demo.py +++ b/gallery/tutorials/pipeline_demo.py @@ -223,7 +223,7 @@ # Compare with known true rotations mean_ang_dist = mean_aligned_angular_distance(oriented_src.rotations, true_rotations) -mean_ang_dist +print(f"Mean aligned angular distance: {mean_ang_dist} degrees") # %% diff --git a/src/aspire/utils/coor_trans.py b/src/aspire/utils/coor_trans.py index 2b7f432111..2c38275521 100644 --- a/src/aspire/utils/coor_trans.py +++ b/src/aspire/utils/coor_trans.py @@ -304,7 +304,6 @@ def mean_aligned_angular_distance(rots_est, rots_gt, degree_tol=None): if degree_tol is not None: np.testing.assert_array_less(mean_ang_dist, degree_tol) - return return mean_ang_dist diff --git a/tests/test_utils.py b/tests/test_utils.py index 8e6a61b28c..5dc2226ca4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -355,7 +355,7 @@ def test_fuzzy_mask(): # Smoke test for 1D, 2D, and 3D fuzzy_mask. for dim in range(1, 4): - _ = fuzzy_mask((8,) * dim, np.float32) + _ = fuzzy_mask((32,) * dim, np.float32) # Check that we raise an error for bad dimension. with pytest.raises(RuntimeError, match=r"Only 1D, 2D, or 3D fuzzy_mask*"): diff --git a/tox.ini b/tox.ini index 6969a99719..3f6c88b440 100644 --- a/tox.ini +++ b/tox.ini @@ -68,6 +68,7 @@ per-file-ignores = __init__.py: F401 gallery/tutorials/aspire_introduction.py: T201, F401, E402 gallery/tutorials/configuration.py: T201, E402 + gallery/tutorials/pipeline_demo.py: T201 gallery/tutorials/turorials/data_downloader.py: E402 gallery/tutorials/tutorials/ctf.py: T201, E402 gallery/tutorials/tutorials/micrograph_source.py: T201, E402 From 36b61877e9eaa5e159f91b1ed84984a2c1f485cb Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 21 Sep 2023 10:16:04 -0400 Subject: [PATCH 042/294] check square/cube dimensions. --- src/aspire/utils/misc.py | 10 +++++++++- tests/test_utils.py | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/aspire/utils/misc.py b/src/aspire/utils/misc.py index 7598556b49..8551840520 100644 --- a/src/aspire/utils/misc.py +++ b/src/aspire/utils/misc.py @@ -280,7 +280,8 @@ def fuzzy_mask(L, dtype, r0=None, risetime=None): Made with an error function with effective rise time. - :param L: The sizes of image in tuple structure. + :param L: The sizes of image in tuple structure. Must be 1D, 2D square, + or 3D cube. :param dtype: dtype for fuzzy mask. :param r0: The specified radius. Defaults to floor(0.45 * L) :param risetime: The rise time for `erf` function. Defaults to floor(0.05 * L) @@ -299,12 +300,19 @@ def fuzzy_mask(L, dtype, r0=None, risetime=None): if dim == 1: grid = grid_1d(**grid_kwargs) + elif dim == 2: + if not (L[0] == L[1]): + raise ValueError(f"A 2D fuzzy_mask must be square, found L={L}.") grid = grid_2d(**grid_kwargs) axes.append("y") + elif dim == 3: + if not (L[0] == L[1] == L[2]): + raise ValueError(f"A 3D fuzzy_mask must be cubic, found L={L}.") grid = grid_3d(**grid_kwargs) axes.extend(["y", "z"]) + else: raise RuntimeError( f"Only 1D, 2D, or 3D fuzzy_mask supported. Found {dim}-dimensional `L`." diff --git a/tests/test_utils.py b/tests/test_utils.py index 5dc2226ca4..ecd87c0122 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -351,7 +351,7 @@ def test_fuzzy_mask(): ] ) fmask = fuzzy_mask((8, 8), results.dtype, r0=2, risetime=2) - assert np.allclose(results, fmask, atol=1e-7) + np.testing.assert_allclose(results, fmask, atol=1e-7) # Smoke test for 1D, 2D, and 3D fuzzy_mask. for dim in range(1, 4): @@ -361,6 +361,14 @@ def test_fuzzy_mask(): with pytest.raises(RuntimeError, match=r"Only 1D, 2D, or 3D fuzzy_mask*"): _ = fuzzy_mask((8,) * 4, np.float32) + # Check we raise for bad 2D shape. + with pytest.raises(ValueError, match=r"A 2D fuzzy_mask must be square*"): + _ = fuzzy_mask((2, 3), np.float32) + + # Check we raise for bad 3D shape. + with pytest.raises(ValueError, match=r"A 3D fuzzy_mask must be cubic*"): + _ = fuzzy_mask((2, 3, 3), np.float32) + def test_multiprocessing_utils(): """ From 0f68d597e8931be4f74a9beb83471ea769814b70 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 21 Sep 2023 10:25:38 -0400 Subject: [PATCH 043/294] replace questionable testing logic. --- tests/test_orient_sync_voting.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index b5d97d9d03..bd91e98e71 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -144,7 +144,11 @@ def test_estimate_rotations_fuzzy_mask(): orient_est_fuzzy.rotations, noisy_src.rotations ) - assert mean_angle_dist_fuzzy < mean_angle_dist < 10 + # Check that the estimate is reasonable, ie. mean_angle_dist < 10 degrees. + np.testing.assert_array_less(mean_angle_dist, 10) + + # Check that fuzzy_mask improves the estimate. + np.testing.assert_array_less(mean_angle_dist_fuzzy, mean_angle_dist) def test_theta_error(): From 58faf2cdb62641fd6590141b98b128e9080436d6 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 21 Sep 2023 14:17:18 -0400 Subject: [PATCH 044/294] noise estimator 2nd moment, abs(x)**2 vs abs(x**2) --- src/aspire/noise/noise.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aspire/noise/noise.py b/src/aspire/noise/noise.py index 154e77f624..88e03caffa 100644 --- a/src/aspire/noise/noise.py +++ b/src/aspire/noise/noise.py @@ -292,7 +292,7 @@ def _estimate_noise_variance(self): _denominator = self.src.n * np.sum(mask) first_moment += np.sum(images_masked) / _denominator - second_moment += np.sum(np.abs(images_masked**2)) / _denominator + second_moment += np.sum(np.abs(images_masked) ** 2) / _denominator return second_moment - first_moment**2 @@ -338,7 +338,7 @@ def estimate_noise_psd(self): _denominator = self.src.n * np.sum(mask) mean_est += np.sum(images_masked) / _denominator im_masked_f = xp.asnumpy(fft.centered_fft2(xp.asarray(images_masked))) - noise_psd_est += np.sum(np.abs(im_masked_f**2), axis=0) / _denominator + noise_psd_est += np.sum(np.abs(im_masked_f) ** 2, axis=0) / _denominator mid = self.src.L // 2 noise_psd_est[mid, mid] -= mean_est**2 From ece96f54842168c18f6690ed85500650a9f45e2c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 29 Sep 2023 12:23:49 -0400 Subject: [PATCH 045/294] Enforce single blank line between docstring body and parameters. --- src/aspire/abinitio/commonline_base.py | 3 +++ src/aspire/abinitio/commonline_c3_c4.py | 1 + src/aspire/abinitio/commonline_sync.py | 1 + src/aspire/abinitio/sync_voting.py | 2 ++ src/aspire/basis/fle_2d.py | 4 ++++ src/aspire/denoising/denoised_src.py | 1 + src/aspire/image/image.py | 2 ++ src/aspire/image/xform.py | 2 ++ src/aspire/noise/noise.py | 1 + src/aspire/nufft/__init__.py | 2 ++ src/aspire/nufft/pynfft.py | 1 + src/aspire/operators/blk_diag_matrix.py | 1 + src/aspire/operators/filters.py | 1 + src/aspire/reconstruction/estimator.py | 1 + src/aspire/source/coordinates.py | 7 +++++++ src/aspire/source/image.py | 5 +++++ src/aspire/source/relion.py | 1 + src/aspire/source/simulation.py | 2 ++ src/aspire/utils/coor_trans.py | 1 - src/aspire/utils/misc.py | 1 + src/aspire/utils/resolution_estimation.py | 1 + 21 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/aspire/abinitio/commonline_base.py b/src/aspire/abinitio/commonline_base.py index 0074d4b562..ca79b05ec6 100644 --- a/src/aspire/abinitio/commonline_base.py +++ b/src/aspire/abinitio/commonline_base.py @@ -395,6 +395,7 @@ def _estimate_num_shift_equations(self, n_img, equations_factor=1, max_memory=40 The function computes total number of shift equations based on number of images and preselected memory factor. + :param n_img: The total number of input images :param equations_factor: The factor to rescale the number of shift equations (=1 in default) @@ -442,6 +443,7 @@ def _generate_shift_phase_and_filter(self, r_max, max_shift, shift_step): The shift phases are pre-defined in a range of max_shift that can be applied to maximize the common line calculation. The common-line filter is also applied to the radial direction for easier detection. + :param r_max: Maximum index for common line detection :param max_shift: Maximum value of 1D shift (in pixels) to search :param shift_step: Resolution of shift estimation in pixels @@ -511,6 +513,7 @@ def _apply_filter_and_norm(self, subscripts, pf, r_max, h): :subscripts: Specifies the subscripts for summation of Numpy `einsum` function + :param pf: Fourier transform of images :param r_max: Maximum index for common line detection :param h: common lines filter diff --git a/src/aspire/abinitio/commonline_c3_c4.py b/src/aspire/abinitio/commonline_c3_c4.py index b3c6e3df9d..63ac41ac88 100644 --- a/src/aspire/abinitio/commonline_c3_c4.py +++ b/src/aspire/abinitio/commonline_c3_c4.py @@ -553,6 +553,7 @@ def _syncmatrix_ij_vote_3n(self, clmatrix, i, j, k_list, n_theta): Given the common lines matrix `clmatrix`, a list of images specified in k_list and the number of common lines n_theta, find the (i, j) rotation block Rij. + :param clmatrix: The common lines matrix :param i: The i image :param j: The j image diff --git a/src/aspire/abinitio/commonline_sync.py b/src/aspire/abinitio/commonline_sync.py index 6fbec4c8ef..8cef5b6e40 100644 --- a/src/aspire/abinitio/commonline_sync.py +++ b/src/aspire/abinitio/commonline_sync.py @@ -180,6 +180,7 @@ def _syncmatrix_ij_vote(self, clmatrix, i, j, k_list, n_theta): Given the common lines matrix `clmatrix`, a list of images specified in k_list and the number of common lines n_theta, find the (i, j) rotation block (in X and Y) of the synchronization matrix. + :param clmatrix: The common lines matrix :param i: The i image :param j: The j image diff --git a/src/aspire/abinitio/sync_voting.py b/src/aspire/abinitio/sync_voting.py index 0902c57059..abc11ef6e1 100644 --- a/src/aspire/abinitio/sync_voting.py +++ b/src/aspire/abinitio/sync_voting.py @@ -19,6 +19,7 @@ def _rotratio_eulerangle_vec(self, clmatrix, i, j, good_k, n_theta): Given a common lines matrix, where the index of each common line is in the range of n_theta and a list of good image k from voting results. + :param clmatrix: The common lines matrix :param i: The i image :param j: The j image @@ -61,6 +62,7 @@ def _vote_ij(self, clmatrix, n_theta, i, j, k_list): clmatrix is the common lines matrix, constructed using angular resolution, n_theta. k_list are the images to be used for voting of the pair of images (i ,j). + :param clmatrix: The common lines matrix :param n_theta: The number of points in the theta direction (common lines) :param i: The i image diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 6f37cddc00..86b1626064 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -396,6 +396,7 @@ def _threshold_basis_functions(self): """ Implements the bandlimit threshold which caps the number of basis functions that are actually required. + :return: The final overall number of basis functions to be used. """ # Maximum bandlimit @@ -630,6 +631,7 @@ def _create_dense_matrix(self): """ Directly computes the transformation matrix from Cartesian coordinates to FLE coordinates without any shortcuts. + :return: A NumPy array of size `(self.nres**2, self.count)` containing the matrix entries. """ @@ -649,6 +651,7 @@ def _create_dense_matrix(self): def lowpass(self, coeffs, bandlimit): """ Apply a low-pass filter to FLE coefficients `coeffs` with threshold `bandlimit`. + :param coeffs: A NumPy array of FLE coefficients, of shape (num_images, self.count) :param bandlimit: Integer bandlimit (max frequency). :return: Band-limited coefficient array. @@ -673,6 +676,7 @@ def lowpass(self, coeffs, bandlimit): def radial_convolve(self, coeffs, radial_img): """ Convolve a stack of FLE coefficients with a 2D radial function. + :param coeffs: A NumPy array of FLE coefficients of size (num_images, self.count). :param radial_img: A 2D NumPy array of size (self.nres, self.nres). :return: Convolved FLE coefficients. diff --git a/src/aspire/denoising/denoised_src.py b/src/aspire/denoising/denoised_src.py index 94d4c1e4e9..d89cb412cc 100644 --- a/src/aspire/denoising/denoised_src.py +++ b/src/aspire/denoising/denoised_src.py @@ -34,6 +34,7 @@ def _images(self, indices): """ Internal function to return a set of images after denoising, when accessed via the `ImageSource.images` property. + :param indices: The indices of images to return as a 1-D NumPy array. :return: an `Image` object after denoisng. """ diff --git a/src/aspire/image/image.py b/src/aspire/image/image.py index 8d8b7652b0..4713266b54 100644 --- a/src/aspire/image/image.py +++ b/src/aspire/image/image.py @@ -437,6 +437,7 @@ def load(filepath, dtype=None): def _im_translate(self, shifts): """ Translate image by shifts + :param im: An array of size n-by-L-by-L containing images to be translated. :param shifts: An array of size n-by-2 specifying the shifts in pixels. Alternatively, it can be a row vector of length 2, in which case the same shifts is applied to each image. @@ -493,6 +494,7 @@ def size(self): def backproject(self, rot_matrices): """ Backproject images along rotation + :param im: An Image (stack) to backproject. :param rot_matrices: An n-by-3-by-3 array of rotation matrices \ corresponding to viewing directions. diff --git a/src/aspire/image/xform.py b/src/aspire/image/xform.py index 189bc33ce4..0759c819e8 100644 --- a/src/aspire/image/xform.py +++ b/src/aspire/image/xform.py @@ -76,6 +76,7 @@ def _forward(self, im, indices): def enabled(self): """ Enable this Xform in a context manager, regardless of its `active` attribute value. + :return: A context manager in which this Xform is enabled. """ return Xform.XformActiveContextManager(self, active=True) @@ -83,6 +84,7 @@ def enabled(self): def disabled(self): """ Disable this Xform in a context manager, regardless of its `active` attribute value. + :return: A context manager in which this Xform is disabled. """ return Xform.XformActiveContextManager(self, active=False) diff --git a/src/aspire/noise/noise.py b/src/aspire/noise/noise.py index 88e03caffa..642ef0f5c5 100644 --- a/src/aspire/noise/noise.py +++ b/src/aspire/noise/noise.py @@ -277,6 +277,7 @@ def _create_filter(self, noise_variance=None): def _estimate_noise_variance(self): """ Any additional arguments/keyword-arguments are passed on to the Source's 'images' method + :return: The estimated noise variance of the images in the Source used to create this estimator. TODO: How's this initial estimate of variance different from the 'estimate' method? """ diff --git a/src/aspire/nufft/__init__.py b/src/aspire/nufft/__init__.py index 6452956ba5..aa7c3a4adf 100644 --- a/src/aspire/nufft/__init__.py +++ b/src/aspire/nufft/__init__.py @@ -18,6 +18,7 @@ def check_backends(raise_errors=True): """ Check all NFFT backends in package configuration + :param raise_errors: Whether to raise a RuntimeError if no backends detected. :return: On return, the global names 'backends'/'default_plan_class' have been populated @@ -28,6 +29,7 @@ def check_backends(raise_errors=True): def _try_backend(backend): """ This function tries out a particular NFFT backend by name. + :param backend: A string representing the NFFT backend we want to try. Currently one of: 'cufinufft' The Python wrapper for the CUDA variant of FINUFFT library diff --git a/src/aspire/nufft/pynfft.py b/src/aspire/nufft/pynfft.py index fdeb831956..fec853098f 100644 --- a/src/aspire/nufft/pynfft.py +++ b/src/aspire/nufft/pynfft.py @@ -17,6 +17,7 @@ def epsilon_to_nfft_cutoff(epsilon): def __init__(self, sz, fourier_pts, epsilon=1e-15, **kwargs): """ A plan for non-uniform FFT (3D) + :param sz: A tuple indicating the geometry of the signal :param fourier_pts: The points in Fourier space where the Fourier transform is to be calculated, arranged as a 3-by-K array. These need to be in the range [-pi, pi] in each dimension. diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index a92dd078b0..65f95b4774 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -754,6 +754,7 @@ def rapply(self, X): def eigvals(self): """ Compute the eigenvalues of a BlkDiagMatrix. + :return: Array of eigvals, with length equal to the fully expanded matrix diagonal. """ diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index d838f6387d..d39b475656 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -52,6 +52,7 @@ def __mul__(self, other): def __str__(self): """ Show class name of Filter + :return: A string of class name """ return self.__class__.__name__ diff --git a/src/aspire/reconstruction/estimator.py b/src/aspire/reconstruction/estimator.py index c2a07e7a78..a5cc73e917 100644 --- a/src/aspire/reconstruction/estimator.py +++ b/src/aspire/reconstruction/estimator.py @@ -70,6 +70,7 @@ def estimate(self, b_coeff=None, tol=1e-5, regularizer=0): def apply_kernel(self, vol_coeff, kernel=None): """ Applies the kernel represented by convolution + :param vol_coeff: The volume to be convolved, stored in the basis coefficients. :param kernel: a Kernel object. If None, the kernel for this Estimator is used. :return: The result of evaluating `vol_coeff` in the given basis, convolving with the kernel given by diff --git a/src/aspire/source/coordinates.py b/src/aspire/source/coordinates.py index 027490711b..dca7aaf873 100644 --- a/src/aspire/source/coordinates.py +++ b/src/aspire/source/coordinates.py @@ -197,6 +197,7 @@ def _box_coord_from_center(center, particle_size): to a list `[lower left x, lower left y, particle_size, particle_size]` representing the box around the particle in box format. + :param center: a list of length two representing a center :param particle_size: the size of the box around the particle """ @@ -219,6 +220,7 @@ def _center_from_box_coord(box_coord): `[lower left x, lower left y, particle_size, particle_size]` representing a particle in the box format to a list `[x, y]` representing the particle center. + :param box_coord: a list of length 4 representing the particle box """ # Get lower left corner x and y coordinates @@ -232,6 +234,7 @@ def _coords_list_from_star(self, star_file): """ Given a Relion STAR coordinate file (generally containing particle centers) return a list of coordinates in box format. + :param star_file: A path to a STAR file containing particle centers """ data_block = StarFile(star_file).get_block_by_index(0) @@ -245,6 +248,7 @@ def _populate_local_metadata(self): """ Called during ImageSource.save(), populates metadata columns specific to `CoordinateSource` when saving to STAR file. + :return: A list of the names of the columns added. """ # Insert stored particle coordinates (centers) into metadata @@ -268,6 +272,7 @@ def _exclude_boundary_particles(self): """ Remove particles boxes which do not fit in the micrograph with the given `particle_size`. + :return: Number of particles removed """ out_of_range = [] @@ -414,6 +419,7 @@ def _crop_micrograph(data, coord): Crops a particle box defined by `coord` out of `data`. According to MRC 2014 convention, the origin represents the bottom-left corner of the image. + :param data: A 2D numpy array representing a micrograph :param coord: A list of integers: (lower left X, lower left Y, X, Y) """ @@ -430,6 +436,7 @@ def _images(self, indices): particles were excluded due to their box not fitting into the mrc dimensions. Thus, the exact particles returned are a function of the `particle_size`. + :param indices: A 1-D NumPy array of integer indices. :return: An `Image` object. """ diff --git a/src/aspire/source/image.py b/src/aspire/source/image.py index e5fb7a4350..aa127bfb66 100644 --- a/src/aspire/source/image.py +++ b/src/aspire/source/image.py @@ -53,6 +53,7 @@ def __getitem__(self, indices): ImageAccessor can be indexed via Python slice object, 1-D NumPy array, list, or a single integer, corresponding to the indices of the requested images. By default, slices default to a start of 0, an end of self.num_imgs, and a step of 1. + :return: An Image object containing the requested images. """ if isinstance(indices, Iterable) and not isinstance(indices, np.ndarray): @@ -304,6 +305,7 @@ def __len__(self): def _metadata_as_dict(self, metadata_fields, indices, default_value=None): """ Return a dictionary of selected metadata fields at selected indices. + :param metadata_fields: An iterable of strings specifying metadata fields. :param indices: An ndarray of 0-indexed locations we're interested in. :param default_value: A scalar default value to use if a metadata_field is not found. @@ -324,6 +326,7 @@ def _metadata_as_dict(self, metadata_fields, indices, default_value=None): def _metadata_as_ndarray(self, metadata_fields, indices, default_value=None): """ Return a numpy array of selected metadata fields at selected indices. + :param metadata_fields: An iterable of strings specifying metadata fields. :param indices: An ndarray of 0-indexed locations we're interested in. :param default_value: A scalar default value to use if a metadata_field is not found. @@ -1078,6 +1081,7 @@ def _populate_local_metadata(self): """ Populate metadata columns specific to the `ImageSource` subclass being saved. Subclasses optionally override, but must return a list of strings. + :return: A list of the names of the columns added. """ return [] @@ -1687,6 +1691,7 @@ def _images(self, indices): """ Returns images corresponding to `indices` after being accessed via the `ImageSource.images` property + :param indices: A 1-D NumPy array of indices. :return: An `Image` object. """ diff --git a/src/aspire/source/relion.py b/src/aspire/source/relion.py index 6106bf4389..02b8ab9363 100644 --- a/src/aspire/source/relion.py +++ b/src/aspire/source/relion.py @@ -203,6 +203,7 @@ def _images(self, indices): """ Returns particle images when accessed via the `ImageSource.images` property. Loads particle images corresponding to `indices` from StarFile and .mrcs stacks. + :param indices: A 1-D NumPy array of integer indices. :return: An `Image` object. """ diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index 2aa544abcf..46a01fdfe9 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -272,6 +272,7 @@ def _clean_images(self, indices): def _images(self, indices, clean_images=False): """ Returns particle images when accessed via the `ImageSource.images` property. + :param indices: A 1-D NumPy array of integer indices. :param clean_images: Only used internally, toggled on when `clean_images` requested. Will skip accessing cache, skip noise, while applying CTF to projections. @@ -366,6 +367,7 @@ def covar_true(self): def eigs(self): """ Eigendecomposition of volume covariance matrix of simulation + :return: A 2-tuple: eigs_true: The eigenvectors of the volume covariance matrix in the form of Volume instance. lambdas_true: The eigenvalues of the covariance matrix in the form of a (C-1)-by-(C-1) diagonal matrix. diff --git a/src/aspire/utils/coor_trans.py b/src/aspire/utils/coor_trans.py index 2c38275521..e909e2f394 100644 --- a/src/aspire/utils/coor_trans.py +++ b/src/aspire/utils/coor_trans.py @@ -238,7 +238,6 @@ def get_aligned_rotations(rots, Q_mat, flag): Calculated aligned rotation matrices from the orthogonal transformation that best aligns the estimated rotations to the reference rotations. - :param rots: The reference rotations to which we would like to align in the form of a n-by-3-by-3 array. :param Q_mat: optimal orthogonal 3x3 transformation matrix diff --git a/src/aspire/utils/misc.py b/src/aspire/utils/misc.py index 8551840520..f8d3a98c3f 100644 --- a/src/aspire/utils/misc.py +++ b/src/aspire/utils/misc.py @@ -206,6 +206,7 @@ def gaussian_3d(size, mu=(0, 0, 0), sigma=(1, 1, 1), indexing="zyx", dtype=np.fl def bump_3d(size, spread=1, dtype=np.float64): """ Returns a centered 3D bump function in a (size)x(size)x(size) numpy array. + :param size: The length of the dimensions of the array (pixels. :param spread: A factor controling the spread of the bump function. :param dtype: dtype of returned array diff --git a/src/aspire/utils/resolution_estimation.py b/src/aspire/utils/resolution_estimation.py index 73780e36be..5a54c269b8 100644 --- a/src/aspire/utils/resolution_estimation.py +++ b/src/aspire/utils/resolution_estimation.py @@ -248,6 +248,7 @@ def _nufft_correlations(self): def analyze_correlations(self, cutoff): """ Convert from the Fourier correlations to frequencies and resolution. + :param cutoff: Cutoff value, traditionally `.143`. """ From c2f201bb8f4207c7f533433bdd109c00e732853d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 14:59:24 -0400 Subject: [PATCH 046/294] initial add coef class --- src/aspire/basis/__init__.py | 1 + src/aspire/basis/basis.py | 30 +++-- src/aspire/basis/coef.py | 252 +++++++++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+), 11 deletions(-) create mode 100644 src/aspire/basis/coef.py diff --git a/src/aspire/basis/__init__.py b/src/aspire/basis/__init__.py index 7d57b57295..2d34e951bc 100644 --- a/src/aspire/basis/__init__.py +++ b/src/aspire/basis/__init__.py @@ -2,6 +2,7 @@ # isort: off from .basis import Basis +from .coef import Coef from .steerable import SteerableBasis2D from .fb import FBBasisMixin diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 372e13f241..d05749c713 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -95,9 +95,16 @@ def evaluate(self, v): f" Inconsistent dtypes v: {v.dtype} self coefficient dtype: {self.coefficient_dtype}" ) + from .coef import Coef + + if not isinstance(v, Coef): + # v = Coef(self, v) + raise TypeError(f"`evaluate` should be passed a `Coef`, received {type(v)}") + # Flatten stack, ndim is wrt Basis (2 or 3) - stack_shape = v.shape[:-1] v = v.reshape(-1, self.count) + v = v.stack_reshape(-1).asnumpy() + # Compute the transform x = self._evaluate(v) # Restore stack shape @@ -141,8 +148,10 @@ def evaluate_t(self, v): # Restore stack shape x = x.reshape(*stack_shape, self.count) - # Return an ndarray - return x + # Avoid circular dependence + from .coef import Coef + + return Coef(self, x) def _evaluate_t(self, v): raise NotImplementedError("Subclasses should implement this") @@ -190,6 +199,9 @@ def expand(self, x): those first dimensions of `x`. """ + + from .coef import Coef + if isinstance(x, Image) or isinstance(x, Volume): x = x.asnumpy() @@ -199,6 +211,7 @@ def expand(self, x): f" Inconsistent dtypes x: {x.dtype} self: {self.dtype}" ) + # TODO: We should only need to do this block when we are not passed Image/Volume. # check that last ndim values of input shape match # the shape of this basis assert ( @@ -211,7 +224,7 @@ def expand(self, x): operator = LinearOperator( shape=(self.count, self.count), - matvec=lambda v: self.evaluate_t(self.evaluate(v)), + matvec=lambda v: self.evaluate_t(self.evaluate(Coef(self, v))), dtype=self.dtype, ) @@ -224,7 +237,7 @@ def expand(self, x): v = np.zeros((n_data, self.count), dtype=self.coefficient_dtype) for isample in range(0, n_data): - b = self.evaluate_t(self._cls(x[isample])).T + b = self.evaluate_t(self._cls(x[isample])).asnumpy().T # TODO: need check the initial condition x0 can improve the results or not. v[isample], info = cg(operator, b, tol=tol, atol=0) if info != 0: @@ -232,10 +245,5 @@ def expand(self, x): # return v coefficients with the last dimension of self.count v = v.reshape((*sz_roll, self.count)) - return v - @property - def blk_diag_cov_shape(self): - raise NotImplementedError( - "This method should be implemented for any steerable 2D basis." - ) + return Coef(self, v) diff --git a/src/aspire/basis/coef.py b/src/aspire/basis/coef.py new file mode 100644 index 0000000000..46e6ec2cc2 --- /dev/null +++ b/src/aspire/basis/coef.py @@ -0,0 +1,252 @@ +import logging + +import numpy as np + +from .basis import Basis +from .steerable import SteerableBasis2D + +logger = logging.getLogger(__name__) + + +class Coef: + """ + Numpy interoperable basis stacks. + """ + + def __init__(self, basis, data, dtype=None): + """ + A stack of one or more coefficient arrays. + + The stack can be multidimensional with `stack_size` equal + to the product of the stack dimensions. Singletons will be + expanded into a 1D stack of length one. + + The last axes always represents the coefficient `count`. + + :param data: Numpy array containing image data with shape + `(..., count)`. + :param dtype: Optionally cast `data` to this dtype. + Defaults to `data.dtype`. + + :return: Image instance holding `data`. + """ + + if isinstance(data, Coef): + data = data.asnumpy() + elif not isinstance(data, np.ndarray): + raise ValueError("Coef should be instantiated with an ndarray") + + if data.ndim < 1: + raise ValueError( + "Coef data should be ndarray with shape (N1...) x count or (count)." + ) + elif data.ndim == 1: + data = np.expand_dims(data, axis=0) + + if dtype is None: + self.dtype = data.dtype + else: + self.dtype = np.dtype(dtype) + + if not isinstance(basis, Basis): + raise TypeError( + f"`basis` is required to be a `Basis` instance, received {type(basis)}" + ) + self.basis = basis + + self._data = data.astype(self.dtype, copy=False) + self.ndim = self._data.ndim + self.shape = self._data.shape + self.stack_ndim = self._data.ndim - 1 + self.stack_shape = self._data.shape[:-1] + self.stack_size = np.prod(self.stack_shape) + self.count = self._data.shape[-1] + + if self.count != self.basis.count: + raise RuntimeError( + f"Provided data count of {self.count} does not match basis count of {self.basis.count}." + ) + + # Numpy interop + # https://numpy.org/devdocs/user/basics.interoperability.html#the-array-interface-protocol + self.__array_interface__ = self.asnumpy().__array_interface__ + self.__array__ = self.asnumpy() + + def __len__(self): + """ + Return stack length. + + Note this is product of all stack dimensions. + """ + return self.stack_size + + def asnumpy(self): + """ + Return image data as a (, count) + read-only array view. + + :return: read-only ndarray view + """ + + view = self._data.view() + view.flags.writeable = False + return view + + def _check_key_dims(self, key): + if isinstance(key, tuple) and (len(key) > self._data.ndim): + raise ValueError( + f"Coef stack_dim is {self.stack_ndim}, slice length must be =< {self.ndim}" + ) + + def __getitem__(self, key): + self._check_key_dims(key) + return self.__class__(self.basis, self._data[key]) + + def __setitem__(self, key, value): + self._check_key_dims(key) + self._data[key] = value + + def stack_reshape(self, *args): + """ + Reshape the stack axis. + + :*args: Integer(s) or tuple describing the intended shape. + + :returns: Coef instance + """ + + # If we're passed a tuple, use that + if len(args) == 1 and isinstance(args[0], tuple): + shape = args[0] + else: + # Otherwise use the variadic args + shape = args + + # Sanity check the size + if shape != (-1,) and np.prod(shape) != self.stack_size: + raise ValueError( + f"Number of images {self.stack_size} cannot be reshaped to {shape}." + ) + + return self.__class__( + self.basis, self._data.reshape(*shape, self._data.shape[-1]) + ) + + def copy(self): + """ + Return a new `Coef` instance with a deep copy of the data. + """ + return self.__class__(self.basis, self._data.copy()) + + def evaluate(self): + """ + Return the evaluation of coefficients in the associated `basis`. + """ + return self.basis.evaluate(self) + + def rotate(self, radians, refl=None): + """ + Returns coefs rotated counter-clockwise by `radians`. + + Raises error if underlying coef basis does not support rotations. + + :param radians: Rotation in radians. + :param refl: Optional reflect image (about y=0) (bool) + :return: rotated coefs. + """ + if not isinstance(self.basis, SteerableBasis2D): + raise RuntimeError(f"self.basis={self.basis} is not SteerableBasis.") + + return self.basis.rotate(self, radians, refl) + + def shift(self, shifts): + """ + Returns coefs shifted by `shifts`. + + This will transform to real cartesian space, shift, + and transform back to Polar Fourier-Bessel space. + + :param coef: Basis coefs. + :param shifts: Shifts in pixels (x,y). Shape (1,2) or (len(coef), 2). + :return: coefs of shifted images. + """ + + if not callable(getattr(self.basis, "shift", None)): + raise RuntimeError( + f"self.basis={self.basis} does not provide `shift` method." + ) + + return self.basis.shift(self, shifts) + + def __mul__(self, other): + """ + Overload operator for multiplication. + + :param other: `Coef` instance to multiply with. + Also allows for multiplication by Numpy arrays and scalars. + :return: `Coef` instance. + """ + + if isinstance(other, Coef): + other = other._data + + return self.__class__(self.basis, self._data * other) + + def __add__(self, other): + """ + Overload operator for addition. + + :param other: `Coef` instance to add. + Also allows for addition by Numpy arrays and scalars. + :return: `Coef` instance. + """ + + if isinstance(other, Coef): + other = other._data + + return self.__class__(self.basis, self._data + other) + + def __sub__(self, other): + """ + Overload operator for subtraction. + + :param other: `Coef` instance to subtract. + Also allows for subtraction by Numpy arrays and scalars. + :return: `Coef` instance. + """ + + if isinstance(other, Coef): + other = other._data + + return self.__class__(self.basis, self._data - other) + + def __neg__(self): + """ + Overload operator for negation. + + :return: `Coef` instance. + """ + + return self.__class__(self.basis, -self._data) + + def size(self): + """ + Return np.size of underlying data. + + This should be `stack_size * count`, + or `len(self) * count`. + """ + return np.size(self._data) + + # This is included for completion, but is not being adopted yet. + def by_indices(self, **kwargs): + """ + Select coefficients by indices (`radial`, `angular`). + + See `SteerableBasis.indices_mask` for argument details. + + :return: `Coef` vector. + """ + + mask = self.basis.indices_mask(**kwargs) + return self._data[:, mask] From e3af0437304203d7606bf7eab6015445d7814044 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 15:35:10 -0400 Subject: [PATCH 047/294] stash partial basis migration --- src/aspire/basis/fb_2d.py | 98 +------------------------------------- src/aspire/basis/fle_2d.py | 35 +++++++++----- 2 files changed, 25 insertions(+), 108 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index a05ac12fd9..8deaa8f593 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -6,7 +6,7 @@ from aspire.basis import FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import unique_coords_nd from aspire.operators import BlkDiagMatrix -from aspire.utils import complex_type, real_type, roll_dim, unroll_dim +from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape logger = logging.getLogger(__name__) @@ -110,9 +110,6 @@ def _compute_indices(self): self.angular_indices = indices_ells self.radial_indices = indices_ks self.signs_indices = indices_sgns - # Relating to paper: a[i] = a_ell_ks = a_angularindices[i]_radialindices[i] - self.complex_angular_indices = indices_ells[self._pos] # k - self.complex_radial_indices = indices_ks[self._pos] # q def indices(self): """ @@ -283,99 +280,6 @@ def _evaluate_t(self, v): v = roll_dim(v, sz_roll) return v.T # RCOPT - def to_complex(self, coef): - """ - Return complex valued representation of coefficients. - This can be useful when comparing or implementing methods - from literature. - - There is a corresponding method, to_real. - - :param coef: Coefficients from this basis. - :return: Complex coefficent representation from this basis. - """ - - if coef.ndim == 1: - coef = coef.reshape(1, -1) - - if coef.dtype not in (np.float64, np.float32): - raise TypeError("coef provided to to_complex should be real.") - - # Pass through dtype precions, but check and warn if mismatched. - dtype = complex_type(coef.dtype) - if coef.dtype != self.dtype: - logger.warning( - f"coef dtype {coef.dtype} does not match precision of basis.dtype {self.dtype}, returning {dtype}." - ) - - # Return the same precision as coef - imaginary = dtype(1j) - - ccoef = np.zeros((coef.shape[0], self.complex_count), dtype=dtype) - - ind = 0 - idx = np.arange(self.k_max[0], dtype=int) - ind += np.size(idx) - - ccoef[:, idx] = coef[:, idx] - - for ell in range(1, self.ell_max + 1): - idx = ind + np.arange(self.k_max[ell], dtype=int) - ccoef[:, idx] = ( - coef[:, self._pos[idx]] - imaginary * coef[:, self._neg[idx]] - ) / 2.0 - - ind += np.size(idx) - - return ccoef - - def to_real(self, complex_coef): - """ - Return real valued representation of complex coefficients. - This can be useful when comparing or implementing methods - from literature. - - There is a corresponding method, to_complex. - - :param complex_coef: Complex coefficients from this basis. - :return: Real coefficent representation from this basis. - """ - if complex_coef.ndim == 1: - complex_coef = complex_coef.reshape(1, -1) - - if complex_coef.dtype not in (np.complex128, np.complex64): - raise TypeError("coef provided to to_real should be complex.") - - # Pass through dtype precions, but check and warn if mismatched. - dtype = real_type(complex_coef.dtype) - if dtype != self.dtype: - logger.warning( - f"Complex coef dtype {complex_coef.dtype} does not match precision of basis.dtype {self.dtype}, returning {dtype}." - ) - - coef = np.zeros((complex_coef.shape[0], self.count), dtype=dtype) - - ind = 0 - idx = np.arange(self.k_max[0], dtype=int) - ind += np.size(idx) - ind_pos = ind - - coef[:, idx] = complex_coef[:, idx].real - - for ell in range(1, self.ell_max + 1): - idx = ind + np.arange(self.k_max[ell], dtype=int) - idx_pos = ind_pos + np.arange(self.k_max[ell], dtype=int) - idx_neg = idx_pos + self.k_max[ell] - - c = complex_coef[:, idx] - coef[:, idx_pos] = 2.0 * np.real(c) - coef[:, idx_neg] = -2.0 * np.imag(c) - - ind += np.size(idx) - ind_pos += 2 * self.k_max[ell] - - return coef - def calculate_bispectrum( self, coef, flatten=False, filter_nonzero_freqs=False, freq_cutoff=None ): diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 86b1626064..b427a36234 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -5,7 +5,7 @@ from scipy.fft import dct, idct from scipy.special import jv -from aspire.basis import FBBasisMixin, SteerableBasis2D +from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import besselj_zeros from aspire.basis.fle_2d_utils import ( barycentric_interp_sparse, @@ -656,14 +656,14 @@ def lowpass(self, coeffs, bandlimit): :param bandlimit: Integer bandlimit (max frequency). :return: Band-limited coefficient array. """ - if len(coeffs.shape) == 1: - coeffs = coeffs.reshape((1, coeffs.shape[0])) - assert ( - len(coeffs.shape) == 2 - ), "Input a stack of coefficients of dimension (num_images, self.count)." - assert ( - coeffs.shape[1] == self.count - ), "Number of coefficients must match self.count." + + if not isinstance(coeffs, Coef): + raise TypeError( + f"`coeffs` should be a `Coef` instance, received {type(coeffs)}." + ) + + # Copy to mutate the coeffs. + coeffs = coeffs.asnumpy().copy() k = self.count - 1 for _ in range(self.count): @@ -671,7 +671,7 @@ def lowpass(self, coeffs, bandlimit): k = k - 1 coeffs[:, k + 1 :] = 0 - return coeffs + return Coef(self, coeffs) def radial_convolve(self, coeffs, radial_img): """ @@ -681,6 +681,19 @@ def radial_convolve(self, coeffs, radial_img): :param radial_img: A 2D NumPy array of size (self.nres, self.nres). :return: Convolved FLE coefficients. """ + + if not isinstance(coeffs, Coef): + raise TypeError( + f"`coeffs` should be a `Coef` instance, received {type(coeffs)}." + ) + + if len(coeffs.stack_shape) > 1: + raise NotImplementedError( + "`radial_convolve` currently only implemented for 1D stacks." + ) + + coeffs = coeffs.asnumpy() + num_img = coeffs.shape[0] coeffs_conv = np.zeros(coeffs.shape) @@ -699,7 +712,7 @@ def radial_convolve(self, coeffs, radial_img): # Convert from internal FLE ordering to FB convention coeffs_conv = coeffs_conv[..., self._fle_to_fb_indices] - return coeffs_conv + return Coef(self, coeffs_conv) def _radial_convolve_weights(self, b): """ From 5e84991b6f967ba6905d84086f50a165a8df72be Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 15:35:34 -0400 Subject: [PATCH 048/294] fspca basis conversion --- src/aspire/basis/fspca.py | 40 +++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index 31a926472a..c29d48d4a3 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -3,7 +3,7 @@ import numpy as np -from aspire.basis import FFBBasis2D, SteerableBasis2D +from aspire.basis import Coef, FFBBasis2D, SteerableBasis2D from aspire.covariance import BatchedRotCov2D from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type, fix_signs, real_type @@ -168,7 +168,7 @@ def build(self): ) # Create the arrays to be packed by _compute_spca - self.eigvals = np.zeros(self.basis.count, dtype=self.dtype) + self._eigvals = np.zeros(self.basis.count, dtype=self.dtype) self.eigvecs = BlkDiagMatrix.empty(2 * self.basis.ell_max + 1, dtype=self.dtype) @@ -211,7 +211,7 @@ def _compute_spca(self): basis_inds.append(_basis_inds) # Store the eigvals for this block, note this is a flat array. - self.eigvals[_basis_inds] = eigvals_k + self._eigvals[_basis_inds] = eigvals_k # Store the eigvecs, note this is a BlkDiagMatrix and is assigned incrementally. self.eigvecs[angular_index] = eigvecs_k @@ -231,7 +231,7 @@ def _compute_spca(self): # # We can pass a full or truncated slice of sorted_indices to any array indexed by # the coefs. This is used later for compression and index re-generation. - self.sorted_indices = np.argsort(-np.abs(self.eigvals)) + self.sorted_indices = np.argsort(-np.abs(self._eigvals)) compressed_indices = self._get_compressed_indices() @@ -252,6 +252,7 @@ def _compute_spca(self): start = i * self.batch_size finish = min((i + 1) * self.batch_size, self.src.n) batch_coef = self.basis.evaluate_t(self.src.images[start:finish]) + batch_coef = batch_coef.asnumpy() # Make the Data matrix (A_k) # # Construct A_k, matrix of expansion coefficients a^i_k_q @@ -330,13 +331,15 @@ def expand(self, x): Fourier Bessel basis. :return: Stack of coefs in the FSPCABasis. """ + if not isinstance(x, Coef): + raise TypeError(f"'x' should be `Coef` instance, received {type(x)}.") # Apply linear combination defined by FSPCA (eigvecs) - c_fspca = x @ self.eigvecs + c_fspca = x.asnumpy() @ self.eigvecs assert c_fspca.shape == (x.shape[0], self.count) - return c_fspca + return Coef(self, c_fspca) def evaluate_to_image_basis(self, c): """ @@ -346,6 +349,9 @@ def evaluate_to_image_basis(self, c): :return: The Image instance representing a stack of images in the standard 2D coordinate basis.. """ + if not isinstance(c, Coef): + raise TypeError(f"'c' should be `Coef` instance, received {type(c)}.") + c_fb = self.evaluate(c) return self.basis.evaluate(c_fb) @@ -358,6 +364,9 @@ def evaluate(self, c): :return: The (real) coefs representing a stack of images in self.basis """ + if not isinstance(c, Coef): + raise TypeError(f"'c' should be `Coef` instance, received {type(c)}.") + # apply FSPCA eigenvector to coefs c, yields coefs in self.basis eigvecs = self.eigvecs if isinstance(eigvecs, BlkDiagMatrix): @@ -368,7 +377,7 @@ def evaluate(self, c): # corrected_c[:, self.angular_indices!=0] *= 2 # return corrected_c @ eigvecs.T - return c @ eigvecs.T + return Coef(c.basis, c @ eigvecs.T) # TODO: Python>=3.8 @cached_property def _get_compressed_indices(self): @@ -435,7 +444,7 @@ def _compress(self): compressed_indices = self._get_compressed_indices() self.count = len(compressed_indices) - self.eigvals = self.eigvals[compressed_indices] + self._eigvals = self._eigvals[compressed_indices] if isinstance(self.eigvecs, BlkDiagMatrix): self.eigvecs = self.eigvecs.dense() self.eigvecs = self.eigvecs[:, compressed_indices] @@ -464,9 +473,9 @@ def to_complex(self, coef): :param coef: Coefficients from this basis. :return: Complex coefficent representation from this basis. """ - - if coef.ndim == 1: - coef = coef.reshape(1, -1) + if not isinstance(coef, Coef): + raise TypeError(f"'coef' should be `Coef` instance, received {type(coef)}.") + coef = coef.asnumpy() if coef.dtype not in (np.float64, np.float32): raise TypeError("coef provided to to_complex should be real.") @@ -557,6 +566,13 @@ def calculate_bispectrum( freq_cutoff=freq_cutoff, ) + @property + def eigvals(self): + """ + Return the eigenvals as a Coef instance of FSPCABasis. + """ + return Coef(self, self._eigvals) + def eigen_images(self): """ Return the eigen images of the FSPCA basis, evaluated to image space. @@ -570,7 +586,7 @@ def eigen_images(self): if isinstance(eigvecs, BlkDiagMatrix): eigvecs = eigvecs.dense() - return self.basis.evaluate(eigvecs.T) + return Coef(self.basis, eigvecs.T).evaluate() def shift(self, coef, shifts): """ From 37899b444261ea447e7b4907ff963b9dc8bda47c Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 15:40:48 -0400 Subject: [PATCH 049/294] stash initial pswf Coef conversion --- src/aspire/basis/fpswf_2d.py | 22 ++++---- src/aspire/basis/pswf_2d.py | 102 ++++++++++++++++++++++++++++------- 2 files changed, 97 insertions(+), 27 deletions(-) diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index f030d12e20..a0896ce602 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -97,7 +97,9 @@ def _precomp(self): n_max, ) = self._pswf_integration_sub_routine() - self.us_fft_pts = us_fft_pts + self.us_fft_pts = us_fft_pts.astype( + self.dtype, copy=False + ) # xxx find where this is incorrect self.blk_r = blk_r self.num_angular_pts = num_angular_pts self.r_quad_indices = r_quad_indices @@ -123,7 +125,7 @@ def _evaluate_t(self, images): # Accumulate coefficients coefficients = self._pswf_integration(nfft_res) - return coefficients + return self.to_real(coefficients).asnumpy() def _generate_pswf_quad( self, n, bandlimit, phi_approximate_error, lambda_max, epsilon @@ -214,7 +216,9 @@ def _generate_pswf_radial_quad( if k % 2 == 0: k = k + 1 - range_array = np.arange(approx_length).reshape((1, approx_length)) + range_array = np.arange(approx_length, dtype=self.dtype).reshape( + (1, approx_length) + ) idx_for_quad_nodes = int((k + 1) / 2) num_quad_pts = idx_for_quad_nodes - 1 @@ -277,14 +281,13 @@ def phi_for_quad_nodes(t): fun_vec = phi_for_quad_nodes(x) sign_flipping_vec = np.where(np.sign(fun_vec[:-1]) != np.sign(fun_vec[1:]))[0] - phi_zeros = np.zeros(idx_for_quad_nodes - 1) + phi_zeros = np.zeros(idx_for_quad_nodes - 1, dtype=self.dtype) tmp = phi_for_quad_nodes(x) for i, j in enumerate(sign_flipping_vec[: idx_for_quad_nodes - 1]): new_zero = x[j] - tmp[j] * (x[j + 1] - x[j]) / (tmp[j + 1] - tmp[j]) phi_zeros[i] = new_zero - phi_zeros = np.array(phi_zeros) return phi_zeros def _sum_minus_cumsum_smaller_eps(self, x, eps): @@ -299,17 +302,17 @@ def _pswf_integration_sub_routine(self): r_quad_indices.extend(num_angular_pts) r_quad_indices = np.cumsum(r_quad_indices, dtype="int") - n_max = int(max(self.ang_freqs) + 1) + n_max = int(max(self.complex_angular_indices) + 1) numel_for_n = np.zeros(n_max, dtype="int") for i in range(n_max): - numel_for_n[i] = np.count_nonzero(self.ang_freqs == i) + numel_for_n[i] = np.count_nonzero(self.complex_angular_indices == i) indices_for_n = [0] indices_for_n.extend(numel_for_n) indices_for_n = np.cumsum(indices_for_n, dtype="int") - blk_r = [0] * n_max + blk_r = [0] * n_max # xxx array? temp_const = self.bandlimit / (2 * np.pi * self.rcut) for i in range(n_max): blk_r[i] = ( @@ -352,7 +355,8 @@ def _pswf_integration(self, images_nufft): (len(self.radial_quad_pts) * self.n_max, num_images), order="F" ) coeff_vec_quad = np.zeros( - (num_images, len(self.ang_freqs)), dtype=complex_type(self.dtype) + (num_images, len(self.complex_angular_indices)), + dtype=complex_type(self.dtype), ) m = self.pswf_radial_quad.shape[1] for i in range(self.n_max): diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 6fda8661f5..405089e7bc 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -2,7 +2,7 @@ import numpy as np -from aspire.basis import Basis +from aspire.basis import Coef, SteerableBasis2D from aspire.basis.basis_utils import ( d_decay_approx_fun, k_operator, @@ -12,12 +12,13 @@ t_x_mat, ) from aspire.basis.pswf_utils import BNMatrix +from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type logger = logging.getLogger(__name__) -class PSWFBasis2D(Basis): +class PSWFBasis2D(SteerableBasis2D): """ Define a derived class for direct Prolate Spheroidal Wave Function (PSWF) expanding 2D images @@ -33,6 +34,8 @@ class PSWFBasis2D(Basis): Comput. Harmon. Anal. 22, 235-256 (2007). """ + matrix_type = BlkDiagMatrix + def __init__(self, size, gamma_trunc=1.0, beta=1.0, dtype=np.float32): """ Initialize an object for 2D PSWF basis expansion using direct method @@ -58,9 +61,6 @@ def __init__(self, size, gamma_trunc=1.0, beta=1.0, dtype=np.float32): self.beta = beta super().__init__(size, dtype=dtype) - # this basis has complex coefficients - self.coefficient_dtype = complex_type(self.dtype) - def _build(self): """ Build internal data structures for the direct 2D PSWF method @@ -105,12 +105,12 @@ def _precomp(self): """ self._generate_samples() - self.non_neg_freq_inds = slice(0, len(self.ang_freqs)) + self.non_neg_freq_inds = slice(0, len(self.complex_angular_indices)) - tmp = np.nonzero(self.ang_freqs == 0)[0] + tmp = np.nonzero(self.complex_angular_indices == 0)[0] self.zero_freq_inds = slice(tmp[0], tmp[-1] + 1) - tmp = np.nonzero(self.ang_freqs > 0)[0] + tmp = np.nonzero(self.complex_angular_indices > 0)[0] self.pos_freq_inds = slice(tmp[0], tmp[-1] + 1) def _generate_samples(self): @@ -138,18 +138,81 @@ def _generate_samples(self): alpha_all.extend(alpha[:n_end]) m += 1 - self.alpha_nn = np.array(alpha_all).reshape(-1, 1) + self.alpha_nn = np.array(alpha_all, dtype=complex_type(self.dtype)).reshape( + -1, 1 + ) self.max_ns = max_ns self.samples = self._evaluate_pswf2d_all(self._r_disk, self._theta_disk, max_ns) - self.ang_freqs = np.repeat(np.arange(len(max_ns)), max_ns).astype("float") - self.rad_freqs = np.concatenate([range(1, i + 1) for i in max_ns]).astype( - "float" + self.complex_angular_indices = np.repeat( + np.arange(len(max_ns), dtype=int), max_ns ) + self.complex_radial_indices = np.concatenate( + [np.arange(1, i + 1, dtype=int) for i in max_ns] + ) + + # Added to support subclassing SteerableBasis + self.complex_signs_indices = np.sign(self.complex_angular_indices) + self.samples = (self.beta / 2.0) * self.samples * self.alpha_nn self.samples_conj_transpose = self.samples.conj().transpose() # the column dimension of samples_conj_transpose is the number of basis coefficients - self.count = self.samples_conj_transpose.shape[1] + self.complex_count = self.samples_conj_transpose.shape[1] + + # hack + nz = np.sum(self.complex_signs_indices == 0) + nnz = self.complex_count - nz + + self.real_count = nz + 2 * nnz + self.count = self.real_count + + self.radial_indices = np.empty(self.real_count, dtype=int) + self.angular_indices = np.empty(self.real_count, dtype=int) + self.signs_indices = np.empty(self.real_count, dtype=int) + + # hackity hack + self._pos = np.zeros(self.complex_count, dtype=int) + self._neg = np.zeros(self.complex_count, dtype=int) + + i = 0 + ci = 0 + self.k_max = [] + self.ell_max = np.max(self.complex_angular_indices) + for ell in range(self.ell_max + 1): + sgns = (1,) if ell == 0 else (1, -1) + k_max = np.sum(self.complex_angular_indices == ell) + self.k_max.append(k_max) + ks = np.arange(0, k_max) + + for sgn in sgns: + rng = np.arange(i, i + len(ks)) + self.angular_indices[rng] = ell + self.radial_indices[rng] = ks + self.signs_indices[rng] = sgn + + # hackity hack + if sgn == 1: + self._pos[ci + ks] = rng + elif sgn == -1: + self._neg[ci + ks] = rng + # /hackity hack + + i += len(ks) + + ci += len(ks) + + # /hack + + # for tmp compat, probably can remove `indices` or clean it up later. + def indices(self): + """ + Return the precomputed indices for each basis function. + """ + return { + "ells": self.angular_indices, + "ks": self.radial_indices, + "sgns": self.signs_indices, + } def _evaluate_t(self, images): """ @@ -161,23 +224,26 @@ def _evaluate_t(self, images): """ flattened_images = images[:, self._disk_mask] - return flattened_images @ self.samples_conj_transpose + return self.to_real(flattened_images @ self.samples_conj_transpose).asnumpy() def _evaluate(self, coefficients): """ Evaluate coefficients in standard 2D coordinate basis from those in PSWF basis - :param coeffcients: A coefficient vector (or an array of coefficient + :param coefficients: A coefficient vector (or an array of coefficient vectors) in PSWF basis to be evaluated. (n_image, count) :return : Image in standard 2D coordinate basis. """ + # hack, convert to complex + coefficients = self.to_complex(Coef(self, coefficients)) + # Handle a single coefficient vector or stack of vectors. coefficients = np.atleast_2d(coefficients) n_images = coefficients.shape[0] - angular_is_zero = np.absolute(self.ang_freqs) == 0 + angular_is_zero = np.absolute(self.complex_angular_indices) == 0 flatten_images = coefficients[:, angular_is_zero] @ self.samples[ angular_is_zero @@ -259,7 +325,7 @@ def _evaluate_pswf2d_all(self, r, theta, max_ns): d_vec = self.d_vec_all[i] phase_part = np.exp(1j * i * theta) / np.sqrt(2 * np.pi) - range_array = np.arange(len(d_vec)) + range_array = np.arange(len(d_vec), dtype=self.dtype) r_radial_part_mat = t_radial_part_mat(r, i, range_array, len(d_vec)).dot( d_vec[:, :max_n] ) @@ -364,5 +430,5 @@ def _pswf_2d_minor_computations(self, big_n, n, bandlimit, phi_approximate_error d_vec, _ = BNMatrix(big_n, bandlimit, approx_length).get_eig_vectors() - range_array = np.array(range(approx_length)) + range_array = np.array(range(approx_length), dtype=self.dtype) return d_vec, approx_length, range_array From 6e59026f83eaf3183083b0cea016cc22bd8da139 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Sep 2023 15:44:58 -0400 Subject: [PATCH 050/294] stash initial steerable Coef conversion [skip ci] --- src/aspire/basis/steerable.py | 200 +++++++++++++++++++++++++++++++--- 1 file changed, 182 insertions(+), 18 deletions(-) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 2dca059a36..039a83d8c8 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -29,6 +29,14 @@ def __init__(self, *args, **kwargs): self._zero_angular_inds = self.angular_indices == 0 self._pos_angular_inds = (self.signs_indices == 1) & (self.angular_indices != 0) self._neg_angular_inds = self.signs_indices == -1 + self._non_neg_angular_inds = self.signs_indices >= 0 + self._blk_diag_cov_shape = None + + # Try to centralize inds between FB/FFB and FLE in SteerableBasis2D + self._indices = self.indices() + self.complex_count = self.count - sum(self._neg_angular_inds) + self.complex_angular_indices = self.angular_indices[self._non_neg_angular_inds] + self.complex_radial_indices = self.radial_indices[self._non_neg_angular_inds] # Attribute for caching the blk_diag shape once known. self._blk_diag_cov_shape = None @@ -158,17 +166,25 @@ def rotate(self, coef, radians, refl=None): :param refl: Optional reflect image (about y=0) (bool) :return: rotated coefs. """ + from .coef import Coef - # Enforce a stack axis to support sanity checks - coef = np.atleast_2d(coef) + if not isinstance(coef, Coef): + raise TypeError(f"`coef` must be `Coef` instance, received {type(coef)}.") + + coef = coef.asnumpy() # Covert radians to a broadcastable shape if isinstance(radians, Iterable): - radians = np.fromiter(radians, dtype=self.dtype).reshape(-1, 1) - if len(radians) != len(coef): + radians = np.array(radians, dtype=self.dtype) + if radians.ndim < 2: + radians = radians.reshape(-1, 1) + else: + radians = np.atleast_3d(radians) + + if radians.size != np.prod(coef.shape[:-1]): raise RuntimeError( - "`rotate` call `radians` length cannot broadcast with" - f" `coef` {len(coef)} != {len(radians)}" + f"`rotate` call `radians` {radians.shape} does not match" + f" `coef` {coef.shape[:-1]}." ) # else: radians can be a constant @@ -180,17 +196,17 @@ def rotate(self, coef, radians, refl=None): # For all coef in stack, # compute the ks * radian used in the trig functions ks_rad = np.atleast_2d(self.angular_indices * radians) - ks_pos = ks_rad[:, self._pos_angular_inds] - ks_neg = ks_rad[:, self._neg_angular_inds] + ks_pos = ks_rad[..., self._pos_angular_inds] + ks_neg = ks_rad[..., self._neg_angular_inds] # Slice the coef on postive and negative ells - coef_zer = coef[:, self._zero_angular_inds] - coef_pos = coef[:, self._pos_angular_inds] - coef_neg = coef[:, self._neg_angular_inds] + coef_zer = coef[..., self._zero_angular_inds] + coef_pos = coef[..., self._pos_angular_inds] + coef_neg = coef[..., self._neg_angular_inds] # Handle zero case and avoid mutating the original array coef = np.empty_like(coef) - coef[:, self._zero_angular_inds] = coef_zer + coef[..., self._zero_angular_inds] = coef_zer # refl if refl is not None: @@ -201,14 +217,14 @@ def rotate(self, coef, radians, refl=None): coef_neg[refl] = coef_neg[refl] * -1 # Apply formula - coef[:, self._pos_angular_inds] = coef_pos * np.cos(ks_pos) + coef_neg * np.sin( - ks_neg - ) - coef[:, self._neg_angular_inds] = coef_neg * np.cos(ks_neg) - coef_pos * np.sin( + coef[..., self._pos_angular_inds] = coef_pos * np.cos( ks_pos - ) + ) + coef_neg * np.sin(ks_neg) + coef[..., self._neg_angular_inds] = coef_neg * np.cos( + ks_neg + ) - coef_pos * np.sin(ks_pos) - return coef + return Coef(self, coef) def complex_rotate(self, complex_coef, radians, refl=None): """ @@ -319,3 +335,151 @@ def blk_diag_cov_shape(self): # Return the cached shape return self._blk_diag_cov_shape + + def indices_mask(self, **kwargs): + """ + Given `radial=` or `angular=` expressions, return (`count`,) + shaped mask where values satisfying the expression are `True`. + + Examples: + No args yield all indices. + `angular>=0` selects coefficients with non negative angular indices. + `angular=1, radial=2` selects coefficients satisfying angular index of 1 _and_ radial index of 2. + + + :return: Boolen mask of shape (`count`,). + Intended to be broadcast with `Coef` containers. + """ + + radial = kwargs.get("radial", None) + angular = kwargs.get("angular", None) + signs = kwargs.get("signs", None) + + # slowly construct the map + signs_mask = np.zeros(self.count, dtype=bool) + radial_mask = signs_mask.copy() + angular_mask = signs_mask.copy() + + if radial is None: + radial_mask[:] = True + else: + for k in np.atleast_1d(radial): + radial_mask[self.radial_indices == k] = True + + if angular is None: + angular_mask[:] = True + else: + for el in np.atleast_1d(angular): + angular_mask[self.angular_indices == el] = True + + if signs is None: + signs_mask[:] = True + else: + for s in np.atleast_1d(signs): + signs_mask[self.signs_indices == s] = True + + mask = radial_mask & angular_mask & signs_mask + + return mask + + def to_real(self, complex_coef): + """ + Return real valued representation of complex coefficients. + This can be useful when comparing or implementing methods + from literature. + + There is a corresponding method, to_complex. + + :param complex_coef: Complex coefficients from this basis. + :return: Real coefficent representation from this basis. + """ + from .coef import Coef + + if complex_coef.ndim == 1: + complex_coef = complex_coef.reshape(1, -1) + + if complex_coef.dtype not in (np.complex128, np.complex64): + raise TypeError("coef provided to to_real should be complex.") + + # Pass through dtype precisions, but check and warn if mismatched. + dtype = real_type(complex_coef.dtype) + if dtype != self.dtype: + logger.warning( + f"Complex coef dtype {complex_coef.dtype} does not match precision of basis.dtype {self.dtype}, returning {dtype}." + ) + + coef = np.zeros((complex_coef.shape[0], self.count), dtype=dtype) + + ind = 0 + idx = np.arange(self.k_max[0], dtype=int) + ind += np.size(idx) + ind_pos = ind + + coef[:, idx] = complex_coef[:, idx].real + + for ell in range(1, self.ell_max + 1): + idx = ind + np.arange(self.k_max[ell], dtype=int) + idx_pos = ind_pos + np.arange(self.k_max[ell], dtype=int) + idx_neg = idx_pos + self.k_max[ell] + + c = complex_coef[:, idx] + coef[:, idx_pos] = 2.0 * np.real(c) + coef[:, idx_neg] = -2.0 * np.imag(c) + + ind += np.size(idx) + ind_pos += 2 * self.k_max[ell] + + return Coef(self, coef) + + def to_complex(self, coef): + """ + Return complex valued representation of coefficients. + This can be useful when comparing or implementing methods + from literature. + + There is a corresponding method, to_real. + + :param coef: Coefficients from this basis. + :return: Complex coefficent representation from this basis. + """ + from .coef import Coef + + if not isinstance(coef, Coef): + raise TypeError( + f"coef should be instanace of `Coef`, received {type(coef)}." + ) + + # if coef.ndim == 1: + # coef = coef.reshape(1, -1) + + if coef.dtype not in (np.float64, np.float32): + raise TypeError("coef provided to to_complex should be real.") + + # Pass through dtype precions, but check and warn if mismatched. + dtype = complex_type(coef.dtype) + if coef.dtype != self.dtype: + logger.warning( + f"coef dtype {coef.dtype} does not match precision of basis.dtype {self.dtype}, returning {dtype}." + ) + + # Return the same precision as coef + imaginary = dtype(1j) + + ccoef = np.zeros((*coef.stack_shape, self.complex_count), dtype=dtype) + coef = coef.asnumpy() + + ind = 0 + idx = np.arange(self.k_max[0], dtype=int) + ind += np.size(idx) + + ccoef[..., idx] = coef[..., idx] + + for ell in range(1, self.ell_max + 1): + idx = ind + np.arange(self.k_max[ell], dtype=int) + ccoef[..., idx] = ( + coef[..., self._pos[idx]] - imaginary * coef[..., self._neg[idx]] + ) / 2.0 + + ind += np.size(idx) + + return ccoef From a608a5baf988b96a1187c282acaea00c0fab3148 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 11:03:38 -0400 Subject: [PATCH 051/294] initial migration classification towards Coef --- src/aspire/classification/averager2d.py | 13 +++++++------ src/aspire/classification/rir_class2d.py | 19 ++++++++++++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/aspire/classification/averager2d.py b/src/aspire/classification/averager2d.py index 4700a7a230..70ade2c6b7 100644 --- a/src/aspire/classification/averager2d.py +++ b/src/aspire/classification/averager2d.py @@ -11,6 +11,7 @@ from ray.util.multiprocessing import Pool from aspire import config +from aspire.basis import COef from aspire.classification.reddy_chatterji import reddy_chatterji_register from aspire.image import Image, ImageStacker, MeanImageStacker from aspire.utils import trange @@ -230,7 +231,7 @@ def _innerloop(i): ) # Averaging in composite_basis - return self.image_stacker(neighbors_coefs) + return self.image_stacker(neighbors_coefs.asnumpy()) if self.num_procs <= 1: for i in trange(n_classes): @@ -253,7 +254,7 @@ def _innerloop(i): b_avgs[i] = result # Now we convert the averaged images from Basis to Cartesian. - return self.composite_basis.evaluate(b_avgs) + return Coef(self.composite_basis, b_avgs).evaluate() def _shift_search_grid(self, L, radius, roll_zero=False): """ @@ -362,12 +363,12 @@ def _innerloop(k): ) # then store dot between class base image (0) and each nbor - for j, nbor in enumerate(rotated_nbrs): + for j, nbor in enumerate(rotated_nbrs.asnumpy()): # Skip the base image. if j == 0: continue norm_nbor = np.linalg.norm(nbor) - _correlations[j, i] = np.dot(nbr_coef[0], nbor) / ( + _correlations[j, i] = np.dot(nbr_coef.asnumpy()[0], nbor) / ( norm_nbor * norm_0 ) @@ -681,7 +682,7 @@ def _innerloop(i): ) # Averaging in composite_basis - return self.image_stacker(neighbors_coefs) + return self.image_stacker(neighbors_coefs.asnumpy()) if self.num_procs <= 1: for i in trange(n_classes): @@ -704,7 +705,7 @@ def _innerloop(i): b_avgs[i] = result # Now we convert the averaged images from Basis to Cartesian. - return self.composite_basis.evaluate(b_avgs) + return Coef(self.composite_basis, b_avgs).evaluate() class BFSReddyChatterjiAverager2D(ReddyChatterjiAverager2D): diff --git a/src/aspire/classification/rir_class2d.py b/src/aspire/classification/rir_class2d.py index a20ce3c0d2..9e3915b0ca 100644 --- a/src/aspire/classification/rir_class2d.py +++ b/src/aspire/classification/rir_class2d.py @@ -4,7 +4,7 @@ import numpy as np from sklearn.neighbors import NearestNeighbors -from aspire.basis import FSPCABasis +from aspire.basis import Coef, FSPCABasis from aspire.classification import Class2D from aspire.classification.legacy_implementations import bispec_2drot_large, pca_y from aspire.numeric import ComplexPCA @@ -181,7 +181,7 @@ def classify(self, diagnostics=False): self.fspca_coef = self.pca_basis.spca_coef # Compute Bispectrum - coef_b, coef_b_r = self.bispectrum(self.fspca_coef) + coef_b, coef_b_r = self.bispectrum(Coef(self.pca_basis, self.fspca_coef)) # # Stage 2: Compute Nearest Neighbors logger.info(f"Calculate Nearest Neighbors using {self._nn_implementation}.") @@ -251,6 +251,12 @@ def bispectrum(self, coef): :param coef: complex steerable coefficients (eg. from FSPCABasis). :returns: tuple of arrays (coef_b, coef_b_r) """ + + if not isinstance(coef, Coef): + raise TypeError( + f"`coef` should be a `Coef` instance, received {type(coef)}" + ) + # _bispectrum is assigned during initialization. return self._bispectrum(coef) @@ -405,7 +411,9 @@ def _devel_bispectrum(self, coef): coef = self.pca_basis.to_complex(coef) # Take just positive frequencies, corresponds to complex indices. # Original implementation used norm of Complex values, here abs of Real. - eigvals = np.abs(self.pca_basis.eigvals[self.pca_basis.signs_indices >= 0]) + eigvals = np.abs( + self.pca_basis.eigvals.asnumpy()[0, self.pca_basis.signs_indices >= 0] + ) # Legacy code included a sanity check: # non_zero_freqs = self.pca_basis.complex_angular_indices != 0 @@ -480,6 +488,11 @@ def _legacy_bispectrum(self, coef, retry_attempts=3): :return: Compressed feature and reflected feature vectors. """ + if not isinstance(coef, Coef): + raise TypeError( + f"`coef` should be a `Coef` instance, received {type(coef)}" + ) + # The legacy code expects the complex representation coef = self.pca_basis.to_complex(coef) complex_eigvals = self.pca_basis.to_complex(self.pca_basis.eigvals).reshape( From e49c145c5f3d5fa01dfbf8b145d37e57d7157de2 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 11:05:51 -0400 Subject: [PATCH 052/294] initial migration ctf towards Coef --- src/aspire/ctf/ctf_estimator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aspire/ctf/ctf_estimator.py b/src/aspire/ctf/ctf_estimator.py index 7f9fab3571..ca2c3fed5d 100644 --- a/src/aspire/ctf/ctf_estimator.py +++ b/src/aspire/ctf/ctf_estimator.py @@ -15,7 +15,7 @@ from scipy.optimize import linprog from scipy.signal.windows import dpss -from aspire.basis.ffb_2d import FFBBasis2D +from aspire.basis import Coef, FFBBasis2D from aspire.image import Image from aspire.numeric import fft from aspire.storage import StarFile @@ -266,7 +266,7 @@ def elliptical_average(self, ffbbasis, amplitude_spectrum, circular): """ # RCOPT, come back and change the indices for this method - coeffs_s = ffbbasis.evaluate_t(amplitude_spectrum).T + coeffs_s = ffbbasis.evaluate_t(amplitude_spectrum).asnumpy().copy().T coeffs_n = coeffs_s.copy() coeffs_s[np.argwhere(ffbbasis._indices["ells"] == 1)] = 0 @@ -276,9 +276,9 @@ def elliptical_average(self, ffbbasis, amplitude_spectrum, circular): else: coeffs_n[np.argwhere(ffbbasis._indices["ells"] == 0)] = 0 coeffs_n[np.argwhere(ffbbasis._indices["ells"] == 2)] = 0 - noise = ffbbasis.evaluate(coeffs_n.T) + noise = Coef(ffbbasis, coeffs_n.T).evaluate() - psd = ffbbasis.evaluate(coeffs_s.T) + psd = Coef(ffbbasis, coeffs_s.T).evaluate() return psd, noise From 127fc77971c14eee33092cd48e05fec9a09b2429 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 11:13:12 -0400 Subject: [PATCH 053/294] initial migration reconstruction towards Coef --- src/aspire/reconstruction/estimator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/aspire/reconstruction/estimator.py b/src/aspire/reconstruction/estimator.py index a5cc73e917..5c8bbf1a0a 100644 --- a/src/aspire/reconstruction/estimator.py +++ b/src/aspire/reconstruction/estimator.py @@ -1,5 +1,6 @@ import logging +from aspire.basis import Coef from aspire.reconstruction.kernel import FourierKernel logger = logging.getLogger(__name__) @@ -63,7 +64,7 @@ def estimate(self, b_coeff=None, tol=1e-5, regularizer=0): if b_coeff is None: b_coeff = self.src_backward() est_coeff = self.conj_grad(b_coeff, tol=tol, regularizer=regularizer) - est = self.basis.evaluate(est_coeff).T + est = Coef(self.basis, est_coeff).evaluate().T return est @@ -76,9 +77,11 @@ def apply_kernel(self, vol_coeff, kernel=None): :return: The result of evaluating `vol_coeff` in the given basis, convolving with the kernel given by kernel, and backprojecting into the basis. """ + if kernel is None: kernel = self.kernel + vol = self.basis.evaluate(vol_coeff) # returns a Volume vol = kernel.convolve_volume(vol) # returns a Volume - vol_coef = self.basis.evaluate_t(vol) + vol_coef = Coef(self.basis, vol_coeff).evaluate() return vol_coef From ee7410fb95003d03e148f7a7f27263ff98f330af Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 11:30:00 -0400 Subject: [PATCH 054/294] migration some tests towards using Ceof [skip ci] --- tests/_basis_util.py | 30 +++++++++++++++------------- tests/test_FBbasis2D.py | 16 +++++++-------- tests/test_FBbasis3D.py | 4 ++-- tests/test_FFBbasis2D.py | 4 ++-- tests/test_FFBbasis3D.py | 4 ++-- tests/test_FLEbasis2D.py | 42 ++++++++-------------------------------- 6 files changed, 38 insertions(+), 62 deletions(-) diff --git a/tests/_basis_util.py b/tests/_basis_util.py index fc4cea74a2..5cc0bb5237 100644 --- a/tests/_basis_util.py +++ b/tests/_basis_util.py @@ -1,6 +1,7 @@ import numpy as np import pytest +from aspire.basis import Coef from aspire.image import Image from aspire.utils import gaussian_2d, utest_tolerance from aspire.utils.coor_trans import grid_2d @@ -96,12 +97,12 @@ def testIsotropic(self, basis): sigma = L / 8 im = gaussian_2d(L, sigma=sigma, dtype=basis.dtype) - coef = basis.expand(im) + coef_np = basis.expand(im).asnumpy() ells = basis.indices()["ells"] - energy_outside = np.sum(np.abs(coef[ells != 0]) ** 2) - energy_total = np.sum(np.abs(coef) ** 2) + energy_outside = np.sum(np.abs(coef_np[..., ells != 0]) ** 2) + energy_total = np.sum(np.abs(coef_np) ** 2) energy_ratio = energy_outside / energy_total @@ -122,12 +123,12 @@ def testModulated(self, basis): for trig_fun in (np.sin, np.cos): im1 = im * trig_fun(ell * g2d["phi"]) - coef = basis.expand(im1) + coef_np = basis.expand(im1).asnumpy() ells = basis.indices()["ells"] - energy_outside = np.sum(np.abs(coef[ells != ell]) ** 2) - energy_total = np.sum(np.abs(coef) ** 2) + energy_outside = np.sum(np.abs(coef_np[..., ells != ell]) ** 2) + energy_total = np.sum(np.abs(coef_np) ** 2) energy_ratio = energy_outside / energy_total @@ -135,19 +136,21 @@ def testModulated(self, basis): def testEvaluateExpand(self, basis): coef1 = randn(basis.count, seed=self.seed) - coef1 = coef1.astype(basis.dtype) + coef1 = Coef(basis, coef1.astype(basis.dtype)) im = basis.evaluate(coef1) if isinstance(im, Image): im = im.asnumpy() - coef2 = basis.expand(im)[0] + coef2 = basis.expand(im) - assert coef1.shape == coef2.shape + assert ( + coef1.shape == coef2.shape + ), f"shape mismatch {coef1.shape} != {coef2.shape}" assert np.allclose(coef1, coef2, atol=utest_tolerance(basis.dtype)) def testAdjoint(self, basis): u = randn(basis.count, seed=self.seed) - u = u.astype(basis.dtype) + u = Coef(basis, u, dtype=basis.dtype) Au = basis.evaluate(u) if isinstance(Au, Image): @@ -180,7 +183,8 @@ def testEvaluate(self, basis): # evaluate should take a NumPy array of type basis.coefficient_dtype # and return an Image/Volume _class = self.getClass(basis) - result = basis.evaluate(np.zeros((basis.count), dtype=basis.coefficient_dtype)) + coef = Coef(basis, np.zeros((basis.count)), dtype=basis.coefficient_dtype) + result = basis.evaluate(coef) assert isinstance(result, _class) def testEvaluate_t(self, basis): @@ -190,7 +194,7 @@ def testEvaluate_t(self, basis): result = basis.evaluate_t( _class(np.zeros((basis.nres,) * basis.ndim, dtype=basis.dtype)) ) - assert isinstance(result, np.ndarray) + assert isinstance(result, Coef) assert result.dtype == basis.coefficient_dtype def testExpand(self, basis): @@ -200,7 +204,7 @@ def testExpand(self, basis): result = basis.expand( _class(np.zeros((basis.nres,) * basis.ndim, dtype=basis.dtype)) ) - assert isinstance(result, np.ndarray) + assert isinstance(result, Coef) assert result.dtype == basis.coefficient_dtype def testInitWithIntSize(self, basis): diff --git a/tests/test_FBbasis2D.py b/tests/test_FBbasis2D.py index c5e9c555ce..dc74d395b9 100644 --- a/tests/test_FBbasis2D.py +++ b/tests/test_FBbasis2D.py @@ -5,7 +5,7 @@ from pytest import raises from scipy.special import jv -from aspire.basis import FBBasis2D +from aspire.basis import Coef, FBBasis2D from aspire.image import Image from aspire.source import Simulation from aspire.utils import complex_type, real_type @@ -56,7 +56,7 @@ def _testElement(self, basis, ell, k, sgn): coef_ref = np.zeros(basis.count, dtype=basis.dtype) coef_ref[(ells == ell) & (sgns == sgn) & (ks == k)] = 1 - im_ref = basis.evaluate(coef_ref) + im_ref = basis.evaluate(Coef(basis, coef_ref)) coef = basis.expand(im) @@ -87,15 +87,16 @@ def testComplexCoversion(self, basis): assert np.allclose(v1, v2) def testComplexCoversionErrorsToComplex(self, basis): - x = randn(*basis.sz, seed=self.seed) + x = randn(*basis.sz, seed=self.seed).astype(basis.dtype) # Express in an FB basis - v1 = basis.expand(x.astype(basis.dtype)) + v1 = basis.expand(x) # Test catching Errors with raises(TypeError): # Pass complex into `to_complex` - _ = basis.to_complex(v1.astype(np.complex64)) + v1_cpx = Coef(basis, v1, dtype=np.complex64) + _ = basis.to_complex(v1_cpx) # Test casting case, where basis and coef don't match if basis.dtype == np.float32: @@ -105,12 +106,9 @@ def testComplexCoversionErrorsToComplex(self, basis): # Result should be same precision as coef input, just complex. result_dtype = complex_type(test_dtype) - v3 = basis.to_complex(v1.astype(test_dtype)) + v3 = basis.to_complex(Coef(basis, v1, dtype=test_dtype)) assert v3.dtype == result_dtype - # Try 0d vector, should not crash. - _ = basis.to_complex(v1.reshape(-1)) - def testComplexCoversionErrorsToReal(self, basis): x = randn(*basis.sz, seed=self.seed) diff --git a/tests/test_FBbasis3D.py b/tests/test_FBbasis3D.py index 02ba88109a..c32175d9c8 100644 --- a/tests/test_FBbasis3D.py +++ b/tests/test_FBbasis3D.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from aspire.basis import FBBasis3D +from aspire.basis import Coef, FBBasis3D from aspire.utils import grid_3d, utest_tolerance from aspire.volume import AsymmetricVolume, Volume @@ -458,7 +458,7 @@ def testFBBasis3DEvaluate(self, basis): ], dtype=basis.dtype, ) - result = basis.evaluate(coeffs) + result = Coef(basis, coeffs).evaluate() assert np.allclose( result.asnumpy(), diff --git a/tests/test_FFBbasis2D.py b/tests/test_FFBbasis2D.py index 1796bbf0e7..cd9301c5db 100644 --- a/tests/test_FFBbasis2D.py +++ b/tests/test_FFBbasis2D.py @@ -5,7 +5,7 @@ import pytest from scipy.special import jv -from aspire.basis import FFBBasis2D +from aspire.basis import Coef, FFBBasis2D from aspire.image import Image from aspire.source import Simulation from aspire.utils.misc import grid_2d @@ -64,7 +64,7 @@ def _testElement(self, basis, ell, k, sgn): coef_ref = np.zeros(basis.count, dtype=basis.dtype) coef_ref[(ells == ell) & (sgns == sgn) & (ks == k)] = 1 - im_ref = basis.evaluate(coef_ref).asnumpy()[0] + im_ref = basis.evaluate(Coef(basis, coef_ref)).asnumpy()[0] coef = basis.expand(im) diff --git a/tests/test_FFBbasis3D.py b/tests/test_FFBbasis3D.py index c8879ab2a8..c9259dc05b 100644 --- a/tests/test_FFBbasis3D.py +++ b/tests/test_FFBbasis3D.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from aspire.basis import FFBBasis3D +from aspire.basis import Coef, FFBBasis3D from aspire.utils import grid_3d from aspire.volume import AsymmetricVolume, Volume @@ -460,7 +460,7 @@ def testFFBBasis3DEvaluate(self, basis): dtype=basis.dtype, ) - result = basis.evaluate(coeffs) + result = Coef(basis, coeffs).evaluate() ref = np.load( os.path.join(DATA_DIR, "ffbbasis3d_xcoeff_out_8_8_8.npy") diff --git a/tests/test_FLEbasis2D.py b/tests/test_FLEbasis2D.py index 6af33580cb..b56c7aaa7c 100644 --- a/tests/test_FLEbasis2D.py +++ b/tests/test_FLEbasis2D.py @@ -4,7 +4,7 @@ import numpy as np import pytest -from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D +from aspire.basis import Coef, FBBasis2D, FFBBasis2D, FLEBasis2D from aspire.image import Image from aspire.nufft import backend_available from aspire.numeric import fft @@ -103,7 +103,7 @@ def testFastVDense(self, basis): basis.nres, epsilon=1e-4, dtype=np.float64, match_fb=False ).evaluate_t(x) - result_dense = dense_b @ coeffs.T + result_dense = dense_b @ coeffs.asnumpy().T result_fast = basis.evaluate(coeffs).asnumpy() assert relerr(result_dense, result_fast) < (self.test_eps * basis.epsilon) @@ -136,7 +136,7 @@ def testMatchFBEvaluate(basis): fb_basis = FBBasis2D(basis.nres, dtype=np.float64) # in match_fb, count is the same for both bases - coeffs = np.eye(basis.count) + coeffs = Coef(basis, np.eye(basis.count)) fb_images = fb_basis.evaluate(coeffs) fle_images = basis.evaluate(coeffs) @@ -151,7 +151,7 @@ def testMatchFBDenseEvaluate(basis): fb_basis = FBBasis2D(basis.nres, dtype=np.float64) - coeffs = np.eye(basis.count) + coeffs = Coef(basis, np.eye(basis.count)) fb_images = fb_basis.evaluate(coeffs).asnumpy() fle_out = basis._create_dense_matrix() @ coeffs @@ -171,7 +171,7 @@ def testMatchFBEvaluate_t(basis): fb_basis = FBBasis2D(basis.nres, dtype=np.float64) # test images to evaluate - images = fb_basis.evaluate(np.eye(basis.count)) + images = fb_basis.evaluate(Coef(basis, np.eye(basis.count))) fb_coeffs = fb_basis.evaluate_t(images) fle_coeffs = basis.evaluate_t(images) @@ -188,7 +188,7 @@ def testMatchFBDenseEvaluate_t(basis): # test images to evaluate # gets a stack of shape (basis.count, L, L) - images = fb_basis.evaluate(np.eye(basis.count)) + images = fb_basis.evaluate(Coef(basis, np.eye(basis.count))) # reshape to a stack of basis.count vectors of length L**2 vec = images.asnumpy().reshape((-1, basis.nres**2)) @@ -213,7 +213,7 @@ def testLowPass(): nonzero_coeffs = [] for i in range(4): bandlimit = L // (2**i) - coeffs_lowpassed = basis.lowpass(coeffs, bandlimit) + coeffs_lowpassed = basis.lowpass(coeffs, bandlimit).asnumpy() nonzero_coeffs.append(np.sum(coeffs_lowpassed != 0)) # for bandlimit == L, no frequencies should be removed @@ -225,19 +225,6 @@ def testLowPass(): # make sure you can pass in a 1-D array if you want _ = basis.lowpass(coeffs[0, :], L) - # cannot pass in the wrong number of coefficients - with pytest.raises( - AssertionError, match="Number of coefficients must match self.count." - ): - _ = basis.lowpass(coeffs[:, :1000], L) - - # cannot pass in wrong shape - with pytest.raises( - AssertionError, - match="Input a stack of coefficients of dimension", - ): - _ = basis.lowpass(np.zeros((3, 3, 3)), L) - def testRotate(): # test ability to accurately rotate images via @@ -280,20 +267,7 @@ def testRotate(): assert np.allclose(np.flipud(ims.asnumpy()[0]), ims_fle_pi[0], atol=1e-4) # make sure you can pass in a 1-D array if you want - _ = basis.lowpass(np.zeros((basis.count,)), np.pi) - - # cannot pass in the wrong number of coefficients - with pytest.raises( - AssertionError, match="Number of coefficients must match self.count." - ): - _ = basis.rotate(np.zeros((1, 10)), np.pi) - - # cannot pass in wrong shape - with pytest.raises( - AssertionError, - match="Input a stack of coefficients of dimension", - ): - _ = basis.lowpass(np.zeros((3, 3, 3)), np.pi) + _ = basis.lowpass(Coef(basis, np.zeros((basis.count,))), np.pi) def testRotate45(): From cc797821f891ba20d150e051559138f49b1932d2 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 11:35:09 -0400 Subject: [PATCH 055/294] cleanup typos --- src/aspire/basis/basis.py | 1 + src/aspire/basis/fb_2d.py | 1 - src/aspire/basis/steerable.py | 1 - src/aspire/classification/averager2d.py | 2 +- 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index d05749c713..50ab6e1227 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -102,6 +102,7 @@ def evaluate(self, v): raise TypeError(f"`evaluate` should be passed a `Coef`, received {type(v)}") # Flatten stack, ndim is wrt Basis (2 or 3) + stack_shape = v.stack_shape v = v.reshape(-1, self.count) v = v.stack_reshape(-1).asnumpy() diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 8deaa8f593..84c57e3dba 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -5,7 +5,6 @@ from aspire.basis import FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import unique_coords_nd -from aspire.operators import BlkDiagMatrix from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 039a83d8c8..85a012c3b3 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -1,4 +1,3 @@ -import abc import logging from collections.abc import Iterable diff --git a/src/aspire/classification/averager2d.py b/src/aspire/classification/averager2d.py index 70ade2c6b7..3c487070ab 100644 --- a/src/aspire/classification/averager2d.py +++ b/src/aspire/classification/averager2d.py @@ -11,7 +11,7 @@ from ray.util.multiprocessing import Pool from aspire import config -from aspire.basis import COef +from aspire.basis import Coef from aspire.classification.reddy_chatterji import reddy_chatterji_register from aspire.image import Image, ImageStacker, MeanImageStacker from aspire.utils import trange From 99fc3feb0872e39d07789e8341b15be2e4f0c7ff Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 13:50:30 -0400 Subject: [PATCH 056/294] migrate PSWF FPSWF tests towards Coef [skip ci] --- tests/test_FPSWFbasis2D.py | 21 ++++++++++++++------- tests/test_PSWFbasis2D.py | 18 +++++++++++------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/tests/test_FPSWFbasis2D.py b/tests/test_FPSWFbasis2D.py index 2ca6e3e213..02602ecd2f 100644 --- a/tests/test_FPSWFbasis2D.py +++ b/tests/test_FPSWFbasis2D.py @@ -20,20 +20,27 @@ def testFPSWFBasis2DEvaluate_t(self, basis): os.path.join(DATA_DIR, "ffbbasis2d_xcoeff_in_8_8.npy") ).T # RCOPT images = Image(img_ary) + result = basis.evaluate_t(images) - coeffs = np.load( + # Historically, FPSWF returned complex values. + # Load and convert them for this hard coded test. + ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT + coeffs = basis.to_real(ccoeffs) - # make sure both real and imaginary parts are consistent. - assert np.allclose(np.real(result), np.real(coeffs)) and np.allclose( - np.imag(result) * 1j, np.imag(coeffs) * 1j - ) + np.testing.assert_allclose(result, coeffs, atol=utest_tolerance(basis.dtype)) def testFPSWFBasis2DEvaluate(self, basis): - coeffs = np.load( + # Historically, FPSWF returned complex values. + # Load and convert them for this hard coded test. + ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT + coeffs = basis.to_real(ccoeffs) + result = coeffs.evaluate() + result = basis.evaluate(coeffs) images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoeff_out_8_8.npy")).T # RCOPT - assert np.allclose(result.asnumpy(), images) + + np.testing.assert_allclose(result, images) diff --git a/tests/test_PSWFbasis2D.py b/tests/test_PSWFbasis2D.py index 8908d3173b..cf34ebdf71 100644 --- a/tests/test_PSWFbasis2D.py +++ b/tests/test_PSWFbasis2D.py @@ -23,19 +23,23 @@ def testPSWFBasis2DEvaluate_t(self, basis): result = basis.evaluate_t(images) - coeffs = np.load( + # Historically, PSWF returned complex values. + # Load and convert them for this hard coded test. + ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT + coeffs = basis.to_real(ccoeffs) - # make sure both real and imaginary parts are consistent. - assert np.allclose(np.real(result), np.real(coeffs)) and np.allclose( - np.imag(result) * 1j, np.imag(coeffs) * 1j - ) + np.testing.assert_allclose(result, coeffs) def testPSWFBasis2DEvaluate(self, basis): - coeffs = np.load( + # Historically, PSWF returned complex values. + # Load and convert them for this hard coded test. + ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT - result = basis.evaluate(coeffs) + coeffs = basis.to_real(ccoeffs) + + result = coeffs.evaluate() images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoeff_out_8_8.npy")).T # RCOPT assert np.allclose(result.asnumpy(), images) From da080b19e5c534e6f478ca4b07a639c5a924d37d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 14 Sep 2023 09:03:07 -0400 Subject: [PATCH 057/294] class2D updates --- src/aspire/basis/steerable.py | 5 +++++ tests/test_class2D.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 85a012c3b3..d54b2600a4 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -394,6 +394,11 @@ def to_real(self, complex_coef): """ from .coef import Coef + if not isinstance(coef, Coef): + raise TypeError( + f"coef should be instanace of `Coef`, received {type(coef)}." + ) + if complex_coef.ndim == 1: complex_coef = complex_coef.reshape(1, -1) diff --git a/tests/test_class2D.py b/tests/test_class2D.py index 40074c3c80..f683394bbf 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -101,12 +101,16 @@ def test_complex_conversions_errors(sim_fixture): with pytest.raises(TypeError, match="coef provided to to_complex should be real."): _ = fspca_basis.to_complex( - np.arange(fspca_basis.count), + Coef( + fspca_basis, + np.arange(fspca_basis.count), + dtype=np.complex64, + ) ) with pytest.raises(TypeError, match="coef provided to to_real should be complex."): _ = fspca_basis.to_real( - np.arange(fspca_basis.count, dtype=np.float32).flatten() + Coef(fspca_basis, np.arange(fspca_basis.count), dtype=np.float32) ) From 4402a8f69840e071dd3860626cc658027eb25101 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 14 Sep 2023 09:25:54 -0400 Subject: [PATCH 058/294] initial cov2d Coef updates --- src/aspire/covariance/covar2d.py | 37 ++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index c6d380afa4..934f7ef0f4 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -4,7 +4,7 @@ from numpy.linalg import eig, inv from scipy.linalg import solve, sqrtm -from aspire.basis import FFBBasis2D +from aspire.basis import Coef, FFBBasis2D from aspire.operators import BlkDiagMatrix, DiagMatrix from aspire.optimization import conj_grad, fill_struct from aspire.utils import make_symmat @@ -197,6 +197,13 @@ def get_mean(self, coeffs, ctf_basis=None, ctf_idx=None): :return: The mean value vector for all images. """ + if not isinstance(coeffs, Coef): + raise TypeError( + f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + ) + + coeffs = coeffs.asnumpy() + # TODO: Redundant, remove? if coeffs.size == 0: raise RuntimeError("The coefficients need to be calculated!") @@ -220,7 +227,7 @@ def get_mean(self, coeffs, ctf_basis=None, ctf_idx=None): A += weight * (ctf_basis_k_t @ ctf_basis_k) mean_coeff = A.solve(b) - return mean_coeff + return Coef(self.basis, mean_coeff) def get_covar( self, @@ -254,6 +261,12 @@ def get_covar( are accounted for and inverted to yield a covariance estimate of the unfiltered images. """ + if not isinstance(coeffs, Coef): + raise TypeError( + f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + ) + coeffs = coeffs.asnumpy() + if coeffs.size == 0: raise RuntimeError("The coefficients need to be calculated!") @@ -278,7 +291,7 @@ def identity(x): covar_est_opt = fill_struct(covar_est_opt, default_est_opt) if mean_coeff is None: - mean_coeff = self.get_mean(coeffs, ctf_basis, ctf_idx) + mean_coeff = self.get_mean(Coef(self.basis, coeffs), ctf_basis, ctf_idx) b_coeff = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) b_noise = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) @@ -370,7 +383,7 @@ def apply(A, x): logger.info("Convert matrices to positive semidefinite.") covar_coeff = covar_coeff.make_psd() - return covar_coeff + return Coef(self.basis, covar_coeff) def shrink_covar_backward(self, b, b_noise, n, noise_var, shrinker): """ @@ -421,6 +434,12 @@ def get_cwf_coeffs( and white noise of variance `noise_var` for the noise. """ + if not isinstance(coeffs, Coef): + raise TypeError( + f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + ) + coeffs = coeffs.asnumpy() + if mean_coeff is None: mean_coeff = self.get_mean(coeffs, ctf_basis, ctf_idx) @@ -465,7 +484,7 @@ def get_cwf_coeffs( coeff_est_k = coeff_est_k + mean_coeff coeffs_est[ctf_idx == k] = coeff_est_k - return coeffs_est + return Coef(self.basis, coeffs_est) class BatchedRotCov2D(RotCov2D): @@ -536,7 +555,7 @@ def _calc_rhs(self): batch = np.arange(start, min(start + self.batch_size, src.n)) im = src.images[batch[0] : batch[0] + len(batch)] - coeff = basis.evaluate_t(im) + coeff = basis.evaluate_t(im).asnumpy() for k in np.unique(ctf_idx[batch]): coeff_k = coeff[ctf_idx[batch] == k] @@ -701,7 +720,7 @@ def get_mean(self): b_mean_all = np.stack(self.b_mean).sum(axis=0) mean_coeff = self.A_mean.solve(b_mean_all) - return mean_coeff + return Coef(self.basis, mean_coeff) def get_covar( self, noise_var=0, mean_coeff=None, covar_est_opt=None, make_psd=True @@ -789,7 +808,7 @@ def identity(x): logger.info("Convert matrices to positive semidefinite.") covar_coeff = covar_coeff.make_psd() - return covar_coeff + return Coef(self.basis, covar_coeff) def get_cwf_coeffs( self, coeffs, ctf_basis, ctf_idx, mean_coeff, covar_coeff, noise_var=0 @@ -852,4 +871,4 @@ def get_cwf_coeffs( coeff_est_k = coeff_est_k + mean_coeff coeffs_est[ctf_idx == k] = coeff_est_k - return coeffs_est + return Coef(self.basis, coeffs_est) From 7807e6df3e94dafc5e9947aa6a9855bf165f5367 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 14 Sep 2023 11:45:31 -0400 Subject: [PATCH 059/294] pass FLE2d and FB2D tests --- src/aspire/basis/basis.py | 3 +-- src/aspire/basis/coef.py | 8 ++++++-- src/aspire/basis/fb_2d.py | 1 + src/aspire/basis/steerable.py | 18 +++++++----------- tests/test_FBbasis2D.py | 11 +++++------ tests/test_FPSWFbasis2D.py | 1 + tests/test_class2D.py | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 50ab6e1227..2439f12b6c 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -101,9 +101,8 @@ def evaluate(self, v): # v = Coef(self, v) raise TypeError(f"`evaluate` should be passed a `Coef`, received {type(v)}") - # Flatten stack, ndim is wrt Basis (2 or 3) + # Flatten stack stack_shape = v.stack_shape - v = v.reshape(-1, self.count) v = v.stack_reshape(-1).asnumpy() # Compute the transform diff --git a/src/aspire/basis/coef.py b/src/aspire/basis/coef.py index 46e6ec2cc2..9df4000015 100644 --- a/src/aspire/basis/coef.py +++ b/src/aspire/basis/coef.py @@ -62,9 +62,13 @@ def __init__(self, basis, data, dtype=None): self.stack_size = np.prod(self.stack_shape) self.count = self._data.shape[-1] - if self.count != self.basis.count: + basis_count = self.basis.count + if np.iscomplexobj(data): + basis_count = self.basis.complex_count + + if self.count != basis_count: raise RuntimeError( - f"Provided data count of {self.count} does not match basis count of {self.basis.count}." + f"Provided data count of {self.count} does not match basis count of {basis_count}." ) # Numpy interop diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 84c57e3dba..8deaa8f593 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -5,6 +5,7 @@ from aspire.basis import FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import unique_coords_nd +from aspire.operators import BlkDiagMatrix from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index d54b2600a4..d0a528ac44 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -1,3 +1,4 @@ +import abc import logging from collections.abc import Iterable @@ -5,7 +6,7 @@ from aspire.basis import Basis from aspire.operators import BlkDiagMatrix -from aspire.utils import complex_type +from aspire.utils import complex_type, real_type logger = logging.getLogger(__name__) @@ -394,14 +395,11 @@ def to_real(self, complex_coef): """ from .coef import Coef - if not isinstance(coef, Coef): + if not isinstance(complex_coef, Coef): raise TypeError( - f"coef should be instanace of `Coef`, received {type(coef)}." + f"complex_coef should be instanace of `Coef`, received {type(complex_coef)}." ) - if complex_coef.ndim == 1: - complex_coef = complex_coef.reshape(1, -1) - if complex_coef.dtype not in (np.complex128, np.complex64): raise TypeError("coef provided to to_real should be complex.") @@ -412,7 +410,8 @@ def to_real(self, complex_coef): f"Complex coef dtype {complex_coef.dtype} does not match precision of basis.dtype {self.dtype}, returning {dtype}." ) - coef = np.zeros((complex_coef.shape[0], self.count), dtype=dtype) + coef = np.zeros((*complex_coef.stack_shape, self.count), dtype=dtype) + complex_coef = complex_coef.asnumpy() ind = 0 idx = np.arange(self.k_max[0], dtype=int) @@ -453,9 +452,6 @@ def to_complex(self, coef): f"coef should be instanace of `Coef`, received {type(coef)}." ) - # if coef.ndim == 1: - # coef = coef.reshape(1, -1) - if coef.dtype not in (np.float64, np.float32): raise TypeError("coef provided to to_complex should be real.") @@ -486,4 +482,4 @@ def to_complex(self, coef): ind += np.size(idx) - return ccoef + return Coef(self, ccoef) diff --git a/tests/test_FBbasis2D.py b/tests/test_FBbasis2D.py index dc74d395b9..250a2b4f6e 100644 --- a/tests/test_FBbasis2D.py +++ b/tests/test_FBbasis2D.py @@ -113,12 +113,13 @@ def testComplexCoversionErrorsToReal(self, basis): x = randn(*basis.sz, seed=self.seed) # Express in an FB basis - cv1 = basis.to_complex(basis.expand(x.astype(basis.dtype))) + cv = basis.expand(x.astype(basis.dtype)) + ccv = basis.to_complex(cv) # Test catching Errors with raises(TypeError): # Pass real into `to_real` - _ = basis.to_real(cv1.real.astype(np.float32)) + _ = basis.to_real(cv) # Test casting case, where basis and coef precision don't match if basis.dtype == np.float32: @@ -128,12 +129,10 @@ def testComplexCoversionErrorsToReal(self, basis): # Result should be same precision as coef input, just real. result_dtype = real_type(test_dtype) - v3 = basis.to_real(cv1.astype(test_dtype)) + x = Coef(basis, ccv.asnumpy().astype(test_dtype)) + v3 = basis.to_real(x) assert v3.dtype == result_dtype - # Try a 0d vector, should not crash. - _ = basis.to_real(cv1.reshape(-1)) - params = [pytest.param(256, np.float32, marks=pytest.mark.expensive)] diff --git a/tests/test_FPSWFbasis2D.py b/tests/test_FPSWFbasis2D.py index 02602ecd2f..1bd818de3d 100644 --- a/tests/test_FPSWFbasis2D.py +++ b/tests/test_FPSWFbasis2D.py @@ -5,6 +5,7 @@ from aspire.basis import FPSWFBasis2D from aspire.image import Image +from aspire.utils import utest_tolerance from ._basis_util import UniversalBasisMixin, pswf_params_2d, show_basis_params diff --git a/tests/test_class2D.py b/tests/test_class2D.py index f683394bbf..47fe2bb1db 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -5,7 +5,7 @@ import pytest from sklearn import datasets -from aspire.basis import FBBasis2D, FFBBasis2D, FSPCABasis +from aspire.basis import Coef, FBBasis2D, FFBBasis2D, FSPCABasis from aspire.classification import RIRClass2D from aspire.classification.legacy_implementations import bispec_2drot_large, pca_y from aspire.noise import WhiteNoiseAdder From 93482fc3e9bbc9a98748d09ea8ab7d919b44c91e Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 14 Sep 2023 14:35:20 -0400 Subject: [PATCH 060/294] fixup covar2d test cases (Coef) --- tests/test_covar2d.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index dd7850a1ed..89631cb4bd 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -120,7 +120,7 @@ def test_get_mean(cov2d_fixture): results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_mean.npy")) cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] - mean_coeff = cov2d._get_mean(coeff_clean) + mean_coeff = cov2d._get_mean(coeff_clean.asnumpy()) np.testing.assert_allclose(results, mean_coeff, atol=utest_tolerance(cov2d.dtype)) @@ -149,10 +149,10 @@ def test_get_mean_ctf(cov2d_fixture, ctf_enabled): if ctf_enabled: result = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_meanctf.npy")) else: - result = cov2d._get_mean(coeff_clean) + result = cov2d._get_mean(coeff_clean.asnumpy()) tol = 0.002 - np.testing.assert_allclose(mean_coeff_ctf, result, atol=tol) + np.testing.assert_allclose(mean_coeff_ctf.asnumpy()[0], result, atol=tol) def test_get_cwf_coeffs_clean(cov2d_fixture): From b5ce0130a95079567cd15a7da88f9eee306c0f5e Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 14 Sep 2023 15:30:46 -0400 Subject: [PATCH 061/294] restore cov2d and batched cov2d tests after Coef changes [skip ci] --- src/aspire/covariance/covar2d.py | 25 ++++++++++++++++--------- src/aspire/operators/blk_diag_matrix.py | 7 ++++--- tests/test_batched_covar2d.py | 4 ++-- tests/test_covar2d.py | 3 ++- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 934f7ef0f4..69f6f6336c 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -133,8 +133,8 @@ def _get_covar(self, coeffs, mean_coeff=None, do_refl=True): """ Calculate the covariance matrix from the expansion coefficients without CTF information. - :param coeffs: A coefficient vector (or an array of coefficient vectors) calculated from 2D images. - :param mean_coeff: The mean vector calculated from the `coeffs`. + :param coeffs: A coefficient vector (an array of coefficient vectors) calculated from 2D images. + :param mean_coeff: The mean vector calculated from the `coeffs`. (array) :param do_refl: If true, enforce invariance to reflection (default false). :return: The covariance matrix of coefficients for all images. """ @@ -307,7 +307,7 @@ def identity(x): ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff) + mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) covar_coeff_k = self._get_covar(coeff_k, mean_coeff_k) b_coeff += weight * (ctf_basis_k_t @ covar_coeff_k @ ctf_basis_k) @@ -383,7 +383,7 @@ def apply(A, x): logger.info("Convert matrices to positive semidefinite.") covar_coeff = covar_coeff.make_psd() - return Coef(self.basis, covar_coeff) + return covar_coeff def shrink_covar_backward(self, b, b_noise, n, noise_var, shrinker): """ @@ -438,7 +438,6 @@ def get_cwf_coeffs( raise TypeError( f"`coeffs` should be instance of `Coef`, received {type(Coef)}." ) - coeffs = coeffs.asnumpy() if mean_coeff is None: mean_coeff = self.get_mean(coeffs, ctf_basis, ctf_idx) @@ -448,6 +447,8 @@ def get_cwf_coeffs( coeffs, ctf_basis, ctf_idx, mean_coeff, noise_var=noise_var ) + coeffs = coeffs.asnumpy() + # Handle CTF arguments. if (ctf_basis is None) ^ (ctf_idx is None): raise RuntimeError( @@ -469,7 +470,7 @@ def get_cwf_coeffs( ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff) + mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) coeff_est_k = coeff_k - mean_coeff_k if noise_var == 0: @@ -628,7 +629,7 @@ def _mean_correct_covar_rhs(self, b_covar, b_mean, mean_coeff): ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff) + mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) mean_coeff_k = ctf_basis_k_t.apply(mean_coeff_k) mean_coeff_k = mean_coeff_k[: partition[0][0]] @@ -808,7 +809,7 @@ def identity(x): logger.info("Convert matrices to positive semidefinite.") covar_coeff = covar_coeff.make_psd() - return Coef(self.basis, covar_coeff) + return covar_coeff def get_cwf_coeffs( self, coeffs, ctf_basis, ctf_idx, mean_coeff, covar_coeff, noise_var=0 @@ -829,6 +830,12 @@ def get_cwf_coeffs( and white noise of variance `noise_var` for the noise. """ + if not isinstance(coeffs, Coef): + raise TypeError( + f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + ) + coeffs = coeffs.asnumpy() + if mean_coeff is None: mean_coeff = self.get_mean() @@ -856,7 +863,7 @@ def get_cwf_coeffs( ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff) + mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) coeff_est_k = coeff_k - mean_coeff_k if noise_var == 0: diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index 65f95b4774..ebcbe941bd 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -703,17 +703,18 @@ def apply(self, X): :param X: Coefficient matrix, each column is a coefficient vector. :return: A matrix with new coefficient vectors. """ + import aspire cols = self.partition[:, 1] - if np.sum(cols) != np.size(X, 0): - raise RuntimeError("Sizes of matrix `self` and `X` are not compatible.") - vector = False if np.ndim(X) == 1: X = X[:, np.newaxis] vector = True + if np.sum(cols) != np.size(X, 0): + raise RuntimeError("Sizes of matrix `self` and `X` are not compatible.") + rows = np.array( [ np.size(X, 1), diff --git a/tests/test_batched_covar2d.py b/tests/test_batched_covar2d.py index 801ff8b7c6..075359972c 100644 --- a/tests/test_batched_covar2d.py +++ b/tests/test_batched_covar2d.py @@ -2,7 +2,7 @@ import numpy as np -from aspire.basis import FFBBasis2D +from aspire.basis import Coef, FFBBasis2D from aspire.covariance import BatchedRotCov2D, RotCov2D from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter @@ -85,7 +85,7 @@ def testMeanCovar(self): def testZeroMean(self): # Make sure it works with zero mean (pure second moment). - zero_coeff = np.zeros((self.basis.count,), dtype=self.dtype) + zero_coeff = Coef(self.basis, np.zeros((self.basis.count,), dtype=self.dtype)) covar_cov2d = self.cov2d.get_covar( self.coeff, diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index 89631cb4bd..1e092c0f35 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -131,7 +131,8 @@ def test_get_covar(cov2d_fixture): ) cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] - covar_coeff = cov2d._get_covar(coeff_clean) + + covar_coeff = cov2d._get_covar(coeff_clean.asnumpy()) for im, mat in enumerate(results.tolist()): np.testing.assert_allclose(mat, covar_coeff[im], rtol=1e-05) From 0e78be8014b834c9c96289d5e4ac79344f59cea4 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 15 Sep 2023 10:20:17 -0400 Subject: [PATCH 062/294] restore pswf and fpswf tests --- src/aspire/basis/fpswf_2d.py | 3 ++- src/aspire/basis/pswf_2d.py | 4 ++-- tests/test_FPSWFbasis2D.py | 8 ++++---- tests/test_PSWFbasis2D.py | 8 ++++---- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index a0896ce602..c9759c386a 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -5,6 +5,7 @@ from scipy.optimize import least_squares from scipy.special import jn +from aspire.basis import Coef from aspire.basis.basis_utils import lgwt, t_x_mat, t_x_mat_dot from aspire.basis.pswf_2d import PSWFBasis2D from aspire.nufft import nufft @@ -123,7 +124,7 @@ def _evaluate_t(self, images): nfft_res = nufft(images_disk, self.us_fft_pts) # Accumulate coefficients - coefficients = self._pswf_integration(nfft_res) + coefficients = Coef(self, self._pswf_integration(nfft_res)) return self.to_real(coefficients).asnumpy() diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 405089e7bc..e14411b312 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -223,8 +223,8 @@ def _evaluate_t(self, images): :return: The evaluation of the coefficient array in the PSWF basis. """ flattened_images = images[:, self._disk_mask] - - return self.to_real(flattened_images @ self.samples_conj_transpose).asnumpy() + ccoef = Coef(self, flattened_images @ self.samples_conj_transpose) + return self.to_real(ccoef).asnumpy() def _evaluate(self, coefficients): """ diff --git a/tests/test_FPSWFbasis2D.py b/tests/test_FPSWFbasis2D.py index 1bd818de3d..78fe0b0cee 100644 --- a/tests/test_FPSWFbasis2D.py +++ b/tests/test_FPSWFbasis2D.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from aspire.basis import FPSWFBasis2D +from aspire.basis import Coef, FPSWFBasis2D from aspire.image import Image from aspire.utils import utest_tolerance @@ -28,7 +28,7 @@ def testFPSWFBasis2DEvaluate_t(self, basis): ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT - coeffs = basis.to_real(ccoeffs) + coeffs = basis.to_real(Coef(basis, ccoeffs)) np.testing.assert_allclose(result, coeffs, atol=utest_tolerance(basis.dtype)) @@ -38,10 +38,10 @@ def testFPSWFBasis2DEvaluate(self, basis): ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT - coeffs = basis.to_real(ccoeffs) + coeffs = basis.to_real(Coef(basis, ccoeffs)) result = coeffs.evaluate() result = basis.evaluate(coeffs) images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoeff_out_8_8.npy")).T # RCOPT - np.testing.assert_allclose(result, images) + np.testing.assert_allclose(result, images, rtol=1e-05, atol=1e-08) diff --git a/tests/test_PSWFbasis2D.py b/tests/test_PSWFbasis2D.py index cf34ebdf71..aaf8bc738c 100644 --- a/tests/test_PSWFbasis2D.py +++ b/tests/test_PSWFbasis2D.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from aspire.basis import PSWFBasis2D +from aspire.basis import Coef, PSWFBasis2D from aspire.image import Image from ._basis_util import UniversalBasisMixin, pswf_params_2d, show_basis_params @@ -28,9 +28,9 @@ def testPSWFBasis2DEvaluate_t(self, basis): ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT - coeffs = basis.to_real(ccoeffs) + coeffs = basis.to_real(Coef(basis, ccoeffs)) - np.testing.assert_allclose(result, coeffs) + np.testing.assert_allclose(result, coeffs, rtol=1e-05, atol=1e-08) def testPSWFBasis2DEvaluate(self, basis): # Historically, PSWF returned complex values. @@ -38,7 +38,7 @@ def testPSWFBasis2DEvaluate(self, basis): ccoeffs = np.load( os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") ).T # RCOPT - coeffs = basis.to_real(ccoeffs) + coeffs = basis.to_real(Coef(basis, ccoeffs)) result = coeffs.evaluate() images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoeff_out_8_8.npy")).T # RCOPT From 8f5971cafcf57aed69cf420eef482411066f9ba8 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 15 Sep 2023 11:00:27 -0400 Subject: [PATCH 063/294] restore class2d tests --- src/aspire/basis/fspca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index c29d48d4a3..750767da32 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -240,7 +240,7 @@ def _compute_spca(self): ) # Compute coefficient vector of mean image at zeroth component - self.mean_coef_zero = self.mean_coef_est[self.angular_indices == 0] + self.mean_coef_zero = self.mean_coef_est.asnumpy()[0][self.angular_indices == 0] # Define mask for zero angular mode, used in loop below zero_ell_mask = self.basis._indices["ells"] == 0 From 2ad3d899f06542600e5e44cefdbdb20ab35f9a97 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 15 Sep 2023 11:21:16 -0400 Subject: [PATCH 064/294] restore more class2d tests --- src/aspire/basis/fspca.py | 4 ++-- src/aspire/basis/steerable.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index 750767da32..a2c6d93999 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -512,7 +512,7 @@ def to_complex(self, coef): for i, k in enumerate(ccoef_d.keys()): ccoef[:, i] = ccoef_d[k] - return ccoef + return Coef(self, ccoef) def to_real(self, complex_coef): """ @@ -552,7 +552,7 @@ def to_real(self, complex_coef): coef[:, pos_i] = 2.0 * complex_coef[:, i].real coef[:, neg_i] = -2.0 * complex_coef[:, i].imag - return coef + return Coef(self, coef) def calculate_bispectrum( self, coef, flatten=False, filter_nonzero_freqs=False, freq_cutoff=None diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index d0a528ac44..8695639bfb 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -59,6 +59,14 @@ def calculate_bispectrum( :return: Bispectum matrix (complex valued). """ + # Avoids circular import + from aspire.basis import Coef + + if not isinstance(complex_coef, Coef): + raise TypeError(f"Expect `Coef` received {type(complex_coef)}.") + complex_coef = complex_coef.asnumpy() + + # TODO, can clean this up after enforcing Coef. # Check shape if complex_coef.shape[0] != 1: raise ValueError( From 6fc7b30663f79f943129f0f3776406cd30f0d2f5 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 15 Sep 2023 11:39:41 -0400 Subject: [PATCH 065/294] restore more class2d tests again [skip ci] --- src/aspire/classification/rir_class2d.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/aspire/classification/rir_class2d.py b/src/aspire/classification/rir_class2d.py index 9e3915b0ca..a8445e5932 100644 --- a/src/aspire/classification/rir_class2d.py +++ b/src/aspire/classification/rir_class2d.py @@ -440,7 +440,7 @@ def _devel_bispectrum(self, coef): for i in trange(self.src.n): B = self.pca_basis.calculate_bispectrum( - coef_normed[i, np.newaxis], + Coef(self.pca_basis, coef_normed[i]), filter_nonzero_freqs=True, freq_cutoff=self.bispectrum_freq_cutoff, ) @@ -494,9 +494,11 @@ def _legacy_bispectrum(self, coef, retry_attempts=3): ) # The legacy code expects the complex representation - coef = self.pca_basis.to_complex(coef) - complex_eigvals = self.pca_basis.to_complex(self.pca_basis.eigvals).reshape( - self.pca_basis.complex_count + coef = self.pca_basis.to_complex(coef).asnumpy() + complex_eigvals = ( + self.pca_basis.to_complex(self.pca_basis.eigvals) + .asnumpy() + .reshape(self.pca_basis.complex_count) ) # flatten # bispec_2drot_large has a random selection component. From 43d60dc03851b1d6f8f9addba4902599d96611e9 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 15 Sep 2023 14:53:46 -0400 Subject: [PATCH 066/294] fix rebase bug, missing _t --- src/aspire/operators/blk_diag_matrix.py | 1 - src/aspire/reconstruction/estimator.py | 4 ++-- src/aspire/reconstruction/mean.py | 7 +++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index ebcbe941bd..14d64a762a 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -703,7 +703,6 @@ def apply(self, X): :param X: Coefficient matrix, each column is a coefficient vector. :return: A matrix with new coefficient vectors. """ - import aspire cols = self.partition[:, 1] diff --git a/src/aspire/reconstruction/estimator.py b/src/aspire/reconstruction/estimator.py index 5c8bbf1a0a..62004da1d4 100644 --- a/src/aspire/reconstruction/estimator.py +++ b/src/aspire/reconstruction/estimator.py @@ -81,7 +81,7 @@ def apply_kernel(self, vol_coeff, kernel=None): if kernel is None: kernel = self.kernel - vol = self.basis.evaluate(vol_coeff) # returns a Volume + vol = Coef(self.basis, vol_coeff).evaluate() # returns a Volume vol = kernel.convolve_volume(vol) # returns a Volume - vol_coef = Coef(self.basis, vol_coeff).evaluate() + vol_coef = self.basis.evaluate_t(vol) return vol_coef diff --git a/src/aspire/reconstruction/mean.py b/src/aspire/reconstruction/mean.py index 96dc811080..dc8fb4d335 100644 --- a/src/aspire/reconstruction/mean.py +++ b/src/aspire/reconstruction/mean.py @@ -6,6 +6,7 @@ from scipy.sparse.linalg import LinearOperator, cg from aspire import config +from aspire.basis import Coef from aspire.nufft import anufft from aspire.numeric import fft from aspire.operators import evaluate_src_filters_on_grid @@ -231,11 +232,13 @@ def apply_kernel(self, vol_coeff, kernel=None): np.zeros((self.r, self.src.L, self.src.L, self.src.L), dtype=self.dtype) ) - vol = self.basis.evaluate(vol_coeff) + vol = Coef(self.basis, vol_coeff).evaluate() for k in range(self.r): for j in range(self.r): - vols_out[k] = vols_out[k] + kernel.convolve_volume(vol[j], j, k) + vols_out[k] = vols_out[k] + kernel.convolve_volume( + vol.asnumpy()[j], j, k + ) # Note this is where we would add mask_gamma vol_coeff = self.basis.evaluate_t(vols_out) From e3690cd92dce4676196161c131b5b27107c809bd Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 18 Sep 2023 09:24:50 -0400 Subject: [PATCH 067/294] Fixup circular dependence --- src/aspire/basis/__init__.py | 3 +- src/aspire/basis/basis.py | 258 +++++++++++++++++++++++++++++++++- src/aspire/basis/coef.py | 256 --------------------------------- src/aspire/basis/steerable.py | 8 +- 4 files changed, 253 insertions(+), 272 deletions(-) delete mode 100644 src/aspire/basis/coef.py diff --git a/src/aspire/basis/__init__.py b/src/aspire/basis/__init__.py index 2d34e951bc..c150692f32 100644 --- a/src/aspire/basis/__init__.py +++ b/src/aspire/basis/__init__.py @@ -1,8 +1,7 @@ # We'll tell isort not to sort these base classes # isort: off -from .basis import Basis -from .coef import Coef +from .basis import Basis, Coef from .steerable import SteerableBasis2D from .fb import FBBasisMixin diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 2439f12b6c..73f34e841b 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -10,6 +10,257 @@ logger = logging.getLogger(__name__) +class Coef: + """ + Numpy interoperable basis stacks. + """ + + def __init__(self, basis, data, dtype=None): + """ + A stack of one or more coefficient arrays. + + The stack can be multidimensional with `stack_size` equal + to the product of the stack dimensions. Singletons will be + expanded into a 1D stack of length one. + + The last axes always represents the coefficient `count`. + + :param data: Numpy array containing image data with shape + `(..., count)`. + :param dtype: Optionally cast `data` to this dtype. + Defaults to `data.dtype`. + + :return: Image instance holding `data`. + """ + + if isinstance(data, Coef): + data = data.asnumpy() + elif not isinstance(data, np.ndarray): + raise ValueError("Coef should be instantiated with an ndarray") + + if data.ndim < 1: + raise ValueError( + "Coef data should be ndarray with shape (N1...) x count or (count)." + ) + elif data.ndim == 1: + data = np.expand_dims(data, axis=0) + + if dtype is None: + self.dtype = data.dtype + else: + self.dtype = np.dtype(dtype) + + if not isinstance(basis, Basis): + raise TypeError( + f"`basis` is required to be a `Basis` instance, received {type(basis)}" + ) + self.basis = basis + + self._data = data.astype(self.dtype, copy=False) + self.ndim = self._data.ndim + self.shape = self._data.shape + self.stack_ndim = self._data.ndim - 1 + self.stack_shape = self._data.shape[:-1] + self.stack_size = np.prod(self.stack_shape) + self.count = self._data.shape[-1] + + basis_count = self.basis.count + if np.iscomplexobj(data): + basis_count = self.basis.complex_count + + if self.count != basis_count: + raise RuntimeError( + f"Provided data count of {self.count} does not match basis count of {basis_count}." + ) + + # Numpy interop + # https://numpy.org/devdocs/user/basics.interoperability.html#the-array-interface-protocol + self.__array_interface__ = self.asnumpy().__array_interface__ + self.__array__ = self.asnumpy() + + def __len__(self): + """ + Return stack length. + + Note this is product of all stack dimensions. + """ + return self.stack_size + + def asnumpy(self): + """ + Return image data as a (, count) + read-only array view. + + :return: read-only ndarray view + """ + + view = self._data.view() + view.flags.writeable = False + return view + + def _check_key_dims(self, key): + if isinstance(key, tuple) and (len(key) > self._data.ndim): + raise ValueError( + f"Coef stack_dim is {self.stack_ndim}, slice length must be =< {self.ndim}" + ) + + def __getitem__(self, key): + self._check_key_dims(key) + return self.__class__(self.basis, self._data[key]) + + def __setitem__(self, key, value): + self._check_key_dims(key) + self._data[key] = value + + def stack_reshape(self, *args): + """ + Reshape the stack axis. + + :*args: Integer(s) or tuple describing the intended shape. + + :returns: Coef instance + """ + + # If we're passed a tuple, use that + if len(args) == 1 and isinstance(args[0], tuple): + shape = args[0] + else: + # Otherwise use the variadic args + shape = args + + # Sanity check the size + if shape != (-1,) and np.prod(shape) != self.stack_size: + raise ValueError( + f"Number of images {self.stack_size} cannot be reshaped to {shape}." + ) + + return self.__class__( + self.basis, self._data.reshape(*shape, self._data.shape[-1]) + ) + + def copy(self): + """ + Return a new `Coef` instance with a deep copy of the data. + """ + return self.__class__(self.basis, self._data.copy()) + + def evaluate(self): + """ + Return the evaluation of coefficients in the associated `basis`. + """ + return self.basis.evaluate(self) + + def rotate(self, radians, refl=None): + """ + Returns coefs rotated counter-clockwise by `radians`. + + Raises error if underlying coef basis does not support rotations. + + :param radians: Rotation in radians. + :param refl: Optional reflect image (about y=0) (bool) + :return: rotated coefs. + """ + + if not callable(getattr(self.basis, "rotate", None)): + raise RuntimeError( + f"self.basis={self.basis} does not provide `rotate` method." + ) + + return self.basis.rotate(self, radians, refl) + + def shift(self, shifts): + """ + Returns coefs shifted by `shifts`. + + This will transform to real cartesian space, shift, + and transform back to Polar Fourier-Bessel space. + + :param coef: Basis coefs. + :param shifts: Shifts in pixels (x,y). Shape (1,2) or (len(coef), 2). + :return: coefs of shifted images. + """ + + if not callable(getattr(self.basis, "shift", None)): + raise RuntimeError( + f"self.basis={self.basis} does not provide `shift` method." + ) + + return self.basis.shift(self, shifts) + + def __mul__(self, other): + """ + Overload operator for multiplication. + + :param other: `Coef` instance to multiply with. + Also allows for multiplication by Numpy arrays and scalars. + :return: `Coef` instance. + """ + + if isinstance(other, Coef): + other = other._data + + return self.__class__(self.basis, self._data * other) + + def __add__(self, other): + """ + Overload operator for addition. + + :param other: `Coef` instance to add. + Also allows for addition by Numpy arrays and scalars. + :return: `Coef` instance. + """ + + if isinstance(other, Coef): + other = other._data + + return self.__class__(self.basis, self._data + other) + + def __sub__(self, other): + """ + Overload operator for subtraction. + + :param other: `Coef` instance to subtract. + Also allows for subtraction by Numpy arrays and scalars. + :return: `Coef` instance. + """ + + if isinstance(other, Coef): + other = other._data + + return self.__class__(self.basis, self._data - other) + + def __neg__(self): + """ + Overload operator for negation. + + :return: `Coef` instance. + """ + + return self.__class__(self.basis, -self._data) + + def size(self): + """ + Return np.size of underlying data. + + This should be `stack_size * count`, + or `len(self) * count`. + """ + return np.size(self._data) + + # This is included for completion, but is not being adopted yet. + def by_indices(self, **kwargs): + """ + Select coefficients by indices (`radial`, `angular`). + + See `SteerableBasis.indices_mask` for argument details. + + :return: `Coef` vector. + """ + + mask = self.basis.indices_mask(**kwargs) + return self._data[:, mask] + + class Basis: """ Define a base class for expanding 2D particle images and 3D structure volumes @@ -95,8 +346,6 @@ def evaluate(self, v): f" Inconsistent dtypes v: {v.dtype} self coefficient dtype: {self.coefficient_dtype}" ) - from .coef import Coef - if not isinstance(v, Coef): # v = Coef(self, v) raise TypeError(f"`evaluate` should be passed a `Coef`, received {type(v)}") @@ -148,9 +397,6 @@ def evaluate_t(self, v): # Restore stack shape x = x.reshape(*stack_shape, self.count) - # Avoid circular dependence - from .coef import Coef - return Coef(self, x) def _evaluate_t(self, v): @@ -200,8 +446,6 @@ def expand(self, x): """ - from .coef import Coef - if isinstance(x, Image) or isinstance(x, Volume): x = x.asnumpy() diff --git a/src/aspire/basis/coef.py b/src/aspire/basis/coef.py deleted file mode 100644 index 9df4000015..0000000000 --- a/src/aspire/basis/coef.py +++ /dev/null @@ -1,256 +0,0 @@ -import logging - -import numpy as np - -from .basis import Basis -from .steerable import SteerableBasis2D - -logger = logging.getLogger(__name__) - - -class Coef: - """ - Numpy interoperable basis stacks. - """ - - def __init__(self, basis, data, dtype=None): - """ - A stack of one or more coefficient arrays. - - The stack can be multidimensional with `stack_size` equal - to the product of the stack dimensions. Singletons will be - expanded into a 1D stack of length one. - - The last axes always represents the coefficient `count`. - - :param data: Numpy array containing image data with shape - `(..., count)`. - :param dtype: Optionally cast `data` to this dtype. - Defaults to `data.dtype`. - - :return: Image instance holding `data`. - """ - - if isinstance(data, Coef): - data = data.asnumpy() - elif not isinstance(data, np.ndarray): - raise ValueError("Coef should be instantiated with an ndarray") - - if data.ndim < 1: - raise ValueError( - "Coef data should be ndarray with shape (N1...) x count or (count)." - ) - elif data.ndim == 1: - data = np.expand_dims(data, axis=0) - - if dtype is None: - self.dtype = data.dtype - else: - self.dtype = np.dtype(dtype) - - if not isinstance(basis, Basis): - raise TypeError( - f"`basis` is required to be a `Basis` instance, received {type(basis)}" - ) - self.basis = basis - - self._data = data.astype(self.dtype, copy=False) - self.ndim = self._data.ndim - self.shape = self._data.shape - self.stack_ndim = self._data.ndim - 1 - self.stack_shape = self._data.shape[:-1] - self.stack_size = np.prod(self.stack_shape) - self.count = self._data.shape[-1] - - basis_count = self.basis.count - if np.iscomplexobj(data): - basis_count = self.basis.complex_count - - if self.count != basis_count: - raise RuntimeError( - f"Provided data count of {self.count} does not match basis count of {basis_count}." - ) - - # Numpy interop - # https://numpy.org/devdocs/user/basics.interoperability.html#the-array-interface-protocol - self.__array_interface__ = self.asnumpy().__array_interface__ - self.__array__ = self.asnumpy() - - def __len__(self): - """ - Return stack length. - - Note this is product of all stack dimensions. - """ - return self.stack_size - - def asnumpy(self): - """ - Return image data as a (, count) - read-only array view. - - :return: read-only ndarray view - """ - - view = self._data.view() - view.flags.writeable = False - return view - - def _check_key_dims(self, key): - if isinstance(key, tuple) and (len(key) > self._data.ndim): - raise ValueError( - f"Coef stack_dim is {self.stack_ndim}, slice length must be =< {self.ndim}" - ) - - def __getitem__(self, key): - self._check_key_dims(key) - return self.__class__(self.basis, self._data[key]) - - def __setitem__(self, key, value): - self._check_key_dims(key) - self._data[key] = value - - def stack_reshape(self, *args): - """ - Reshape the stack axis. - - :*args: Integer(s) or tuple describing the intended shape. - - :returns: Coef instance - """ - - # If we're passed a tuple, use that - if len(args) == 1 and isinstance(args[0], tuple): - shape = args[0] - else: - # Otherwise use the variadic args - shape = args - - # Sanity check the size - if shape != (-1,) and np.prod(shape) != self.stack_size: - raise ValueError( - f"Number of images {self.stack_size} cannot be reshaped to {shape}." - ) - - return self.__class__( - self.basis, self._data.reshape(*shape, self._data.shape[-1]) - ) - - def copy(self): - """ - Return a new `Coef` instance with a deep copy of the data. - """ - return self.__class__(self.basis, self._data.copy()) - - def evaluate(self): - """ - Return the evaluation of coefficients in the associated `basis`. - """ - return self.basis.evaluate(self) - - def rotate(self, radians, refl=None): - """ - Returns coefs rotated counter-clockwise by `radians`. - - Raises error if underlying coef basis does not support rotations. - - :param radians: Rotation in radians. - :param refl: Optional reflect image (about y=0) (bool) - :return: rotated coefs. - """ - if not isinstance(self.basis, SteerableBasis2D): - raise RuntimeError(f"self.basis={self.basis} is not SteerableBasis.") - - return self.basis.rotate(self, radians, refl) - - def shift(self, shifts): - """ - Returns coefs shifted by `shifts`. - - This will transform to real cartesian space, shift, - and transform back to Polar Fourier-Bessel space. - - :param coef: Basis coefs. - :param shifts: Shifts in pixels (x,y). Shape (1,2) or (len(coef), 2). - :return: coefs of shifted images. - """ - - if not callable(getattr(self.basis, "shift", None)): - raise RuntimeError( - f"self.basis={self.basis} does not provide `shift` method." - ) - - return self.basis.shift(self, shifts) - - def __mul__(self, other): - """ - Overload operator for multiplication. - - :param other: `Coef` instance to multiply with. - Also allows for multiplication by Numpy arrays and scalars. - :return: `Coef` instance. - """ - - if isinstance(other, Coef): - other = other._data - - return self.__class__(self.basis, self._data * other) - - def __add__(self, other): - """ - Overload operator for addition. - - :param other: `Coef` instance to add. - Also allows for addition by Numpy arrays and scalars. - :return: `Coef` instance. - """ - - if isinstance(other, Coef): - other = other._data - - return self.__class__(self.basis, self._data + other) - - def __sub__(self, other): - """ - Overload operator for subtraction. - - :param other: `Coef` instance to subtract. - Also allows for subtraction by Numpy arrays and scalars. - :return: `Coef` instance. - """ - - if isinstance(other, Coef): - other = other._data - - return self.__class__(self.basis, self._data - other) - - def __neg__(self): - """ - Overload operator for negation. - - :return: `Coef` instance. - """ - - return self.__class__(self.basis, -self._data) - - def size(self): - """ - Return np.size of underlying data. - - This should be `stack_size * count`, - or `len(self) * count`. - """ - return np.size(self._data) - - # This is included for completion, but is not being adopted yet. - def by_indices(self, **kwargs): - """ - Select coefficients by indices (`radial`, `angular`). - - See `SteerableBasis.indices_mask` for argument details. - - :return: `Coef` vector. - """ - - mask = self.basis.indices_mask(**kwargs) - return self._data[:, mask] diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 8695639bfb..2a7feb45a4 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -4,7 +4,7 @@ import numpy as np -from aspire.basis import Basis +from aspire.basis import Basis, Coef from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type, real_type @@ -59,9 +59,6 @@ def calculate_bispectrum( :return: Bispectum matrix (complex valued). """ - # Avoids circular import - from aspire.basis import Coef - if not isinstance(complex_coef, Coef): raise TypeError(f"Expect `Coef` received {type(complex_coef)}.") complex_coef = complex_coef.asnumpy() @@ -174,7 +171,6 @@ def rotate(self, coef, radians, refl=None): :param refl: Optional reflect image (about y=0) (bool) :return: rotated coefs. """ - from .coef import Coef if not isinstance(coef, Coef): raise TypeError(f"`coef` must be `Coef` instance, received {type(coef)}.") @@ -401,7 +397,6 @@ def to_real(self, complex_coef): :param complex_coef: Complex coefficients from this basis. :return: Real coefficent representation from this basis. """ - from .coef import Coef if not isinstance(complex_coef, Coef): raise TypeError( @@ -453,7 +448,6 @@ def to_complex(self, coef): :param coef: Coefficients from this basis. :return: Complex coefficent representation from this basis. """ - from .coef import Coef if not isinstance(coef, Coef): raise TypeError( From aa785d617d0361ae6edec144b75648496ac64889 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 18 Sep 2023 09:25:02 -0400 Subject: [PATCH 068/294] Add Coef unit test --- tests/test_coef.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/test_coef.py diff --git a/tests/test_coef.py b/tests/test_coef.py new file mode 100644 index 0000000000..2baf33413e --- /dev/null +++ b/tests/test_coef.py @@ -0,0 +1,95 @@ +import numpy as np +import pytest + +from aspire.basis import Coef, FFBBasis2D + +IMG_SIZE = [ + 32, + pytest.param(31, marks=pytest.mark.expensive), +] +DTYPES = [ + np.float64, + pytest.param(np.float32, marks=pytest.mark.expensive), +] +STACKS = [ + (), + (1,), + (2,), + (3, 4), +] + + +def sim_fixture_id(params): + stack, count, dtype = params + return f"stack={stack}, count={count}, dtype={dtype}" + + +# Dtypes for coef array +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +def dtype(request): + return request.param + + +# Dtypes for basis +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +def basis_dtype(request): + return request.param + + +@pytest.fixture(params=IMG_SIZE, ids=lambda x: f"count={x}") +def img_size(request): + return request.param + + +@pytest.fixture(params=STACKS, ids=lambda x: f"stack={x}") +def stack(request): + return request.param + + +ALLYOURBASES = [FFBBasis2D] + + +@pytest.fixture(params=ALLYOURBASES, ids=lambda x: f"basis={x}") +def basis(request, img_size, basis_dtype): + cls = request.param + return cls(img_size, dtype=basis_dtype) + + +@pytest.fixture +def coef_fixture(basis, stack, dtype): + """ + Construct testing coefficient array. + """ + # Combine the stack and coefficent counts into multidimensional + # shape. + size = stack + (basis.count,) + + coef_np = np.random.random(size=size).astype(dtype, copy=False) + + return Coef(basis, coef_np, dtype=dtype) + + +def test_coef_evalute(coef_fixture, basis): + assert np.allclose(coef_fixture.evaluate(), basis.evaluate(coef_fixture)) + + +def test_coef_rotate(coef_fixture, basis): + # Rotations + rots = np.linspace(-np.pi, np.pi, coef_fixture.stack_size).reshape( + coef_fixture.stack_shape + ) + + # Refl + refl = ( + np.random.rand(coef_fixture.stack_size).reshape(coef_fixture.stack_shape) > 0.5 + ) # Random bool + + assert np.allclose(coef_fixture.rotate(rots), basis.rotate(coef_fixture, rots)) + + assert np.allclose( + coef_fixture.rotate(rots, refl), basis.rotate(coef_fixture, rots, refl) + ) + + +def test_coef_shift(coef_fixture): + pass From b3addca6c4829d83613b7c1bffd1bda46cdcfdcf Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 18 Sep 2023 09:59:04 -0400 Subject: [PATCH 069/294] Minor cleanup, mostly strings. --- src/aspire/basis/fpswf_2d.py | 4 ++-- src/aspire/basis/pswf_2d.py | 16 +++++++--------- src/aspire/basis/steerable.py | 34 +++++++++++++++++----------------- tests/test_FFBbasis2D.py | 2 +- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index c9759c386a..daaf0a914b 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -100,7 +100,7 @@ def _precomp(self): self.us_fft_pts = us_fft_pts.astype( self.dtype, copy=False - ) # xxx find where this is incorrect + ) # TODO, debug where this is incorrect dtype self.blk_r = blk_r self.num_angular_pts = num_angular_pts self.r_quad_indices = r_quad_indices @@ -313,7 +313,7 @@ def _pswf_integration_sub_routine(self): indices_for_n.extend(numel_for_n) indices_for_n = np.cumsum(indices_for_n, dtype="int") - blk_r = [0] * n_max # xxx array? + blk_r = [0] * n_max # TODO, consider array here temp_const = self.bandlimit / (2 * np.pi * self.rcut) for i in range(n_max): blk_r[i] = ( diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index e14411b312..e62bee3f81 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -159,7 +159,9 @@ def _generate_samples(self): # the column dimension of samples_conj_transpose is the number of basis coefficients self.complex_count = self.samples_conj_transpose.shape[1] - # hack + # Add required real indices attributes and maps + # TODO, this block of code can probably be consolidated with + # FB basis. For now, just get everything working together. nz = np.sum(self.complex_signs_indices == 0) nnz = self.complex_count - nz @@ -170,7 +172,6 @@ def _generate_samples(self): self.angular_indices = np.empty(self.real_count, dtype=int) self.signs_indices = np.empty(self.real_count, dtype=int) - # hackity hack self._pos = np.zeros(self.complex_count, dtype=int) self._neg = np.zeros(self.complex_count, dtype=int) @@ -190,20 +191,17 @@ def _generate_samples(self): self.radial_indices[rng] = ks self.signs_indices[rng] = sgn - # hackity hack if sgn == 1: self._pos[ci + ks] = rng elif sgn == -1: self._neg[ci + ks] = rng - # /hackity hack i += len(ks) ci += len(ks) - # /hack - - # for tmp compat, probably can remove `indices` or clean it up later. + # Added for compatibility. + # Probably can remove `indices` dict wholesale later (MATLAB holdover). def indices(self): """ Return the precomputed indices for each basis function. @@ -236,7 +234,7 @@ def _evaluate(self, coefficients): """ - # hack, convert to complex + # Convert real coefficient to complex. coefficients = self.to_complex(Coef(self, coefficients)) # Handle a single coefficient vector or stack of vectors. @@ -430,5 +428,5 @@ def _pswf_2d_minor_computations(self, big_n, n, bandlimit, phi_approximate_error d_vec, _ = BNMatrix(big_n, bandlimit, approx_length).get_eig_vectors() - range_array = np.array(range(approx_length), dtype=self.dtype) + range_array = np.arange(approx_length, dtype=self.dtype) return d_vec, approx_length, range_array diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 2a7feb45a4..3db5d5f082 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -32,7 +32,7 @@ def __init__(self, *args, **kwargs): self._non_neg_angular_inds = self.signs_indices >= 0 self._blk_diag_cov_shape = None - # Try to centralize inds between FB/FFB and FLE in SteerableBasis2D + # Centralize indices attributes between FB/PSWF/FLE in SteerableBasis2D self._indices = self.indices() self.complex_count = self.count - sum(self._neg_angular_inds) self.complex_angular_indices = self.angular_indices[self._non_neg_angular_inds] @@ -63,7 +63,6 @@ def calculate_bispectrum( raise TypeError(f"Expect `Coef` received {type(complex_coef)}.") complex_coef = complex_coef.asnumpy() - # TODO, can clean this up after enforcing Coef. # Check shape if complex_coef.shape[0] != 1: raise ValueError( @@ -340,6 +339,7 @@ def blk_diag_cov_shape(self): # Return the cached shape return self._blk_diag_cov_shape + # This is included for completion, but is not being adopted yet. def indices_mask(self, **kwargs): """ Given `radial=` or `angular=` expressions, return (`count`,) @@ -389,13 +389,13 @@ def indices_mask(self, **kwargs): def to_real(self, complex_coef): """ Return real valued representation of complex coefficients. - This can be useful when comparing or implementing methods - from literature. + This can be useful when comparing, prototyping, or + implementing methods from literature. - There is a corresponding method, to_complex. + There is a corresponding method, `to_complex`. - :param complex_coef: Complex coefficients from this basis. - :return: Real coefficent representation from this basis. + :param complex_coef: Complex `Coef` from this basis. + :return: Real `Ceof` representation from this basis. """ if not isinstance(complex_coef, Coef): @@ -439,14 +439,14 @@ def to_real(self, complex_coef): def to_complex(self, coef): """ - Return complex valued representation of coefficients. - This can be useful when comparing or implementing methods - from literature. + Return complex valued representation of complex coefficients. + This can be useful when comparing, prototyping, or + implementing methods from literature. - There is a corresponding method, to_real. + There is a corresponding method, `to_real`. - :param coef: Coefficients from this basis. - :return: Complex coefficent representation from this basis. + :param coef: Real `Coef` from this basis. + :return: Complex `Coef` representation from this basis. """ if not isinstance(coef, Coef): @@ -467,21 +467,21 @@ def to_complex(self, coef): # Return the same precision as coef imaginary = dtype(1j) - ccoef = np.zeros((*coef.stack_shape, self.complex_count), dtype=dtype) + complex_coef = np.zeros((*coef.stack_shape, self.complex_count), dtype=dtype) coef = coef.asnumpy() ind = 0 idx = np.arange(self.k_max[0], dtype=int) ind += np.size(idx) - ccoef[..., idx] = coef[..., idx] + complex_coef[..., idx] = coef[..., idx] for ell in range(1, self.ell_max + 1): idx = ind + np.arange(self.k_max[ell], dtype=int) - ccoef[..., idx] = ( + complex_coef[..., idx] = ( coef[..., self._pos[idx]] - imaginary * coef[..., self._neg[idx]] ) / 2.0 ind += np.size(idx) - return Coef(self, ccoef) + return Coef(self, complex_coef) diff --git a/tests/test_FFBbasis2D.py b/tests/test_FFBbasis2D.py index cd9301c5db..58343863b9 100644 --- a/tests/test_FFBbasis2D.py +++ b/tests/test_FFBbasis2D.py @@ -64,7 +64,7 @@ def _testElement(self, basis, ell, k, sgn): coef_ref = np.zeros(basis.count, dtype=basis.dtype) coef_ref[(ells == ell) & (sgns == sgn) & (ks == k)] = 1 - im_ref = basis.evaluate(Coef(basis, coef_ref)).asnumpy()[0] + im_ref = Coef(basis, coef_ref).evaluate().asnumpy()[0] coef = basis.expand(im) From 27abfa1e5569de095ef03609c622d887163dc474 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 18 Sep 2023 10:00:32 -0400 Subject: [PATCH 070/294] revert unnessecary volume asnumpy conversion --- src/aspire/reconstruction/mean.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/aspire/reconstruction/mean.py b/src/aspire/reconstruction/mean.py index dc8fb4d335..9aaef7d08b 100644 --- a/src/aspire/reconstruction/mean.py +++ b/src/aspire/reconstruction/mean.py @@ -236,9 +236,7 @@ def apply_kernel(self, vol_coeff, kernel=None): for k in range(self.r): for j in range(self.r): - vols_out[k] = vols_out[k] + kernel.convolve_volume( - vol.asnumpy()[j], j, k - ) + vols_out[k] = vols_out[k] + kernel.convolve_volume(vol[j], j, k) # Note this is where we would add mask_gamma vol_coeff = self.basis.evaluate_t(vols_out) From 75313f0fcc6d8ef9147eaeac34ed8b1b1bb354ca Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 19 Sep 2023 12:55:05 -0400 Subject: [PATCH 071/294] add missing tests and misc string cleanup --- src/aspire/basis/basis.py | 13 +- src/aspire/basis/steerable.py | 9 +- tests/test_FBbasis2D.py | 4 +- tests/test_coef.py | 230 ++++++++++++++++++++++++++++++++-- 4 files changed, 237 insertions(+), 19 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 73f34e841b..d42e1887a6 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -12,7 +12,8 @@ class Coef: """ - Numpy interoperable basis stacks. + Numpy interoperable container for stacks of coefficient vectors. + Each `Coef` instance has an associated `Basis`. """ def __init__(self, basis, data, dtype=None): @@ -33,9 +34,7 @@ def __init__(self, basis, data, dtype=None): :return: Image instance holding `data`. """ - if isinstance(data, Coef): - data = data.asnumpy() - elif not isinstance(data, np.ndarray): + if not isinstance(data, np.ndarray): raise ValueError("Coef should be instantiated with an ndarray") if data.ndim < 1: @@ -64,6 +63,7 @@ def __init__(self, basis, data, dtype=None): self.stack_size = np.prod(self.stack_shape) self.count = self._data.shape[-1] + # Derive count based on real/complex coefficients. basis_count = self.basis.count if np.iscomplexobj(data): basis_count = self.basis.complex_count @@ -238,6 +238,7 @@ def __neg__(self): return self.__class__(self.basis, -self._data) + @property def size(self): """ Return np.size of underlying data. @@ -254,11 +255,11 @@ def by_indices(self, **kwargs): See `SteerableBasis.indices_mask` for argument details. - :return: `Coef` vector. + :return: Numpy array. """ mask = self.basis.indices_mask(**kwargs) - return self._data[:, mask] + return self._data[..., mask] class Basis: diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 3db5d5f082..e13b922b4d 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -347,8 +347,11 @@ def indices_mask(self, **kwargs): Examples: No args yield all indices. - `angular>=0` selects coefficients with non negative angular indices. + `angular=0 creates a mask for selecting coefficients with zero angular indices. `angular=1, radial=2` selects coefficients satisfying angular index of 1 _and_ radial index of 2. + More advanced operations can combine indices attributes. + `angular=self.angular_indices>=0, radial=r` selects coefficients with non negative angular indices and some radial index `r`. + :return: Boolen mask of shape (`count`,). @@ -400,7 +403,7 @@ def to_real(self, complex_coef): if not isinstance(complex_coef, Coef): raise TypeError( - f"complex_coef should be instanace of `Coef`, received {type(complex_coef)}." + f"complex_coef should be instance of `Coef`, received {type(complex_coef)}." ) if complex_coef.dtype not in (np.complex128, np.complex64): @@ -451,7 +454,7 @@ def to_complex(self, coef): if not isinstance(coef, Coef): raise TypeError( - f"coef should be instanace of `Coef`, received {type(coef)}." + f"coef should be instance of `Coef`, received {type(coef)}." ) if coef.dtype not in (np.float64, np.float32): diff --git a/tests/test_FBbasis2D.py b/tests/test_FBbasis2D.py index 250a2b4f6e..635548b91d 100644 --- a/tests/test_FBbasis2D.py +++ b/tests/test_FBbasis2D.py @@ -89,8 +89,8 @@ def testComplexCoversion(self, basis): def testComplexCoversionErrorsToComplex(self, basis): x = randn(*basis.sz, seed=self.seed).astype(basis.dtype) - # Express in an FB basis - v1 = basis.expand(x) + # Express in an FB basis, cast to array. + v1 = basis.expand(x).asnumpy() # Test catching Errors with raises(TypeError): diff --git a/tests/test_coef.py b/tests/test_coef.py index 2baf33413e..990595e78b 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -24,25 +24,35 @@ def sim_fixture_id(params): return f"stack={stack}, count={count}, dtype={dtype}" -# Dtypes for coef array @pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") def dtype(request): + """ + Dtypes for coef array + """ return request.param -# Dtypes for basis @pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") def basis_dtype(request): + """ + Dtypes for basis + """ return request.param @pytest.fixture(params=IMG_SIZE, ids=lambda x: f"count={x}") def img_size(request): + """ + Image size for basis. + """ return request.param @pytest.fixture(params=STACKS, ids=lambda x: f"stack={x}") def stack(request): + """ + Stack dimensions. + """ return request.param @@ -51,6 +61,9 @@ def stack(request): @pytest.fixture(params=ALLYOURBASES, ids=lambda x: f"basis={x}") def basis(request, img_size, basis_dtype): + """ + Parameterized `Basis` instantiation. + """ cls = request.param return cls(img_size, dtype=basis_dtype) @@ -58,7 +71,7 @@ def basis(request, img_size, basis_dtype): @pytest.fixture def coef_fixture(basis, stack, dtype): """ - Construct testing coefficient array. + Construct parameterized testing coefficient array as `Coef`. """ # Combine the stack and coefficent counts into multidimensional # shape. @@ -69,11 +82,179 @@ def coef_fixture(basis, stack, dtype): return Coef(basis, coef_np, dtype=dtype) +def test_mismatch_count(basis): + """ + Confirm raises when instantiated with incorrect coef vector len. + """ + # Derive an incorrect Coef + x = np.empty(basis.count + 1, basis.dtype) + with pytest.raises(RuntimeError, match=r".*does not match basis count.*"): + _ = Coef(basis, x) + + +def test_incorrect_coef_type(basis): + """ + Confirm raises when instantiated with incorrect coef type. + """ + # Construct incorrect Coef type (list) + x = list(range(basis.count + 1)) + with pytest.raises(ValueError, match=r".*should be instantiated with an.*"): + _ = Coef(basis, x) + + +def test_0dim(basis): + """ + Confirm raises when instantiated with 0dim scalar. + """ + # Construct 0dim scalar object + x = np.array(1) + with pytest.raises(ValueError, match=r".*with shape.*"): + _ = Coef(basis, x) + + +def test_not_a_basis(): + """ + Confirm raises when instantiated with something that is not a Basis. + """ + # Derive an incorrect Coef + x = np.empty(10) + with pytest.raises(TypeError, match=r".*required to be a `Basis`.*"): + _ = Coef(None, x) + + +def test_coef_key_dims(coef_fixture): + """ + Test key lookup out of bounds dimension raises. + """ + dim = coef_fixture.ndim + # Construct a key with too many dims + key = (0,) * (dim + 1) + with pytest.raises(ValueError, match=r".*stack_dim is.*"): + _ = coef_fixture[key] + + +def test_stack_reshape(basis): + """ + Test stack_reshape matches corresponding pure Numpy reshape. + """ + # create a multi dim coef array. + x = np.empty((2, 3, 4, basis.count)) + c = Coef(basis, x) + + # Test -1 flatten + ref = x.reshape(-1, basis.count) + np.testing.assert_allclose(c.stack_reshape(-1).asnumpy(), ref) + # Test 1d flatten + np.testing.assert_allclose(c.stack_reshape(np.prod(x.shape[:-1])).asnumpy(), ref) + # Test 2d flatten tuple + ref = x.reshape(np.prod(x.shape[:-2]), x.shape[-2], basis.count) + np.testing.assert_allclose( + c.stack_reshape((np.prod(x.shape[:-2]), x.shape[-2])).asnumpy(), ref + ) + # Test 2d flatten args + ref = x.reshape(np.prod(x.shape[:-2]), x.shape[-2], basis.count) + np.testing.assert_allclose( + c.stack_reshape(np.prod(x.shape[:-2]), x.shape[-2]).asnumpy(), ref + ) + + +def test_size(coef_fixture): + """ + Confirm size matches. + """ + np.testing.assert_equal(coef_fixture.size, coef_fixture.asnumpy().size) + np.testing.assert_equal(coef_fixture.size, coef_fixture._data.size) + + +# Test basic arithmetic functions + + +def test_add(basis, coef_fixture): + """ + Tests addition operation against pure Numpy. + """ + # Make array + x = np.random.random(size=coef_fixture.shape).astype(coef_fixture.dtype, copy=False) + # Construct Coef + c = Coef(basis, x) + + # Perform operation as array for reference + ref = coef_fixture.asnumpy() + x + + # Perform operation as `Coef` for result + res = coef_fixture + c + + # Compare result with reference + np.testing.assert_allclose(res, ref) + + +def test_sub(basis, coef_fixture): + """ + Tests subtraction operation against pure Numpy. + """ + # Make array + x = np.random.random(size=coef_fixture.shape).astype(coef_fixture.dtype, copy=False) + # Construct Coef + c = Coef(basis, x) + + # Perform operation as array for reference + ref = coef_fixture.asnumpy() - x + + # Perform operation as `Coef` for result + res = coef_fixture - c + + # Compare result with reference + np.testing.assert_allclose(res, ref) + + +def test_neg(basis, coef_fixture): + """ + Tests negation operation against pure Numpy. + """ + # Perform operation as array for reference + ref = -coef_fixture.asnumpy() + + # Perform operation as `Coef` for result + res = -coef_fixture + + # Compare result with reference + np.testing.assert_allclose(res, ref) + + +# Test Passthrough Functions + + +def test_by_indices(coef_fixture, basis): + """ + Test indice passthrough. + """ + keys = [ + dict(), + dict(angular=1), + dict(radial=2), + dict(angular=1, radial=2), + dict(angular=basis.angular_indices > 0), + ] + + for key in keys: + np.testing.assert_allclose( + coef_fixture.by_indices(**key), + coef_fixture.asnumpy()[..., basis.indices_mask(**key)], + ) + + def test_coef_evalute(coef_fixture, basis): - assert np.allclose(coef_fixture.evaluate(), basis.evaluate(coef_fixture)) + """ + Test evaluate pass through. + """ + np.testing.assert_allclose(coef_fixture.evaluate(), basis.evaluate(coef_fixture)) def test_coef_rotate(coef_fixture, basis): + """ + Test rotation pass through. + """ + # Rotations rots = np.linspace(-np.pi, np.pi, coef_fixture.stack_size).reshape( coef_fixture.stack_shape @@ -84,12 +265,45 @@ def test_coef_rotate(coef_fixture, basis): np.random.rand(coef_fixture.stack_size).reshape(coef_fixture.stack_shape) > 0.5 ) # Random bool - assert np.allclose(coef_fixture.rotate(rots), basis.rotate(coef_fixture, rots)) + np.testing.assert_allclose( + coef_fixture.rotate(rots), basis.rotate(coef_fixture, rots) + ) - assert np.allclose( + np.testing.assert_allclose( coef_fixture.rotate(rots, refl), basis.rotate(coef_fixture, rots, refl) ) -def test_coef_shift(coef_fixture): - pass +# Test related Basis Coef checks. +# These are easier to test here via parameterization. +def test_evaluate_incorrect_type(coef_fixture, basis): + """ + Test that evaluate raises when passed non Coef type. + """ + with pytest.raises(TypeError, match=r".*should be passed a `Coef`.*"): + # Pass something that is not a Coef, eg Numpy array. + basis.evaluate(coef_fixture.asnumpy()) + + +def test_to_real_incorrect_type(coef_fixture, basis): + """ + Test to_real conversion raises on non `Coef` type. + """ + # Convert Coef to complex, then to Numpy. + x = basis.to_complex(coef_fixture).asnumpy() + + # Call to_real with Numpy array + with pytest.raises(TypeError, match=r".*should be instance of `Coef`.*"): + _ = basis.to_real(x) + + +def test_to_complex_incorrect_type(coef_fixture, basis): + """ + Test to_complex conversion raises on non `Coef` type. + """ + # Convert Coef to Numpy. + x = coef_fixture.asnumpy() + + # Call to_complex with Numpy array + with pytest.raises(TypeError, match=r".*should be instance of `Coef`.*"): + _ = basis.to_complex(x) From c4eb963f647b27c7906bca88febc11eea02aeb52 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 19 Sep 2023 13:19:31 -0400 Subject: [PATCH 072/294] Extend Coef unit tests to all steerable basis subclass --- src/aspire/basis/fle_2d.py | 20 ++++++++++++++++++++ src/aspire/basis/steerable.py | 8 ++++---- tests/test_coef.py | 30 +++++++++++++++++++++++++----- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index b427a36234..c1690b4d27 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -122,6 +122,26 @@ def _build_indices(self): self.radial_indices = self._fle_radial_indices[self._fle_to_fb_indices] # Note we negate the FLE signs? self.signs_indices = self._fle_signs_indices[self._fle_to_fb_indices] + # These map indices in complex array to pair of indices in real array + self.complex_count = sum(self.k_max) + self._pos = np.zeros(self.complex_count, dtype=int) + self._neg = np.zeros(self.complex_count, dtype=int) + i = 0 + ci = 0 + for ell in range(self.ell_max + 1): + sgns = (1,) if ell == 0 else (1, -1) + ks = np.arange(0, self.k_max[ell]) + + for sgn in sgns: + rng = np.arange(i, i + len(ks)) + if sgn == 1: + self._pos[ci + ks] = rng + elif sgn == -1: + self._neg[ci + ks] = rng + + i += len(ks) + + ci += len(ks) def indices(self): """ diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index e13b922b4d..50307b0d67 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -424,16 +424,16 @@ def to_real(self, complex_coef): ind += np.size(idx) ind_pos = ind - coef[:, idx] = complex_coef[:, idx].real + coef[..., idx] = complex_coef[..., idx].real for ell in range(1, self.ell_max + 1): idx = ind + np.arange(self.k_max[ell], dtype=int) idx_pos = ind_pos + np.arange(self.k_max[ell], dtype=int) idx_neg = idx_pos + self.k_max[ell] - c = complex_coef[:, idx] - coef[:, idx_pos] = 2.0 * np.real(c) - coef[:, idx_neg] = -2.0 * np.imag(c) + c = complex_coef[..., idx] + coef[..., idx_pos] = 2.0 * np.real(c) + coef[..., idx_neg] = -2.0 * np.imag(c) ind += np.size(idx) ind_pos += 2 * self.k_max[ell] diff --git a/tests/test_coef.py b/tests/test_coef.py index 990595e78b..eaf292f461 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -1,7 +1,14 @@ import numpy as np import pytest -from aspire.basis import Coef, FFBBasis2D +from aspire.basis import ( + Coef, + FBBasis2D, + FFBBasis2D, + FLEBasis2D, + FPSWFBasis2D, + PSWFBasis2D, +) IMG_SIZE = [ 32, @@ -18,6 +25,14 @@ (3, 4), ] +ALLYOURBASES = [ + FBBasis2D, + FFBBasis2D, + PSWFBasis2D, + FPSWFBasis2D, + FLEBasis2D, +] + def sim_fixture_id(params): stack, count, dtype = params @@ -56,9 +71,6 @@ def stack(request): return request.param -ALLYOURBASES = [FFBBasis2D] - - @pytest.fixture(params=ALLYOURBASES, ids=lambda x: f"basis={x}") def basis(request, img_size, basis_dtype): """ @@ -247,7 +259,9 @@ def test_coef_evalute(coef_fixture, basis): """ Test evaluate pass through. """ - np.testing.assert_allclose(coef_fixture.evaluate(), basis.evaluate(coef_fixture)) + np.testing.assert_allclose( + coef_fixture.evaluate(), basis.evaluate(coef_fixture), rtol=1e-05, atol=1e-08 + ) def test_coef_rotate(coef_fixture, basis): @@ -307,3 +321,9 @@ def test_to_complex_incorrect_type(coef_fixture, basis): # Call to_complex with Numpy array with pytest.raises(TypeError, match=r".*should be instance of `Coef`.*"): _ = basis.to_complex(x) + + +def test_real_complex_real_roundtrip(coef_fixture, basis): + rcoef = basis.to_real(basis.to_complex(coef_fixture)) + + np.testing.assert_allclose(rcoef, coef_fixture, rtol=1e-05, atol=1e-08) From a7af697efed5f16d612fca2d9f19e2c21065ba8f Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 19 Sep 2023 14:15:00 -0400 Subject: [PATCH 073/294] more touchups --- src/aspire/basis/basis.py | 3 ++- tests/test_coef.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index d42e1887a6..e0bf37f802 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -26,12 +26,13 @@ def __init__(self, basis, data, dtype=None): The last axes always represents the coefficient `count`. + :param basis: `Basis` associated with `data` coefficients. :param data: Numpy array containing image data with shape `(..., count)`. :param dtype: Optionally cast `data` to this dtype. Defaults to `data.dtype`. - :return: Image instance holding `data`. + :return: `Coef` instance holding `data`. """ if not isinstance(data, np.ndarray): diff --git a/tests/test_coef.py b/tests/test_coef.py index eaf292f461..6f5d618e98 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -145,6 +145,23 @@ def test_coef_key_dims(coef_fixture): _ = coef_fixture[key] +def test_incorrect_reshape(basis): + """ + Confirm raises when attempting incorrect stack reshape. + """ + + # create a multi dim coef array. + x = np.empty((2, 3, 4, basis.count)) + c = Coef(basis, x) + + # Alter the stack shape, creating an incorrect shape. + shp = list(c.stack_shape) + shp[0] = shp[0] + 1 + + with pytest.raises(ValueError, match=r".*cannot be reshaped to.*"): + _ = c.stack_reshape(*shp) + + def test_stack_reshape(basis): """ Test stack_reshape matches corresponding pure Numpy reshape. @@ -233,6 +250,25 @@ def test_neg(basis, coef_fixture): np.testing.assert_allclose(res, ref) +def test_mul(basis, coef_fixture): + """ + Tests multiplication operation against pure Numpy. + """ + # Make array + x = np.random.random(size=coef_fixture.shape).astype(coef_fixture.dtype, copy=False) + # Construct Coef + c = Coef(basis, x) + + # Perform operation as array for reference + ref = coef_fixture.asnumpy() * x + + # Perform operation as `Coef` for result + res = coef_fixture * c + + # Compare result with reference + np.testing.assert_allclose(res, ref) + + # Test Passthrough Functions From b4c94a63bc8621d60c7fdce9868abd175bf8f508 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 19 Sep 2023 14:26:28 -0400 Subject: [PATCH 074/294] evaluate test tolerance (floats) --- tests/test_coef.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_coef.py b/tests/test_coef.py index 6f5d618e98..dc29b4701a 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -9,6 +9,7 @@ FPSWFBasis2D, PSWFBasis2D, ) +from aspire.utils import utest_tolerance IMG_SIZE = [ 32, @@ -296,7 +297,10 @@ def test_coef_evalute(coef_fixture, basis): Test evaluate pass through. """ np.testing.assert_allclose( - coef_fixture.evaluate(), basis.evaluate(coef_fixture), rtol=1e-05, atol=1e-08 + coef_fixture.evaluate(), + basis.evaluate(coef_fixture), + rtol=1e-05, + atol=utest_tolerance(basis.dtype), ) From bed8b5e36ecd889294dd13d4f26c7eef0e648fd8 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 19 Sep 2023 14:30:31 -0400 Subject: [PATCH 075/294] scope the fixtures to resuse instead of parameterizing --- tests/test_coef.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_coef.py b/tests/test_coef.py index dc29b4701a..d24d0ccdac 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -12,12 +12,12 @@ from aspire.utils import utest_tolerance IMG_SIZE = [ + 31, 32, - pytest.param(31, marks=pytest.mark.expensive), ] DTYPES = [ + np.float32, np.float64, - pytest.param(np.float32, marks=pytest.mark.expensive), ] STACKS = [ (), @@ -40,7 +40,7 @@ def sim_fixture_id(params): return f"stack={stack}, count={count}, dtype={dtype}" -@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}", scope="module") def dtype(request): """ Dtypes for coef array @@ -48,7 +48,7 @@ def dtype(request): return request.param -@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}", scope="module") def basis_dtype(request): """ Dtypes for basis @@ -56,7 +56,7 @@ def basis_dtype(request): return request.param -@pytest.fixture(params=IMG_SIZE, ids=lambda x: f"count={x}") +@pytest.fixture(params=IMG_SIZE, ids=lambda x: f"count={x}", scope="module") def img_size(request): """ Image size for basis. @@ -64,7 +64,7 @@ def img_size(request): return request.param -@pytest.fixture(params=STACKS, ids=lambda x: f"stack={x}") +@pytest.fixture(params=STACKS, ids=lambda x: f"stack={x}", scope="module") def stack(request): """ Stack dimensions. @@ -72,7 +72,7 @@ def stack(request): return request.param -@pytest.fixture(params=ALLYOURBASES, ids=lambda x: f"basis={x}") +@pytest.fixture(params=ALLYOURBASES, ids=lambda x: f"basis={x}", scope="module") def basis(request, img_size, basis_dtype): """ Parameterized `Basis` instantiation. @@ -81,7 +81,7 @@ def basis(request, img_size, basis_dtype): return cls(img_size, dtype=basis_dtype) -@pytest.fixture +@pytest.fixture(scope="module") def coef_fixture(basis, stack, dtype): """ Construct parameterized testing coefficient array as `Coef`. From bb4b0be4bc6aea3ac9f2d013f7ad2adc99270e62 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 21 Sep 2023 09:52:05 -0400 Subject: [PATCH 076/294] strings --- src/aspire/basis/basis.py | 11 ++++++----- src/aspire/basis/fle_2d.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index e0bf37f802..f7d11260ce 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -132,7 +132,7 @@ def stack_reshape(self, *args): # Sanity check the size if shape != (-1,) and np.prod(shape) != self.stack_size: raise ValueError( - f"Number of images {self.stack_size} cannot be reshaped to {shape}." + f"Number of coefficient vectors {self.stack_size} cannot be reshaped to {shape}." ) return self.__class__( @@ -335,13 +335,15 @@ def evaluate(self, v): """ Evaluate coefficient vector in basis - :param v: A coefficient vector (or an array of coefficient vectors) - to be evaluated. The first dimension must correspond to the number of - coefficient vectors, while the second must correspond to `self.count` + :param v: `Coef` instance containing the coefficients to be + evaluated. The first dimension must correspond to the + number of coefficient vectors, while the second must + correspond to `self.count`. :return: The evaluation of the coefficient vector(s) `v` for this basis. This is an Image or a Volume object containing one image/volume for each coefficient vector, and of size `self.sz`. """ + if v.dtype != self.coefficient_dtype: logger.warning( f"{self.__class__.__name__}::evaluate" @@ -349,7 +351,6 @@ def evaluate(self, v): ) if not isinstance(v, Coef): - # v = Coef(self, v) raise TypeError(f"`evaluate` should be passed a `Coef`, received {type(v)}") # Flatten stack diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index c1690b4d27..4683f88276 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -672,7 +672,7 @@ def lowpass(self, coeffs, bandlimit): """ Apply a low-pass filter to FLE coefficients `coeffs` with threshold `bandlimit`. - :param coeffs: A NumPy array of FLE coefficients, of shape (num_images, self.count) + :param coeffs: A `Coef` instance containing FLE coefficients. :param bandlimit: Integer bandlimit (max frequency). :return: Band-limited coefficient array. """ @@ -697,7 +697,7 @@ def radial_convolve(self, coeffs, radial_img): """ Convolve a stack of FLE coefficients with a 2D radial function. - :param coeffs: A NumPy array of FLE coefficients of size (num_images, self.count). + :param coeffs: A `Coef` instance containing FLE coefficients. :param radial_img: A 2D NumPy array of size (self.nres, self.nres). :return: Convolved FLE coefficients. """ From bc5656696ffddf43b8668793e9a1d74cb6d3e55d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 21 Sep 2023 09:52:58 -0400 Subject: [PATCH 077/294] tmp variable name change. --- src/aspire/basis/pswf_2d.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index e62bee3f81..dce38536c9 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -221,8 +221,8 @@ def _evaluate_t(self, images): :return: The evaluation of the coefficient array in the PSWF basis. """ flattened_images = images[:, self._disk_mask] - ccoef = Coef(self, flattened_images @ self.samples_conj_transpose) - return self.to_real(ccoef).asnumpy() + complex_coef = Coef(self, flattened_images @ self.samples_conj_transpose) + return self.to_real(complex_coef).asnumpy() def _evaluate(self, coefficients): """ From 4e47256278e6b61f8634f92f7b4d47298ab32080 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 21 Sep 2023 11:31:47 -0400 Subject: [PATCH 078/294] change dim expansion expression --- src/aspire/basis/steerable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 50307b0d67..148d756efe 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -182,7 +182,7 @@ def rotate(self, coef, radians, refl=None): if radians.ndim < 2: radians = radians.reshape(-1, 1) else: - radians = np.atleast_3d(radians) + radians = np.expand_dims(radians, axis=-1) if radians.size != np.prod(coef.shape[:-1]): raise RuntimeError( From 511282b27bf86fcfebde7ddf64021c9be7e9ae28 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 21 Sep 2023 13:16:58 -0400 Subject: [PATCH 079/294] spell out sizes in unit test comment --- tests/test_coef.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_coef.py b/tests/test_coef.py index d24d0ccdac..97d963639e 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -176,12 +176,12 @@ def test_stack_reshape(basis): np.testing.assert_allclose(c.stack_reshape(-1).asnumpy(), ref) # Test 1d flatten np.testing.assert_allclose(c.stack_reshape(np.prod(x.shape[:-1])).asnumpy(), ref) - # Test 2d flatten tuple + # Test 2d reshape tuple (2,3,4) ~> ((6,4)) ref = x.reshape(np.prod(x.shape[:-2]), x.shape[-2], basis.count) np.testing.assert_allclose( c.stack_reshape((np.prod(x.shape[:-2]), x.shape[-2])).asnumpy(), ref ) - # Test 2d flatten args + # Test 2d reshape args (2,3,4) ~> (6,4) ref = x.reshape(np.prod(x.shape[:-2]), x.shape[-2], basis.count) np.testing.assert_allclose( c.stack_reshape(np.prod(x.shape[:-2]), x.shape[-2]).asnumpy(), ref From 466621eef23ad5f185b2db8667f07e2509167b34 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 27 Sep 2023 10:58:47 -0400 Subject: [PATCH 080/294] change __len__ behavior --- src/aspire/basis/basis.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index f7d11260ce..6f75e9da29 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -81,11 +81,9 @@ def __init__(self, basis, data, dtype=None): def __len__(self): """ - Return stack length. - - Note this is product of all stack dimensions. + Return length of slowest stack axis. """ - return self.stack_size + return self.stack_shape[0] def asnumpy(self): """ From 6e2317ad3b293922f3d22054f0b988cef26a43e5 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 27 Sep 2023 11:02:27 -0400 Subject: [PATCH 081/294] string changes --- src/aspire/basis/fle_2d.py | 76 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 4683f88276..034c5a0f33 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -482,7 +482,7 @@ def _create_basis_functions(self): self.norm_constants = norm_constants self.basis_functions = basis_functions - def _evaluate(self, coeffs): + def _evaluate(self, coefs): """ Evaluates FLE coefficients and return in standard 2D Cartesian coordinates. @@ -491,10 +491,10 @@ def _evaluate(self, coeffs): :return: An Image object containing the corresponding images. """ # convert from FB order - coeffs = coeffs[..., self._fb_to_fle_indices] + coefs = coefs[..., self._fb_to_fle_indices] # See Remark 3.3 and Section 3.4 - betas = self._step3(coeffs) + betas = self._step3(coefs) z = self._step2(betas) im = self._step1(z) return im.astype(self.dtype) @@ -512,11 +512,11 @@ def _evaluate_t(self, imgs): imgs[:, self.radial_mask] = 0 z = self._step1_t(imgs) b = self._step2_t(z) - coeffs = self._step3_t(b) + coefs = self._step3_t(b) # return in FB order - coeffs = coeffs[..., self._fle_to_fb_indices] - return coeffs.astype(self.coefficient_dtype, copy=False) + coefs = coefs[..., self._fle_to_fb_indices] + return coefs.astype(self.coefficient_dtype, copy=False) def _step1_t(self, im): """ @@ -570,31 +570,31 @@ def _step3_t(self, betas): betas = idct(betas, axis=1, type=2) * 2 * betas.shape[1] betas = np.moveaxis(betas, 0, -1) - coeffs = np.zeros((self.count, num_img), dtype=np.float64) + coefs = np.zeros((self.count, num_img), dtype=np.float64) for i in range(self.ell_p_max + 1): - coeffs[self.idx_list[i]] = self.A3[i] @ betas[:, i, :] - coeffs = coeffs.T + coefs[self.idx_list[i]] = self.A3[i] @ betas[:, i, :] + coefs = coefs.T - return coeffs * self.norm_constants / self.h + return coefs * self.norm_constants / self.h - def _step3(self, coeffs): + def _step3(self, coefs): """ Adjoint of _step3_t and Step 1 of the forward transformation (coefficients to images). Uses barycenteric interpolation in reverse to compute values of Betas at Chebyshev nodes, given an array of FLE coefficients. """ - coeffs = coeffs.copy().reshape(-1, self.count) - num_img = coeffs.shape[0] - coeffs *= self.h * self.norm_constants - coeffs = coeffs.T + coefs = coefs.copy().reshape(-1, self.count) + num_img = coefs.shape[0] + coefs *= self.h * self.norm_constants + coefs = coefs.T out = np.zeros( (self.num_interp, 2 * self.max_ell + 1, num_img), dtype=np.float64, ) for i in range(self.ell_p_max + 1): - out[:, i, :] = self.A3_T[i] @ coeffs[self.idx_list[i]] + out[:, i, :] = self.A3_T[i] @ coefs[self.idx_list[i]] out = np.moveaxis(out, -1, 0) if self.num_interp > self.num_radial_nodes: out = dct(out, axis=1, type=2) @@ -668,71 +668,71 @@ def _create_dense_matrix(self): return B - def lowpass(self, coeffs, bandlimit): + def lowpass(self, coefs, bandlimit): """ - Apply a low-pass filter to FLE coefficients `coeffs` with threshold `bandlimit`. + Apply a low-pass filter to FLE coefficients `coefs` with threshold `bandlimit`. - :param coeffs: A `Coef` instance containing FLE coefficients. + :param coefs: A `Coef` instance containing FLE coefficients. :param bandlimit: Integer bandlimit (max frequency). :return: Band-limited coefficient array. """ - if not isinstance(coeffs, Coef): + if not isinstance(coefs, Coef): raise TypeError( - f"`coeffs` should be a `Coef` instance, received {type(coeffs)}." + f"`coefs` should be a `Coef` instance, received {type(coefs)}." ) - # Copy to mutate the coeffs. - coeffs = coeffs.asnumpy().copy() + # Copy to mutate the coefs. + coefs = coefs.asnumpy().copy() k = self.count - 1 for _ in range(self.count): if self.bessel_zeros[k] / (np.pi) > (bandlimit - 1) // 2: k = k - 1 - coeffs[:, k + 1 :] = 0 + coefs[:, k + 1 :] = 0 - return Coef(self, coeffs) + return Coef(self, coefs) - def radial_convolve(self, coeffs, radial_img): + def radial_convolve(self, coefs, radial_img): """ Convolve a stack of FLE coefficients with a 2D radial function. - :param coeffs: A `Coef` instance containing FLE coefficients. + :param coefs: A `Coef` instance containing FLE coefficients. :param radial_img: A 2D NumPy array of size (self.nres, self.nres). :return: Convolved FLE coefficients. """ - if not isinstance(coeffs, Coef): + if not isinstance(coefs, Coef): raise TypeError( - f"`coeffs` should be a `Coef` instance, received {type(coeffs)}." + f"`coefs` should be a `Coef` instance, received {type(coefs)}." ) - if len(coeffs.stack_shape) > 1: + if len(coefs.stack_shape) > 1: raise NotImplementedError( "`radial_convolve` currently only implemented for 1D stacks." ) - coeffs = coeffs.asnumpy() + coefs = coefs.asnumpy() - num_img = coeffs.shape[0] - coeffs_conv = np.zeros(coeffs.shape) + num_img = coefs.shape[0] + coefs_conv = np.zeros(coefs.shape) # Convert to internal FLE indices ordering - coeffs = coeffs[..., self._fb_to_fle_indices] + coefs = coefs[..., self._fb_to_fle_indices] for k in range(num_img): - _coeffs = coeffs[k, :] + _coefs = coefs[k, :] z = self._step1_t(radial_img) b = self._step2_t(z) weights = self._radial_convolve_weights(b) b = weights / (self.h**2) b = b.reshape(self.count) - coeffs_conv[k, :] = np.real(self.c2r @ (b * (self.r2c @ _coeffs).flatten())) + coefs_conv[k, :] = np.real(self.c2r @ (b * (self.r2c @ _coefs).flatten())) # Convert from internal FLE ordering to FB convention - coeffs_conv = coeffs_conv[..., self._fle_to_fb_indices] + coefs_conv = coefs_conv[..., self._fle_to_fb_indices] - return Coef(self, coeffs_conv) + return Coef(self, coefs_conv) def _radial_convolve_weights(self, b): """ From 717ef413f1cba7508a7ddd6a9e9509ff0f3178cd Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 27 Sep 2023 11:05:53 -0400 Subject: [PATCH 082/294] revert eigval return type --- src/aspire/basis/fspca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index a2c6d93999..e57611e476 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -571,7 +571,7 @@ def eigvals(self): """ Return the eigenvals as a Coef instance of FSPCABasis. """ - return Coef(self, self._eigvals) + return self._eigvals def eigen_images(self): """ From b58af53631933b415134c2178b2f737ed8f29158 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 27 Sep 2023 11:13:24 -0400 Subject: [PATCH 083/294] string changes --- .../tutorials/tutorials/cov2d_simulation.py | 32 +- .../tutorials/tutorials/image_expansion.py | 16 +- src/aspire/abinitio/commonline_base.py | 4 +- src/aspire/basis/fpswf_2d.py | 6 +- src/aspire/basis/fspca.py | 8 +- .../classification/legacy_implementations.py | 22 +- src/aspire/classification/rir_class2d.py | 22 +- src/aspire/covariance/covar.py | 44 +-- src/aspire/covariance/covar2d.py | 326 +++++++++--------- src/aspire/ctf/ctf_estimator.py | 16 +- src/aspire/denoising/denoiser_cov2d.py | 14 +- src/aspire/image/image.py | 4 +- src/aspire/reconstruction/estimator.py | 18 +- src/aspire/reconstruction/mean.py | 28 +- tests/test_BlkDiagMatrix.py | 54 +-- tests/test_FBbasis2D.py | 4 +- tests/test_FBbasis3D.py | 10 +- tests/test_FFBbasis2D.py | 8 +- tests/test_FFBbasis3D.py | 18 +- tests/test_FLEbasis2D.py | 84 ++--- tests/test_FPSWFbasis2D.py | 22 +- tests/test_PSWFbasis2D.py | 20 +- tests/test_batched_covar2d.py | 62 ++-- tests/test_class2D.py | 6 +- tests/test_coef.py | 2 +- tests/test_covar2d.py | 102 +++--- tests/test_mean_estimator.py | 12 +- tests/test_weighted_mean_estimator.py | 12 +- 28 files changed, 483 insertions(+), 493 deletions(-) diff --git a/gallery/tutorials/tutorials/cov2d_simulation.py b/gallery/tutorials/tutorials/cov2d_simulation.py index b75450e138..e0ed8328e0 100644 --- a/gallery/tutorials/tutorials/cov2d_simulation.py +++ b/gallery/tutorials/tutorials/cov2d_simulation.py @@ -140,8 +140,8 @@ # ``basis.evaluate_t``. logger.info("Get coefficients of clean and noisy images in FFB basis.") -coeff_clean = ffbbasis.evaluate_t(imgs_clean) -coeff_noise = ffbbasis.evaluate_t(imgs_noise) +coef_clean = ffbbasis.evaluate_t(imgs_clean) +coef_noise = ffbbasis.evaluate_t(imgs_noise) # %% # Create Cov2D Object and Calculate Mean and Variance for Clean Images @@ -161,8 +161,8 @@ "Get 2D covariance matrices of clean and noisy images using FB coefficients." ) cov2d = RotCov2D(ffbbasis) -mean_coeff = cov2d.get_mean(coeff_clean) -covar_coeff = cov2d.get_covar(coeff_clean, mean_coeff, noise_var=0) +mean_coef = cov2d.get_mean(coef_clean) +covar_coef = cov2d.get_covar(coef_clean, mean_coef, noise_var=0) # %% # Estimate mean and covariance for noisy images with CTF and shrink method @@ -184,12 +184,12 @@ "precision": "float64", "preconditioner": "identity", } -mean_coeff_est = cov2d.get_mean(coeff_noise, h_ctf_fb, h_idx) -covar_coeff_est = cov2d.get_covar( - coeff_noise, +mean_coef_est = cov2d.get_mean(coef_noise, h_ctf_fb, h_idx) +covar_coef_est = cov2d.get_covar( + coef_noise, h_ctf_fb, h_idx, - mean_coeff_est, + mean_coef_est, noise_var=noise_var, covar_est_opt=covar_opt, ) @@ -203,17 +203,17 @@ # the lowest expected mean square error out of all linear estimators. logger.info("Get the CWF coefficients of noising images.") -coeff_est = cov2d.get_cwf_coeffs( - coeff_noise, +coef_est = cov2d.get_cwf_coefs( + coef_noise, h_ctf_fb, h_idx, - mean_coeff=mean_coeff_est, - covar_coeff=covar_coeff_est, + mean_coef=mean_coef_est, + covar_coef=covar_coef_est, noise_var=noise_var, ) # Convert Fourier-Bessel coefficients back into 2D images -imgs_est = ffbbasis.evaluate(coeff_est) +imgs_est = ffbbasis.evaluate(coef_est) # %% # Evaluate the Results @@ -221,12 +221,12 @@ # Calculate the difference between the estimated covariance and the "true" # covariance estimated from the clean Fourier-Bessel coefficients. -covar_coeff_diff = covar_coeff - covar_coeff_est +covar_coef_diff = covar_coef - covar_coef_est # Calculate the deviation between the clean estimates and those obtained from # the noisy, filtered images. -diff_mean = anorm(mean_coeff_est - mean_coeff) / anorm(mean_coeff) -diff_covar = covar_coeff_diff.norm() / covar_coeff.norm() +diff_mean = anorm(mean_coef_est - mean_coef) / anorm(mean_coef) +diff_covar = covar_coef_diff.norm() / covar_coef.norm() # Calculate the normalized RMSE of the estimated images. nrmse_ims = (imgs_est - imgs_clean).norm() / imgs_clean.norm() diff --git a/gallery/tutorials/tutorials/image_expansion.py b/gallery/tutorials/tutorials/image_expansion.py index 93adaec129..4c3f36aac4 100644 --- a/gallery/tutorials/tutorials/image_expansion.py +++ b/gallery/tutorials/tutorials/image_expansion.py @@ -51,13 +51,13 @@ # Get the expansion coefficients based on FB basis logger.info("Start normal FB expansion of original images.") tstart = timeit.default_timer() -fb_coeffs = fb_basis.evaluate_t(org_images) +fb_coefs = fb_basis.evaluate_t(org_images) tstop = timeit.default_timer() dtime = tstop - tstart logger.info(f"Finish normal FB expansion of original images in {dtime:.4f} seconds.") # Reconstruct images from the expansion coefficients based on FB basis -fb_images = fb_basis.evaluate(fb_coeffs).asnumpy() +fb_images = fb_basis.evaluate(fb_coefs).asnumpy() logger.info("Finish reconstruction of images from normal FB expansion coefficients.") # Calculate the mean value of maximum differences between the FB estimated images and the original images @@ -94,13 +94,13 @@ # Get the expansion coefficients based on fast FB basis logger.info("start fast FB expansion of original images.") tstart = timeit.default_timer() -ffb_coeffs = ffb_basis.evaluate_t(org_images) +ffb_coefs = ffb_basis.evaluate_t(org_images) tstop = timeit.default_timer() dtime = tstop - tstart logger.info(f"Finish fast FB expansion of original images in {dtime:.4f} seconds.") # Reconstruct images from the expansion coefficients based on fast FB basis -ffb_images = ffb_basis.evaluate(ffb_coeffs).asnumpy() +ffb_images = ffb_basis.evaluate(ffb_coefs).asnumpy() logger.info("Finish reconstruction of images from fast FB expansion coefficients.") # Calculate the mean value of maximum differences between the fast FB estimated images to the original images @@ -138,13 +138,13 @@ # Get the expansion coefficients based on direct PSWF basis logger.info("Start direct PSWF expansion of original images.") tstart = timeit.default_timer() -pswf_coeffs = pswf_basis.evaluate_t(org_images) +pswf_coefs = pswf_basis.evaluate_t(org_images) tstop = timeit.default_timer() dtime = tstop - tstart logger.info(f"Finish direct PSWF expansion of original images in {dtime:.4f} seconds.") # Reconstruct images from the expansion coefficients based on direct PSWF basis -pswf_images = pswf_basis.evaluate(pswf_coeffs).asnumpy() +pswf_images = pswf_basis.evaluate(pswf_coefs).asnumpy() logger.info("Finish reconstruction of images from direct PSWF expansion coefficients.") # Calculate the mean value of maximum differences between direct PSWF estimated images and original images @@ -182,13 +182,13 @@ # Get the expansion coefficients based on fast PSWF basis logger.info("Start fast PSWF expansion of original images.") tstart = timeit.default_timer() -fpswf_coeffs = fpswf_basis.evaluate_t(org_images) +fpswf_coefs = fpswf_basis.evaluate_t(org_images) tstop = timeit.default_timer() dtime = tstop - tstart logger.info(f"Finish fast PSWF expansion of original images in {dtime:.4f} seconds.") # Reconstruct images from the expansion coefficients based on direct PSWF basis -fpswf_images = fpswf_basis.evaluate(fpswf_coeffs).asnumpy() +fpswf_images = fpswf_basis.evaluate(fpswf_coefs).asnumpy() logger.info("Finish reconstruction of images from fast PSWF expansion coefficients.") # Calculate mean value of maximum differences between the fast PSWF estimated images and the original images diff --git a/src/aspire/abinitio/commonline_base.py b/src/aspire/abinitio/commonline_base.py index ca79b05ec6..485cbe95a8 100644 --- a/src/aspire/abinitio/commonline_base.py +++ b/src/aspire/abinitio/commonline_base.py @@ -370,7 +370,7 @@ def _get_shift_equations_approx(self, equations_factor=1, max_memory=4000): shift_b[shift_eq_idx] = dx # Compute the coefficients of the current equation - coeffs = np.array( + coefs = np.array( [ np.sin(shift_alpha), np.cos(shift_alpha), @@ -378,7 +378,7 @@ def _get_shift_equations_approx(self, equations_factor=1, max_memory=4000): -np.cos(shift_beta), ] ) - shift_eq[idx] = -1 * coeffs if is_pf_j_flipped else coeffs + shift_eq[idx] = -1 * coefs if is_pf_j_flipped else coefs # create sparse matrix object only containing non-zero elements shift_equations = sparse.csr_matrix( diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index daaf0a914b..deed21bb61 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -355,14 +355,14 @@ def _pswf_integration(self, images_nufft): r_n_eval_mat = r_n_eval_mat.reshape( (len(self.radial_quad_pts) * self.n_max, num_images), order="F" ) - coeff_vec_quad = np.zeros( + coef_vec_quad = np.zeros( (num_images, len(self.complex_angular_indices)), dtype=complex_type(self.dtype), ) m = self.pswf_radial_quad.shape[1] for i in range(self.n_max): - coeff_vec_quad[ + coef_vec_quad[ :, self.indices_for_n[i] + np.arange(self.numel_for_n[i]) ] = np.dot(self.blk_r[i], r_n_eval_mat[i * m : (i + 1) * m, :]).T - return coeff_vec_quad + return coef_vec_quad diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index e57611e476..1974ed2757 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -42,7 +42,7 @@ def __init__( Default value of `None` will use `self.basis.count`. :param noise_var: Optionally assign noise variance. Default value of `None` will estimate noise with WhiteNoiseEstimator. - Use 0 when using clean images so cov2d skips applying noisy covar coeffs.. + Use 0 when using clean images so cov2d skips applying noisy covar coefs.. :param batch_size: Batch size for computing basis coefficients. `batch_size` is also passed to BatchedRotCov2D. """ @@ -162,7 +162,7 @@ def build(self): } self.mean_coef_est = cov2d.get_mean() self.covar_coef_est = cov2d.get_covar( - mean_coeff=self.mean_coef_est, + mean_coef=self.mean_coef_est, noise_var=self.noise_var, covar_est_opt=covar_opt, ) @@ -471,7 +471,7 @@ def to_complex(self, coef): There is a corresponding method, to_real. :param coef: Coefficients from this basis. - :return: Complex coefficent representation from this basis. + :return: Complex coeficent representation from this basis. """ if not isinstance(coef, Coef): raise TypeError(f"'coef' should be `Coef` instance, received {type(coef)}.") @@ -523,7 +523,7 @@ def to_real(self, complex_coef): There is a corresponding method, to_complex. :param complex_coef: Complex coefficients from this basis. - :return: Real coefficent representation from this basis. + :return: Real coeficent representation from this basis. """ if complex_coef.ndim == 1: diff --git a/src/aspire/classification/legacy_implementations.py b/src/aspire/classification/legacy_implementations.py index e627c19742..5fb94649bf 100644 --- a/src/aspire/classification/legacy_implementations.py +++ b/src/aspire/classification/legacy_implementations.py @@ -125,18 +125,18 @@ def bispec_operator_1(freqs): return o1, o2 -def bispec_2drot_large(coeff, freqs, eigval, alpha, sample_n, seed=None): +def bispec_2drot_large(coef, freqs, eigval, alpha, sample_n, seed=None): """ alpha 1/3 sample_n 4000 """ freqs_not_zero = freqs != 0 - coeff_norm = np.log(np.power(np.absolute(coeff[freqs_not_zero]), alpha)) - if np.any(coeff_norm == float("-inf")): - raise ValueError("coeff_norm should not be -inf") + coef_norm = np.log(np.power(np.absolute(coef[freqs_not_zero]), alpha)) + if np.any(coef_norm == float("-inf")): + raise ValueError("coef_norm should not be -inf") - phase = coeff[freqs_not_zero] / np.absolute(coeff[freqs_not_zero]) + phase = coef[freqs_not_zero] / np.absolute(coef[freqs_not_zero]) phase = np.arctan2(np.imag(phase), np.real(phase)) eigval = eigval[freqs_not_zero] o1, o2 = bispec_operator_1(freqs[freqs_not_zero]) @@ -151,15 +151,15 @@ def bispec_2drot_large(coeff, freqs, eigval, alpha, sample_n, seed=None): m_id = np.where(x < sample_n * p_m)[0] o1 = o1[m_id] o2 = o2[m_id] - m = np.exp(o1 * coeff_norm + 1j * o2 * phase) + m = np.exp(o1 * coef_norm + 1j * o2 * phase) # svd of the reduced bispectrum u, s, v = pca_y(m, 300, seed=seed) - coeff_b = np.einsum("i, ij -> ij", s, np.conjugate(v)) - coeff_b_r = np.conjugate(u.T).dot(np.conjugate(m)) + coef_b = np.einsum("i, ij -> ij", s, np.conjugate(v)) + coef_b_r = np.conjugate(u.T).dot(np.conjugate(m)) - coeff_b = coeff_b / np.linalg.norm(coeff_b, axis=0) - coeff_b_r = coeff_b_r / np.linalg.norm(coeff_b_r, axis=0) + coef_b = coef_b / np.linalg.norm(coef_b, axis=0) + coef_b_r = coef_b_r / np.linalg.norm(coef_b_r, axis=0) - return coeff_b, coeff_b_r + return coef_b, coef_b_r diff --git a/src/aspire/classification/rir_class2d.py b/src/aspire/classification/rir_class2d.py index a8445e5932..e0db1bf8a5 100644 --- a/src/aspire/classification/rir_class2d.py +++ b/src/aspire/classification/rir_class2d.py @@ -260,7 +260,7 @@ def bispectrum(self, coef): # _bispectrum is assigned during initialization. return self._bispectrum(coef) - def _sk_nn_classification(self, coeff_b, coeff_b_r): + def _sk_nn_classification(self, coef_b, coef_b_r): """ Perform nearest neighbor classification using scikit learn. @@ -275,10 +275,10 @@ def _sk_nn_classification(self, coeff_b, coeff_b_r): # so we'll pretend we have 2*n_features of real values. # Don't worry about the copy because NearestNeighbors wants # C-contiguous anyway... (it would copy internally otherwise) - X = np.column_stack((coeff_b.real, coeff_b.imag)) + X = np.column_stack((coef_b.real, coef_b.imag)) # We'll also want to consider the neighbors under reflection. - # These coefficients should be provided by coeff_b_r - X_r = np.column_stack((coeff_b_r.real, coeff_b_r.imag)) + # These coefficients should be provided by coef_b_r + X_r = np.column_stack((coef_b_r.real, coef_b_r.imag)) # We can compare both non-reflected and reflected representations as one large set by # taking care later that we store refl=True where indices>=n_img @@ -297,7 +297,7 @@ def _sk_nn_classification(self, coeff_b, coeff_b_r): return classes, refl, distances - def _legacy_nn_classification(self, coeff_b, coeff_b_r): + def _legacy_nn_classification(self, coef_b, coef_b_r): """ Perform nearest neighbor classification using port of ASPIRE legacy MATLAB code. @@ -305,8 +305,8 @@ def _legacy_nn_classification(self, coeff_b, coeff_b_r): """ # Note kept ordering from legacy code (n_features, n_img) - coeff_b = coeff_b.T - coeff_b_r = coeff_b_r.T + coef_b = coef_b.T + coef_b_r = coef_b_r.T n_im = self.src.n # Shouldn't have more neighbors than images @@ -317,7 +317,7 @@ def _legacy_nn_classification(self, coeff_b, coeff_b_r): ) n_nbor = n_im - 1 - concat_coeff = np.concatenate((coeff_b, coeff_b_r), axis=1) + concat_coef = np.concatenate((coef_b, coef_b_r), axis=1) num_batches = (n_im + self.batch_size - 1) // self.batch_size @@ -327,8 +327,8 @@ def _legacy_nn_classification(self, coeff_b, coeff_b_r): for i in trange(num_batches): start = i * self.batch_size finish = min((i + 1) * self.batch_size, n_im) - batch = np.conjugate(coeff_b[:, start:finish]) - corr = np.real(np.dot(batch.T, concat_coeff)) + batch = np.conjugate(coef_b[:, start:finish]) + corr = np.real(np.dot(batch.T, concat_coef)) assert np.all( np.abs(corr) <= 1.01 # Allow some numerical wiggle @@ -511,7 +511,7 @@ def _legacy_bispectrum(self, coef, retry_attempts=3): _seed = self.seed or 0 while attempt < retry_attempts: coef_b, coef_b_r = bispec_2drot_large( - coeff=coef.T, # Note F style transpose here and in return + coef=coef.T, # Note F style transpose here and in return freqs=self.pca_basis.complex_angular_indices, eigval=complex_eigvals, alpha=self.alpha, diff --git a/src/aspire/covariance/covar.py b/src/aspire/covariance/covar.py index ad72849221..3e9a64f545 100644 --- a/src/aspire/covariance/covar.py +++ b/src/aspire/covariance/covar.py @@ -87,15 +87,15 @@ def compute_kernel(self): def estimate(self, mean_vol, noise_variance, tol=1e-5, regularizer=0): logger.info("Running Covariance Estimator") - b_coeff = self.src_backward(mean_vol, noise_variance) - est_coeff = self.conj_grad(b_coeff, tol=tol, regularizer=regularizer) - covar_est = self.basis.mat_evaluate(est_coeff) + b_coef = self.src_backward(mean_vol, noise_variance) + est_coef = self.conj_grad(b_coef, tol=tol, regularizer=regularizer) + covar_est = self.basis.mat_evaluate(est_coef) covar_est = vecmat_to_volmat(make_symmat(volmat_to_vecmat(covar_est))) return covar_est - def conj_grad(self, b_coeff, tol=1e-5, regularizer=0): - b_coeff = symmat_to_vec_iso(b_coeff) - N = b_coeff.shape[0] + def conj_grad(self, b_coef, tol=1e-5, regularizer=0): + b_coef = symmat_to_vec_iso(b_coef) + N = b_coef.shape[0] kernel = self.kernel if regularizer > 0: @@ -118,41 +118,41 @@ def conj_grad(self, b_coeff, tol=1e-5, regularizer=0): dtype=self.dtype, ) - target_residual = tol * norm(b_coeff) + target_residual = tol * norm(b_coef) def cb(xk): logger.info( - f"Delta {norm(b_coeff - self.apply_kernel(xk, packed=True))} (target {target_residual})" + f"Delta {norm(b_coef - self.apply_kernel(xk, packed=True))} (target {target_residual})" ) x, info = scipy.sparse.linalg.cg( - operator, b_coeff, M=M, callback=cb, tol=tol, atol=0 + operator, b_coef, M=M, callback=cb, tol=tol, atol=0 ) if info != 0: raise RuntimeError("Unable to converge!") return vec_to_symmat_iso(x) - def apply_kernel(self, coeff, kernel=None, packed=False): + def apply_kernel(self, coef, kernel=None, packed=False): """ Applies the kernel represented by convolution - :param coeff: The volume matrix (6 dimensions) to be convolved (but see the `packed` argument below). + :param coef: The volume matrix (6 dimensions) to be convolved (but see the `packed` argument below). :param kernel: a Kernel object. If None, the kernel for this Estimator is used. - :param packed: whether the `coeff` matrix represents an isometrically mapped packed vector, - through the `symmat_to_vec_iso` function. In this case, the function expands `coeff` into a symmetric + :param packed: whether the `coef` matrix represents an isometrically mapped packed vector, + through the `symmat_to_vec_iso` function. In this case, the function expands `coef` into a symmetric matrix internally, and returns a packed vector in return. - :return: The result of evaluating `coeff` in the given basis, convolving with the kernel given by + :return: The result of evaluating `coef` in the given basis, convolving with the kernel given by kernel, and backprojecting into the basis. If `packed` is True, then the isometrically mapped packed vector is returned instead. """ if kernel is None: kernel = self.kernel if packed: - coeff = vec_to_symmat_iso(coeff) + coef = vec_to_symmat_iso(coef) result = self.basis.mat_evaluate_t( - kernel.convolve_volume_matrix(self.basis.mat_evaluate(coeff)) + kernel.convolve_volume_matrix(self.basis.mat_evaluate(coef)) ) return symmat_to_vec_iso(result) if packed else result @@ -182,14 +182,14 @@ def src_backward(self, mean_vol, noise_variance, shrink_method=None): covar_b += vecmat_to_volmat(im_centered_b.T @ im_centered_b) / self.src.n - covar_b_coeff = self.basis.mat_evaluate_t(covar_b) - return self._shrink(covar_b_coeff, noise_variance, shrink_method) + covar_b_coef = self.basis.mat_evaluate_t(covar_b) + return self._shrink(covar_b_coef, noise_variance, shrink_method) - def _shrink(self, covar_b_coeff, noise_variance, method=None): + def _shrink(self, covar_b_coef, noise_variance, method=None): """ Shrink covariance matrix - :param covar_b_coeff: Outer products of the mean-subtracted images + :param covar_b_coef: Outer products of the mean-subtracted images :param noise_variance: Noise variance :param method: One of None/'frobenius_norm'/'operator_norm'/'soft_threshold' :return: Shrunk covariance matrix @@ -203,8 +203,8 @@ def _shrink(self, covar_b_coeff, noise_variance, method=None): An = self.basis.mat_evaluate_t(self.mean_kernel.toeplitz()) if method is None: - covar_b_coeff -= noise_variance * An + covar_b_coef -= noise_variance * An else: raise NotImplementedError("Only default shrink method supported.") - return covar_b_coeff + return covar_b_coef diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 69f6f6336c..aebbcc1dd0 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -112,46 +112,46 @@ def _ctf_identity_mat(self): else: return BlkDiagMatrix.eye(self.basis.blk_diag_cov_shape, dtype=self.dtype) - def _get_mean(self, coeffs): + def _get_mean(self, coefs): """ Calculate the mean vector from the expansion coefficients of 2D images without CTF information. - :param coeffs: A coefficient vector (or an array of coefficient vectors) to be averaged. + :param coefs: A coefficient vector (or an array of coefficient vectors) to be averaged. :return: The mean value vector for all images. """ - if coeffs.size == 0: + if coefs.size == 0: raise RuntimeError("The coefficients need to be calculated first!") mask = self.basis._indices["ells"] == 0 - mean_coeff = np.zeros(self.basis.count, dtype=coeffs.dtype) - mean_coeff[mask] = np.mean(coeffs[..., mask], axis=0) + mean_coef = np.zeros(self.basis.count, dtype=coefs.dtype) + mean_coef[mask] = np.mean(coefs[..., mask], axis=0) - return mean_coeff + return mean_coef - def _get_covar(self, coeffs, mean_coeff=None, do_refl=True): + def _get_covar(self, coefs, mean_coef=None, do_refl=True): """ Calculate the covariance matrix from the expansion coefficients without CTF information. - :param coeffs: A coefficient vector (an array of coefficient vectors) calculated from 2D images. - :param mean_coeff: The mean vector calculated from the `coeffs`. (array) + :param coefs: A coefficient vector (an array of coefficient vectors) calculated from 2D images. + :param mean_coef: The mean vector calculated from the `coefs`. :param do_refl: If true, enforce invariance to reflection (default false). :return: The covariance matrix of coefficients for all images. """ - if coeffs.size == 0: + if coefs.size == 0: raise RuntimeError("The coefficients need to be calculated first!") - if mean_coeff is None: - mean_coeff = self._get_mean(coeffs) + if mean_coef is None: + mean_coef = self._get_mean(coefs) # Initialize a totally empty BlkDiagMatrix, build incrementally. - covar_coeff = BlkDiagMatrix.empty(0, dtype=coeffs.dtype) + covar_coef = BlkDiagMatrix.empty(0, dtype=coefs.dtype) ell = 0 mask = self.basis._indices["ells"] == ell - coeff_ell = coeffs[..., mask] - mean_coeff[mask] - covar_ell = np.array(coeff_ell.T @ coeff_ell / coeffs.shape[0]) - covar_coeff.append(covar_ell) + coef_ell = coefs[..., mask] - mean_coef[mask] + covar_ell = np.array(coef_ell.T @ coef_ell / coefs.shape[0]) + covar_coef.append(covar_ell) for ell in range(1, self.basis.ell_max + 1): mask_ell = self.basis._indices["ells"] == ell @@ -159,82 +159,82 @@ def _get_covar(self, coeffs, mean_coeff=None, do_refl=True): mask_neg = mask_ell & (self.basis._indices["sgns"] == -1) covar_ell_diag = np.array( - coeffs[:, mask_pos].T @ coeffs[:, mask_pos] - + coeffs[:, mask_neg].T @ coeffs[:, mask_neg] - ) / (2 * coeffs.shape[0]) + coefs[:, mask_pos].T @ coefs[:, mask_pos] + + coefs[:, mask_neg].T @ coefs[:, mask_neg] + ) / (2 * coefs.shape[0]) if do_refl: - covar_coeff.append(covar_ell_diag) - covar_coeff.append(covar_ell_diag) + covar_coef.append(covar_ell_diag) + covar_coef.append(covar_ell_diag) else: covar_ell_off = np.array( ( - coeffs[:, mask_pos] @ coeffs[:, mask_neg].T / coeffs.shape[0] - - coeffs[:, mask_pos].T @ coeffs[:, mask_neg] + coefs[:, mask_pos] @ coefs[:, mask_neg].T / coefs.shape[0] + - coefs[:, mask_pos].T @ coefs[:, mask_neg] ) - / (2 * coeffs.shape[0]) + / (2 * coefs.shape[0]) ) hsize = covar_ell_diag.shape[0] - covar_coeff_blk = np.zeros((2, hsize, 2, hsize)) + covar_coef_blk = np.zeros((2, hsize, 2, hsize)) - covar_coeff_blk[0:2, :, 0:2, :] = covar_ell_diag[:hsize, :hsize] - covar_coeff_blk[0, :, 1, :] = covar_ell_off[:hsize, :hsize] - covar_coeff_blk[1, :, 0, :] = covar_ell_off.T[:hsize, :hsize] + covar_coef_blk[0:2, :, 0:2, :] = covar_ell_diag[:hsize, :hsize] + covar_coef_blk[0, :, 1, :] = covar_ell_off[:hsize, :hsize] + covar_coef_blk[1, :, 0, :] = covar_ell_off.T[:hsize, :hsize] - covar_coeff.append(covar_coeff_blk.reshape(2 * hsize, 2 * hsize)) + covar_coef.append(covar_coef_blk.reshape(2 * hsize, 2 * hsize)) - return covar_coeff + return covar_coef - def get_mean(self, coeffs, ctf_basis=None, ctf_idx=None): + def get_mean(self, coefs, ctf_basis=None, ctf_idx=None): """ Calculate the mean vector from the expansion coefficients with CTF information. - :param coeffs: A coefficient vector (or an array of coefficient vectors) to be averaged. + :param coefs: A coefficient vector (or an array of coefficient vectors) to be averaged. :param ctf_basis: The CTF functions in the Basis expansion. :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. :return: The mean value vector for all images. """ - if not isinstance(coeffs, Coef): + if not isinstance(coefs, Coef): raise TypeError( - f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + f"`coefs` should be instance of `Coef`, received {type(Coef)}." ) - coeffs = coeffs.asnumpy() + coefs = coefs.asnumpy() # TODO: Redundant, remove? - if coeffs.size == 0: + if coefs.size == 0: raise RuntimeError("The coefficients need to be calculated!") # should assert we require none or both... if (ctf_basis is None) or (ctf_idx is None): - ctf_idx = np.zeros(coeffs.shape[0], dtype=int) + ctf_idx = np.zeros(coefs.shape[0], dtype=int) ctf_basis = [self._ctf_identity_mat()] - b = np.zeros(self.basis.count, dtype=coeffs.dtype) + b = np.zeros(self.basis.count, dtype=coefs.dtype) A = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) for k in np.unique(ctf_idx[:]).T: - coeff_k = coeffs[ctf_idx == k] - weight = coeff_k.shape[0] / coeffs.shape[0] - mean_coeff_k = self._get_mean(coeff_k) + coef_k = coefs[ctf_idx == k] + weight = coef_k.shape[0] / coefs.shape[0] + mean_coef_k = self._get_mean(coef_k) ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - b += weight * ctf_basis_k_t.apply(mean_coeff_k) + b += weight * ctf_basis_k_t.apply(mean_coef_k) A += weight * (ctf_basis_k_t @ ctf_basis_k) - mean_coeff = A.solve(b) - return Coef(self.basis, mean_coeff) + mean_coef = A.solve(b) + return Coef(self.basis, mean_coef) def get_covar( self, - coeffs, + coefs, ctf_basis=None, ctf_idx=None, - mean_coeff=None, + mean_coef=None, do_refl=True, noise_var=0, covar_est_opt=None, @@ -243,35 +243,35 @@ def get_covar( """ Calculate the covariance matrix from the expansion coefficients and CTF information. - :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. + :param coefs: A coefficient vector (or an array of coefficient vectors) to be calculated. :param ctf_basis: The CTF functions in the Basis expansion. :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. - :param mean_coeff: The mean value vector from all images. - :param noise_var: The estimated variance of noise. The value should be zero for `coeffs` + :param mean_coef: The mean value vector from all images. + :param noise_var: The estimated variance of noise. The value should be zero for `coefs` from clean images of simulation data. :param covar_est_opt: The optimization parameter list for obtaining the Cov2D matrix. :param make_psd: If True, make the covariance matrix positive semidefinite :return: The basis coefficients of the covariance matrix in the form of cell array representing a block diagonal matrix. These block diagonal matrices are implemented as BlkDiagMatrix instances. - The covariance is calculated from the images represented by the coeffs array, + The covariance is calculated from the images represented by the coefs array, along with all possible rotations and reflections. As a result, the computed covariance matrix is invariant to both reflection and rotation. The effect of the filters in ctf_basis are accounted for and inverted to yield a covariance estimate of the unfiltered images. """ - if not isinstance(coeffs, Coef): + if not isinstance(coefs, Coef): raise TypeError( - f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + f"`coefs` should be instance of `Coef`, received {type(Coef)}." ) - coeffs = coeffs.asnumpy() + coefs = coefs.asnumpy() - if coeffs.size == 0: + if coefs.size == 0: raise RuntimeError("The coefficients need to be calculated!") if (ctf_basis is None) or (ctf_idx is None): - ctf_idx = np.zeros(coeffs.shape[0], dtype=int) + ctf_idx = np.zeros(coefs.shape[0], dtype=int) ctf_basis = [self._ctf_identity_mat()] def identity(x): @@ -290,10 +290,10 @@ def identity(x): covar_est_opt = fill_struct(covar_est_opt, default_est_opt) - if mean_coeff is None: - mean_coeff = self.get_mean(Coef(self.basis, coeffs), ctf_basis, ctf_idx) + if mean_coef is None: + mean_coef = self.get_mean(Coef(self.basis, coefs), ctf_basis, ctf_idx) - b_coeff = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) + b_coef = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) b_noise = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) A = [] for _ in range(len(ctf_basis)): @@ -302,15 +302,15 @@ def identity(x): M = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) for k in np.unique(ctf_idx[:]): - coeff_k = coeffs[ctf_idx == k].astype(self.dtype) - weight = coeff_k.shape[0] / coeffs.shape[0] + coef_k = coefs[ctf_idx == k].astype(self.dtype) + weight = coef_k.shape[0] / coefs.shape[0] ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) - covar_coeff_k = self._get_covar(coeff_k, mean_coeff_k) + mean_coef_k = ctf_basis_k.apply(mean_coef.asnumpy()[0]) + covar_coef_k = self._get_covar(coef_k, mean_coef_k) - b_coeff += weight * (ctf_basis_k_t @ covar_coeff_k @ ctf_basis_k) + b_coef += weight * (ctf_basis_k_t @ covar_coef_k @ ctf_basis_k) ctf_basis_k_sq = ctf_basis_k_t @ ctf_basis_k b_noise += weight * ctf_basis_k_sq @@ -322,16 +322,16 @@ def identity(x): A[k] = A_k M += A[k] - if not b_coeff.check_psd(): + if not b_coef.check_psd(): logger.warning("Left side b in Cov2D is not positive semidefinite.") if covar_est_opt["shrinker"] is None: - b = b_coeff - noise_var * b_noise + b = b_coef - noise_var * b_noise else: b = self.shrink_covar_backward( - b_coeff, + b_coef, b_noise, - np.size(coeffs, 0), + np.size(coefs, 0), noise_var, covar_est_opt["shrinker"], ) @@ -345,7 +345,7 @@ def identity(x): cg_opt = covar_est_opt - covar_coeff = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) + covar_coef = BlkDiagMatrix.zeros(self.basis.blk_diag_cov_shape) def precond_fun(S, x): p = np.size(S, 0) @@ -372,18 +372,18 @@ def apply(A, x): b_ell = m_reshape(b[ell], (p**2,)) S = inv(M[ell]) cg_opt["preconditioner"] = lambda x, S=S: precond_fun(S, x) - covar_coeff_ell, _, _ = conj_grad( + covar_coef_ell, _, _ = conj_grad( lambda x, A_ell=A_ell: apply(A_ell, x), b_ell, cg_opt ) - covar_coeff[ell] = m_reshape(covar_coeff_ell, (p, p)) + covar_coef[ell] = m_reshape(covar_coef_ell, (p, p)) - if not covar_coeff.check_psd(): + if not covar_coef.check_psd(): logger.warning("Covariance matrix in Cov2D is not positive semidefinite.") if make_psd: logger.info("Convert matrices to positive semidefinite.") - covar_coeff = covar_coeff.make_psd() + covar_coef = covar_coef.make_psd() - return covar_coeff + return covar_coef def shrink_covar_backward(self, b, b_noise, n, noise_var, shrinker): """ @@ -409,45 +409,45 @@ def shrink_covar_backward(self, b, b_noise, n, noise_var, shrinker): b_out[ell] = b_ell return b_out - def get_cwf_coeffs( + def get_cwf_coefs( self, - coeffs, + coefs, ctf_basis=None, ctf_idx=None, - mean_coeff=None, - covar_coeff=None, + mean_coef=None, + covar_coef=None, noise_var=0, ): """ Estimate the expansion coefficients using the Covariance Wiener Filtering (CWF) method. - :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. + :param coefs: A coefficient vector (or an array of coefficient vectors) to be calculated. :param ctf_basis: The CTF functions in the Basis expansion. :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. - :param mean_coeff: The mean value vector from all images. - :param covar_coeff: The block diagonal covariance matrix of the clean coefficients represented by a cell array. - :param noise_var: The estimated variance of noise. The value should be zero for `coeffs` + :param mean_coef: The mean value vector from all images. + :param covar_coef: The block diagonal covariance matrix of the clean coefficients represented by a cell array. + :param noise_var: The estimated variance of noise. The value should be zero for `coefs` from clean images of simulation data. :return: The estimated coefficients of the unfiltered images in certain math basis. These are obtained using a Wiener filter with the specified covariance for the clean images and white noise of variance `noise_var` for the noise. """ - if not isinstance(coeffs, Coef): + if not isinstance(coefs, Coef): raise TypeError( - f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + f"`coefs` should be instance of `Coef`, received {type(Coef)}." ) - if mean_coeff is None: - mean_coeff = self.get_mean(coeffs, ctf_basis, ctf_idx) + if mean_coef is None: + mean_coef = self.get_mean(coefs, ctf_basis, ctf_idx) - if covar_coeff is None: - covar_coeff = self.get_covar( - coeffs, ctf_basis, ctf_idx, mean_coeff, noise_var=noise_var + if covar_coef is None: + covar_coef = self.get_covar( + coefs, ctf_basis, ctf_idx, mean_coef, noise_var=noise_var ) - coeffs = coeffs.asnumpy() + coefs = coefs.asnumpy() # Handle CTF arguments. if (ctf_basis is None) ^ (ctf_idx is None): @@ -458,34 +458,34 @@ def get_cwf_coeffs( ) elif ctf_basis is None: # Setup defaults for CTF - ctf_idx = np.zeros(coeffs.shape[0], dtype=int) - ctf_basis = [BlkDiagMatrix.eye_like(covar_coeff)] + ctf_idx = np.zeros(coefs.shape[0], dtype=int) + ctf_basis = [BlkDiagMatrix.eye_like(covar_coef)] - noise_covar_coeff = noise_var * BlkDiagMatrix.eye_like(covar_coeff) + noise_covar_coef = noise_var * BlkDiagMatrix.eye_like(covar_coef) - coeffs_est = np.zeros_like(coeffs) + coefs_est = np.zeros_like(coefs) for k in np.unique(ctf_idx[:]): - coeff_k = coeffs[ctf_idx == k] + coef_k = coefs[ctf_idx == k] ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) - coeff_est_k = coeff_k - mean_coeff_k + mean_coef_k = ctf_basis_k.apply(mean_coef.asnumpy()[0]) + coef_est_k = coef_k - mean_coef_k if noise_var == 0: - coeff_est_k = ctf_basis_k.solve(coeff_est_k.T).T + coef_est_k = ctf_basis_k.solve(coef_est_k.T).T else: - sig_covar_coeff = ctf_basis_k @ covar_coeff @ ctf_basis_k_t - sig_noise_covar_coeff = sig_covar_coeff + noise_covar_coeff + sig_covar_coef = ctf_basis_k @ covar_coef @ ctf_basis_k_t + sig_noise_covar_coef = sig_covar_coef + noise_covar_coef - coeff_est_k = sig_noise_covar_coeff.solve(coeff_est_k.T).T - coeff_est_k = (covar_coeff @ ctf_basis_k_t).apply(coeff_est_k.T).T + coef_est_k = sig_noise_covar_coef.solve(coef_est_k.T).T + coef_est_k = (covar_coef @ ctf_basis_k_t).apply(coef_est_k.T).T - coeff_est_k = coeff_est_k + mean_coeff - coeffs_est[ctf_idx == k] = coeff_est_k + coef_est_k = coef_est_k + mean_coef + coefs_est[ctf_idx == k] = coef_est_k - return Coef(self.basis, coeffs_est) + return Coef(self.basis, coefs_est) class BatchedRotCov2D(RotCov2D): @@ -546,7 +546,7 @@ def _calc_rhs(self): ctf_basis = self.ctf_basis ctf_idx = self.ctf_idx - zero_coeff = np.zeros((basis.count,), dtype=self.dtype) + zero_coef = np.zeros((basis.count,), dtype=self.dtype) b_mean = [np.zeros(basis.count, dtype=self.dtype) for _ in ctf_basis] @@ -556,24 +556,24 @@ def _calc_rhs(self): batch = np.arange(start, min(start + self.batch_size, src.n)) im = src.images[batch[0] : batch[0] + len(batch)] - coeff = basis.evaluate_t(im).asnumpy() + coef = basis.evaluate_t(im).asnumpy() for k in np.unique(ctf_idx[batch]): - coeff_k = coeff[ctf_idx[batch] == k] - weight = np.size(coeff_k, 0) / src.n + coef_k = coef[ctf_idx[batch] == k] + weight = np.size(coef_k, 0) / src.n - mean_coeff_k = self._get_mean(coeff_k) + mean_coef_k = self._get_mean(coef_k) ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - b_mean_k = weight * ctf_basis_k_t.apply(mean_coeff_k) + b_mean_k = weight * ctf_basis_k_t.apply(mean_coef_k) b_mean[k] += b_mean_k - covar_coeff_k = self._get_covar(coeff_k, zero_coeff) + covar_coef_k = self._get_covar(coef_k, zero_coef) - b_covar_k = ctf_basis_k_t @ covar_coeff_k + b_covar_k = ctf_basis_k_t @ covar_coef_k b_covar_k = b_covar_k @ ctf_basis_k b_covar_k *= weight @@ -611,7 +611,7 @@ def _calc_op(self): self.A_covar = A_covar self.M_covar = M_covar - def _mean_correct_covar_rhs(self, b_covar, b_mean, mean_coeff): + def _mean_correct_covar_rhs(self, b_covar, b_mean, mean_coef): src = self.src ctf_basis = self.ctf_basis @@ -629,16 +629,16 @@ def _mean_correct_covar_rhs(self, b_covar, b_mean, mean_coeff): ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) - mean_coeff_k = ctf_basis_k_t.apply(mean_coeff_k) + mean_coef_k = ctf_basis_k.apply(mean_coef.asnumpy()[0]) + mean_coef_k = ctf_basis_k_t.apply(mean_coef_k) - mean_coeff_k = mean_coeff_k[: partition[0][0]] + mean_coef_k = mean_coef_k[: partition[0][0]] b_mean_k = b_mean[k][: partition[0][0]] correction = ( - np.outer(mean_coeff_k, b_mean_k) - + np.outer(b_mean_k, mean_coeff_k) - - weight * np.outer(mean_coeff_k, mean_coeff_k) + np.outer(mean_coef_k, b_mean_k) + + np.outer(b_mean_k, mean_coef_k) + - weight * np.outer(mean_coef_k, mean_coef_k) ) b_covar[0] -= correction @@ -685,7 +685,7 @@ def apply(A, x): return y cg_opt = covar_est_opt - covar_coeff = BlkDiagMatrix.zeros( + covar_coef = BlkDiagMatrix.zeros( self.basis.blk_diag_cov_shape, dtype=self.dtype ) @@ -697,12 +697,12 @@ def apply(A, x): b_ell = m_reshape(b_covar[ell], (p**2,)) S = inv(M[ell]) cg_opt["preconditioner"] = lambda x, S=S: precond_fun(S, x) - covar_coeff_ell, _, _ = conj_grad( + covar_coef_ell, _, _ = conj_grad( lambda x, A_ell=A_ell: apply(A_ell, x), b_ell, cg_opt ) - covar_coeff[ell] = m_reshape(covar_coeff_ell, (p, p)) + covar_coef[ell] = m_reshape(covar_coef_ell, (p, p)) - return covar_coeff + return covar_coef def get_mean(self): """ @@ -719,19 +719,17 @@ def get_mean(self): self._calc_op() b_mean_all = np.stack(self.b_mean).sum(axis=0) - mean_coeff = self.A_mean.solve(b_mean_all) + mean_coef = self.A_mean.solve(b_mean_all) - return Coef(self.basis, mean_coeff) + return Coef(self.basis, mean_coef) - def get_covar( - self, noise_var=0, mean_coeff=None, covar_est_opt=None, make_psd=True - ): + def get_covar(self, noise_var=0, mean_coef=None, covar_est_opt=None, make_psd=True): """ Calculate the block diagonal covariance matrix in the basis coefficients. :param noise_var: The variance of the noise in the images (default 1) - :param mean_coeff: If specified, overrides the mean coefficient vector + :param mean_coef: If specified, overrides the mean coefficient vector used to calculate the covariance (default `self.get_mean()`). :param :covar_est_opt: The estimation parameters for obtaining the covariance matrix in the form of a dictionary. Keys include: @@ -780,12 +778,12 @@ def identity(x): if not self.A_covar or self.M_covar: self._calc_op() - if mean_coeff is None: - mean_coeff = self.get_mean() + if mean_coef is None: + mean_coef = self.get_mean() b_covar = self.b_covar - b_covar = self._mean_correct_covar_rhs(b_covar, self.b_mean, mean_coeff) + b_covar = self._mean_correct_covar_rhs(b_covar, self.b_mean, mean_coef) if not b_covar.check_psd(): logger.warning("Left side b in Batched Cov2D is not positive semidefinite.") @@ -798,49 +796,49 @@ def identity(x): "in Batched Cov2D is not positive semidefinite." ) - covar_coeff = self._solve_covar( + covar_coef = self._solve_covar( self.A_covar, b_covar, self.M_covar, covar_est_opt ) - if not covar_coeff.check_psd(): + if not covar_coef.check_psd(): logger.warning( "Covariance matrix in Batched Cov2D is not positive semidefinite." ) if make_psd: logger.info("Convert matrices to positive semidefinite.") - covar_coeff = covar_coeff.make_psd() + covar_coef = covar_coef.make_psd() - return covar_coeff + return covar_coef - def get_cwf_coeffs( - self, coeffs, ctf_basis, ctf_idx, mean_coeff, covar_coeff, noise_var=0 + def get_cwf_coefs( + self, coefs, ctf_basis, ctf_idx, mean_coef, covar_coef, noise_var=0 ): """ Estimate the expansion coefficients using the Covariance Wiener Filtering (CWF) method. - :param coeffs: A coefficient vector (or an array of coefficient vectors) to be calculated. + :param coefs: A coefficient vector (or an array of coefficient vectors) to be calculated. :param ctf_basis: The CTF functions in the Basis expansion. :param ctf_idx: An array of the CTF function indices for all 2D images. If ctf_basis or ctf_idx is None, the identity filter will be applied. - :param mean_coeff: The mean value vector from all images. - :param covar_coeff: The block diagonal covariance matrix of the clean coefficients represented by a cell array. - :param noise_var: The estimated variance of noise. The value should be zero for `coeffs` + :param mean_coef: The mean value vector from all images. + :param covar_coef: The block diagonal covariance matrix of the clean coefficients represented by a cell array. + :param noise_var: The estimated variance of noise. The value should be zero for `coefs` from clean images of simulation data. :return: The estimated coefficients of the unfiltered images in certain math basis. These are obtained using a Wiener filter with the specified covariance for the clean images and white noise of variance `noise_var` for the noise. """ - if not isinstance(coeffs, Coef): + if not isinstance(coefs, Coef): raise TypeError( - f"`coeffs` should be instance of `Coef`, received {type(Coef)}." + f"`coefs` should be instance of `Coef`, received {type(Coef)}." ) - coeffs = coeffs.asnumpy() + coefs = coefs.asnumpy() - if mean_coeff is None: - mean_coeff = self.get_mean() + if mean_coef is None: + mean_coef = self.get_mean() - if covar_coeff is None: - covar_coeff = self.get_covar(noise_var=noise_var, mean_coeff=mean_coeff) + if covar_coef is None: + covar_coef = self.get_covar(noise_var=noise_var, mean_coef=mean_coef) # Handle CTF arguments. if (ctf_basis is None) ^ (ctf_idx is None): @@ -851,31 +849,31 @@ def get_cwf_coeffs( ) elif ctf_basis is None: # Setup defaults for CTF - ctf_idx = np.zeros(coeffs.shape[0], dtype=int) - ctf_basis = [BlkDiagMatrix.eye_like(covar_coeff)] + ctf_idx = np.zeros(coefs.shape[0], dtype=int) + ctf_basis = [BlkDiagMatrix.eye_like(covar_coef)] - noise_covar_coeff = noise_var * BlkDiagMatrix.eye_like(covar_coeff) + noise_covar_coef = noise_var * BlkDiagMatrix.eye_like(covar_coef) - coeffs_est = np.zeros_like(coeffs) + coefs_est = np.zeros_like(coefs) for k in np.unique(ctf_idx[:]): - coeff_k = coeffs[ctf_idx == k] + coef_k = coefs[ctf_idx == k] ctf_basis_k = ctf_basis[k] ctf_basis_k_t = ctf_basis_k.T - mean_coeff_k = ctf_basis_k.apply(mean_coeff.asnumpy()[0]) - coeff_est_k = coeff_k - mean_coeff_k + mean_coef_k = ctf_basis_k.apply(mean_coef.asnumpy()[0]) + coef_est_k = coef_k - mean_coef_k if noise_var == 0: - coeff_est_k = ctf_basis_k.solve(coeff_est_k.T).T + coef_est_k = ctf_basis_k.solve(coef_est_k.T).T else: - sig_covar_coeff = ctf_basis_k @ covar_coeff @ ctf_basis_k_t - sig_noise_covar_coeff = sig_covar_coeff + noise_covar_coeff + sig_covar_coef = ctf_basis_k @ covar_coef @ ctf_basis_k_t + sig_noise_covar_coef = sig_covar_coef + noise_covar_coef - coeff_est_k = sig_noise_covar_coeff.solve(coeff_est_k.T).T - coeff_est_k = (covar_coeff @ ctf_basis_k_t).apply(coeff_est_k.T).T + coef_est_k = sig_noise_covar_coef.solve(coef_est_k.T).T + coef_est_k = (covar_coef @ ctf_basis_k_t).apply(coef_est_k.T).T - coeff_est_k = coeff_est_k + mean_coeff - coeffs_est[ctf_idx == k] = coeff_est_k + coef_est_k = coef_est_k + mean_coef + coefs_est[ctf_idx == k] = coef_est_k - return Coef(self.basis, coeffs_est) + return Coef(self.basis, coefs_est) diff --git a/src/aspire/ctf/ctf_estimator.py b/src/aspire/ctf/ctf_estimator.py index ca2c3fed5d..98d0a2dc37 100644 --- a/src/aspire/ctf/ctf_estimator.py +++ b/src/aspire/ctf/ctf_estimator.py @@ -266,19 +266,19 @@ def elliptical_average(self, ffbbasis, amplitude_spectrum, circular): """ # RCOPT, come back and change the indices for this method - coeffs_s = ffbbasis.evaluate_t(amplitude_spectrum).asnumpy().copy().T - coeffs_n = coeffs_s.copy() + coefs_s = ffbbasis.evaluate_t(amplitude_spectrum).asnumpy().copy().T + coefs_n = coefs_s.copy() - coeffs_s[np.argwhere(ffbbasis._indices["ells"] == 1)] = 0 + coefs_s[np.argwhere(ffbbasis._indices["ells"] == 1)] = 0 if circular: - coeffs_s[np.argwhere(ffbbasis._indices["ells"] == 2)] = 0 + coefs_s[np.argwhere(ffbbasis._indices["ells"] == 2)] = 0 noise = amplitude_spectrum else: - coeffs_n[np.argwhere(ffbbasis._indices["ells"] == 0)] = 0 - coeffs_n[np.argwhere(ffbbasis._indices["ells"] == 2)] = 0 - noise = Coef(ffbbasis, coeffs_n.T).evaluate() + coefs_n[np.argwhere(ffbbasis._indices["ells"] == 0)] = 0 + coefs_n[np.argwhere(ffbbasis._indices["ells"] == 2)] = 0 + noise = Coef(ffbbasis, coefs_n.T).evaluate() - psd = Coef(ffbbasis, coeffs_s.T).evaluate() + psd = Coef(ffbbasis, coefs_s.T).evaluate() return psd, noise diff --git a/src/aspire/denoising/denoiser_cov2d.py b/src/aspire/denoising/denoiser_cov2d.py index a00dcc3078..b776e516ea 100644 --- a/src/aspire/denoising/denoiser_cov2d.py +++ b/src/aspire/denoising/denoiser_cov2d.py @@ -162,7 +162,7 @@ def denoise(self, covar_opt=None, batch_size=512): self.mean_est = self.cov2d.get_mean() self.covar_est = self.cov2d.get_covar( - noise_var=self.var_noise, mean_coeff=self.mean_est, covar_est_opt=covar_opt + noise_var=self.var_noise, mean_coef=self.mean_est, covar_est_opt=covar_opt ) return DenoisedImageSource(self.src, self) @@ -181,21 +181,21 @@ def images(self, istart=0, batch_size=512): img_start = istart img_end = min(istart + batch_size, src.n) imgs_noise = src.images[img_start : img_start + batch_size] - coeffs_noise = self.basis.evaluate_t(imgs_noise) + coefs_noise = self.basis.evaluate_t(imgs_noise) logger.info( f"Estimating Cov2D coefficients for images from {img_start} to {img_end-1}" ) - coeffs_estim = self.cov2d.get_cwf_coeffs( - coeffs_noise, + coefs_estim = self.cov2d.get_cwf_coefs( + coefs_noise, self.cov2d.ctf_basis, self.cov2d.ctf_idx[img_start:img_end], - mean_coeff=self.mean_est, - covar_coeff=self.covar_est, + mean_coef=self.mean_est, + covar_coef=self.covar_est, noise_var=self.var_noise, ) # Convert Fourier-Bessel coefficients back into 2D images logger.info("Converting Cov2D coefficients back to 2D images") - imgs_denoised = self.basis.evaluate(coeffs_estim) + imgs_denoised = self.basis.evaluate(coefs_estim) return imgs_denoised diff --git a/src/aspire/image/image.py b/src/aspire/image/image.py index 4713266b54..c85fc8188f 100644 --- a/src/aspire/image/image.py +++ b/src/aspire/image/image.py @@ -58,8 +58,8 @@ def normalize_bg(imgs, bg_radius=1.0, do_ramp=True): imgs = imgs.reshape((-1, L * L)) # Fit a ramping background and apply to images - coeff = lstsq(ramp_mask, imgs[:, mask_reshape].T)[0] # RCOPT - imgs = imgs - (ramp_all @ coeff).T # RCOPT + coef = lstsq(ramp_mask, imgs[:, mask_reshape].T)[0] # RCOPT + imgs = imgs - (ramp_all @ coef).T # RCOPT imgs = imgs.reshape((-1, L, L)) # Apply mask images and calculate mean and std values of background diff --git a/src/aspire/reconstruction/estimator.py b/src/aspire/reconstruction/estimator.py index 62004da1d4..dec6a901da 100644 --- a/src/aspire/reconstruction/estimator.py +++ b/src/aspire/reconstruction/estimator.py @@ -59,29 +59,29 @@ def __getattr__(self, name): def compute_kernel(self): raise NotImplementedError("Subclasses must implement the compute_kernel method") - def estimate(self, b_coeff=None, tol=1e-5, regularizer=0): + def estimate(self, b_coef=None, tol=1e-5, regularizer=0): """Return an estimate as a Volume instance.""" - if b_coeff is None: - b_coeff = self.src_backward() - est_coeff = self.conj_grad(b_coeff, tol=tol, regularizer=regularizer) - est = Coef(self.basis, est_coeff).evaluate().T + if b_coef is None: + b_coef = self.src_backward() + est_coef = self.conj_grad(b_coef, tol=tol, regularizer=regularizer) + est = Coef(self.basis, est_coef).evaluate().T return est - def apply_kernel(self, vol_coeff, kernel=None): + def apply_kernel(self, vol_coef, kernel=None): """ Applies the kernel represented by convolution - :param vol_coeff: The volume to be convolved, stored in the basis coefficients. + :param vol_coef: The volume to be convolved, stored in the basis coefficients. :param kernel: a Kernel object. If None, the kernel for this Estimator is used. - :return: The result of evaluating `vol_coeff` in the given basis, convolving with the kernel given by + :return: The result of evaluating `vol_coef` in the given basis, convolving with the kernel given by kernel, and backprojecting into the basis. """ if kernel is None: kernel = self.kernel - vol = Coef(self.basis, vol_coeff).evaluate() # returns a Volume + vol = Coef(self.basis, vol_coef).evaluate() # returns a Volume vol = kernel.convolve_volume(vol) # returns a Volume vol_coef = self.basis.evaluate_t(vol) return vol_coef diff --git a/src/aspire/reconstruction/mean.py b/src/aspire/reconstruction/mean.py index 9aaef7d08b..fda258cecd 100644 --- a/src/aspire/reconstruction/mean.py +++ b/src/aspire/reconstruction/mean.py @@ -173,8 +173,8 @@ def src_backward(self): return res - def conj_grad(self, b_coeff, tol=1e-5, regularizer=0): - count = b_coeff.shape[-1] # b_coef should be (r, basis.count) + def conj_grad(self, b_coef, tol=1e-5, regularizer=0): + count = b_coef.shape[-1] # b_coef should be (r, basis.count) kernel = self.kernel if regularizer > 0: @@ -198,50 +198,50 @@ def conj_grad(self, b_coeff, tol=1e-5, regularizer=0): ) tol = tol or config.mean.cg_tol - target_residual = tol * norm(b_coeff) + target_residual = tol * norm(b_coef) def cb(xk): logger.info( - f"Delta {norm(b_coeff - self.apply_kernel(xk))} (target {target_residual})" + f"Delta {norm(b_coef - self.apply_kernel(xk))} (target {target_residual})" ) - x, info = cg(operator, b_coeff.flatten(), M=M, callback=cb, tol=tol, atol=0) + x, info = cg(operator, b_coef.flatten(), M=M, callback=cb, tol=tol, atol=0) if info != 0: raise RuntimeError("Unable to converge!") return x.reshape(self.r, self.basis.count) - def apply_kernel(self, vol_coeff, kernel=None): + def apply_kernel(self, vol_coef, kernel=None): """ Applies the kernel represented by convolution - :param vol_coeff: The volume to be convolved, stored in the basis coefficients. + :param vol_coef: The volume to be convolved, stored in the basis coefficients. :param kernel: a Kernel object. If None, the kernel for this Estimator is used. - :return: The result of evaluating `vol_coeff` in the given basis, convolving with the kernel given by + :return: The result of evaluating `vol_coef` in the given basis, convolving with the kernel given by kernel, and backprojecting into the basis. """ if kernel is None: kernel = self.kernel - assert np.size(vol_coeff) == self.r * self.basis.count - if vol_coeff.ndim == 1: - vol_coeff = vol_coeff.reshape(self.r, self.basis.count) + assert np.size(vol_coef) == self.r * self.basis.count + if vol_coef.ndim == 1: + vol_coef = vol_coef.reshape(self.r, self.basis.count) vols_out = Volume( np.zeros((self.r, self.src.L, self.src.L, self.src.L), dtype=self.dtype) ) - vol = Coef(self.basis, vol_coeff).evaluate() + vol = Coef(self.basis, vol_coef).evaluate() for k in range(self.r): for j in range(self.r): vols_out[k] = vols_out[k] + kernel.convolve_volume(vol[j], j, k) # Note this is where we would add mask_gamma - vol_coeff = self.basis.evaluate_t(vols_out) + vol_coef = self.basis.evaluate_t(vols_out) - return vol_coeff + return vol_coef class MeanEstimator(WeightedVolumesEstimator): diff --git a/tests/test_BlkDiagMatrix.py b/tests/test_BlkDiagMatrix.py index 9f606b9731..3dac18b890 100644 --- a/tests/test_BlkDiagMatrix.py +++ b/tests/test_BlkDiagMatrix.py @@ -125,22 +125,22 @@ def testBlkDiagMatrixSub(self): def testBlkDiagMatrixApply(self): m = np.sum(self.blk_a.partition[:, 1]) k = 3 - coeffm = np.arange(k * m).reshape(m, k).astype(self.blk_a.dtype) + coefm = np.arange(k * m).reshape(m, k).astype(self.blk_a.dtype) # Manually compute ind = 0 - res = np.empty_like(coeffm) + res = np.empty_like(coefm) for b, blk in enumerate(self.blk_a): col = self.blk_a.partition[b, 1] - res[ind : ind + col, :] = blk @ coeffm[ind : ind + col, :] + res[ind : ind + col, :] = blk @ coefm[ind : ind + col, :] ind += col # Check ndim 1 case - c = self.blk_a.apply(coeffm[:, 0]) + c = self.blk_a.apply(coefm[:, 0]) self.allallfunc(c, res[:, 0]) # Check ndim 2 case - d = self.blk_a.apply(coeffm) + d = self.blk_a.apply(coefm) self.allallfunc(res, d) # Here we are checking that the ndim 2 case distributes as described. @@ -148,33 +148,33 @@ def testBlkDiagMatrixApply(self): # should be equivalent to e = [A.apply(r0), ... A.apply(ri)]. e = np.empty((m, k)) for i in range(k): - e[:, i] = self.blk_a.apply(coeffm[:, i]) + e[:, i] = self.blk_a.apply(coefm[:, i]) self.allallfunc(e, d) # We can use syntactic sugar @ for apply as well - f = self.blk_a @ coeffm + f = self.blk_a @ coefm self.allallfunc(f, d) # Test the rapply is also functional - coeffm = coeffm.T # matmul dimensions - res = coeffm @ self.blk_a.dense() - d = self.blk_a.rapply(coeffm) + coefm = coefm.T # matmul dimensions + res = coefm @ self.blk_a.dense() + d = self.blk_a.rapply(coefm) self.allallfunc(res, d) # And the syntactic sugar @ - d = coeffm @ self.blk_a + d = coefm @ self.blk_a self.allallfunc(res, d) # And test some incorrrect invocations: # inplace not supported for matmul of mixed classes. with pytest.raises(RuntimeError, match=r".*method not supported.*"): - self.blk_a @= coeffm + self.blk_a @= coefm # Test left operand of an __rmatmul__ must be an ndarray with pytest.raises( RuntimeError, match=r".*only defined for np.ndarray @ BlkDiagMatrix.*" ): - _ = list(coeffm) @ self.blk_a + _ = list(coefm) @ self.blk_a def testBlkDiagMatrixMatMult(self): result = [np.matmul(*tup) for tup in zip(self.blk_a, self.blk_b)] @@ -285,18 +285,18 @@ def testBlkDiagMatrixSolve(self): m = np.sum(self.blk_a.partition[:, 1]) k = 3 - coeffm = np.arange(k * m).reshape(m, k).astype(self.blk_a.dtype) + coefm = np.arange(k * m).reshape(m, k).astype(self.blk_a.dtype) # Manually compute ind = 0 - res = np.empty_like(coeffm) + res = np.empty_like(coefm) for b, blk in enumerate(B): col = self.blk_a.partition[b, 1] - res[ind : ind + col, :] = solve(blk, coeffm[ind : ind + col, :]) + res[ind : ind + col, :] = solve(blk, coefm[ind : ind + col, :]) ind += col - coeff_est = B.solve(coeffm) - self.allallfunc(res, coeff_est) + coef_est = B.solve(coefm) + self.allallfunc(res, coef_est) def testBlkDiagMatrixTranspose(self): blk_c = [blk.T for blk in self.blk_a] @@ -446,42 +446,42 @@ def testApply(self): n = np.sum(self.blk_x.partition[:, 0]) m = np.sum(self.blk_x.partition[:, 1]) k = 3 - coeffm = np.arange(k * m).reshape(m, k).astype(self.blk_x.dtype) + coefm = np.arange(k * m).reshape(m, k).astype(self.blk_x.dtype) # Manually compute indc = 0 indr = 0 - res = np.empty(shape=(n, k), dtype=coeffm.dtype) + res = np.empty(shape=(n, k), dtype=coefm.dtype) for b, blk in enumerate(self.blk_x): row, col = self.blk_x.partition[b] - res[indr : indr + row, :] = blk @ coeffm[indc : indc + col, :] + res[indr : indr + row, :] = blk @ coefm[indc : indc + col, :] indc += col indr += row # Check ndim 1 case - c = self.blk_x.apply(coeffm[:, 0]) + c = self.blk_x.apply(coefm[:, 0]) self.allallfunc(c, res[:, 0]) # Check ndim 2 case - d = self.blk_x.apply(coeffm) + d = self.blk_x.apply(coefm) self.allallfunc(res, d) # Check against dense numpy matmul - self.allallfunc(d, self.blk_x.dense() @ coeffm) + self.allallfunc(d, self.blk_x.dense() @ coefm) def testSolve(self): """ Test attempts to solve non square BlkDiagMatrix raise error. """ - # Setup a dummy coeff matrix + # Setup a dummy coef matrix n = np.sum(self.blk_x.partition[:, 0]) k = 3 - coeffm = np.arange(n * k).reshape(n, k).astype(self.blk_x.dtype) + coefm = np.arange(n * k).reshape(n, k).astype(self.blk_x.dtype) with pytest.raises( NotImplementedError, match=r"BlkDiagMatrix.solve is only defined for square arrays.*", ): # Attemplt solve using the Block Diagonal implementation - _ = self.blk_x.solve(coeffm) + _ = self.blk_x.solve(coefm) diff --git a/tests/test_FBbasis2D.py b/tests/test_FBbasis2D.py index 635548b91d..a792572c2f 100644 --- a/tests/test_FBbasis2D.py +++ b/tests/test_FBbasis2D.py @@ -155,8 +155,8 @@ def testHighResFBBasis2D(L, dtype): im = sim.images[0] # Round trip - coeff = basis.expand(im) - im_fb = basis.evaluate(coeff) + coef = basis.expand(im) + im_fb = basis.evaluate(coef) # Mask to compare inside disk of radius 1. mask = grid_2d(L, normalized=True)["r"] < 1 diff --git a/tests/test_FBbasis3D.py b/tests/test_FBbasis3D.py index c32175d9c8..7f9a581fbd 100644 --- a/tests/test_FBbasis3D.py +++ b/tests/test_FBbasis3D.py @@ -355,7 +355,7 @@ def testFBBasis3DNorms(self, basis): ) def testFBBasis3DEvaluate(self, basis): - coeffs = np.array( + coefs = np.array( [ 1.07338590e-01, 1.23690941e-01, @@ -458,7 +458,7 @@ def testFBBasis3DEvaluate(self, basis): ], dtype=basis.dtype, ) - result = Coef(basis, coeffs).evaluate() + result = Coef(basis, coefs).evaluate() assert np.allclose( result.asnumpy(), @@ -687,7 +687,7 @@ def testFBBasis3DExpand(self, basis): ) -# NOTE: This test is failing for L=64. `coeff_0` has a few NANs which propogate into `vol_1`. See GH issue #923 +# NOTE: This test is failing for L=64. `coef_0` has a few NANs which propogate into `vol_1`. See GH issue #923 params = [pytest.param(64, np.float32, marks=pytest.mark.expensive)] @@ -702,8 +702,8 @@ def testHighResFBbasis3D(L, dtype): vol = AsymmetricVolume(L=L, C=1, K=64, dtype=dtype, seed=seed).generate() # Round trip - coeff = basis.expand(vol) - vol_fb = basis.evaluate(coeff) + coef = basis.expand(vol) + vol_fb = basis.evaluate(coef) # Mask to compare inside sphere of radius 1. mask = grid_3d(L, normalized=True)["r"] < 1 diff --git a/tests/test_FFBbasis2D.py b/tests/test_FFBbasis2D.py index 58343863b9..d2977f6e4e 100644 --- a/tests/test_FFBbasis2D.py +++ b/tests/test_FFBbasis2D.py @@ -86,7 +86,7 @@ def testRotate(self, basis): # Now low res (8x8) had problems; # better with odd (7x7), but still not good. # We'll use a higher res test image. - # fh = np.load(os.path.join(DATA_DIR, 'ffbbasis2d_xcoeff_in_8_8.npy'))[:7,:7] + # fh = np.load(os.path.join(DATA_DIR, 'ffbbasis2d_xcoef_in_8_8.npy'))[:7,:7] # Use a real data volume to generate a clean test image. v = Volume( np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype( @@ -131,7 +131,7 @@ def testRotateComplex(self, basis): # Now low res (8x8) had problems; # better with odd (7x7), but still not good. # We'll use a higher res test image. - # fh = np.load(os.path.join(DATA_DIR, 'ffbbasis2d_xcoeff_in_8_8.npy'))[:7,:7] + # fh = np.load(os.path.join(DATA_DIR, 'ffbbasis2d_xcoef_in_8_8.npy'))[:7,:7] # Use a real data volume to generate a clean test image. v = Volume( np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype( @@ -236,8 +236,8 @@ def testHighResFFBBasis2D(L, dtype): im = sim.images[0] # Round trip - coeff = basis.evaluate_t(im) - im_ffb = basis.evaluate(coeff) + coef = basis.evaluate_t(im) + im_ffb = basis.evaluate(coef) # Mask to compare inside disk of radius 1. mask = grid_2d(L, normalized=True)["r"] < 1 diff --git a/tests/test_FFBbasis3D.py b/tests/test_FFBbasis3D.py index c9259dc05b..03bdfdd21b 100644 --- a/tests/test_FFBbasis3D.py +++ b/tests/test_FFBbasis3D.py @@ -356,7 +356,7 @@ def testFFBBasis3DNorms(self, basis): ) def testFFBBasis3DEvaluate(self, basis): - coeffs = np.array( + coefs = np.array( [ 1.07338590e-01, 1.23690941e-01, @@ -460,29 +460,29 @@ def testFFBBasis3DEvaluate(self, basis): dtype=basis.dtype, ) - result = Coef(basis, coeffs).evaluate() + result = Coef(basis, coefs).evaluate() ref = np.load( - os.path.join(DATA_DIR, "ffbbasis3d_xcoeff_out_8_8_8.npy") + os.path.join(DATA_DIR, "ffbbasis3d_xcoef_out_8_8_8.npy") ).T # RCOPT assert np.allclose(result, ref, atol=1e-2) def testFFBBasis3DEvaluate_t(self, basis): - x = np.load(os.path.join(DATA_DIR, "ffbbasis3d_xcoeff_in_8_8_8.npy")).T # RCOPT + x = np.load(os.path.join(DATA_DIR, "ffbbasis3d_xcoef_in_8_8_8.npy")).T # RCOPT x = x.astype(basis.dtype, copy=False) result = basis.evaluate_t(Volume(x)) - ref = np.load(os.path.join(DATA_DIR, "ffbbasis3d_vcoeff_out_8_8_8.npy"))[..., 0] + ref = np.load(os.path.join(DATA_DIR, "ffbbasis3d_vcoef_out_8_8_8.npy"))[..., 0] assert np.allclose(result, ref, atol=1e-2) def testFFBBasis3DExpand(self, basis): - x = np.load(os.path.join(DATA_DIR, "ffbbasis3d_xcoeff_in_8_8_8.npy")).T # RCOPT + x = np.load(os.path.join(DATA_DIR, "ffbbasis3d_xcoef_in_8_8_8.npy")).T # RCOPT x = x.astype(basis.dtype, copy=False) result = basis.expand(x) - ref = np.load(os.path.join(DATA_DIR, "ffbbasis3d_vcoeff_out_exp_8_8_8.npy"))[ + ref = np.load(os.path.join(DATA_DIR, "ffbbasis3d_vcoef_out_exp_8_8_8.npy"))[ ..., 0 ] @@ -502,8 +502,8 @@ def testHighResFFBbasis3D(L, dtype): vol = AsymmetricVolume(L=L, C=1, K=64, dtype=dtype, seed=seed).generate() # Round trip - coeff = basis.evaluate_t(vol) - vol_ffb = basis.evaluate(coeff) + coef = basis.evaluate_t(vol) + vol_ffb = basis.evaluate(coef) # Mask to compare inside sphere of radius 1. mask = grid_3d(L, normalized=True)["r"] < 1 diff --git a/tests/test_FLEbasis2D.py b/tests/test_FLEbasis2D.py index b56c7aaa7c..14eb8faf5b 100644 --- a/tests/test_FLEbasis2D.py +++ b/tests/test_FLEbasis2D.py @@ -99,12 +99,12 @@ def testFastVDense(self, basis): # get sample coefficients x = create_images(basis.nres, 1) # hold input test data constant (would depend on epsilon parameter) - coeffs = FLEBasis2D( + coefs = FLEBasis2D( basis.nres, epsilon=1e-4, dtype=np.float64, match_fb=False ).evaluate_t(x) - result_dense = dense_b @ coeffs.asnumpy().T - result_fast = basis.evaluate(coeffs).asnumpy() + result_dense = dense_b @ coefs.asnumpy().T + result_fast = basis.evaluate(coefs).asnumpy() assert relerr(result_dense, result_fast) < (self.test_eps * basis.epsilon) @@ -136,10 +136,10 @@ def testMatchFBEvaluate(basis): fb_basis = FBBasis2D(basis.nres, dtype=np.float64) # in match_fb, count is the same for both bases - coeffs = Coef(basis, np.eye(basis.count)) + coefs = Coef(basis, np.eye(basis.count)) - fb_images = fb_basis.evaluate(coeffs) - fle_images = basis.evaluate(coeffs) + fb_images = fb_basis.evaluate(coefs) + fle_images = basis.evaluate(coefs) assert np.allclose(fb_images._data, fle_images._data, atol=1e-4) @@ -151,10 +151,10 @@ def testMatchFBDenseEvaluate(basis): fb_basis = FBBasis2D(basis.nres, dtype=np.float64) - coeffs = Coef(basis, np.eye(basis.count)) + coefs = Coef(basis, np.eye(basis.count)) - fb_images = fb_basis.evaluate(coeffs).asnumpy() - fle_out = basis._create_dense_matrix() @ coeffs + fb_images = fb_basis.evaluate(coefs).asnumpy() + fle_out = basis._create_dense_matrix() @ coefs fle_images = Image(fle_out.T.reshape(-1, basis.nres, basis.nres)).asnumpy() # Matrix column reording in match_fb mode flips signs of some of the basis functions @@ -173,10 +173,10 @@ def testMatchFBEvaluate_t(basis): # test images to evaluate images = fb_basis.evaluate(Coef(basis, np.eye(basis.count))) - fb_coeffs = fb_basis.evaluate_t(images) - fle_coeffs = basis.evaluate_t(images) + fb_coefs = fb_basis.evaluate_t(images) + fle_coefs = basis.evaluate_t(images) - assert np.allclose(fb_coeffs, fle_coeffs, atol=1e-4) + assert np.allclose(fb_coefs, fle_coefs, atol=1e-4) @pytest.mark.parametrize("basis", test_bases_match_fb, ids=show_fle_params) @@ -192,11 +192,11 @@ def testMatchFBDenseEvaluate_t(basis): # reshape to a stack of basis.count vectors of length L**2 vec = images.asnumpy().reshape((-1, basis.nres**2)) - fb_coeffs = fb_basis.evaluate_t(images) - fle_coeffs = basis._create_dense_matrix().T @ vec.T + fb_coefs = fb_basis.evaluate_t(images) + fle_coefs = basis._create_dense_matrix().T @ vec.T # Matrix column reording in match_fb mode flips signs of some of the basis coefficients - assert np.allclose(np.abs(fb_coeffs), np.abs(fle_coeffs), atol=1e-4) + assert np.allclose(np.abs(fb_coefs), np.abs(fle_coefs), atol=1e-4) def testLowPass(): @@ -208,22 +208,22 @@ def testLowPass(): # sample coefficients ims = create_images(L, 1) - coeffs = basis.evaluate_t(ims) + coefs = basis.evaluate_t(ims) - nonzero_coeffs = [] + nonzero_coefs = [] for i in range(4): bandlimit = L // (2**i) - coeffs_lowpassed = basis.lowpass(coeffs, bandlimit).asnumpy() - nonzero_coeffs.append(np.sum(coeffs_lowpassed != 0)) + coefs_lowpassed = basis.lowpass(coefs, bandlimit).asnumpy() + nonzero_coefs.append(np.sum(coefs_lowpassed != 0)) # for bandlimit == L, no frequencies should be removed - assert nonzero_coeffs[0] == basis.count + assert nonzero_coefs[0] == basis.count - # for lower bandlimits, there should be fewer and fewer nonzero coeffs - assert nonzero_coeffs[0] > nonzero_coeffs[1] > nonzero_coeffs[2] > nonzero_coeffs[3] + # for lower bandlimits, there should be fewer and fewer nonzero coefs + assert nonzero_coefs[0] > nonzero_coefs[1] > nonzero_coefs[2] > nonzero_coefs[3] # make sure you can pass in a 1-D array if you want - _ = basis.lowpass(coeffs[0, :], L) + _ = basis.lowpass(coefs[0, :], L) def testRotate(): @@ -239,29 +239,29 @@ def testRotate(): ims_90 = Image(np.rot90(ims.asnumpy(), axes=(1, 2))) # get FLE coefficients - coeffs = basis.evaluate_t(ims) - coeffs_cart_rot = basis.evaluate_t(ims_90) + coefs = basis.evaluate_t(ims) + coefs_cart_rot = basis.evaluate_t(ims_90) # rotate original image in FLE space using Steerable rotate method - coeffs_fle_rot = basis.rotate(coeffs, np.pi / 2) + coefs_fle_rot = basis.rotate(coefs, np.pi / 2) # back to cartesian - ims_cart_rot = basis.evaluate(coeffs_cart_rot) - ims_fle_rot = basis.evaluate(coeffs_fle_rot) + ims_cart_rot = basis.evaluate(coefs_cart_rot) + ims_fle_rot = basis.evaluate(coefs_fle_rot) # test rot90 close assert np.allclose(ims_cart_rot[0], ims_fle_rot[0], atol=1e-4) # 2Pi identity in FLE space (rotate by 2Pi) - coeffs_fle_2pi = basis.rotate(coeffs, 2 * np.pi) - ims_fle_2pi = basis.evaluate(coeffs_fle_2pi) + coefs_fle_2pi = basis.rotate(coefs, 2 * np.pi) + ims_fle_2pi = basis.evaluate(coefs_fle_2pi) # test 2Pi identity assert np.allclose(ims[0], ims_fle_2pi[0], atol=utest_tolerance(basis.dtype)) # Reflect in FLE space (rotate by Pi) - coeffs_fle_pi = basis.rotate(coeffs, np.pi) - ims_fle_pi = basis.evaluate(coeffs_fle_pi) + coefs_fle_pi = basis.rotate(coefs, np.pi) + ims_fle_pi = basis.evaluate(coefs_fle_pi) # test reflection assert np.allclose(np.flipud(ims.asnumpy()[0]), ims_fle_pi[0], atol=1e-4) @@ -283,16 +283,16 @@ def testRotate45(): ims = create_images(L, 1) # get FLE coefficients - fb_coeffs = fb_basis.evaluate_t(ims) - coeffs = basis.evaluate_t(ims) + fb_coefs = fb_basis.evaluate_t(ims) + coefs = basis.evaluate_t(ims) # rotate original image in FLE space using Steerable rotate method - fb_coeffs_rot = fb_basis.rotate(fb_coeffs, np.pi / 4) - coeffs_rot = basis.rotate(coeffs, np.pi / 4) + fb_coefs_rot = fb_basis.rotate(fb_coefs, np.pi / 4) + coefs_rot = basis.rotate(coefs, np.pi / 4) # back to cartesian - fb_ims_rot = fb_basis.evaluate(fb_coeffs_rot) - ims_rot = basis.evaluate(coeffs_rot) + fb_ims_rot = fb_basis.evaluate(fb_coefs_rot) + ims_rot = basis.evaluate(coefs_rot) # test close assert np.allclose(ims_rot[0], fb_ims_rot[0], atol=1e-4) @@ -311,13 +311,13 @@ def testRadialConvolution(): # get sample images ims = create_images(L, 10) # convolve using coefficients - coeffs = basis.evaluate_t(ims) - coeffs_convolved = basis.radial_convolve(coeffs, x) - imgs_convolved_fle = basis.evaluate(coeffs_convolved).asnumpy() + coefs = basis.evaluate_t(ims) + coefs_convolved = basis.radial_convolve(coefs, x) + imgs_convolved_fle = basis.evaluate(coefs_convolved).asnumpy() # convolve using FFT x = basis.evaluate(basis.evaluate_t(x)).asnumpy() - ims = basis.evaluate(coeffs).asnumpy() + ims = basis.evaluate(coefs).asnumpy() imgs_convolved_slow = np.zeros((10, L, L)) for i in range(10): diff --git a/tests/test_FPSWFbasis2D.py b/tests/test_FPSWFbasis2D.py index 78fe0b0cee..eb2df64626 100644 --- a/tests/test_FPSWFbasis2D.py +++ b/tests/test_FPSWFbasis2D.py @@ -18,30 +18,26 @@ class TestFPSWFBasis2D(UniversalBasisMixin): def testFPSWFBasis2DEvaluate_t(self, basis): img_ary = np.load( - os.path.join(DATA_DIR, "ffbbasis2d_xcoeff_in_8_8.npy") + os.path.join(DATA_DIR, "ffbbasis2d_xcoef_in_8_8.npy") ).T # RCOPT images = Image(img_ary) result = basis.evaluate_t(images) # Historically, FPSWF returned complex values. # Load and convert them for this hard coded test. - ccoeffs = np.load( - os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") - ).T # RCOPT - coeffs = basis.to_real(Coef(basis, ccoeffs)) + ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT + coefs = basis.to_real(Coef(basis, ccoefs)) - np.testing.assert_allclose(result, coeffs, atol=utest_tolerance(basis.dtype)) + np.testing.assert_allclose(result, coefs, atol=utest_tolerance(basis.dtype)) def testFPSWFBasis2DEvaluate(self, basis): # Historically, FPSWF returned complex values. # Load and convert them for this hard coded test. - ccoeffs = np.load( - os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") - ).T # RCOPT - coeffs = basis.to_real(Coef(basis, ccoeffs)) - result = coeffs.evaluate() + ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT + coefs = basis.to_real(Coef(basis, ccoefs)) + result = coefs.evaluate() - result = basis.evaluate(coeffs) - images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoeff_out_8_8.npy")).T # RCOPT + result = basis.evaluate(coefs) + images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoef_out_8_8.npy")).T # RCOPT np.testing.assert_allclose(result, images, rtol=1e-05, atol=1e-08) diff --git a/tests/test_PSWFbasis2D.py b/tests/test_PSWFbasis2D.py index aaf8bc738c..8b2160391f 100644 --- a/tests/test_PSWFbasis2D.py +++ b/tests/test_PSWFbasis2D.py @@ -17,7 +17,7 @@ class TestPSWFBasis2D(UniversalBasisMixin): def testPSWFBasis2DEvaluate_t(self, basis): img_ary = np.load( - os.path.join(DATA_DIR, "ffbbasis2d_xcoeff_in_8_8.npy") + os.path.join(DATA_DIR, "ffbbasis2d_xcoef_in_8_8.npy") ).T # RCOPT images = Image(img_ary) @@ -25,21 +25,17 @@ def testPSWFBasis2DEvaluate_t(self, basis): # Historically, PSWF returned complex values. # Load and convert them for this hard coded test. - ccoeffs = np.load( - os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") - ).T # RCOPT - coeffs = basis.to_real(Coef(basis, ccoeffs)) + ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT + coefs = basis.to_real(Coef(basis, ccoefs)) - np.testing.assert_allclose(result, coeffs, rtol=1e-05, atol=1e-08) + np.testing.assert_allclose(result, coefs, rtol=1e-05, atol=1e-08) def testPSWFBasis2DEvaluate(self, basis): # Historically, PSWF returned complex values. # Load and convert them for this hard coded test. - ccoeffs = np.load( - os.path.join(DATA_DIR, "pswf2d_vcoeffs_out_8_8.npy") - ).T # RCOPT - coeffs = basis.to_real(Coef(basis, ccoeffs)) + ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT + coefs = basis.to_real(Coef(basis, ccoefs)) - result = coeffs.evaluate() - images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoeff_out_8_8.npy")).T # RCOPT + result = coefs.evaluate() + images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoef_out_8_8.npy")).T # RCOPT assert np.allclose(result.asnumpy(), images) diff --git a/tests/test_batched_covar2d.py b/tests/test_batched_covar2d.py index 075359972c..fbc6d3d7dc 100644 --- a/tests/test_batched_covar2d.py +++ b/tests/test_batched_covar2d.py @@ -39,7 +39,7 @@ def setUp(self): noise_adder=noise_adder, ) self.basis = FFBBasis2D((L, L), dtype=self.dtype) - self.coeff = self.basis.evaluate_t(self.src.images[:]) + self.coef = self.basis.evaluate_t(self.src.images[:]) self.cov2d = RotCov2D(self.basis) self.bcov2d = BatchedRotCov2D(self.src, self.basis, batch_size=7) @@ -60,11 +60,11 @@ def testMeanCovar(self): # Test basic functionality against RotCov2D. mean_cov2d = self.cov2d.get_mean( - self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx + self.coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_cov2d = self.cov2d.get_covar( - self.coeff, - mean_coeff=mean_cov2d, + self.coef, + mean_coef=mean_cov2d, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, noise_var=self.noise_var, @@ -85,16 +85,16 @@ def testMeanCovar(self): def testZeroMean(self): # Make sure it works with zero mean (pure second moment). - zero_coeff = Coef(self.basis, np.zeros((self.basis.count,), dtype=self.dtype)) + zero_coef = Coef(self.basis, np.zeros((self.basis.count,), dtype=self.dtype)) covar_cov2d = self.cov2d.get_covar( - self.coeff, - mean_coeff=zero_coeff, + self.coef, + mean_coef=zero_coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, ) - covar_bcov2d = self.bcov2d.get_covar(mean_coeff=zero_coeff) + covar_bcov2d = self.bcov2d.get_covar(mean_coef=zero_coef) self.assertTrue( self.blk_diag_allclose( @@ -105,7 +105,7 @@ def testZeroMean(self): def testAutoMean(self): # Make sure it automatically calls get_mean if needed. covar_cov2d = self.cov2d.get_covar( - self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx + self.coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_bcov2d = self.bcov2d.get_covar() @@ -129,7 +129,7 @@ def testShrink(self): } covar_cov2d = self.cov2d.get_covar( - self.coeff, + self.coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, covar_est_opt=covar_est_opt, @@ -155,22 +155,22 @@ def testAutoBasis(self): def testCWFCoeff(self): # Calculate CWF coefficients using Cov2D base class mean_cov2d = self.cov2d.get_mean( - self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx + self.coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_cov2d = self.cov2d.get_covar( - self.coeff, + self.coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, noise_var=self.noise_var, make_psd=True, ) - coeff_cov2d = self.cov2d.get_cwf_coeffs( - self.coeff, + coef_cov2d = self.cov2d.get_cwf_coefs( + self.coef, self.ctf_basis, self.ctf_idx, - mean_coeff=mean_cov2d, - covar_coeff=covar_cov2d, + mean_coef=mean_cov2d, + covar_coef=covar_cov2d, noise_var=self.noise_var, ) @@ -178,8 +178,8 @@ def testCWFCoeff(self): mean_bcov2d = self.bcov2d.get_mean() covar_bcov2d = self.bcov2d.get_covar(noise_var=self.noise_var, make_psd=True) - coeff_bcov2d = self.bcov2d.get_cwf_coeffs( - self.coeff, + coef_bcov2d = self.bcov2d.get_cwf_coefs( + self.coef, self.ctf_basis, self.ctf_idx, mean_bcov2d, @@ -188,15 +188,15 @@ def testCWFCoeff(self): ) self.assertTrue( self.blk_diag_allclose( - coeff_cov2d, - coeff_bcov2d, + coef_cov2d, + coef_bcov2d, atol=utest_tolerance(self.dtype), ) ) def testCWFCoeffCleanCTF(self): """ - Test case of clean images (coeff_clean and noise_var=0) + Test case of clean images (coef_clean and noise_var=0) while using a non Identity CTF. This case may come up when a developer switches between @@ -205,22 +205,22 @@ def testCWFCoeffCleanCTF(self): # Calculate CWF coefficients using Cov2D base class mean_cov2d = self.cov2d.get_mean( - self.coeff, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx + self.coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx ) covar_cov2d = self.cov2d.get_covar( - self.coeff, + self.coef, ctf_basis=self.ctf_basis, ctf_idx=self.ctf_idx, noise_var=self.noise_var, make_psd=True, ) - coeff_cov2d = self.cov2d.get_cwf_coeffs( - self.coeff, + coef_cov2d = self.cov2d.get_cwf_coefs( + self.coef, self.ctf_basis, self.ctf_idx, - mean_coeff=mean_cov2d, - covar_coeff=covar_cov2d, + mean_coef=mean_cov2d, + covar_coef=covar_cov2d, noise_var=0, ) @@ -228,8 +228,8 @@ def testCWFCoeffCleanCTF(self): mean_bcov2d = self.bcov2d.get_mean() covar_bcov2d = self.bcov2d.get_covar(noise_var=self.noise_var, make_psd=True) - coeff_bcov2d = self.bcov2d.get_cwf_coeffs( - self.coeff, + coef_bcov2d = self.bcov2d.get_cwf_coefs( + self.coef, self.ctf_basis, self.ctf_idx, mean_bcov2d, @@ -238,8 +238,8 @@ def testCWFCoeffCleanCTF(self): ) self.assertTrue( self.blk_diag_allclose( - coeff_cov2d, - coeff_bcov2d, + coef_cov2d, + coef_bcov2d, atol=utest_tolerance(self.dtype), ) ) diff --git a/tests/test_class2D.py b/tests/test_class2D.py index 47fe2bb1db..92735ee807 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -421,15 +421,15 @@ def test_pca_y(): def test_bispect_overflow(): """ - A zero value coeff will cause a div0 error in log call. + A zero value coef will cause a div0 error in log call. Check it is raised. """ - with pytest.raises(ValueError, match="coeff_norm should not be -inf"): + with pytest.raises(ValueError, match="coef_norm should not be -inf"): # This should emit a warning before raising with pytest.warns(RuntimeWarning): bispec_2drot_large( - coeff=np.arange(10), + coef=np.arange(10), freqs=np.arange(1, 11), eigval=np.arange(10), alpha=1 / 3, diff --git a/tests/test_coef.py b/tests/test_coef.py index 97d963639e..c5e9c2bb70 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -86,7 +86,7 @@ def coef_fixture(basis, stack, dtype): """ Construct parameterized testing coefficient array as `Coef`. """ - # Combine the stack and coefficent counts into multidimensional + # Combine the stack and coeficent counts into multidimensional # shape. size = stack + (basis.count,) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index 1e092c0f35..b064179efb 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -110,18 +110,18 @@ def cov2d_fixture(volume, basis, ctf_enabled): sim.cache() cov2d = RotCov2D(basis) - coeff_clean = basis.evaluate_t(sim.projections[:]) - coeff = basis.evaluate_t(sim.images[:]) + coef_clean = basis.evaluate_t(sim.projections[:]) + coef = basis.evaluate_t(sim.images[:]) - return sim, cov2d, coeff_clean, coeff, h_ctf_fb, h_idx + return sim, cov2d, coef_clean, coef, h_ctf_fb, h_idx def test_get_mean(cov2d_fixture): results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_mean.npy")) - cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + cov2d, coef_clean = cov2d_fixture[1], cov2d_fixture[2] - mean_coeff = cov2d._get_mean(coeff_clean.asnumpy()) - np.testing.assert_allclose(results, mean_coeff, atol=utest_tolerance(cov2d.dtype)) + mean_coef = cov2d._get_mean(coef_clean.asnumpy()) + np.testing.assert_allclose(results, mean_coef, atol=utest_tolerance(cov2d.dtype)) def test_get_covar(cov2d_fixture): @@ -130,59 +130,59 @@ def test_get_covar(cov2d_fixture): allow_pickle=True, ) - cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + cov2d, coef_clean = cov2d_fixture[1], cov2d_fixture[2] - covar_coeff = cov2d._get_covar(coeff_clean.asnumpy()) + covar_coef = cov2d._get_covar(coef_clean.asnumpy()) for im, mat in enumerate(results.tolist()): - np.testing.assert_allclose(mat, covar_coeff[im], rtol=1e-05) + np.testing.assert_allclose(mat, covar_coef[im], rtol=1e-05) def test_get_mean_ctf(cov2d_fixture, ctf_enabled): """ Compare `get_mean` (no CTF args) with `_get_mean` (no CTF model). """ - sim, cov2d, coeff_clean, coeff, h_ctf_fb, h_idx = cov2d_fixture + sim, cov2d, coef_clean, coef, h_ctf_fb, h_idx = cov2d_fixture - mean_coeff_ctf = cov2d.get_mean(coeff, h_ctf_fb, h_idx) + mean_coef_ctf = cov2d.get_mean(coef, h_ctf_fb, h_idx) tol = utest_tolerance(sim.dtype) if ctf_enabled: result = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_meanctf.npy")) else: - result = cov2d._get_mean(coeff_clean.asnumpy()) + result = cov2d._get_mean(coef_clean.asnumpy()) tol = 0.002 - np.testing.assert_allclose(mean_coeff_ctf.asnumpy()[0], result, atol=tol) + np.testing.assert_allclose(mean_coef_ctf.asnumpy()[0], result, atol=tol) -def test_get_cwf_coeffs_clean(cov2d_fixture): +def test_get_cwf_coefs_clean(cov2d_fixture): results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff_clean.npy") + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coef_clean.npy") ) - cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + cov2d, coef_clean = cov2d_fixture[1], cov2d_fixture[2] - coeff_cwf_clean = cov2d.get_cwf_coeffs(coeff_clean, noise_var=0) + coef_cwf_clean = cov2d.get_cwf_coefs(coef_clean, noise_var=0) np.testing.assert_allclose( - results, coeff_cwf_clean, atol=utest_tolerance(cov2d.dtype) + results, coef_cwf_clean, atol=utest_tolerance(cov2d.dtype) ) -def test_get_cwf_coeffs_clean_ctf(cov2d_fixture): +def test_get_cwf_coefs_clean_ctf(cov2d_fixture): """ - Test case of clean images (coeff_clean and noise_var=0) + Test case of clean images (coef_clean and noise_var=0) while using a non Identity CTF. This case may come up when a developer switches between clean and dirty images. """ - sim, cov2d, coeff_clean, _, h_ctf_fb, h_idx = cov2d_fixture + sim, cov2d, coef_clean, _, h_ctf_fb, h_idx = cov2d_fixture - coeff_cwf = cov2d.get_cwf_coeffs(coeff_clean, h_ctf_fb, h_idx, noise_var=0) + coef_cwf = cov2d.get_cwf_coefs(coef_clean, h_ctf_fb, h_idx, noise_var=0) - # Reconstruct images from output of get_cwf_coeffs - img_est = cov2d.basis.evaluate(coeff_cwf) + # Reconstruct images from output of get_cwf_coefs + img_est = cov2d.basis.evaluate(coef_cwf) # Compare with clean images delta = np.mean(np.square((sim.clean_images[:] - img_est).asnumpy())) np.testing.assert_array_less(delta, 0.01) @@ -192,40 +192,40 @@ def test_shrinker_inputs(cov2d_fixture): """ Check we raise with specific message for unsupporting shrinker arg. """ - cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + cov2d, coef_clean = cov2d_fixture[1], cov2d_fixture[2] bad_shrinker_inputs = ["None", "notashrinker", ""] for shrinker in bad_shrinker_inputs: with raises(AssertionError, match="Unsupported shrink method"): - _ = cov2d.get_covar(coeff_clean, covar_est_opt={"shrinker": shrinker}) + _ = cov2d.get_covar(coef_clean, covar_est_opt={"shrinker": shrinker}) def test_shrinkage(cov2d_fixture, shrinker): """ Test all the shrinkers we know about run without crashing, """ - cov2d, coeff_clean = cov2d_fixture[1], cov2d_fixture[2] + cov2d, coef_clean = cov2d_fixture[1], cov2d_fixture[2] results = np.load( os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covar.npy"), allow_pickle=True, ) - covar_coeff = cov2d.get_covar(coeff_clean, covar_est_opt={"shrinker": shrinker}) + covar_coef = cov2d.get_covar(coef_clean, covar_est_opt={"shrinker": shrinker}) for im, mat in enumerate(results.tolist()): np.testing.assert_allclose( - mat, covar_coeff[im], atol=utest_tolerance(cov2d.dtype) + mat, covar_coef[im], atol=utest_tolerance(cov2d.dtype) ) -def test_get_cwf_coeffs_ctf_args(cov2d_fixture): +def test_get_cwf_coefs_ctf_args(cov2d_fixture): """ Test we raise when user supplies incorrect CTF arguments, and that the error message matches. """ - sim, cov2d, _, coeff, h_ctf_fb, _ = cov2d_fixture + sim, cov2d, _, coef, h_ctf_fb, _ = cov2d_fixture # When half the ctf info (h_ctf_fb) is populated, # set the other ctf param (h_idx) to none. @@ -236,32 +236,32 @@ def test_get_cwf_coeffs_ctf_args(cov2d_fixture): # Both the above situations should be an error. with raises(RuntimeError, match=r".*Given ctf_.*"): - _ = cov2d.get_cwf_coeffs(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) + _ = cov2d.get_cwf_coefs(coef, h_ctf_fb, h_idx, noise_var=NOISE_VAR) -def test_get_cwf_coeffs(cov2d_fixture, ctf_enabled): +def test_get_cwf_coefs(cov2d_fixture, ctf_enabled): """ - Tests `get_cwf_coeffs` with poulated CTF. + Tests `get_cwf_coefs` with poulated CTF. """ - _, cov2d, coeff_clean, coeff, h_ctf_fb, h_idx = cov2d_fixture + _, cov2d, coef_clean, coef, h_ctf_fb, h_idx = cov2d_fixture # Hard coded file expects sim with ctf. if not ctf_enabled: pytest.skip(reason="Reference file n/a.") - results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff.npy")) + results = np.load(os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coef.npy")) - coeff_cwf = cov2d.get_cwf_coeffs(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) + coef_cwf = cov2d.get_cwf_coefs(coef, h_ctf_fb, h_idx, noise_var=NOISE_VAR) - np.testing.assert_allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) + np.testing.assert_allclose(results, coef_cwf, atol=utest_tolerance(cov2d.dtype)) -def test_get_cwf_coeffs_without_ctf_args(cov2d_fixture, ctf_enabled): +def test_get_cwf_coefs_without_ctf_args(cov2d_fixture, ctf_enabled): """ - Tests `get_cwf_coeffs` with poulated CTF. + Tests `get_cwf_coefs` with poulated CTF. """ - _, cov2d, _, coeff, _, _ = cov2d_fixture + _, cov2d, _, coef, _, _ = cov2d_fixture # Hard coded file expects sim with ctf. if not ctf_enabled: @@ -270,12 +270,12 @@ def test_get_cwf_coeffs_without_ctf_args(cov2d_fixture, ctf_enabled): # Note, I think this file is incorrectly named... # It appears to have come from operations on images with ctf applied. results = np.load( - os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coeff_noCTF.npy") + os.path.join(DATA_DIR, "clean70SRibosome_cov2d_cwf_coef_noCTF.npy") ) - coeff_cwf = cov2d.get_cwf_coeffs(coeff, noise_var=NOISE_VAR) + coef_cwf = cov2d.get_cwf_coefs(coef, noise_var=NOISE_VAR) - np.testing.assert_allclose(results, coeff_cwf, atol=utest_tolerance(cov2d.dtype)) + np.testing.assert_allclose(results, coef_cwf, atol=utest_tolerance(cov2d.dtype)) def test_get_covar_ctf(cov2d_fixture, ctf_enabled): @@ -283,20 +283,20 @@ def test_get_covar_ctf(cov2d_fixture, ctf_enabled): if not ctf_enabled: pytest.skip(reason="Reference file n/a.") - sim, cov2d, _, coeff, h_ctf_fb, h_idx = cov2d_fixture + sim, cov2d, _, coef, h_ctf_fb, h_idx = cov2d_fixture results = np.load( os.path.join(DATA_DIR, "clean70SRibosome_cov2d_covarctf.npy"), allow_pickle=True, ) - covar_coeff_ctf = cov2d.get_covar(coeff, h_ctf_fb, h_idx, noise_var=NOISE_VAR) + covar_coef_ctf = cov2d.get_covar(coef, h_ctf_fb, h_idx, noise_var=NOISE_VAR) for im, mat in enumerate(results.tolist()): - np.testing.assert_allclose(mat, covar_coeff_ctf[im], rtol=1e-05, atol=1e-08) + np.testing.assert_allclose(mat, covar_coef_ctf[im], rtol=1e-05, atol=1e-08) def test_get_covar_ctf_shrink(cov2d_fixture, ctf_enabled): - sim, cov2d, _, coeff, h_ctf_fb, h_idx = cov2d_fixture + sim, cov2d, _, coef, h_ctf_fb, h_idx = cov2d_fixture # Hard coded file expects sim with ctf. if not ctf_enabled: @@ -317,8 +317,8 @@ def test_get_covar_ctf_shrink(cov2d_fixture, ctf_enabled): "precision": cov2d.dtype, } - covar_coeff_ctf_shrink = cov2d.get_covar( - coeff, + covar_coef_ctf_shrink = cov2d.get_covar( + coef, h_ctf_fb, h_idx, noise_var=NOISE_VAR, @@ -326,4 +326,4 @@ def test_get_covar_ctf_shrink(cov2d_fixture, ctf_enabled): ) for im, mat in enumerate(results.tolist()): - np.testing.assert_allclose(mat, covar_coeff_ctf_shrink[im]) + np.testing.assert_allclose(mat, covar_coef_ctf_shrink[im]) diff --git a/tests/test_mean_estimator.py b/tests/test_mean_estimator.py index ad73bb2716..a3bb9a3a0f 100644 --- a/tests/test_mean_estimator.py +++ b/tests/test_mean_estimator.py @@ -140,10 +140,10 @@ def testEstimate(self): ) def testAdjoint(self): - mean_b_coeff = self.estimator.src_backward().squeeze() + mean_b_coef = self.estimator.src_backward().squeeze() self.assertTrue( np.allclose( - mean_b_coeff, + mean_b_coef, [ 1.07338590e-01, 1.23690941e-01, @@ -249,7 +249,7 @@ def testAdjoint(self): ) def testOptimize1(self): - mean_b_coeff = np.array( + mean_b_coef = np.array( [ [ 1.07338590e-01, @@ -354,7 +354,7 @@ def testOptimize1(self): ] ) - x = self.estimator.conj_grad(mean_b_coeff) + x = self.estimator.conj_grad(mean_b_coef) self.assertTrue( np.allclose( x, @@ -463,7 +463,7 @@ def testOptimize1(self): ) def testOptimize2(self): - mean_b_coeff = np.array( + mean_b_coef = np.array( [ [ 1.07338590e-01, @@ -568,7 +568,7 @@ def testOptimize2(self): ] ) - x = self.estimator_with_preconditioner.conj_grad(mean_b_coeff) + x = self.estimator_with_preconditioner.conj_grad(mean_b_coef) self.assertTrue( np.allclose( x, diff --git a/tests/test_weighted_mean_estimator.py b/tests/test_weighted_mean_estimator.py index e7be5cf77a..a66440a330 100644 --- a/tests/test_weighted_mean_estimator.py +++ b/tests/test_weighted_mean_estimator.py @@ -135,10 +135,10 @@ def testPositiveWeightedEstimates(self): self.assertTrue(np.allclose(a, b, atol=1e-5)) def testAdjoint(self): - mean_b_coeff = self.estimator.src_backward().squeeze() + mean_b_coef = self.estimator.src_backward().squeeze() self.assertTrue( np.allclose( - mean_b_coeff, + mean_b_coef, [ 1.07338590e-01, 1.23690941e-01, @@ -244,7 +244,7 @@ def testAdjoint(self): ) def testOptimize1(self): - mean_b_coeff = np.array( + mean_b_coef = np.array( [ [ 1.07338590e-01, @@ -351,7 +351,7 @@ def testOptimize1(self): ) # Given equal weighting we should get the same result for all self.r volumes. - x = self.estimator.conj_grad(mean_b_coeff) + x = self.estimator.conj_grad(mean_b_coef) ref = np.array( [ @@ -461,7 +461,7 @@ def testOptimize1(self): self.assertTrue(np.allclose(x.flatten(), ref, atol=1e-4)) def testOptimize2(self): - mean_b_coeff = np.array( + mean_b_coef = np.array( [ [ 1.07338590e-01, @@ -567,7 +567,7 @@ def testOptimize2(self): * self.r ) - x = self.estimator_with_preconditioner.conj_grad(mean_b_coeff) + x = self.estimator_with_preconditioner.conj_grad(mean_b_coef) self.assertTrue( np.allclose( x, From b1549afd84026f8c0b530a758ab4d69683457056 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 08:43:10 -0400 Subject: [PATCH 084/294] coeff to coef bin filename conversion --- ...oeff.npy => clean70SRibosome_cov2d_cwf_coef.npy} | Bin ...py => clean70SRibosome_cov2d_cwf_coef_clean.npy} | Bin ...py => clean70SRibosome_cov2d_cwf_coef_noCTF.npy} | Bin ...ficients_8_8.npy => fbbasis_coeficients_8_8.npy} | Bin ...eff_out_8_8.npy => ffbbasis2d_vcoef_out_8_8.npy} | Bin ...exp_8_8.npy => ffbbasis2d_vcoef_out_exp_8_8.npy} | Bin ...coeff_in_8_8.npy => ffbbasis2d_xcoef_in_8_8.npy} | Bin ...eff_out_8_8.npy => ffbbasis2d_xcoef_out_8_8.npy} | Bin ...out_8_8_8.npy => ffbbasis3d_vcoef_out_8_8_8.npy} | Bin ...8_8_8.npy => ffbbasis3d_vcoef_out_exp_8_8_8.npy} | Bin ...f_in_8_8_8.npy => ffbbasis3d_xcoef_in_8_8_8.npy} | Bin ...out_8_8_8.npy => ffbbasis3d_xcoef_out_8_8_8.npy} | Bin ...ts_8_4_32.npy => pfbasis_coeficients_8_4_32.npy} | Bin ...coeffs_out_8_8.npy => pswf2d_vcoefs_out_8_8.npy} | Bin ..._xcoeff_out_8_8.npy => pswf2d_xcoef_out_8_8.npy} | Bin 15 files changed, 0 insertions(+), 0 deletions(-) rename tests/saved_test_data/{clean70SRibosome_cov2d_cwf_coeff.npy => clean70SRibosome_cov2d_cwf_coef.npy} (100%) rename tests/saved_test_data/{clean70SRibosome_cov2d_cwf_coeff_clean.npy => clean70SRibosome_cov2d_cwf_coef_clean.npy} (100%) rename tests/saved_test_data/{clean70SRibosome_cov2d_cwf_coeff_noCTF.npy => clean70SRibosome_cov2d_cwf_coef_noCTF.npy} (100%) rename tests/saved_test_data/{fbbasis_coefficients_8_8.npy => fbbasis_coeficients_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis2d_vcoeff_out_8_8.npy => ffbbasis2d_vcoef_out_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis2d_vcoeff_out_exp_8_8.npy => ffbbasis2d_vcoef_out_exp_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis2d_xcoeff_in_8_8.npy => ffbbasis2d_xcoef_in_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis2d_xcoeff_out_8_8.npy => ffbbasis2d_xcoef_out_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis3d_vcoeff_out_8_8_8.npy => ffbbasis3d_vcoef_out_8_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis3d_vcoeff_out_exp_8_8_8.npy => ffbbasis3d_vcoef_out_exp_8_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis3d_xcoeff_in_8_8_8.npy => ffbbasis3d_xcoef_in_8_8_8.npy} (100%) rename tests/saved_test_data/{ffbbasis3d_xcoeff_out_8_8_8.npy => ffbbasis3d_xcoef_out_8_8_8.npy} (100%) rename tests/saved_test_data/{pfbasis_coefficients_8_4_32.npy => pfbasis_coeficients_8_4_32.npy} (100%) rename tests/saved_test_data/{pswf2d_vcoeffs_out_8_8.npy => pswf2d_vcoefs_out_8_8.npy} (100%) rename tests/saved_test_data/{pswf2d_xcoeff_out_8_8.npy => pswf2d_xcoef_out_8_8.npy} (100%) diff --git a/tests/saved_test_data/clean70SRibosome_cov2d_cwf_coeff.npy b/tests/saved_test_data/clean70SRibosome_cov2d_cwf_coef.npy similarity index 100% rename from tests/saved_test_data/clean70SRibosome_cov2d_cwf_coeff.npy rename to tests/saved_test_data/clean70SRibosome_cov2d_cwf_coef.npy diff --git a/tests/saved_test_data/clean70SRibosome_cov2d_cwf_coeff_clean.npy b/tests/saved_test_data/clean70SRibosome_cov2d_cwf_coef_clean.npy similarity index 100% rename from tests/saved_test_data/clean70SRibosome_cov2d_cwf_coeff_clean.npy rename to tests/saved_test_data/clean70SRibosome_cov2d_cwf_coef_clean.npy diff --git a/tests/saved_test_data/clean70SRibosome_cov2d_cwf_coeff_noCTF.npy b/tests/saved_test_data/clean70SRibosome_cov2d_cwf_coef_noCTF.npy similarity index 100% rename from tests/saved_test_data/clean70SRibosome_cov2d_cwf_coeff_noCTF.npy rename to tests/saved_test_data/clean70SRibosome_cov2d_cwf_coef_noCTF.npy diff --git a/tests/saved_test_data/fbbasis_coefficients_8_8.npy b/tests/saved_test_data/fbbasis_coeficients_8_8.npy similarity index 100% rename from tests/saved_test_data/fbbasis_coefficients_8_8.npy rename to tests/saved_test_data/fbbasis_coeficients_8_8.npy diff --git a/tests/saved_test_data/ffbbasis2d_vcoeff_out_8_8.npy b/tests/saved_test_data/ffbbasis2d_vcoef_out_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis2d_vcoeff_out_8_8.npy rename to tests/saved_test_data/ffbbasis2d_vcoef_out_8_8.npy diff --git a/tests/saved_test_data/ffbbasis2d_vcoeff_out_exp_8_8.npy b/tests/saved_test_data/ffbbasis2d_vcoef_out_exp_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis2d_vcoeff_out_exp_8_8.npy rename to tests/saved_test_data/ffbbasis2d_vcoef_out_exp_8_8.npy diff --git a/tests/saved_test_data/ffbbasis2d_xcoeff_in_8_8.npy b/tests/saved_test_data/ffbbasis2d_xcoef_in_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis2d_xcoeff_in_8_8.npy rename to tests/saved_test_data/ffbbasis2d_xcoef_in_8_8.npy diff --git a/tests/saved_test_data/ffbbasis2d_xcoeff_out_8_8.npy b/tests/saved_test_data/ffbbasis2d_xcoef_out_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis2d_xcoeff_out_8_8.npy rename to tests/saved_test_data/ffbbasis2d_xcoef_out_8_8.npy diff --git a/tests/saved_test_data/ffbbasis3d_vcoeff_out_8_8_8.npy b/tests/saved_test_data/ffbbasis3d_vcoef_out_8_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis3d_vcoeff_out_8_8_8.npy rename to tests/saved_test_data/ffbbasis3d_vcoef_out_8_8_8.npy diff --git a/tests/saved_test_data/ffbbasis3d_vcoeff_out_exp_8_8_8.npy b/tests/saved_test_data/ffbbasis3d_vcoef_out_exp_8_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis3d_vcoeff_out_exp_8_8_8.npy rename to tests/saved_test_data/ffbbasis3d_vcoef_out_exp_8_8_8.npy diff --git a/tests/saved_test_data/ffbbasis3d_xcoeff_in_8_8_8.npy b/tests/saved_test_data/ffbbasis3d_xcoef_in_8_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis3d_xcoeff_in_8_8_8.npy rename to tests/saved_test_data/ffbbasis3d_xcoef_in_8_8_8.npy diff --git a/tests/saved_test_data/ffbbasis3d_xcoeff_out_8_8_8.npy b/tests/saved_test_data/ffbbasis3d_xcoef_out_8_8_8.npy similarity index 100% rename from tests/saved_test_data/ffbbasis3d_xcoeff_out_8_8_8.npy rename to tests/saved_test_data/ffbbasis3d_xcoef_out_8_8_8.npy diff --git a/tests/saved_test_data/pfbasis_coefficients_8_4_32.npy b/tests/saved_test_data/pfbasis_coeficients_8_4_32.npy similarity index 100% rename from tests/saved_test_data/pfbasis_coefficients_8_4_32.npy rename to tests/saved_test_data/pfbasis_coeficients_8_4_32.npy diff --git a/tests/saved_test_data/pswf2d_vcoeffs_out_8_8.npy b/tests/saved_test_data/pswf2d_vcoefs_out_8_8.npy similarity index 100% rename from tests/saved_test_data/pswf2d_vcoeffs_out_8_8.npy rename to tests/saved_test_data/pswf2d_vcoefs_out_8_8.npy diff --git a/tests/saved_test_data/pswf2d_xcoeff_out_8_8.npy b/tests/saved_test_data/pswf2d_xcoef_out_8_8.npy similarity index 100% rename from tests/saved_test_data/pswf2d_xcoeff_out_8_8.npy rename to tests/saved_test_data/pswf2d_xcoef_out_8_8.npy From f047bdee28fa5f736a93d63896d6f1bfe11754d2 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 27 Sep 2023 14:54:06 -0400 Subject: [PATCH 085/294] initial stub ComplexCoef --- src/aspire/basis/__init__.py | 2 +- src/aspire/basis/basis.py | 115 ++++++++++++++++++++++++++++++++-- src/aspire/basis/steerable.py | 8 +-- 3 files changed, 114 insertions(+), 11 deletions(-) diff --git a/src/aspire/basis/__init__.py b/src/aspire/basis/__init__.py index c150692f32..8f292a31bc 100644 --- a/src/aspire/basis/__init__.py +++ b/src/aspire/basis/__init__.py @@ -1,7 +1,7 @@ # We'll tell isort not to sort these base classes # isort: off -from .basis import Basis, Coef +from .basis import Basis, Coef, ComplexCoef from .steerable import SteerableBasis2D from .fb import FBBasisMixin diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 6f75e9da29..894457eff1 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -12,10 +12,12 @@ class Coef: """ - Numpy interoperable container for stacks of coefficient vectors. + Numpy interoperable container for stacks of real coefficient vectors. Each `Coef` instance has an associated `Basis`. """ + _allowed_dtypes = (np.float32, np.float64) + def __init__(self, basis, data, dtype=None): """ A stack of one or more coefficient arrays. @@ -50,6 +52,9 @@ def __init__(self, basis, data, dtype=None): else: self.dtype = np.dtype(dtype) + # Check real/complex dtype basis on class. + self._check_dtype() + if not isinstance(basis, Basis): raise TypeError( f"`basis` is required to be a `Basis` instance, received {type(basis)}" @@ -64,10 +69,8 @@ def __init__(self, basis, data, dtype=None): self.stack_size = np.prod(self.stack_shape) self.count = self._data.shape[-1] - # Derive count based on real/complex coefficients. - basis_count = self.basis.count - if np.iscomplexobj(data): - basis_count = self.basis.complex_count + # Derive count from basis. + basis_count = self._get_basis_count() if self.count != basis_count: raise RuntimeError( @@ -79,6 +82,26 @@ def __init__(self, basis, data, dtype=None): self.__array_interface__ = self.asnumpy().__array_interface__ self.__array__ = self.asnumpy() + def _check_dtype(self): + """ + Private helper method to check real/complex dtype based on class `_allowed_dtypes`. + + Raises on mismatch. + """ + + if self.dtype not in self._allowed_dtypes: + raise TypeError( + f"{self.__class__.__name__} requires {self._allowed_dtypes} coefficients, attempted {self.dtype}." + ) + + def _get_basis_count(self): + """ + Private helper method to return coefficient count from basis. + + :return: Basis count (integer). + """ + return int(self.basis.count) + def __len__(self): """ Return length of slowest stack axis. @@ -172,7 +195,7 @@ def shift(self, shifts): Returns coefs shifted by `shifts`. This will transform to real cartesian space, shift, - and transform back to Polar Fourier-Bessel space. + and transform back to basis space. :param coef: Basis coefs. :param shifts: Shifts in pixels (x,y). Shape (1,2) or (len(coef), 2). @@ -260,6 +283,86 @@ def by_indices(self, **kwargs): mask = self.basis.indices_mask(**kwargs) return self._data[..., mask] + def to_complex(self): + """ + Return `ComplexCoef` of real coefficients. + """ + return self.basis, self.basis.to_complex() + + def to_real(self): + """ + Not implemented for real Coef. + """ + raise TypeError("Coef already real.") + + +class ComplexCoef(Coef): + """ + Numpy interoperable container for stacks of complex coefficient vectors. + Each `ComplexCoef` instance has an associated `Basis`. + """ + + _allowed_dtypes = (np.complex64, np.complex128) + + def _get_basis_count(self): + """ + Private helper method to return coefficient complex count from basis. + + :return: Basis complex count (integer). + """ + + return int(self.basis.complex_count) + + def evaluate(self): + """ + Return the evaluation of coefficients in the associated `basis`. + """ + super().evaluate(self.to_real()) + + def rotate(self, radians, refl=None): + """ + Returns coefs rotated counter-clockwise by `radians`. + + Raises error if underlying coef basis does not support rotations. + + :param radians: Rotation in radians. + :param refl: Optional reflect image (about y=0) (bool) + :return: Rotated ComplexCoefs. + """ + + return self.basis.rotate(self.to_real(), radians, refl).to_complex() + + def shift(self, shifts): + """ + Returns complex coefs shifted by `shifts`. + + This will transform to real cartesian space, shift, + and transform back to basis space. + + :param coef: Basis coefs. + :param shifts: Shifts in pixels (x,y). Shape (1,2) or (len(coef), 2). + :return: Complex coefs of shifted images. + """ + + if not callable(getattr(self.basis, "shift", None)): + raise RuntimeError( + f"self.basis={self.basis} does not provide `shift` method." + ) + + return self.basis.shift(self.to_real(), shifts).to_complex() + + def to_real(self): + """ + Return `Coef` of complex coefficients. + """ + return self.basis.to_real() + + def to_complex(self): + """ + Not implemented for ComplexCoef. + """ + raise TypeError("Coef already complex.") + class Basis: """ diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 148d756efe..321c2b47bc 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -4,7 +4,7 @@ import numpy as np -from aspire.basis import Basis, Coef +from aspire.basis import Basis, Coef, ComplexCoef from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type, real_type @@ -401,7 +401,7 @@ def to_real(self, complex_coef): :return: Real `Ceof` representation from this basis. """ - if not isinstance(complex_coef, Coef): + if not isinstance(complex_coef, ComplexCoef): raise TypeError( f"complex_coef should be instance of `Coef`, received {type(complex_coef)}." ) @@ -449,7 +449,7 @@ def to_complex(self, coef): There is a corresponding method, `to_real`. :param coef: Real `Coef` from this basis. - :return: Complex `Coef` representation from this basis. + :return: `ComplexCoef` representation from this basis. """ if not isinstance(coef, Coef): @@ -487,4 +487,4 @@ def to_complex(self, coef): ind += np.size(idx) - return Coef(self, complex_coef) + return ComplexCoef(self, complex_coef) From 9cc65a16c3c909ffd9c7c9a45b0246cb0114695c Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 10:37:36 -0400 Subject: [PATCH 086/294] minimal test updates to support ComplexCoef addition --- src/aspire/basis/basis.py | 4 +-- src/aspire/basis/fpswf_2d.py | 6 ++--- src/aspire/basis/fspca.py | 4 +-- src/aspire/basis/pswf_2d.py | 8 +++--- src/aspire/classification/rir_class2d.py | 11 ++++----- tests/test_FBbasis2D.py | 31 +++++++++++++++++++++--- tests/test_FPSWFbasis2D.py | 6 ++--- tests/test_PSWFbasis2D.py | 6 ++--- tests/test_class2D.py | 4 +-- 9 files changed, 51 insertions(+), 29 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 894457eff1..f1c38515e8 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -287,7 +287,7 @@ def to_complex(self): """ Return `ComplexCoef` of real coefficients. """ - return self.basis, self.basis.to_complex() + return self.basis.to_complex(self) def to_real(self): """ @@ -355,7 +355,7 @@ def to_real(self): """ Return `Coef` of complex coefficients. """ - return self.basis.to_real() + return self.basis.to_real(self) def to_complex(self): """ diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index deed21bb61..393a03fddc 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -5,7 +5,7 @@ from scipy.optimize import least_squares from scipy.special import jn -from aspire.basis import Coef +from aspire.basis import ComplexCoef from aspire.basis.basis_utils import lgwt, t_x_mat, t_x_mat_dot from aspire.basis.pswf_2d import PSWFBasis2D from aspire.nufft import nufft @@ -124,9 +124,9 @@ def _evaluate_t(self, images): nfft_res = nufft(images_disk, self.us_fft_pts) # Accumulate coefficients - coefficients = Coef(self, self._pswf_integration(nfft_res)) + coefficients = ComplexCoef(self, self._pswf_integration(nfft_res)) - return self.to_real(coefficients).asnumpy() + return coefficients.to_real().asnumpy() def _generate_pswf_quad( self, n, bandlimit, phi_approximate_error, lambda_max, epsilon diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index 1974ed2757..0989ed4f73 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -3,7 +3,7 @@ import numpy as np -from aspire.basis import Coef, FFBBasis2D, SteerableBasis2D +from aspire.basis import Coef, ComplexCoef, FFBBasis2D, SteerableBasis2D from aspire.covariance import BatchedRotCov2D from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type, fix_signs, real_type @@ -512,7 +512,7 @@ def to_complex(self, coef): for i, k in enumerate(ccoef_d.keys()): ccoef[:, i] = ccoef_d[k] - return Coef(self, ccoef) + return ComplexCoef(self, ccoef) def to_real(self, complex_coef): """ diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index dce38536c9..c1deb92c21 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -2,7 +2,7 @@ import numpy as np -from aspire.basis import Coef, SteerableBasis2D +from aspire.basis import Coef, ComplexCoef, SteerableBasis2D from aspire.basis.basis_utils import ( d_decay_approx_fun, k_operator, @@ -221,8 +221,8 @@ def _evaluate_t(self, images): :return: The evaluation of the coefficient array in the PSWF basis. """ flattened_images = images[:, self._disk_mask] - complex_coef = Coef(self, flattened_images @ self.samples_conj_transpose) - return self.to_real(complex_coef).asnumpy() + complex_coef = ComplexCoef(self, flattened_images @ self.samples_conj_transpose) + return complex_coef.to_real().asnumpy() def _evaluate(self, coefficients): """ @@ -235,7 +235,7 @@ def _evaluate(self, coefficients): """ # Convert real coefficient to complex. - coefficients = self.to_complex(Coef(self, coefficients)) + coefficients = Coef(self, coefficients).to_complex() # Handle a single coefficient vector or stack of vectors. coefficients = np.atleast_2d(coefficients) diff --git a/src/aspire/classification/rir_class2d.py b/src/aspire/classification/rir_class2d.py index e0db1bf8a5..d989efd84c 100644 --- a/src/aspire/classification/rir_class2d.py +++ b/src/aspire/classification/rir_class2d.py @@ -4,7 +4,7 @@ import numpy as np from sklearn.neighbors import NearestNeighbors -from aspire.basis import Coef, FSPCABasis +from aspire.basis import Coef, ComplexCoef, FSPCABasis from aspire.classification import Class2D from aspire.classification.legacy_implementations import bispec_2drot_large, pca_y from aspire.numeric import ComplexPCA @@ -411,9 +411,7 @@ def _devel_bispectrum(self, coef): coef = self.pca_basis.to_complex(coef) # Take just positive frequencies, corresponds to complex indices. # Original implementation used norm of Complex values, here abs of Real. - eigvals = np.abs( - self.pca_basis.eigvals.asnumpy()[0, self.pca_basis.signs_indices >= 0] - ) + eigvals = np.abs(self.pca_basis.eigvals[self.pca_basis.signs_indices >= 0]) # Legacy code included a sanity check: # non_zero_freqs = self.pca_basis.complex_angular_indices != 0 @@ -440,7 +438,7 @@ def _devel_bispectrum(self, coef): for i in trange(self.src.n): B = self.pca_basis.calculate_bispectrum( - Coef(self.pca_basis, coef_normed[i]), + ComplexCoef(self.pca_basis, coef_normed[i]), filter_nonzero_freqs=True, freq_cutoff=self.bispectrum_freq_cutoff, ) @@ -496,7 +494,8 @@ def _legacy_bispectrum(self, coef, retry_attempts=3): # The legacy code expects the complex representation coef = self.pca_basis.to_complex(coef).asnumpy() complex_eigvals = ( - self.pca_basis.to_complex(self.pca_basis.eigvals) + Coef(self.pca_basis, self.pca_basis.eigvals) + .to_complex() .asnumpy() .reshape(self.pca_basis.complex_count) ) # flatten diff --git a/tests/test_FBbasis2D.py b/tests/test_FBbasis2D.py index a792572c2f..b24c14568d 100644 --- a/tests/test_FBbasis2D.py +++ b/tests/test_FBbasis2D.py @@ -5,7 +5,7 @@ from pytest import raises from scipy.special import jv -from aspire.basis import Coef, FBBasis2D +from aspire.basis import Coef, ComplexCoef, FBBasis2D from aspire.image import Image from aspire.source import Simulation from aspire.utils import complex_type, real_type @@ -86,6 +86,14 @@ def testComplexCoversion(self, basis): # The round trip should be equivalent up to machine precision assert np.allclose(v1, v2) + # Convert real FB coef to complex coef using Coef class + cv = v1.to_complex() + # then convert back to real coef representation. + v2 = cv.to_real() + + # The round trip should be equivalent up to machine precision + assert np.allclose(v1, v2) + def testComplexCoversionErrorsToComplex(self, basis): x = randn(*basis.sz, seed=self.seed).astype(basis.dtype) @@ -98,6 +106,16 @@ def testComplexCoversionErrorsToComplex(self, basis): v1_cpx = Coef(basis, v1, dtype=np.complex64) _ = basis.to_complex(v1_cpx) + # Test catching Errors + with raises(TypeError): + # Pass complex into `to_complex` + v1_cpx = Coef(basis, v1, dtype=np.complex64) + + with raises(TypeError): + # Pass complex into `to_complex` + v1_cpx = Coef(basis, v1).to_complex() + _ = v1_cpx.to_complex() + # Test casting case, where basis and coef don't match if basis.dtype == np.float32: test_dtype = np.float64 @@ -114,13 +132,18 @@ def testComplexCoversionErrorsToReal(self, basis): # Express in an FB basis cv = basis.expand(x.astype(basis.dtype)) - ccv = basis.to_complex(cv) + ccv = cv.to_complex() # Test catching Errors with raises(TypeError): # Pass real into `to_real` _ = basis.to_real(cv) + # Test catching Errors + with raises(TypeError): + # Pass real into `to_real` + _ = cv.to_real() + # Test casting case, where basis and coef precision don't match if basis.dtype == np.float32: test_dtype = np.complex128 @@ -129,8 +152,8 @@ def testComplexCoversionErrorsToReal(self, basis): # Result should be same precision as coef input, just real. result_dtype = real_type(test_dtype) - x = Coef(basis, ccv.asnumpy().astype(test_dtype)) - v3 = basis.to_real(x) + x = ComplexCoef(basis, ccv.asnumpy().astype(test_dtype)) + v3 = x.to_real() assert v3.dtype == result_dtype diff --git a/tests/test_FPSWFbasis2D.py b/tests/test_FPSWFbasis2D.py index eb2df64626..7c9a7133fb 100644 --- a/tests/test_FPSWFbasis2D.py +++ b/tests/test_FPSWFbasis2D.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from aspire.basis import Coef, FPSWFBasis2D +from aspire.basis import ComplexCoef, FPSWFBasis2D from aspire.image import Image from aspire.utils import utest_tolerance @@ -26,7 +26,7 @@ def testFPSWFBasis2DEvaluate_t(self, basis): # Historically, FPSWF returned complex values. # Load and convert them for this hard coded test. ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT - coefs = basis.to_real(Coef(basis, ccoefs)) + coefs = ComplexCoef(basis, ccoefs).to_real() np.testing.assert_allclose(result, coefs, atol=utest_tolerance(basis.dtype)) @@ -34,7 +34,7 @@ def testFPSWFBasis2DEvaluate(self, basis): # Historically, FPSWF returned complex values. # Load and convert them for this hard coded test. ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT - coefs = basis.to_real(Coef(basis, ccoefs)) + coefs = ComplexCoef(basis, ccoefs).to_real() result = coefs.evaluate() result = basis.evaluate(coefs) diff --git a/tests/test_PSWFbasis2D.py b/tests/test_PSWFbasis2D.py index 8b2160391f..21c182d6da 100644 --- a/tests/test_PSWFbasis2D.py +++ b/tests/test_PSWFbasis2D.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from aspire.basis import Coef, PSWFBasis2D +from aspire.basis import ComplexCoef, PSWFBasis2D from aspire.image import Image from ._basis_util import UniversalBasisMixin, pswf_params_2d, show_basis_params @@ -26,7 +26,7 @@ def testPSWFBasis2DEvaluate_t(self, basis): # Historically, PSWF returned complex values. # Load and convert them for this hard coded test. ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT - coefs = basis.to_real(Coef(basis, ccoefs)) + coefs = ComplexCoef(basis, ccoefs).to_real() np.testing.assert_allclose(result, coefs, rtol=1e-05, atol=1e-08) @@ -34,7 +34,7 @@ def testPSWFBasis2DEvaluate(self, basis): # Historically, PSWF returned complex values. # Load and convert them for this hard coded test. ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT - coefs = basis.to_real(Coef(basis, ccoefs)) + coefs = ComplexCoef(basis, ccoefs).to_real() result = coefs.evaluate() images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoef_out_8_8.npy")).T # RCOPT diff --git a/tests/test_class2D.py b/tests/test_class2D.py index 92735ee807..db58a914aa 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -99,7 +99,7 @@ def test_complex_conversions_errors(sim_fixture): """ imgs, _, fspca_basis = sim_fixture - with pytest.raises(TypeError, match="coef provided to to_complex should be real."): + with pytest.raises(TypeError): _ = fspca_basis.to_complex( Coef( fspca_basis, @@ -108,7 +108,7 @@ def test_complex_conversions_errors(sim_fixture): ) ) - with pytest.raises(TypeError, match="coef provided to to_real should be complex."): + with pytest.raises(TypeError): _ = fspca_basis.to_real( Coef(fspca_basis, np.arange(fspca_basis.count), dtype=np.float32) ) From 11214ff070f7ef8f63396e62eeea114d98d4d68d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 13:25:46 -0400 Subject: [PATCH 087/294] address recent numpy deprecation warning in related code --- src/aspire/operators/blk_diag_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index 14d64a762a..eeba0a508e 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -766,7 +766,7 @@ def check_psd(self): :return: True if all blocks have non-negative eigenvalues. """ - return np.alltrue(self.eigvals() > 0.0) + return np.all(self.eigvals() > 0.0) def make_psd(self): """ From 624baf0e7bf11376f1ed4db3e129b13c83264203 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 15:21:12 -0400 Subject: [PATCH 088/294] touch ups --- src/aspire/basis/basis.py | 10 +++++----- src/aspire/basis/fspca.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index f1c38515e8..9902a54ed3 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -52,7 +52,7 @@ def __init__(self, basis, data, dtype=None): else: self.dtype = np.dtype(dtype) - # Check real/complex dtype basis on class. + # Check real/complex dtype based on class. self._check_dtype() if not isinstance(basis, Basis): @@ -285,7 +285,7 @@ def by_indices(self, **kwargs): def to_complex(self): """ - Return `ComplexCoef` of real coefficients. + Convert and return real coefficients as `ComplexCoef`. """ return self.basis.to_complex(self) @@ -317,7 +317,7 @@ def evaluate(self): """ Return the evaluation of coefficients in the associated `basis`. """ - super().evaluate(self.to_real()) + return super().evaluate(self.to_real()) def rotate(self, radians, refl=None): """ @@ -353,7 +353,7 @@ def shift(self, shifts): def to_real(self): """ - Return `Coef` of complex coefficients. + Convert and return complex coefficients as `Coef`. """ return self.basis.to_real(self) @@ -361,7 +361,7 @@ def to_complex(self): """ Not implemented for ComplexCoef. """ - raise TypeError("Coef already complex.") + raise TypeError("ComplexCoef already complex.") class Basis: diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index 0989ed4f73..cd264e8f60 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -569,7 +569,7 @@ def calculate_bispectrum( @property def eigvals(self): """ - Return the eigenvals as a Coef instance of FSPCABasis. + Return the eigenvals of FSPCABasis as Numpy array. """ return self._eigvals From 3b300c9e1e97e2228b271188dbb7c2d91aab09fc Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 16:07:57 -0400 Subject: [PATCH 089/294] improve test coverage of new ComplexCoef class --- src/aspire/basis/basis.py | 6 +-- tests/test_coef.py | 89 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 9902a54ed3..ed3a1f404d 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -317,7 +317,7 @@ def evaluate(self): """ Return the evaluation of coefficients in the associated `basis`. """ - return super().evaluate(self.to_real()) + return self.to_real().evaluate() def rotate(self, radians, refl=None): """ @@ -330,7 +330,7 @@ def rotate(self, radians, refl=None): :return: Rotated ComplexCoefs. """ - return self.basis.rotate(self.to_real(), radians, refl).to_complex() + return self.to_real().rotate(radians, refl).to_complex() def shift(self, shifts): """ @@ -349,7 +349,7 @@ def shift(self, shifts): f"self.basis={self.basis} does not provide `shift` method." ) - return self.basis.shift(self.to_real(), shifts).to_complex() + return self.to_real().shift(shifts).to_complex() def to_real(self): """ diff --git a/tests/test_coef.py b/tests/test_coef.py index c5e9c2bb70..ab546d9bac 100644 --- a/tests/test_coef.py +++ b/tests/test_coef.py @@ -95,6 +95,14 @@ def coef_fixture(basis, stack, dtype): return Coef(basis, coef_np, dtype=dtype) +@pytest.fixture(scope="module") +def rots(coef_fixture, dtype): + # Rotations + return np.linspace(-np.pi, np.pi, coef_fixture.stack_size).reshape( + coef_fixture.stack_shape + ) + + def test_mismatch_count(basis): """ Confirm raises when instantiated with incorrect coef vector len. @@ -304,16 +312,11 @@ def test_coef_evalute(coef_fixture, basis): ) -def test_coef_rotate(coef_fixture, basis): +def test_coef_rotate(coef_fixture, basis, rots): """ Test rotation pass through. """ - # Rotations - rots = np.linspace(-np.pi, np.pi, coef_fixture.stack_size).reshape( - coef_fixture.stack_shape - ) - # Refl refl = ( np.random.rand(coef_fixture.stack_size).reshape(coef_fixture.stack_shape) > 0.5 @@ -367,3 +370,77 @@ def test_real_complex_real_roundtrip(coef_fixture, basis): rcoef = basis.to_real(basis.to_complex(coef_fixture)) np.testing.assert_allclose(rcoef, coef_fixture, rtol=1e-05, atol=1e-08) + + +def test_complex_evaluate(coef_fixture): + """ + Confirm using `ComplexCoef.evaluate` is equivalent to `Coef.evaluate`. + """ + + # Create a ComplexCoef + complex_coef = coef_fixture.to_complex() + + # Compare + np.testing.assert_allclose( + complex_coef.evaluate(), + coef_fixture.evaluate(), + rtol=1e-05, + atol=utest_tolerance(coef_fixture.basis.dtype), + ) + + +def test_complex_rotate(coef_fixture, rots): + """ + Confirm using `ComplexCoef.rotate` is equivalent to `Coef.rotate`. + """ + # Create a ComplexCoef + complex_coef = coef_fixture.to_complex() + + # Compare + np.testing.assert_allclose( + complex_coef.rotate(rots), + coef_fixture.rotate(rots).to_complex(), + rtol=1e-05, + atol=utest_tolerance(coef_fixture.basis.dtype), + ) + + +def test_shifts(coef_fixture, basis, rots): + """ + Confirm using `Coef.shift` is equivalent to `basis.shift`. + """ + if coef_fixture.stack_ndim > 1: + pytest.xfail(reason="Shifts currently only support 1d stack axis.") + + # Create some shifts, by reusing the `rots` array. + shifts = np.column_stack((rots, rots[::-1])) + + # Compare + np.testing.assert_allclose( + coef_fixture.shift(shifts), + basis.shift(coef_fixture, shifts), + rtol=1e-05, + atol=utest_tolerance(basis.dtype), + ) + + +def test_complex_shift(coef_fixture, rots): + """ + Confirm using `ComplexCoef.shift` is equivalent to `Coef.shift`. + """ + if coef_fixture.stack_ndim > 1: + pytest.xfail(reason="Shifts currently only support 1d stack axis.") + + # Create a ComplexCoef + complex_coef = coef_fixture.to_complex() + + # Create some shifts, by reusing the `rots` array. + shifts = np.column_stack((rots, rots[::-1])) + + # Compare + np.testing.assert_allclose( + complex_coef.shift(shifts), + coef_fixture.shift(shifts).to_complex(), + rtol=1e-05, + atol=utest_tolerance(coef_fixture.basis.dtype), + ) From 8e1b9dcd3a5b5e74eff3a61676786907c227311c Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 17:09:19 -0400 Subject: [PATCH 090/294] fix string spelling typo --- src/aspire/basis/fspca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index cd264e8f60..172fd22184 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -523,7 +523,7 @@ def to_real(self, complex_coef): There is a corresponding method, to_complex. :param complex_coef: Complex coefficients from this basis. - :return: Real coeficent representation from this basis. + :return: Real coefficient representation from this basis. """ if complex_coef.ndim == 1: From f9051a45dee00f86f596b28609c25523ef5891bf Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 17:10:44 -0400 Subject: [PATCH 091/294] remove redundant check --- src/aspire/basis/basis.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index ed3a1f404d..5988ec37ca 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -344,11 +344,6 @@ def shift(self, shifts): :return: Complex coefs of shifted images. """ - if not callable(getattr(self.basis, "shift", None)): - raise RuntimeError( - f"self.basis={self.basis} does not provide `shift` method." - ) - return self.to_real().shift(shifts).to_complex() def to_real(self): From fa3727d3b2023c129ca8f1854c9566a21fcc3f7b Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 8 Sep 2023 09:39:50 -0400 Subject: [PATCH 092/294] stash initial attempt at \#897 and \#904 [skip ci] --- src/aspire/denoising/denoised_src.py | 39 ++++---------- src/aspire/denoising/denoiser.py | 31 +++++++---- src/aspire/denoising/denoiser_cov2d.py | 73 ++++++++++++++------------ 3 files changed, 70 insertions(+), 73 deletions(-) diff --git a/src/aspire/denoising/denoised_src.py b/src/aspire/denoising/denoised_src.py index d89cb412cc..250ee655c6 100644 --- a/src/aspire/denoising/denoised_src.py +++ b/src/aspire/denoising/denoised_src.py @@ -1,8 +1,6 @@ import logging -import numpy as np - -from aspire.image import Image +from aspire.denoising import Denoiser from aspire.source import ImageSource logger = logging.getLogger(__name__) @@ -10,22 +8,22 @@ class DenoisedImageSource(ImageSource): """ - Define a derived ImageSource class to perform operations for denoised 2D images + ImageSource class serving denoised 2D images. """ - def __init__(self, src, denoiser, batch_size=512): + def __init__(self, src, denoiser): """ - Initialize a denoised ImageSource object from original ImageSource of noisy images + Initialize a denoised ImageSource object from an ImageSource. :param src: Original ImageSource object storing noisy images :param denoiser: A Denoiser object for specifying a method for denoising - :param batch_size: Batch size for loading denoised images. """ super().__init__(src.L, src.n, dtype=src.dtype, metadata=src._metadata.copy()) - self._im = None + # TODO, we can probably setup a reasonable default here. self.denoiser = denoiser - self.batch_size = batch_size + if not isinstance(denoiser, Denoiser): + raise TypeError("`denoiser` must be subclass of `Denoiser`") # Any further operations should not mutate this instance. self._mutable = False @@ -36,8 +34,9 @@ def _images(self, indices): `ImageSource.images` property. :param indices: The indices of images to return as a 1-D NumPy array. - :return: an `Image` object after denoisng. + :return: an `Image` object after denoising. """ + # check for cached images first if self._cached_im is not None: logger.info("Loading images from cache") @@ -45,23 +44,7 @@ def _images(self, indices): self._cached_im[indices, :, :], indices ) - # start and end (and indices) refer to the indices in the DenoisedImageSource - # that are being denoised and returned in batches - start = indices.min() - end = indices.max() - - nimgs = len(indices) - im = np.empty((nimgs, self.L, self.L), self.dtype) - - # If we request less than a whole batch, don't crash - batch_size = min(nimgs, self.batch_size) - - logger.info(f"Loading {nimgs} images complete") - for batch_start in range(start, end + 1, batch_size): - imgs_denoised = self.denoiser.images(batch_start, batch_size) - batch_end = min(batch_start + batch_size, end + 1) - # we subtract start here to correct for any offset in the indices - im[batch_start - start : batch_end - start] = imgs_denoised.asnumpy() + imgs_denoised = self.denoiser.denoise[indices] # Finally, apply transforms to resulting Image - return self.generation_pipeline.forward(Image(im), indices) + return self.generation_pipeline.forward(imgs_denoised, indices) diff --git a/src/aspire/denoising/denoiser.py b/src/aspire/denoising/denoiser.py index 5a80f0da39..1cfc7d3947 100644 --- a/src/aspire/denoising/denoiser.py +++ b/src/aspire/denoising/denoiser.py @@ -1,31 +1,42 @@ import logging +from abc import ABC, abstractproperty + +from aspire.source.image import _ImageAccessor logger = logging.getLogger(__name__) -class Denoiser: +class Denoiser(ABC): """ - Define a base class for denoising 2D images + Base class for 2D image denoisers. """ def __init__(self, src): """ - Initialize an object for denoising 2D images from the image source + Initialize an object for denoising 2D images from `src`. - :param src: The source object of 2D images with metadata + :param src: `ImageSource` providing noisy images. """ + self.src = src self.dtype = src.dtype - self.nimg = src.n + self.n = src.n + self._img_accessor = _ImageAccessor(self._denoise, self.n) + @property def denoise(self): """ - Precompute for Denoiser and DenoisedImageSource for 2D images + Subscriptable property returning 2D images after denoising. + + See `_ImageAccessor`. """ - raise NotImplementedError("subclasses must implement this") + self._img_accessor - def image(self, istart=0, batch_size=512): + @abstractproperty + def _denoise(self, indices): """ - Obtain a batch size of 2D images after denosing by a specified method + Subclasses must implement a private `_denoise` method accepting `indices`. + Subclasses handle any caching as well as denoising. + + See `_ImageAccessor`. """ - raise NotImplementedError("subclasses must implement this") diff --git a/src/aspire/denoising/denoiser_cov2d.py b/src/aspire/denoising/denoiser_cov2d.py index b776e516ea..4210d86403 100644 --- a/src/aspire/denoising/denoiser_cov2d.py +++ b/src/aspire/denoising/denoiser_cov2d.py @@ -6,7 +6,6 @@ from aspire.basis import FFBBasis2D from aspire.covariance import BatchedRotCov2D from aspire.denoising import Denoiser -from aspire.denoising.denoised_src import DenoisedImageSource from aspire.noise import WhiteNoiseEstimator from aspire.optimization import fill_struct from aspire.utils import mat_to_vec @@ -104,15 +103,19 @@ class DenoiserCov2D(Denoiser): Define a derived class for denoising 2D images using Cov2D method """ - def __init__(self, src, basis=None, var_noise=None): + def __init__(self, src, basis=None, var_noise=None, batch_size=512, covar_opt=None): """ Initialize an object for denoising 2D images using Cov2D method :param src: The source object of 2D images with metadata :param basis: The basis method to expand 2D images :param var_noise: The estimated variance of noise + :param batch_size: The batch size for processing images + :param covar_opt: The option list for building Cov2D matrix """ + super().__init__(src) + self.batch_size = int(batch_size) # When var_noise is not specfically over-ridden, # recompute it now. See #496. @@ -134,19 +137,6 @@ def __init__(self, src, basis=None, var_noise=None): self.mean_est = None self.covar_est = None - def denoise(self, covar_opt=None, batch_size=512): - """ - Build covariance matrix of 2D images and return a new ImageSource object - - :param covar_opt: The option list for building Cov2D matrix - :param batch_size: The batch size for processing images - :return: A `DenoisedImageSource` object with the specified denoising object - """ - - # Initialize the rotationally invariant covariance matrix of 2D images - # A fixed batch size is used to go through each image - self.cov2d = BatchedRotCov2D(self.src, self.basis, batch_size=batch_size) - default_opt = { "shrinker": "frobenius_norm", "verbose": 0, @@ -157,38 +147,51 @@ def denoise(self, covar_opt=None, batch_size=512): "precision": self.dtype, } - covar_opt = fill_struct(covar_opt, default_opt) - # Calculate the mean and covariance for the rotationally invariant covariance matrix of 2D images + self.covar_opt = fill_struct(covar_opt, default_opt) + + # Initialize the rotationally invariant covariance matrix of 2D images + # A fixed batch_size is used to loop through image stack. + self.cov2d = BatchedRotCov2D(self.src, self.basis, batch_size=batch_size) + + def build_denoiser(self): + """ + Build estimated mean and covariance matrix of 2D images. + + This method should be computed once, on first `images` access. + """ + + if self.cov2d_est is not None: + return + + logger.info(f"Building mean estimate for {len(self.src)} images.") self.mean_est = self.cov2d.get_mean() + logger.info(f"Building covariance estimates for {len(self.src)} images.") self.covar_est = self.cov2d.get_covar( - noise_var=self.var_noise, mean_coef=self.mean_est, covar_est_opt=covar_opt + noise_var=self.var_noise, + mean_coef=self.mean_est, + covar_est_opt=self.covar_opt, ) - return DenoisedImageSource(self.src, self) - - def images(self, istart=0, batch_size=512): + def _denoise(self, indices): """ - Obtain a batch size of 2D images after denosing by Cov2D method + Compute denoised 2D images corresponding to `indices`. - :param istart: the index of starting image - :param batch_size: The batch size for processing images - :return: an `Image` object with denoised images + :return: `Image` object containing denoised images. """ - src = self.src - # Denoise one batch size of 2D images using the SPCAs from the rotationally invariant covariance matrix - img_start = istart - img_end = min(istart + batch_size, src.n) - imgs_noise = src.images[img_start : img_start + batch_size] + # Lazy evaluate estimates on access. + # `build_denoiser` internally guards to compute once. + self.build_denoiser() + + # Denoise requested `indices` selection of 2D images. + imgs_noise = self.src.images[indices] coefs_noise = self.basis.evaluate_t(imgs_noise) - logger.info( - f"Estimating Cov2D coefficients for images from {img_start} to {img_end-1}" - ) + logger.debug(f"Estimating Cov2D coefficients for {len(imgs_noise)} images.") coefs_estim = self.cov2d.get_cwf_coefs( coefs_noise, - self.cov2d.ctf_basis, - self.cov2d.ctf_idx[img_start:img_end], + self.cov2d.ctf_fb, + self.cov2d.ctf_idx[indices], mean_coef=self.mean_est, covar_coef=self.covar_est, noise_var=self.var_noise, From 7c009dd7967ac040ef157807f1b1a4cb737ab9c0 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 8 Sep 2023 10:45:17 -0400 Subject: [PATCH 093/294] cleanup some imports and tests --- src/aspire/denoising/__init__.py | 6 +++++- src/aspire/denoising/denoiser.py | 2 +- src/aspire/denoising/denoiser_cov2d.py | 5 +++-- tests/test_covar2d_denoiser.py | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/aspire/denoising/__init__.py b/src/aspire/denoising/__init__.py index 96bd4a8213..b6a0f3d54c 100644 --- a/src/aspire/denoising/__init__.py +++ b/src/aspire/denoising/__init__.py @@ -1,5 +1,9 @@ from .adaptive_support import adaptive_support from .class_avg import ClassAvgSource, DebugClassAvgSource, DefaultClassAvgSource -from .denoised_src import DenoisedImageSource + +# isort: off from .denoiser import Denoiser from .denoiser_cov2d import DenoiserCov2D, src_wiener_coords +from .denoised_src import DenoisedImageSource + +# isort: on diff --git a/src/aspire/denoising/denoiser.py b/src/aspire/denoising/denoiser.py index 1cfc7d3947..c5acc7f2ee 100644 --- a/src/aspire/denoising/denoiser.py +++ b/src/aspire/denoising/denoiser.py @@ -30,7 +30,7 @@ def denoise(self): See `_ImageAccessor`. """ - self._img_accessor + return self._img_accessor @abstractproperty def _denoise(self, indices): diff --git a/src/aspire/denoising/denoiser_cov2d.py b/src/aspire/denoising/denoiser_cov2d.py index 4210d86403..0ffcc8bf55 100644 --- a/src/aspire/denoising/denoiser_cov2d.py +++ b/src/aspire/denoising/denoiser_cov2d.py @@ -160,7 +160,7 @@ def build_denoiser(self): This method should be computed once, on first `images` access. """ - if self.cov2d_est is not None: + if self.covar_est is not None: return logger.info(f"Building mean estimate for {len(self.src)} images.") @@ -186,8 +186,9 @@ def _denoise(self, indices): # Denoise requested `indices` selection of 2D images. imgs_noise = self.src.images[indices] + coefs_noise = self.basis.evaluate_t(imgs_noise) - logger.debug(f"Estimating Cov2D coefficients for {len(imgs_noise)} images.") + logger.debug(f"Estimating Cov2D coefficients for {imgs_noise.n_images} images.") coefs_estim = self.cov2d.get_cwf_coefs( coefs_noise, self.cov2d.ctf_fb, diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 77883db7ea..d0369fbf63 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -36,8 +36,8 @@ def testMSE(self): # Specify the fast FB basis method for expending the 2D images ffbbasis = FFBBasis2D((img_size, img_size), dtype=dtype) denoiser = DenoiserCov2D(sim, ffbbasis, noise_var) - denoised_src = denoiser.denoise(batch_size=64) - imgs_denoised = denoised_src.images[:] + imgs_denoised = denoiser.denoise[:] + # Calculate the normalized RMSE of the estimated images. nrmse_ims = (imgs_denoised - imgs_clean).norm() / imgs_clean.norm() From 517a3154c6c814fe4069fc78b79cc86b7a61078e Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 8 Sep 2023 15:12:08 -0400 Subject: [PATCH 094/294] address denoise() usage in code/examples --- gallery/experiments/experimental_abinitio_pipeline_10028.py | 4 ++-- gallery/experiments/simulated_abinitio_pipeline.py | 4 ++-- src/aspire/commands/denoise.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gallery/experiments/experimental_abinitio_pipeline_10028.py b/gallery/experiments/experimental_abinitio_pipeline_10028.py index ccd236605a..a475c80602 100644 --- a/gallery/experiments/experimental_abinitio_pipeline_10028.py +++ b/gallery/experiments/experimental_abinitio_pipeline_10028.py @@ -30,7 +30,7 @@ from aspire.abinitio import CLSyncVoting from aspire.basis import FFBBasis3D -from aspire.denoising import DefaultClassAvgSource, DenoiserCov2D +from aspire.denoising import DefaultClassAvgSource, DenoiserCov2D, DenoisedImageSource from aspire.noise import AnisotropicNoiseEstimator from aspire.reconstruction import MeanEstimator from aspire.source import OrientedSource, RelionSource @@ -121,7 +121,7 @@ # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = cwf_denoiser.denoise() + classification_src = DenoisedImageSource(src, cwf_denoiser) # Cache for speedup. Avoids recomputing. classification_src = classification_src.cache() # Peek, what do the denoised images look like... diff --git a/gallery/experiments/simulated_abinitio_pipeline.py b/gallery/experiments/simulated_abinitio_pipeline.py index e4501daf8f..9011fec1c6 100644 --- a/gallery/experiments/simulated_abinitio_pipeline.py +++ b/gallery/experiments/simulated_abinitio_pipeline.py @@ -22,7 +22,7 @@ from aspire.abinitio import CLSyncVoting from aspire.basis import FFBBasis3D -from aspire.denoising import DefaultClassAvgSource, DenoiserCov2D +from aspire.denoising import DefaultClassAvgSource, DenoisedImageSource, DenoiserCov2D from aspire.downloader import emdb_2660 from aspire.noise import AnisotropicNoiseEstimator, CustomNoiseAdder from aspire.operators import FunctionFilter, RadialCTFFilter @@ -145,7 +145,7 @@ def noise_function(x, y): # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = cwf_denoiser.denoise() + classification_src = DenoisedImageSource(src, cwf_denoiser) # Peek, what do the denoised images look like... if interactive: classification_src.images[:10].show() diff --git a/src/aspire/commands/denoise.py b/src/aspire/commands/denoise.py index e0900e69c9..07c84aa4f4 100644 --- a/src/aspire/commands/denoise.py +++ b/src/aspire/commands/denoise.py @@ -4,7 +4,7 @@ from aspire.basis import FFBBasis2D from aspire.commands import log_level_option -from aspire.denoising.denoiser_cov2d import DenoiserCov2D +from aspire.denoising import DenoiserCov2D, DenoisedImageSource from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator from aspire.source.relion import RelionSource from aspire.utils.logging import setConsoleLoggingLevel @@ -101,7 +101,7 @@ def denoise( if denoise_method == "CWF": logger.info("Denoise the images using CWF cov2D method.") denoiser = DenoiserCov2D(source, basis) - denoised_src = denoiser.denoise(batch_size=512) + denoised_src = DenoisedImageSource(source, denoiser) denoised_src.save( starfile_out, batch_size=512, save_mode="single", overwrite=False ) From 84dcb9bd8ed90740e24d4c254d0cdb32173761d5 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 8 Sep 2023 15:12:21 -0400 Subject: [PATCH 095/294] lint cleanup --- gallery/experiments/experimental_abinitio_pipeline_10028.py | 2 +- src/aspire/commands/denoise.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery/experiments/experimental_abinitio_pipeline_10028.py b/gallery/experiments/experimental_abinitio_pipeline_10028.py index a475c80602..81eb984110 100644 --- a/gallery/experiments/experimental_abinitio_pipeline_10028.py +++ b/gallery/experiments/experimental_abinitio_pipeline_10028.py @@ -30,7 +30,7 @@ from aspire.abinitio import CLSyncVoting from aspire.basis import FFBBasis3D -from aspire.denoising import DefaultClassAvgSource, DenoiserCov2D, DenoisedImageSource +from aspire.denoising import DefaultClassAvgSource, DenoisedImageSource, DenoiserCov2D from aspire.noise import AnisotropicNoiseEstimator from aspire.reconstruction import MeanEstimator from aspire.source import OrientedSource, RelionSource diff --git a/src/aspire/commands/denoise.py b/src/aspire/commands/denoise.py index 07c84aa4f4..4d7c2e13f5 100644 --- a/src/aspire/commands/denoise.py +++ b/src/aspire/commands/denoise.py @@ -4,7 +4,7 @@ from aspire.basis import FFBBasis2D from aspire.commands import log_level_option -from aspire.denoising import DenoiserCov2D, DenoisedImageSource +from aspire.denoising import DenoisedImageSource, DenoiserCov2D from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator from aspire.source.relion import RelionSource from aspire.utils.logging import setConsoleLoggingLevel From 27c3ece7be42ffe61a383a9cfa8f79bd724e7dfa Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 11 Sep 2023 10:59:36 -0400 Subject: [PATCH 096/294] self review, dict cleanup --- src/aspire/denoising/denoiser_cov2d.py | 32 ++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/aspire/denoising/denoiser_cov2d.py b/src/aspire/denoising/denoiser_cov2d.py index 0ffcc8bf55..a57d39971b 100644 --- a/src/aspire/denoising/denoiser_cov2d.py +++ b/src/aspire/denoising/denoiser_cov2d.py @@ -1,4 +1,5 @@ import logging +from copy import deepcopy import numpy as np from numpy.linalg import solve @@ -103,6 +104,16 @@ class DenoiserCov2D(Denoiser): Define a derived class for denoising 2D images using Cov2D method """ + # Default options for cov2d configuration. + default_opt = { + "shrinker": "frobenius_norm", + "verbose": 0, + "max_iter": 250, + "iter_callback": [], + "store_iterates": False, + "rel_tolerance": 1e-12, + } + def __init__(self, src, basis=None, var_noise=None, batch_size=512, covar_opt=None): """ Initialize an object for denoising 2D images using Cov2D method @@ -110,8 +121,10 @@ def __init__(self, src, basis=None, var_noise=None, batch_size=512, covar_opt=No :param src: The source object of 2D images with metadata :param basis: The basis method to expand 2D images :param var_noise: The estimated variance of noise - :param batch_size: The batch size for processing images - :param covar_opt: The option list for building Cov2D matrix + :param batch_size: Integer batch size for processing images. + Defaults to 512. + :param covar_opt: Optional dictionary of option overides for Cov2D. + Provided options will supersede defaults in `DenoiserCov2D.default_opt`. """ super().__init__(src) @@ -137,16 +150,11 @@ def __init__(self, src, basis=None, var_noise=None, batch_size=512, covar_opt=No self.mean_est = None self.covar_est = None - default_opt = { - "shrinker": "frobenius_norm", - "verbose": 0, - "max_iter": 250, - "iter_callback": [], - "store_iterates": False, - "rel_tolerance": 1e-12, - "precision": self.dtype, - } - + # Create a local copy of the default options. + default_opt = deepcopy(self.default_opt) + # Assign the dtype corresponding to this instance. + default_opt["precision"] = self.dtype + # Apply any overrides provided by the user. self.covar_opt = fill_struct(covar_opt, default_opt) # Initialize the rotationally invariant covariance matrix of 2D images From d8583df87c62794d8891dba4af122ac44593db5a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 11 Sep 2023 11:01:28 -0400 Subject: [PATCH 097/294] self review strings --- src/aspire/denoising/denoised_src.py | 10 +++++----- src/aspire/denoising/denoiser.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/aspire/denoising/denoised_src.py b/src/aspire/denoising/denoised_src.py index 250ee655c6..a7b939a8da 100644 --- a/src/aspire/denoising/denoised_src.py +++ b/src/aspire/denoising/denoised_src.py @@ -8,15 +8,15 @@ class DenoisedImageSource(ImageSource): """ - ImageSource class serving denoised 2D images. + `ImageSource` class serving denoised 2D images. """ def __init__(self, src, denoiser): """ - Initialize a denoised ImageSource object from an ImageSource. + Initialize a denoised `ImageSource` object from an `ImageSource`. - :param src: Original ImageSource object storing noisy images - :param denoiser: A Denoiser object for specifying a method for denoising + :param src: Original `ImageSource` object storing noisy images + :param denoiser: A `Denoiser` object for specifying a method for denoising """ super().__init__(src.L, src.n, dtype=src.dtype, metadata=src._metadata.copy()) @@ -33,7 +33,7 @@ def _images(self, indices): Internal function to return a set of images after denoising, when accessed via the `ImageSource.images` property. - :param indices: The indices of images to return as a 1-D NumPy array. + :param indices: The indices of images to return as a 1-D Numpy array. :return: an `Image` object after denoising. """ diff --git a/src/aspire/denoising/denoiser.py b/src/aspire/denoising/denoiser.py index c5acc7f2ee..997cf26fd1 100644 --- a/src/aspire/denoising/denoiser.py +++ b/src/aspire/denoising/denoiser.py @@ -1,5 +1,5 @@ import logging -from abc import ABC, abstractproperty +from abc import ABC, abstractmethod from aspire.source.image import _ImageAccessor @@ -32,7 +32,7 @@ def denoise(self): """ return self._img_accessor - @abstractproperty + @abstractmethod def _denoise(self, indices): """ Subclasses must implement a private `_denoise` method accepting `indices`. From 5c093febc3c8af56c59ab1ecfa94d9c1adb55f8a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 11 Sep 2023 11:17:35 -0400 Subject: [PATCH 098/294] cover new code branches --- tests/test_covar2d_denoiser.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index d0369fbf63..1669445a45 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -3,7 +3,7 @@ import numpy as np from aspire.basis.ffb_2d import FFBBasis2D -from aspire.denoising.denoiser_cov2d import DenoiserCov2D +from aspire.denoising import DenoisedImageSource, DenoiserCov2D from aspire.noise import WhiteNoiseAdder from aspire.operators.filters import RadialCTFFilter from aspire.source.simulation import Simulation @@ -42,3 +42,8 @@ def testMSE(self): nrmse_ims = (imgs_denoised - imgs_clean).norm() / imgs_clean.norm() self.assertTrue(nrmse_ims < 0.25) + + # Additionally test the `DenoisedImageSource` and lazy-eval-cache + # of the cov2d estimator. + src = DenoisedImageSource(sim, denoiser) + self.assertTrue(np.allclose(src.images[:], imgs_denoised)) From 5ffe3a85f247fd124f16bcf239bc108dbb6fef58 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 13 Sep 2023 15:17:04 -0400 Subject: [PATCH 099/294] reconcile conflict --- src/aspire/denoising/denoiser_cov2d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/denoising/denoiser_cov2d.py b/src/aspire/denoising/denoiser_cov2d.py index a57d39971b..126b9d4717 100644 --- a/src/aspire/denoising/denoiser_cov2d.py +++ b/src/aspire/denoising/denoiser_cov2d.py @@ -199,7 +199,7 @@ def _denoise(self, indices): logger.debug(f"Estimating Cov2D coefficients for {imgs_noise.n_images} images.") coefs_estim = self.cov2d.get_cwf_coefs( coefs_noise, - self.cov2d.ctf_fb, + self.cov2d.ctf_basis, self.cov2d.ctf_idx[indices], mean_coef=self.mean_est, covar_coef=self.covar_est, From 9fc848a5d0bd3c5265f35904543c71a5da1c2370 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 21 Sep 2023 07:51:09 -0400 Subject: [PATCH 100/294] rename DesnoisedImageSource DesnoisedSource --- gallery/experiments/experimental_abinitio_pipeline_10028.py | 4 ++-- gallery/experiments/simulated_abinitio_pipeline.py | 4 ++-- src/aspire/commands/denoise.py | 4 ++-- src/aspire/denoising/__init__.py | 2 +- src/aspire/denoising/denoised_src.py | 2 +- tests/test_covar2d_denoiser.py | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gallery/experiments/experimental_abinitio_pipeline_10028.py b/gallery/experiments/experimental_abinitio_pipeline_10028.py index 81eb984110..472c21a638 100644 --- a/gallery/experiments/experimental_abinitio_pipeline_10028.py +++ b/gallery/experiments/experimental_abinitio_pipeline_10028.py @@ -30,7 +30,7 @@ from aspire.abinitio import CLSyncVoting from aspire.basis import FFBBasis3D -from aspire.denoising import DefaultClassAvgSource, DenoisedImageSource, DenoiserCov2D +from aspire.denoising import DefaultClassAvgSource, DenoisedSource, DenoiserCov2D from aspire.noise import AnisotropicNoiseEstimator from aspire.reconstruction import MeanEstimator from aspire.source import OrientedSource, RelionSource @@ -121,7 +121,7 @@ # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = DenoisedImageSource(src, cwf_denoiser) + classification_src = DenoisedSource(src, cwf_denoiser) # Cache for speedup. Avoids recomputing. classification_src = classification_src.cache() # Peek, what do the denoised images look like... diff --git a/gallery/experiments/simulated_abinitio_pipeline.py b/gallery/experiments/simulated_abinitio_pipeline.py index 9011fec1c6..63dce7e62a 100644 --- a/gallery/experiments/simulated_abinitio_pipeline.py +++ b/gallery/experiments/simulated_abinitio_pipeline.py @@ -22,7 +22,7 @@ from aspire.abinitio import CLSyncVoting from aspire.basis import FFBBasis3D -from aspire.denoising import DefaultClassAvgSource, DenoisedImageSource, DenoiserCov2D +from aspire.denoising import DefaultClassAvgSource, DenoisedSource, DenoiserCov2D from aspire.downloader import emdb_2660 from aspire.noise import AnisotropicNoiseEstimator, CustomNoiseAdder from aspire.operators import FunctionFilter, RadialCTFFilter @@ -145,7 +145,7 @@ def noise_function(x, y): # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = DenoisedImageSource(src, cwf_denoiser) + classification_src = DenoisedSource(src, cwf_denoiser) # Peek, what do the denoised images look like... if interactive: classification_src.images[:10].show() diff --git a/src/aspire/commands/denoise.py b/src/aspire/commands/denoise.py index 4d7c2e13f5..8a56a18f8e 100644 --- a/src/aspire/commands/denoise.py +++ b/src/aspire/commands/denoise.py @@ -4,7 +4,7 @@ from aspire.basis import FFBBasis2D from aspire.commands import log_level_option -from aspire.denoising import DenoisedImageSource, DenoiserCov2D +from aspire.denoising import DenoisedSource, DenoiserCov2D from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator from aspire.source.relion import RelionSource from aspire.utils.logging import setConsoleLoggingLevel @@ -101,7 +101,7 @@ def denoise( if denoise_method == "CWF": logger.info("Denoise the images using CWF cov2D method.") denoiser = DenoiserCov2D(source, basis) - denoised_src = DenoisedImageSource(source, denoiser) + denoised_src = DenoisedSource(source, denoiser) denoised_src.save( starfile_out, batch_size=512, save_mode="single", overwrite=False ) diff --git a/src/aspire/denoising/__init__.py b/src/aspire/denoising/__init__.py index b6a0f3d54c..920638eb44 100644 --- a/src/aspire/denoising/__init__.py +++ b/src/aspire/denoising/__init__.py @@ -4,6 +4,6 @@ # isort: off from .denoiser import Denoiser from .denoiser_cov2d import DenoiserCov2D, src_wiener_coords -from .denoised_src import DenoisedImageSource +from .denoised_src import DenoisedSource # isort: on diff --git a/src/aspire/denoising/denoised_src.py b/src/aspire/denoising/denoised_src.py index a7b939a8da..93235ae3ec 100644 --- a/src/aspire/denoising/denoised_src.py +++ b/src/aspire/denoising/denoised_src.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) -class DenoisedImageSource(ImageSource): +class DenoisedSource(ImageSource): """ `ImageSource` class serving denoised 2D images. """ diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 1669445a45..10960ceafc 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -3,7 +3,7 @@ import numpy as np from aspire.basis.ffb_2d import FFBBasis2D -from aspire.denoising import DenoisedImageSource, DenoiserCov2D +from aspire.denoising import DenoisedSource, DenoiserCov2D from aspire.noise import WhiteNoiseAdder from aspire.operators.filters import RadialCTFFilter from aspire.source.simulation import Simulation @@ -43,7 +43,7 @@ def testMSE(self): self.assertTrue(nrmse_ims < 0.25) - # Additionally test the `DenoisedImageSource` and lazy-eval-cache + # Additionally test the `DenoisedSource` and lazy-eval-cache # of the cov2d estimator. - src = DenoisedImageSource(sim, denoiser) + src = DenoisedSource(sim, denoiser) self.assertTrue(np.allclose(src.images[:], imgs_denoised)) From 1a79773a9624d42838812166d5f47b4b14d49f26 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 21 Sep 2023 09:11:37 -0400 Subject: [PATCH 101/294] remove src arg from DenoisedSource --- .../experimental_abinitio_pipeline_10028.py | 2 +- gallery/experiments/simulated_abinitio_pipeline.py | 2 +- src/aspire/commands/denoise.py | 2 +- src/aspire/denoising/denoised_src.py | 13 ++++++++----- tests/test_covar2d_denoiser.py | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/gallery/experiments/experimental_abinitio_pipeline_10028.py b/gallery/experiments/experimental_abinitio_pipeline_10028.py index 472c21a638..7e74914d2e 100644 --- a/gallery/experiments/experimental_abinitio_pipeline_10028.py +++ b/gallery/experiments/experimental_abinitio_pipeline_10028.py @@ -121,7 +121,7 @@ # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = DenoisedSource(src, cwf_denoiser) + classification_src = DenoisedSource(cwf_denoiser) # Cache for speedup. Avoids recomputing. classification_src = classification_src.cache() # Peek, what do the denoised images look like... diff --git a/gallery/experiments/simulated_abinitio_pipeline.py b/gallery/experiments/simulated_abinitio_pipeline.py index 63dce7e62a..a35a3ebdec 100644 --- a/gallery/experiments/simulated_abinitio_pipeline.py +++ b/gallery/experiments/simulated_abinitio_pipeline.py @@ -145,7 +145,7 @@ def noise_function(x, y): # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = DenoisedSource(src, cwf_denoiser) + classification_src = DenoisedSource(cwf_denoiser) # Peek, what do the denoised images look like... if interactive: classification_src.images[:10].show() diff --git a/src/aspire/commands/denoise.py b/src/aspire/commands/denoise.py index 8a56a18f8e..ccbdfc6595 100644 --- a/src/aspire/commands/denoise.py +++ b/src/aspire/commands/denoise.py @@ -101,7 +101,7 @@ def denoise( if denoise_method == "CWF": logger.info("Denoise the images using CWF cov2D method.") denoiser = DenoiserCov2D(source, basis) - denoised_src = DenoisedSource(source, denoiser) + denoised_src = DenoisedSource(denoiser) denoised_src.save( starfile_out, batch_size=512, save_mode="single", overwrite=False ) diff --git a/src/aspire/denoising/denoised_src.py b/src/aspire/denoising/denoised_src.py index 93235ae3ec..4c63c8926e 100644 --- a/src/aspire/denoising/denoised_src.py +++ b/src/aspire/denoising/denoised_src.py @@ -11,16 +11,19 @@ class DenoisedSource(ImageSource): `ImageSource` class serving denoised 2D images. """ - def __init__(self, src, denoiser): + def __init__(self, denoiser): """ Initialize a denoised `ImageSource` object from an `ImageSource`. - :param src: Original `ImageSource` object storing noisy images :param denoiser: A `Denoiser` object for specifying a method for denoising """ - - super().__init__(src.L, src.n, dtype=src.dtype, metadata=src._metadata.copy()) - # TODO, we can probably setup a reasonable default here. + self.src = denoiser.src + super().__init__( + self.src.L, + self.src.n, + dtype=self.src.dtype, + metadata=self.src._metadata.copy(), + ) self.denoiser = denoiser if not isinstance(denoiser, Denoiser): raise TypeError("`denoiser` must be subclass of `Denoiser`") diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 10960ceafc..b372421749 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -45,5 +45,5 @@ def testMSE(self): # Additionally test the `DenoisedSource` and lazy-eval-cache # of the cov2d estimator. - src = DenoisedSource(sim, denoiser) + src = DenoisedSource(denoiser) self.assertTrue(np.allclose(src.images[:], imgs_denoised)) From 785f31625dee5f3a140a3523cced1a0719fe42ff Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 10:58:44 -0400 Subject: [PATCH 102/294] Revert "remove src arg from DenoisedSource" This reverts commit 4adc00be44e34ce01560a0acfaa4b6a4dc354155. --- .../experimental_abinitio_pipeline_10028.py | 2 +- gallery/experiments/simulated_abinitio_pipeline.py | 2 +- src/aspire/commands/denoise.py | 2 +- src/aspire/denoising/denoised_src.py | 13 +++++-------- tests/test_covar2d_denoiser.py | 2 +- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/gallery/experiments/experimental_abinitio_pipeline_10028.py b/gallery/experiments/experimental_abinitio_pipeline_10028.py index 7e74914d2e..472c21a638 100644 --- a/gallery/experiments/experimental_abinitio_pipeline_10028.py +++ b/gallery/experiments/experimental_abinitio_pipeline_10028.py @@ -121,7 +121,7 @@ # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = DenoisedSource(cwf_denoiser) + classification_src = DenoisedSource(src, cwf_denoiser) # Cache for speedup. Avoids recomputing. classification_src = classification_src.cache() # Peek, what do the denoised images look like... diff --git a/gallery/experiments/simulated_abinitio_pipeline.py b/gallery/experiments/simulated_abinitio_pipeline.py index a35a3ebdec..63dce7e62a 100644 --- a/gallery/experiments/simulated_abinitio_pipeline.py +++ b/gallery/experiments/simulated_abinitio_pipeline.py @@ -145,7 +145,7 @@ def noise_function(x, y): # Use CWF denoising cwf_denoiser = DenoiserCov2D(src) # Use denoised src for classification - classification_src = DenoisedSource(cwf_denoiser) + classification_src = DenoisedSource(src, cwf_denoiser) # Peek, what do the denoised images look like... if interactive: classification_src.images[:10].show() diff --git a/src/aspire/commands/denoise.py b/src/aspire/commands/denoise.py index ccbdfc6595..8a56a18f8e 100644 --- a/src/aspire/commands/denoise.py +++ b/src/aspire/commands/denoise.py @@ -101,7 +101,7 @@ def denoise( if denoise_method == "CWF": logger.info("Denoise the images using CWF cov2D method.") denoiser = DenoiserCov2D(source, basis) - denoised_src = DenoisedSource(denoiser) + denoised_src = DenoisedSource(source, denoiser) denoised_src.save( starfile_out, batch_size=512, save_mode="single", overwrite=False ) diff --git a/src/aspire/denoising/denoised_src.py b/src/aspire/denoising/denoised_src.py index 4c63c8926e..93235ae3ec 100644 --- a/src/aspire/denoising/denoised_src.py +++ b/src/aspire/denoising/denoised_src.py @@ -11,19 +11,16 @@ class DenoisedSource(ImageSource): `ImageSource` class serving denoised 2D images. """ - def __init__(self, denoiser): + def __init__(self, src, denoiser): """ Initialize a denoised `ImageSource` object from an `ImageSource`. + :param src: Original `ImageSource` object storing noisy images :param denoiser: A `Denoiser` object for specifying a method for denoising """ - self.src = denoiser.src - super().__init__( - self.src.L, - self.src.n, - dtype=self.src.dtype, - metadata=self.src._metadata.copy(), - ) + + super().__init__(src.L, src.n, dtype=src.dtype, metadata=src._metadata.copy()) + # TODO, we can probably setup a reasonable default here. self.denoiser = denoiser if not isinstance(denoiser, Denoiser): raise TypeError("`denoiser` must be subclass of `Denoiser`") diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index b372421749..10960ceafc 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -45,5 +45,5 @@ def testMSE(self): # Additionally test the `DenoisedSource` and lazy-eval-cache # of the cov2d estimator. - src = DenoisedSource(denoiser) + src = DenoisedSource(sim, denoiser) self.assertTrue(np.allclose(src.images[:], imgs_denoised)) From 1bcc9cce6b5667df2ed176426736a2588a297345 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 11:41:11 -0400 Subject: [PATCH 103/294] refactor and add test for denoiser source --- src/aspire/denoising/denoised_src.py | 7 ++ tests/test_covar2d_denoiser.py | 100 ++++++++++++++++----------- 2 files changed, 67 insertions(+), 40 deletions(-) diff --git a/src/aspire/denoising/denoised_src.py b/src/aspire/denoising/denoised_src.py index 93235ae3ec..ac891f20d4 100644 --- a/src/aspire/denoising/denoised_src.py +++ b/src/aspire/denoising/denoised_src.py @@ -25,6 +25,13 @@ def __init__(self, src, denoiser): if not isinstance(denoiser, Denoiser): raise TypeError("`denoiser` must be subclass of `Denoiser`") + # Safety check src and self.denoiser.src are the same. + # See #1020 + if src != self.denoiser.src: + raise NotImplementedError( + "Denoiser `src` and noisy image `src` must match." + ) + # Any further operations should not mutate this instance. self._mutable = False diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 10960ceafc..06eba6e6a4 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -1,6 +1,5 @@ -from unittest import TestCase - import numpy as np +import pytest from aspire.basis.ffb_2d import FFBBasis2D from aspire.denoising import DenoisedSource, DenoiserCov2D @@ -8,42 +7,63 @@ from aspire.operators.filters import RadialCTFFilter from aspire.source.simulation import Simulation +# TODO, parameterize these further. +dtype = np.float32 +img_size = 64 +num_imgs = 1024 +noise_var = 0.1848 +noise_adder = WhiteNoiseAdder(var=noise_var) +filters = [ + RadialCTFFilter(5, 200, defocus=d, Cs=2.0, alpha=0.1) + for d in np.linspace(1.5e4, 2.5e4, 7) +] +basis = FFBBasis2D((img_size, img_size), dtype=dtype) + + +@pytest.fixture(scope="module") +def sim(): + """Simulation source.""" + return Simulation( + L=img_size, + n=num_imgs, + unique_filters=filters, + offsets=0.0, + amplitudes=1.0, + dtype=dtype, + noise_adder=noise_adder, + ) + + +def test_batched_rotcov2d_MSE(sim): + # need larger numbers of images and higher resolution for good MSE + imgs_clean = sim.projections[:] + + # Specify the fast FB basis method for expending the 2D images + denoiser = DenoiserCov2D(sim, basis, noise_var) + imgs_denoised = denoiser.denoise[:] + + # Calculate the normalized RMSE of the estimated images. + nrmse_ims = (imgs_denoised - imgs_clean).norm() / imgs_clean.norm() + + np.testing.assert_array_less(nrmse_ims, 0.25) + + # Additionally test the `DenoisedSource` and lazy-eval-cache + # of the cov2d estimator. + src = DenoisedSource(sim, denoiser) + np.testing.assert_allclose(imgs_denoised, src.images[:]) + + +def test_source_mismatch(sim): + """ " + Assert mismatched sources raises an error. + """ + + # Create a denoiser. + denoiser = DenoiserCov2D(sim, basis, noise_var) + + # Create a different source. + src2 = sim[: sim.n - 1] -class BatchedRotCov2DTestCase(TestCase): - def testMSE(self): - # need larger numbers of images and higher resolution for good MSE - dtype = np.float32 - img_size = 64 - num_imgs = 1024 - noise_var = 0.1848 - noise_adder = WhiteNoiseAdder(var=noise_var) - filters = [ - RadialCTFFilter(5, 200, defocus=d, Cs=2.0, alpha=0.1) - for d in np.linspace(1.5e4, 2.5e4, 7) - ] - # set simulation object - sim = Simulation( - L=img_size, - n=num_imgs, - unique_filters=filters, - offsets=0.0, - amplitudes=1.0, - dtype=dtype, - noise_adder=noise_adder, - ) - imgs_clean = sim.projections[:] - - # Specify the fast FB basis method for expending the 2D images - ffbbasis = FFBBasis2D((img_size, img_size), dtype=dtype) - denoiser = DenoiserCov2D(sim, ffbbasis, noise_var) - imgs_denoised = denoiser.denoise[:] - - # Calculate the normalized RMSE of the estimated images. - nrmse_ims = (imgs_denoised - imgs_clean).norm() / imgs_clean.norm() - - self.assertTrue(nrmse_ims < 0.25) - - # Additionally test the `DenoisedSource` and lazy-eval-cache - # of the cov2d estimator. - src = DenoisedSource(sim, denoiser) - self.assertTrue(np.allclose(src.images[:], imgs_denoised)) + # Raise because src2 not identical to denoiser.src (sim) + with pytest.raises(NotImplementedError, match=r".*must match.*"): + _ = DenoisedSource(src2, denoiser) From bee5baa3a1725182e3c94ac561b882e00352b330 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 11:49:26 -0400 Subject: [PATCH 104/294] Better fixup those strings --- tests/test_covar2d_denoiser.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 06eba6e6a4..0325dfe1f3 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -22,7 +22,7 @@ @pytest.fixture(scope="module") def sim(): - """Simulation source.""" + """Create a reusable Simulation source.""" return Simulation( L=img_size, n=num_imgs, @@ -35,6 +35,9 @@ def sim(): def test_batched_rotcov2d_MSE(sim): + """ + Check calling `DenoiserCov2D` via `DenoiserSource` framework yields acceptable error. + """ # need larger numbers of images and higher resolution for good MSE imgs_clean = sim.projections[:] @@ -44,7 +47,6 @@ def test_batched_rotcov2d_MSE(sim): # Calculate the normalized RMSE of the estimated images. nrmse_ims = (imgs_denoised - imgs_clean).norm() / imgs_clean.norm() - np.testing.assert_array_less(nrmse_ims, 0.25) # Additionally test the `DenoisedSource` and lazy-eval-cache @@ -54,10 +56,9 @@ def test_batched_rotcov2d_MSE(sim): def test_source_mismatch(sim): - """ " + """ Assert mismatched sources raises an error. """ - # Create a denoiser. denoiser = DenoiserCov2D(sim, basis, noise_var) From 990a30b0bc3b1319437d41fa0fd3987b451858eb Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 28 Sep 2023 13:19:26 -0400 Subject: [PATCH 105/294] Use the usual allclose tolerance. --- tests/test_covar2d_denoiser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 0325dfe1f3..07e60a755f 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -52,7 +52,7 @@ def test_batched_rotcov2d_MSE(sim): # Additionally test the `DenoisedSource` and lazy-eval-cache # of the cov2d estimator. src = DenoisedSource(sim, denoiser) - np.testing.assert_allclose(imgs_denoised, src.images[:]) + np.testing.assert_allclose(imgs_denoised, src.images[:], rtol=1e-05, atol=1e-08) def test_source_mismatch(sim): From 217861d393b2c296a6644ad5c5331e004f9fcef5 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 4 Oct 2023 10:07:10 -0400 Subject: [PATCH 106/294] Add utest_tolerance to flaky test. Remove unecessary loop. --- tests/test_volume.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index aa3eaf336e..2db0c9b98c 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -82,6 +82,12 @@ def vols_12(data_12): return Volume(data_12) +@pytest.fixture +def asym_vols(res, dtype): + vols = AsymmetricVolume(L=res, C=N, dtype=dtype, seed=0).generate() + return vols + + @pytest.fixture def random_data(res, dtype): return np.random.randn(res, res, res).astype(dtype) @@ -349,9 +355,10 @@ def test_rotate(L, dtype): assert np.allclose(ref_vol, rot_vol, atol=utest_tolerance(dtype)) -def test_rotate_broadcast_unicast(vols_1, dtype): +def test_rotate_broadcast_unicast(asym_vols): # Build `Rotation` objects. A singleton for broadcasting and a stack for unicasting. # The stack consists of copies of the singleton. + dtype = asym_vols.dtype angles = np.array([pi, pi / 2, 0], dtype=dtype) angles = np.tile(angles, (3, 1)) rot_mat = Rotation.from_euler(angles, dtype=dtype).matrices @@ -359,13 +366,13 @@ def test_rotate_broadcast_unicast(vols_1, dtype): rots = Rotation(rot_mat) # Broadcast the singleton `Rotation` across the `Volume` stack. - vols_broadcast = vols_1.rotate(rot) + vols_broadcast = asym_vols.rotate(rot) # Unicast the `Rotation` stack across the `Volume` stack. - vols_unicast = vols_1.rotate(rots) + vols_unicast = asym_vols.rotate(rots) - for i in range(N): - assert np.allclose(vols_broadcast[i], vols_unicast[i]) + # Tests that all volumes match. + assert np.allclose(vols_broadcast, vols_unicast, atol=utest_tolerance(dtype)) def to_vec(vols_1, vec): From 4d87dd4a086f5e26642f844e7a3db5663fd576c9 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 4 Oct 2023 10:21:50 -0400 Subject: [PATCH 107/294] Use random rotations. --- tests/test_volume.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index 2db0c9b98c..e2254f5e6d 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -359,11 +359,8 @@ def test_rotate_broadcast_unicast(asym_vols): # Build `Rotation` objects. A singleton for broadcasting and a stack for unicasting. # The stack consists of copies of the singleton. dtype = asym_vols.dtype - angles = np.array([pi, pi / 2, 0], dtype=dtype) - angles = np.tile(angles, (3, 1)) - rot_mat = Rotation.from_euler(angles, dtype=dtype).matrices - rot = Rotation(rot_mat[0]) - rots = Rotation(rot_mat) + rot = Rotation.generate_random_rotations(n=1, seed=1234, dtype=dtype) + rots = Rotation(np.broadcast_to(rot.matrices, (3, 3, 3))) # Broadcast the singleton `Rotation` across the `Volume` stack. vols_broadcast = asym_vols.rotate(rot) From 85b5c2941b819860162fead8027e679954665a48 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 5 Oct 2023 09:41:32 -0400 Subject: [PATCH 108/294] Use n_vols instead of hardcoded value. --- tests/test_volume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index e2254f5e6d..7ad409a891 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -360,7 +360,7 @@ def test_rotate_broadcast_unicast(asym_vols): # The stack consists of copies of the singleton. dtype = asym_vols.dtype rot = Rotation.generate_random_rotations(n=1, seed=1234, dtype=dtype) - rots = Rotation(np.broadcast_to(rot.matrices, (3, 3, 3))) + rots = Rotation(np.broadcast_to(rot.matrices, (asym_vols.n_vols, 3, 3))) # Broadcast the singleton `Rotation` across the `Volume` stack. vols_broadcast = asym_vols.rotate(rot) From 19780fefebbdb7603a776671e80243937954b581 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 17 Oct 2023 15:50:53 -0400 Subject: [PATCH 109/294] Add pyshtool dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a78f893a7f..9255c5d6b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "psutil", "pyfftw", "pymanopt", + "pyshtools", "PyWavelets", "pillow", "ray", From d1ed1bb0fd580c5cfe7fe201d4704fc14b4168d3 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 17 Oct 2023 15:59:45 -0400 Subject: [PATCH 110/294] Use pyshtools for higher order sph harm functions in FB --- src/aspire/basis/basis_utils.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/aspire/basis/basis_utils.py b/src/aspire/basis/basis_utils.py index 05366a58e3..fe599e9fdc 100644 --- a/src/aspire/basis/basis_utils.py +++ b/src/aspire/basis/basis_utils.py @@ -4,10 +4,12 @@ """ import logging +import warnings import numpy as np from numpy import diff, exp, log, pi from numpy.polynomial.legendre import leggauss +from pyshtools.expand import spharm_lm from scipy.special import jn, jv, sph_harm from aspire.utils import grid_2d, grid_3d @@ -170,7 +172,29 @@ def real_sph_harmonic(j, m, theta, phi): """ abs_m = abs(m) - y = sph_harm(abs_m, j, phi, theta) + # The `scipy` sph_harm implementation is much faster, + # but incorrectly returns NaN for high orders. + # For higher order use `pyshtools`. + if j < 86: + y = sph_harm(abs_m, j, phi, theta) + else: + warnings.warn( + "Computing higher order spherical harmonics is slow." + " Consider using `FFBBasis3D` or decreasing volume size.", + stacklevel=1, + ) + + y = spharm_lm( + j, + abs_m, + theta, + phi, + kind="complex", + degrees=False, + csphase=-1, + normalization="ortho", + ) + if m < 0: y = np.sqrt(2) * np.imag(y) elif m > 0: From a3660915af40142d162a0b40ac3d18468f4030d7 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 13 Oct 2023 14:55:04 -0400 Subject: [PATCH 111/294] Add files from commonline_sdp branch --- pyproject.toml | 1 + src/aspire/abinitio/__init__.py | 1 + src/aspire/abinitio/commonline_base.py | 4 +- src/aspire/abinitio/commonline_sdp.py | 244 +++++++++++++++++++++++-- tests/test_orient_sdp.py | 195 ++++++++++++++++++++ 5 files changed, 431 insertions(+), 14 deletions(-) create mode 100644 tests/test_orient_sdp.py diff --git a/pyproject.toml b/pyproject.toml index 9255c5d6b6..a83b4251bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ classifiers = [ dependencies = [ "click", "confuse >= 2.0.0", + "cvxpy", "finufft", "gemmi >= 0.4.8", "grpcio >= 1.54.2", diff --git a/src/aspire/abinitio/__init__.py b/src/aspire/abinitio/__init__.py index 69483c9800..ff14cc2d45 100644 --- a/src/aspire/abinitio/__init__.py +++ b/src/aspire/abinitio/__init__.py @@ -1,4 +1,5 @@ from .commonline_base import CLOrient3D +from .commonline_sdp import CommonlineSDP from .sync_voting import SyncVotingMixin # isort: off diff --git a/src/aspire/abinitio/commonline_base.py b/src/aspire/abinitio/commonline_base.py index 485cbe95a8..d1831c8177 100644 --- a/src/aspire/abinitio/commonline_base.py +++ b/src/aspire/abinitio/commonline_base.py @@ -27,9 +27,9 @@ def __init__( mask=True, ): """ - Initialize an object for estimating 3D orientations using common lines + Initialize an object for estimating 3D orientations using common lines. - :param src: The source object of 2D denoised or class-averaged imag + :param src: The source object of 2D denoised or class-averaged images. :param n_rad: The number of points in the radial direction. If None, n_rad will default to the ceiling of half the resolution of the source. :param n_theta: The number of points in the theta direction. This value must be even. diff --git a/src/aspire/abinitio/commonline_sdp.py b/src/aspire/abinitio/commonline_sdp.py index ceb9666267..0176bdfd35 100644 --- a/src/aspire/abinitio/commonline_sdp.py +++ b/src/aspire/abinitio/commonline_sdp.py @@ -1,30 +1,250 @@ import logging +import cvxpy as cp +import numpy as np +from scipy.sparse import csr_array + from aspire.abinitio import CLOrient3D +from aspire.utils.matlab_compat import stable_eigsh logger = logging.getLogger(__name__) -class CommLineSDP(CLOrient3D): +class CommonlineSDP(CLOrient3D): """ - Class to estimate 3D orientations using Semi-Definite Programming - :cite:`DBLP:journals/siamis/SingerS11` + Class to estimate 3D orientations using semi-definite programming. + + See the following publication for more details: + + A. Singer and Y. Shkolnisky, + "Three-Dimensional Structure Determination from Common Lines in Cryo-EM + by Eigenvectors and Semidefinite Programming" + SIAM J. Imaging Sciences, Vol. 4, No. 2, (2011): 543-572. doi:10.1137/090767777 """ - def __init__(self, src): + def estimate_rotations(self): + """ + Estimate rotation matrices using the common lines method with semi-definite programming. + """ + logger.info("Computing the common lines matrix.") + self.build_clmatrix() + + S = self._construct_S(self.clmatrix) + A, b = self._sdp_prep() + gram = self._compute_gram_matrix(S, A, b) + rotations = self._deterministic_rounding(gram) + self.rotations = rotations + + def _construct_S(self, clmatrix): + """ + Construct the 2*n_img x 2*n_img quadratic form matrix S corresponding to the common-lines + matrix 'clmatrix'. + + :param clmatrix: n_img x n_img common-lines matrix. + + :return: 2*n_img x 2*n_img quadratic form matrix S. + """ + logger.info("Constructing the common line quadratic form matrix S.") + + S11 = np.zeros((self.n_img, self.n_img), dtype=self.dtype) + S12 = np.zeros((self.n_img, self.n_img), dtype=self.dtype) + S21 = np.zeros((self.n_img, self.n_img), dtype=self.dtype) + S22 = np.zeros((self.n_img, self.n_img), dtype=self.dtype) + + for i in range(self.n_img): + for j in range(i + 1, self.n_img): + cij = clmatrix[i, j] + cji = clmatrix[j, i] + + xij = np.cos(2 * np.pi * cij / self.n_theta) + yij = np.sin(2 * np.pi * cij / self.n_theta) + xji = np.cos(2 * np.pi * cji / self.n_theta) + yji = np.sin(2 * np.pi * cji / self.n_theta) + + S11[i, j] = xij * xji + S11[j, i] = xji * xij + + S12[i, j] = xij * yji + S12[j, i] = xji * yij + + S21[i, j] = yij * xji + S21[j, i] = yji * xij + + S22[i, j] = yij * yji + S22[j, i] = yji * yij + + S = np.block([[S11, S12], [S21, S22]]) + + return S + + def _sdp_prep(self): + """ + Prepare optimization problem constraints. + + The constraints for the SDP optimization, max tr(SG), performed in `_compute_gram_matrix()` + as min tr(-SG), are that the Gram matrix, G, is semidefinite positive and G11_ii = G22_ii = 1, + G12_ii = G21_ii = 0, i=1,2,...,N, for the block representation of G = [[G11, G12], [G21, G22]]. + + We build a corresponding constraint for CVXPY in the form of tr(A_j @ G) = b_j, j = 1,...,p. + For the constraint G11_ii = G22_ii = 1, we have A_j[i, i] = 1 (zeros elsewhere) and b_j = 1. + For the constraint G12_ii = G21_ii = 0, we have A_j[i, i] = 1 (zeros elsewhere) and b_j = 0. + + :returns: Constraint data A, b. + """ + logger.info("Preparing SDP optimization constraints.") + + n = 2 * self.n_img + A = [] + b = [] + data = np.ones(1, dtype=self.dtype) + for i in range(n): + row_ind = np.array([i]) + col_ind = np.array([i]) + A_i = csr_array((data, (row_ind, col_ind)), shape=(n, n), dtype=self.dtype) + A.append(A_i) + b.append(1) + + for i in range(self.n_img): + row_ind = np.array([i]) + col_ind = np.array([self.n_img + i]) + A_i = csr_array((data, (row_ind, col_ind)), shape=(n, n), dtype=self.dtype) + A.append(A_i) + b.append(0) + + b = np.array(b, dtype=self.dtype) + + return A, b + + def _compute_gram_matrix(self, S, A, b): """ - constructor of an object for estimating 3D orientations + Compute the Gram matrix by solving an SDP optimization. + + The Gram matrix will be of the form G = R.T @ R, where R = [R1 R2] or the concatenation + of the first columns of every rotation, R1, and the second columns of every rotation, R2. + From this Gram matrix, the rotations can be recovered using the deterministic rounding + procedure below. + + Here we optimize over G, max tr(SG), written as min tr(-SG), subject to the constraints + described in `_spd_prep()`. It should be noted that tr(SG) = sum(dot(R_i @ c_ij, R_j @ c_ji)), + and that maximizing this objective function is equivalently to minimizing the L2 norm + of R_i @ c_ij - R_j @ c_ji, ie. finding the best approximation for the rotations R_i. + + :param S: The common-line quadratic form matrix of shape 2 * n_img x 2 * n_img. + :param A: 3 * n_img sparse arrays of constraint data. + :param b: 3 * n_img scalars such that tr(A_i G) = b_i. + + :return: Gram matrix. """ - pass + logger.info("Solving SDP to approximate Gram matrix.") + + n = 2 * self.n_img + # Define and solve the CVXPY problem. + # Create a symmetric matrix variable. + G = cp.Variable((n, n), symmetric=True) + # The operator >> denotes matrix inequality. + constraints = [G >> 0] + constraints += [cp.trace(A[i] @ G) == b[i] for i in range(3 * self.n_img)] + prob = cp.Problem(cp.Minimize(cp.trace(-S @ G)), constraints) + prob.solve() - def estimate(self): + return G.value + + def _deterministic_rounding(self, gram): """ - perform estimation of orientations + Deterministic rounding procedure to recover the rotations from the Gram matrix. + + The Gram matrix contains information about the first two columns of every rotation + matrix. These columns are extracted and used to form the remaining column of every + rotation matrix. + + :param gram: A 2n_img x 2n_img Gram matrix. + + :return: An n_img x 3 x 3 stack of rotation matrices. """ - pass + logger.info("Recovering rotations from Gram matrix.") + + # Obtain top eigenvectors from Gram matrix. + d, v = stable_eigsh(gram, 5) + sort_idx = np.argsort(-d) + logger.info(f"Top 5 eigenvalues from (rank-3) Gram matrix: {d[sort_idx]}") - def output(self): + # Only need the top 3 eigen-vectors. + v = v[:, sort_idx[:3]] + + # According to the structure of the Gram matrix, the first `n_img` rows, denoted v1, + # correspond to the linear combination of the vectors R_{i}^{1}, i=1,...,K, that is of + # column 1 of all rotation matrices. Similarly, the second `n_img` rows of v, + # denoted v2, are linear combinations of R_{i}^{2}, i=1,...,K, that is, the second + # column of all rotation matrices. + v1 = v[: self.n_img].T + v2 = v[self.n_img : 2 * self.n_img].T + + # Use a least-squares method to get A.T*A and a Cholesky decomposition to find A. + A = self._ATA_solver(v1, v2) + + # Recover the rotations. The first two columns of all rotation + # matrices are given by unmixing V1 and V2 using A. The third + # column is the cross product of the first two. + r1 = np.dot(A.T, v1) + r2 = np.dot(A.T, v2) + r3 = np.cross(r1, r2, axis=0) + rotations = np.stack((r1.T, r2.T, r3.T), axis=-1) + + # Make sure that we got rotations by enforcing R to be + # a rotation (in case the error is large) + u, _, v = np.linalg.svd(rotations) + np.einsum("ijk, ikl -> ijl", u, v, out=rotations) + + return rotations + + @staticmethod + def _ATA_solver(v1, v2): """ - Output the 3D orientations + Uses a least squares method to solve for the linear transformation A + such that A*v1=R1 and A*v2=R2 correspond to the first and second columns + of a sequence of rotation matrices. + + :param v1: 3 x n_img array corresponding to linear combinations of the first + columns of all rotation matrices. + :param v2: 3 x n_img array corresponding to linear combinations of the second + columns of all rotation matrices. + + :return: 3x3 linear transformation mapping v1, v2 to first two columns of rotations. """ - pass + # We look for a linear transformation (3 x 3 matrix) A such that + # A*v1'=R1 and A*v2=R2 are the columns of the rotations matrices. + # Therefore: + # v1 * A'*A v1' = 1 + # v2 * A'*A v2' = 1 + # v1 * A'*A v2' = 0 + # These are 3*K linear equations for 9 matrix entries of A'*A + # Actually, there are only 6 unknown variables, because A'*A is symmetric. + # So we will truncate from 9 variables to 6 variables corresponding + # to the upper half of the matrix A'*A + n_img = v1.shape[-1] + truncated_equations = np.zeros((3 * n_img, 9), dtype=v1.dtype) + k = 0 + for i in range(3): + for j in range(3): + truncated_equations[0::3, k] = v1[i] * v1[j] + truncated_equations[1::3, k] = v2[i] * v2[j] + truncated_equations[2::3, k] = v1[i] * v2[j] + k += 1 + + # b = [1 1 0 1 1 0 ...]' is the right hand side vector + b = np.ones(3 * n_img) + b[2::3] = 0 + + # Find the least squares approximation of A'*A in vector form + ATA_vec = np.linalg.lstsq(truncated_equations, b, rcond=None)[0] + + # Construct the matrix A'*A from the vectorized matrix. + # Note, this is only the lower triangle of A'*A. + ATA = ATA_vec.reshape(3, 3) + + # The Cholesky decomposition of A'*A gives A (lower triangle). + # Note, that `np.linalg.cholesky()` only uses the lower-triangular + # and diagonal elements of ATA. + A = np.linalg.cholesky(ATA) + + return A diff --git a/tests/test_orient_sdp.py b/tests/test_orient_sdp.py new file mode 100644 index 0000000000..a161d2fdd7 --- /dev/null +++ b/tests/test_orient_sdp.py @@ -0,0 +1,195 @@ +import numpy as np +import pytest + +from aspire.abinitio import CommonlineSDP +from aspire.nufft import backend_available +from aspire.source import Simulation +from aspire.utils import ( + Rotation, + get_aligned_rotations, + mean_aligned_angular_distance, + register_rotations, + rots_to_clmatrix, +) +from aspire.volume import AsymmetricVolume + +RESOLUTION = [ + 32, + 33, +] + +OFFSETS = [ + None, # Defaults to random offsets. + 0, +] + +DTYPES = [ + np.float32, + pytest.param(np.float64, marks=pytest.mark.expensive), +] + + +@pytest.fixture(params=RESOLUTION, ids=lambda x: f"resolution={x}") +def resolution(request): + return request.param + + +@pytest.fixture(params=OFFSETS, ids=lambda x: f"offsets={x}") +def offsets(request): + return request.param + + +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +def dtype(request): + return request.param + + +@pytest.fixture +def src_orient_est_fixture(resolution, offsets, dtype): + """Fixture for simulation source and orientation estimation object.""" + src = Simulation( + n=50, + L=resolution, + vols=AsymmetricVolume(L=resolution, C=1, K=100, seed=0).generate(), + offsets=offsets, + amplitudes=1, + seed=0, + ) + + # Increase max_shift and set shift_step to be sub-pixel when using + # random offsets in the Simulation. This improves common-line detection. + max_shift = 0.20 + shift_step = 0.25 + + # Set max_shift 1 pixel and shift_step to 1 pixel when using 0 offsets. + if np.all(src.offsets == 0.0): + max_shift = 1 / src.L + shift_step = 1 + + orient_est = CommonlineSDP( + src, max_shift=max_shift, shift_step=shift_step, mask=False + ) + + return src, orient_est + + +def test_estimate_rotations(src_orient_est_fixture): + src, orient_est = src_orient_est_fixture + + if backend_available("cufinufft") and src.dtype == np.float32: + pytest.skip("CI on gpu fails for singles.") + + orient_est.estimate_rotations() + + # Register estimates to ground truth rotations and compute the + # angular distance between them (in degrees). + # Assert that mean aligned angular distance is less than 1 degrees. + mean_aligned_angular_distance(orient_est.rotations, src.rotations, degree_tol=1) + + +def test_construct_S(src_orient_est_fixture): + """Test properties of the common-line quadratic form matrix S.""" + src, orient_est = src_orient_est_fixture + + # Since we are using the ground truth cl_matrix there is no need to test with offsets. + if src.offsets.all() != 0: + pytest.skip("No need to test with offsets.") + + # Construct the matrix S using ground truth common-lines. + gt_cl_matrix = rots_to_clmatrix(src.rotations, orient_est.n_theta) + S = orient_est._construct_S(gt_cl_matrix) + + # Check that S is symmetric. + np.testing.assert_allclose(S, S.T) + + # For uniformly distributed rotations the top eigenvalue should have multiplicity 3. + # As such, we can expect that the top 3 eigenvalues will all be close in value to their mean. + eigs = np.linalg.eigvalsh(S) + eigs_mean = np.mean(eigs[:3]) + + # Check that the top 3 eigenvalues are all within 10% of the their mean. + np.testing.assert_array_less(abs((eigs[:3] - eigs_mean) / eigs_mean), 0.10) + + # Check that the next eigenvalue is not close to the top 3, ie. multiplicity is not greater than 3. + np.testing.assert_array_less(0.25, abs((eigs[4] - eigs_mean) / eigs_mean)) + + +def test_gram_matrix(src_orient_est_fixture): + """Test properties of the common-line Gram matrix.""" + src, orient_est = src_orient_est_fixture + + # Since we are using the ground truth cl_matrix there is no need to test with offsets. + if src.offsets.all() != 0: + pytest.skip("No need to test with offsets.") + + # Construct a ground truth S to pass into Gram computation. + gt_cl_matrix = rots_to_clmatrix(src.rotations, orient_est.n_theta) + S = orient_est._construct_S(gt_cl_matrix) + + # Estimate the Gram matrix + A, b = orient_est._sdp_prep() + gram = orient_est._compute_gram_matrix(S, A, b) + + # Construct the ground truth Gram matrix, G = R @ R.T, where R = [R1, R2] + # with R1 and R2 being the concatenation of the first and second columns + # of all ground truth rotation matrices, respectively. + rots = src.rotations + R1 = rots[:, :, 0] + R2 = rots[:, :, 1] + R = np.concatenate((R1, R2)) + gt_gram = R @ R.T + + # We'll check that the RMSE is within 10% of the mean value of gt_gram + rmse = np.sqrt(np.mean((gram - R @ R.T) ** 2)) + np.testing.assert_array_less(rmse / np.mean(gt_gram), 0.10) + + +def test_ATA_solver(): + # Generate some rotations. + seed = 42 + n_rots = 73 + dtype = np.float32 + rots = Rotation.generate_random_rotations(n=n_rots, seed=seed, dtype=dtype).matrices + + # Create a simple reference linear transformation A that is rank-3. + A_ref = np.diag([1, 2, 3]).astype(dtype, copy=False) + + # Create v1 and v2 such that A_ref*v1=R1 and A_ref*v2=R2, R1 and R2 are the first + # and second columns of all rotations. + R1 = rots[:, :, 0].T + R2 = rots[:, :, 1].T + v1 = np.linalg.inv(A_ref) @ R1 + v2 = np.linalg.inv(A_ref) @ R2 + + # Use ATA_solver to solve for A, given v1 and v2. + A = CommonlineSDP._ATA_solver(v1, v2) + + # Check that A is close to A_ref. + np.testing.assert_allclose(A, A_ref, atol=1e-7) + + +def test_deterministic_rounding(src_orient_est_fixture): + """Test deterministic rounding, which recovers rotations from a Gram matrix.""" + src, orient_est = src_orient_est_fixture + + # Since we are using the ground truth cl_matrix there is no need to test with offsets. + if src.offsets.all() != 0: + pytest.skip("No need to test with offsets.") + + # Construct the ground truth Gram matrix, G = R @ R.T, where R = [R1, R2] + # with R1 and R2 being the concatenation of the first and second columns + # of all ground truth rotation matrices, respectively. + gt_rots = src.rotations + R1 = gt_rots[:, :, 0] + R2 = gt_rots[:, :, 1] + R = np.concatenate((R1, R2)) + gt_gram = R @ R.T + + # Pass the Gram matrix into the deterministic rounding procedure to recover rotations. + est_rots = orient_est._deterministic_rounding(gt_gram) + + # Check that the estimated rotations are close to ground truth after global alignment. + Q_mat, flag = register_rotations(est_rots, gt_rots) + regrot = get_aligned_rotations(est_rots, Q_mat, flag) + + np.testing.assert_allclose(regrot, gt_rots) From 57aabe9b49dc1611a5f36877b723c648403dd107 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 25 Oct 2023 13:23:32 -0400 Subject: [PATCH 112/294] replace manual grid construction with grid_2d in pswf --- src/aspire/basis/pswf_2d.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index c1deb92c21..3bcf6b4024 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -13,7 +13,7 @@ ) from aspire.basis.pswf_utils import BNMatrix from aspire.operators import BlkDiagMatrix -from aspire.utils import complex_type +from aspire.utils import complex_type, grid_2d logger = logging.getLogger(__name__) @@ -82,21 +82,15 @@ def _build(self): def _generate_grid(self): """ Generate the 2D sampling grid - - TODO: need to re-implement to use the similar grid function as FB methods. """ - if self.nres % 2 == 0: - x_1d_grid = range(-self.rcut, self.rcut) - else: - x_1d_grid = range(-self.rcut, self.rcut + 1) - x_2d_grid, y_2d_grid = np.meshgrid(x_1d_grid, x_1d_grid) - r_2d_grid = np.sqrt(np.square(x_2d_grid) + np.square(y_2d_grid)) - points_in_disk = r_2d_grid <= self.rcut - x = y_2d_grid[points_in_disk] - y = x_2d_grid[points_in_disk] + grid = grid_2d(self.nres, normalized=False) + points_in_disk = grid["r"] <= self.rcut + # yuck + x = grid["y"][points_in_disk] + y = grid["x"][points_in_disk] self._r_disk = np.sqrt(np.square(x) + np.square(y)) / self.rcut self._theta_disk = np.angle(x + 1j * y) - self._image_height = len(x_1d_grid) + self._image_height = len(grid["x"][0]) self._disk_mask = points_in_disk def _precomp(self): From e2211d42b68c9a7fcfe5caa31cbbedf20b6f9633 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 25 Oct 2023 13:38:29 -0400 Subject: [PATCH 113/294] swap x,y vars, then remove a bunch of excess code --- src/aspire/basis/pswf_2d.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 3bcf6b4024..352ec32434 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -84,14 +84,9 @@ def _generate_grid(self): Generate the 2D sampling grid """ grid = grid_2d(self.nres, normalized=False) - points_in_disk = grid["r"] <= self.rcut - # yuck - x = grid["y"][points_in_disk] - y = grid["x"][points_in_disk] - self._r_disk = np.sqrt(np.square(x) + np.square(y)) / self.rcut - self._theta_disk = np.angle(x + 1j * y) - self._image_height = len(grid["x"][0]) - self._disk_mask = points_in_disk + self._disk_mask = grid["r"] <= self.rcut + self._r_disk = grid["r"][self._disk_mask] / self.rcut + self._theta_disk = grid["phi"].T[self._disk_mask] def _precomp(self): """ @@ -243,9 +238,7 @@ def _evaluate(self, coefficients): coefficients[:, ~angular_is_zero] @ self.samples[~angular_is_zero] ) - images = np.zeros( - (n_images, self._image_height, self._image_height), dtype=self.dtype - ) + images = np.zeros((n_images, self.nres, self.nres), dtype=self.dtype) images[:, self._disk_mask] = np.real(flatten_images) return images From cef9c5f931f0cbe02932dde857fa14b68a7a6d58 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 26 Oct 2023 09:36:44 -0400 Subject: [PATCH 114/294] Use indexing arg instead of transpose --- src/aspire/basis/pswf_2d.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 352ec32434..ab396b6da0 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -83,10 +83,10 @@ def _generate_grid(self): """ Generate the 2D sampling grid """ - grid = grid_2d(self.nres, normalized=False) + grid = grid_2d(self.nres, normalized=False, indexing="xy") self._disk_mask = grid["r"] <= self.rcut self._r_disk = grid["r"][self._disk_mask] / self.rcut - self._theta_disk = grid["phi"].T[self._disk_mask] + self._theta_disk = grid["phi"][self._disk_mask] def _precomp(self): """ From 528127b5cc7751ecd380d61917495eda993d9e14 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 31 Oct 2023 10:21:55 -0400 Subject: [PATCH 115/294] nearest_rotations method with unit tests. --- src/aspire/abinitio/commonline_sdp.py | 4 +- src/aspire/abinitio/commonline_sync.py | 4 +- src/aspire/utils/__init__.py | 1 + src/aspire/utils/matrix.py | 27 +++++++++++ tests/test_matrix.py | 67 ++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/aspire/abinitio/commonline_sdp.py b/src/aspire/abinitio/commonline_sdp.py index 0176bdfd35..0fdae3c8b6 100644 --- a/src/aspire/abinitio/commonline_sdp.py +++ b/src/aspire/abinitio/commonline_sdp.py @@ -5,6 +5,7 @@ from scipy.sparse import csr_array from aspire.abinitio import CLOrient3D +from aspire.utils import nearest_rotations from aspire.utils.matlab_compat import stable_eigsh logger = logging.getLogger(__name__) @@ -192,8 +193,7 @@ def _deterministic_rounding(self, gram): # Make sure that we got rotations by enforcing R to be # a rotation (in case the error is large) - u, _, v = np.linalg.svd(rotations) - np.einsum("ijk, ikl -> ijl", u, v, out=rotations) + rotations = nearest_rotations(rotations) return rotations diff --git a/src/aspire/abinitio/commonline_sync.py b/src/aspire/abinitio/commonline_sync.py index 8cef5b6e40..5e07181f4a 100644 --- a/src/aspire/abinitio/commonline_sync.py +++ b/src/aspire/abinitio/commonline_sync.py @@ -3,6 +3,7 @@ import numpy as np from aspire.abinitio import CLOrient3D, SyncVotingMixin +from aspire.utils import nearest_rotations from aspire.utils.matlab_compat import stable_eigsh logger = logging.getLogger(__name__) @@ -138,8 +139,7 @@ def estimate_rotations(self): rotations[:, :, 2] = r3.T # Make sure that we got rotations by enforcing R to be # a rotation (in case the error is large) - u, _, v = np.linalg.svd(rotations) - np.einsum("ijk, ikl -> ijl", u, v, out=rotations) + rotations = nearest_rotations(rotations) self.rotations = rotations diff --git a/src/aspire/utils/__init__.py b/src/aspire/utils/__init__.py index 8f07078316..e691e0ba5e 100644 --- a/src/aspire/utils/__init__.py +++ b/src/aspire/utils/__init__.py @@ -46,6 +46,7 @@ make_symmat, mat_to_vec, mdim_mat_fun_conj, + nearest_rotations, roll_dim, symmat_to_vec, symmat_to_vec_iso, diff --git a/src/aspire/utils/matrix.py b/src/aspire/utils/matrix.py index 20d54ee2dd..99b87ffe9d 100644 --- a/src/aspire/utils/matrix.py +++ b/src/aspire/utils/matrix.py @@ -434,6 +434,33 @@ def best_rank1_approximation(A): return (U @ S_rank1 @ V).reshape(og_shape) +def nearest_rotations(A): + """ + Uses the SVD method to compute the set of nearest rotations to the set A of noisy rotations. + + :param A: A 2D array or a 3D array where the first axis is the stack axis. + :return: ndarray of rotations of equal size to A. + """ + og_shape = A.shape + dtype = A.dtype + + if A.ndim == 2: + A = A[np.newaxis] + if A.ndim != 3 or not A.shape[1] == A.shape[2] == 3: + raise ValueError( + f"Array must be of shape (3, 3) or (n, 3, 3). Found shape {A.shape}." + ) + + # For the singular value decomposition A = U @ S @ V, we compute the nearest rotation + # matrices R = U @ V, ensuring first that det(U)*det(V) = 1. + U, _, V = np.linalg.svd(A) + neg_det = np.linalg.det(U) * np.linalg.det(V) < 0 + U[neg_det] = U[neg_det] @ np.diag((1, 1, -1)).astype(dtype, copy=False) + rots = np.einsum("ijk, ikl -> ijl", U, V) + + return rots + + def fix_signs(u): """ Negates columns so the sign of the largest element in the column is positive. diff --git a/tests/test_matrix.py b/tests/test_matrix.py index e0af0e3954..64d70ce036 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -2,15 +2,21 @@ from unittest import TestCase import numpy as np +import pytest from aspire.utils import ( + Rotation, best_rank1_approximation, fix_signs, im_to_vec, mat_to_vec, + mean_aligned_angular_distance, + nearest_rotations, + randn, roll_dim, symmat_to_vec_iso, unroll_dim, + utest_tolerance, vec_to_im, vec_to_symmat, vec_to_symmat_iso, @@ -342,3 +348,64 @@ def testFixSigns(self): x[:, 3] = 0 y[:, 3] = 0 self.assertTrue(np.allclose(fix_signs(x), y)) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +def test_nearest_rotations(dtype): + n_rots = 5 + rots = Rotation.generate_random_rotations(n_rots, seed=0, dtype=dtype).matrices + + # Add some noise to the rotation. + noise = 1e-3 * randn(n_rots * 9, seed=0).astype(dtype, copy=False).reshape( + n_rots, 3, 3 + ) + noisy_rots = rots + noise + + # Find nearest rotations for stack. + nearest_rots = nearest_rotations(noisy_rots) + + # Check that estimates are rotation matrices. + _is_rotation(nearest_rots, dtype) + + # Check that estimates are close to original rotations. + mean_aligned_angular_distance(rots, nearest_rots, degree_tol=1) + + # Check dtype pass-through. + assert nearest_rots.dtype == dtype + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +def test_nearest_rotations_reflection(dtype): + rots = Rotation.generate_random_rotations(1, seed=0, dtype=dtype).matrices + + # Add a reflection and some noise to the rotation. + refl = rots @ np.diag((1, -1, 1)).astype(dtype) + noise = 1e-3 * randn(9, seed=0).astype(dtype, copy=False).reshape(3, 3) + noisy_refl = refl + noise + + # Find nearest rotations + nearest_rot = nearest_rotations(noisy_refl) + + # Check that estimates are rotation matrices. + _is_rotation(nearest_rot, dtype) + + +def _is_rotation(R, dtype): + """ + Helper function to check if a set of 3x3 matrices are rotations + by checking that R.T @ R = I and det(R) = 1. + + :param R: Singleton or stack of 3x3 arrays. + :param dtype: dtype to use for test tolerance. + :return: boolean indicating if all 3x3 arrays are rotations. + """ + if R.ndim == 2: + R = R[np.newaxis] + + n_rots = len(R) + RTR = np.einsum("ikj,ikl->ijl", R, R) + atol = utest_tolerance(dtype) + np.testing.assert_allclose( + RTR, np.broadcast_to(np.eye(3), (n_rots, 3, 3)), atol=atol + ) + np.testing.assert_allclose(np.linalg.det(R), 1, atol=atol) From d008888b219d7c9bfef717d20d9a16dbac1915dc Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 31 Oct 2023 10:32:41 -0400 Subject: [PATCH 116/294] Check we retain singleton shape. --- src/aspire/utils/matrix.py | 2 +- tests/test_matrix.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/aspire/utils/matrix.py b/src/aspire/utils/matrix.py index 99b87ffe9d..506666e4d1 100644 --- a/src/aspire/utils/matrix.py +++ b/src/aspire/utils/matrix.py @@ -458,7 +458,7 @@ def nearest_rotations(A): U[neg_det] = U[neg_det] @ np.diag((1, 1, -1)).astype(dtype, copy=False) rots = np.einsum("ijk, ikl -> ijl", U, V) - return rots + return rots.reshape(og_shape) def fix_signs(u): diff --git a/tests/test_matrix.py b/tests/test_matrix.py index 64d70ce036..d04f096b64 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -376,19 +376,23 @@ def test_nearest_rotations(dtype): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_nearest_rotations_reflection(dtype): - rots = Rotation.generate_random_rotations(1, seed=0, dtype=dtype).matrices + # Generate singleton rotation. + rot = Rotation.generate_random_rotations(1, seed=0, dtype=dtype).matrices[0] # Add a reflection and some noise to the rotation. - refl = rots @ np.diag((1, -1, 1)).astype(dtype) + refl = rot @ np.diag((1, -1, 1)).astype(dtype) noise = 1e-3 * randn(9, seed=0).astype(dtype, copy=False).reshape(3, 3) noisy_refl = refl + noise - # Find nearest rotations + # Find nearest rotation. nearest_rot = nearest_rotations(noisy_refl) # Check that estimates are rotation matrices. _is_rotation(nearest_rot, dtype) + # Check that we retain singleton shape. + assert nearest_rot.shape == rot.shape + def _is_rotation(R, dtype): """ From a001896dd84c2052a1df6f3abe9b9753670bc670 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 31 Oct 2023 10:42:27 -0400 Subject: [PATCH 117/294] Clarifying comment. --- src/aspire/utils/matrix.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/aspire/utils/matrix.py b/src/aspire/utils/matrix.py index 506666e4d1..f223fc1017 100644 --- a/src/aspire/utils/matrix.py +++ b/src/aspire/utils/matrix.py @@ -452,10 +452,11 @@ def nearest_rotations(A): ) # For the singular value decomposition A = U @ S @ V, we compute the nearest rotation - # matrices R = U @ V, ensuring first that det(U)*det(V) = 1. + # matrices R = U @ V. If det(U)*det(V) = -1, we negate the third singular value to ensure + # we have a rotation. U, _, V = np.linalg.svd(A) - neg_det = np.linalg.det(U) * np.linalg.det(V) < 0 - U[neg_det] = U[neg_det] @ np.diag((1, 1, -1)).astype(dtype, copy=False) + neg_det_idx = np.linalg.det(U) * np.linalg.det(V) < 0 + U[neg_det_idx] = U[neg_det_idx] @ np.diag((1, 1, -1)).astype(dtype, copy=False) rots = np.einsum("ijk, ikl -> ijl", U, V) return rots.reshape(og_shape) From 1df6ec24ce8380fdaaa335c26a39e06e2875abdf Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 31 Oct 2023 10:45:51 -0400 Subject: [PATCH 118/294] Edit more comments. --- tests/test_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_matrix.py b/tests/test_matrix.py index d04f096b64..d1dcfa4960 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -355,7 +355,7 @@ def test_nearest_rotations(dtype): n_rots = 5 rots = Rotation.generate_random_rotations(n_rots, seed=0, dtype=dtype).matrices - # Add some noise to the rotation. + # Add some noise to the rotations. noise = 1e-3 * randn(n_rots * 9, seed=0).astype(dtype, copy=False).reshape( n_rots, 3, 3 ) @@ -387,7 +387,7 @@ def test_nearest_rotations_reflection(dtype): # Find nearest rotation. nearest_rot = nearest_rotations(noisy_refl) - # Check that estimates are rotation matrices. + # Check that estimate is a rotation. _is_rotation(nearest_rot, dtype) # Check that we retain singleton shape. From 1dae00cd15635b228173b72cf35fa4888d919e88 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 31 Oct 2023 11:14:10 -0400 Subject: [PATCH 119/294] Test error message. --- tests/test_matrix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_matrix.py b/tests/test_matrix.py index d1dcfa4960..4e1a8ef825 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -394,6 +394,18 @@ def test_nearest_rotations_reflection(dtype): assert nearest_rot.shape == rot.shape +def test_nearest_rotations_error(): + # Check error for bad ndim. + A = randn(2 * 5 * 3 * 3, seed=0).reshape(2, 5, 3, 3) + with pytest.raises(ValueError, match="Array must be of shape"): + _ = nearest_rotations(A) + + # Check error for bad shape. + A = randn(5 * 3 * 2, seed=0).reshape(5, 3, 2) + with pytest.raises(ValueError, match="Array must be of shape"): + _ = nearest_rotations(A) + + def _is_rotation(R, dtype): """ Helper function to check if a set of 3x3 matrices are rotations From 1334ae175336138cfe08ca5924b762100a2cb9fc Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 3 Nov 2023 11:55:26 -0400 Subject: [PATCH 120/294] address review. --- src/aspire/utils/matrix.py | 2 +- tests/test_matrix.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aspire/utils/matrix.py b/src/aspire/utils/matrix.py index f223fc1017..5e56d2e65e 100644 --- a/src/aspire/utils/matrix.py +++ b/src/aspire/utils/matrix.py @@ -457,7 +457,7 @@ def nearest_rotations(A): U, _, V = np.linalg.svd(A) neg_det_idx = np.linalg.det(U) * np.linalg.det(V) < 0 U[neg_det_idx] = U[neg_det_idx] @ np.diag((1, 1, -1)).astype(dtype, copy=False) - rots = np.einsum("ijk, ikl -> ijl", U, V) + rots = U @ V return rots.reshape(og_shape) diff --git a/tests/test_matrix.py b/tests/test_matrix.py index 4e1a8ef825..4fde9f15f4 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -396,12 +396,12 @@ def test_nearest_rotations_reflection(dtype): def test_nearest_rotations_error(): # Check error for bad ndim. - A = randn(2 * 5 * 3 * 3, seed=0).reshape(2, 5, 3, 3) + A = np.empty((2, 5, 3, 3)) with pytest.raises(ValueError, match="Array must be of shape"): _ = nearest_rotations(A) # Check error for bad shape. - A = randn(5 * 3 * 2, seed=0).reshape(5, 3, 2) + A = np.empty((5, 3, 2)) with pytest.raises(ValueError, match="Array must be of shape"): _ = nearest_rotations(A) @@ -419,7 +419,7 @@ def _is_rotation(R, dtype): R = R[np.newaxis] n_rots = len(R) - RTR = np.einsum("ikj,ikl->ijl", R, R) + RTR = np.transpose(R, axes=(0,2,1)) @ R atol = utest_tolerance(dtype) np.testing.assert_allclose( RTR, np.broadcast_to(np.eye(3), (n_rots, 3, 3)), atol=atol From 6c714f1846e6b96fd926e6d6234ab91dc6a3dcd1 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 3 Nov 2023 11:56:30 -0400 Subject: [PATCH 121/294] black --- tests/test_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_matrix.py b/tests/test_matrix.py index 4fde9f15f4..728200b9b3 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -419,7 +419,7 @@ def _is_rotation(R, dtype): R = R[np.newaxis] n_rots = len(R) - RTR = np.transpose(R, axes=(0,2,1)) @ R + RTR = np.transpose(R, axes=(0, 2, 1)) @ R atol = utest_tolerance(dtype) np.testing.assert_allclose( RTR, np.broadcast_to(np.eye(3), (n_rots, 3, 3)), atol=atol From a9cab5b34313fbb45c07111aa4de2a70a533cb26 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 2 Oct 2023 10:34:45 -0400 Subject: [PATCH 122/294] Fill in docstrings in docstring check file. --- docs/check_docstrings.py | 75 ++++++++++++++++++++++++++++++++++++++++ tox.ini | 1 + 2 files changed, 76 insertions(+) create mode 100644 docs/check_docstrings.py diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py new file mode 100644 index 0000000000..e085de3451 --- /dev/null +++ b/docs/check_docstrings.py @@ -0,0 +1,75 @@ +import logging +import os +import re +import sys + +logger = logging.getLogger(__name__) + + +def check_blank_line_above_param_section(file_path): + """ + Check that every docstring with both a body section and a parameter + section separates the two sections with exactly one blank line. Log + errors and return count. + + :param file_path: File path to check for error. + :return: Per file error count. + """ + error_count = 0 + with open(file_path, "r") as file: + content = file.read() + + docstrings = re.findall(r"\"\"\"(.*?)\"\"\"", content, re.DOTALL) + + for docstring in docstrings: + lines = docstring.split("\n") + for i, line in enumerate(lines): + if line.strip().startswith(r":param") or line.strip().startswith( + r":return" + ): + body_section = "\n".join(lines[:i]) + if not body_section: + break + elif body_section != body_section.rstrip() + "\n": + # Get line number of error. + # Using `re.escape` to deal with non-alphanumeric characters. + match = re.search(re.escape(body_section), content) + docstring_start = match.start() + line_number = content.count("\n", 0, docstring_start) + i + + # Log error message. + msg = "Must have exactly 1 blank line between docstring body and parameter sections." + logger.error(f"{file_path}: {line_number}: {msg}") + error_count += 1 + break + else: + break + return error_count + + +def process_directory(directory): + """ + Recursively walk through directories and check for docstring errors. + If any errors found, log error count and exit. + + :param directory: Directory path to walk. + """ + error_count = 0 + for root, _, files in os.walk(directory): + for file in files: + if file.endswith(".py"): + file_path = os.path.join(root, file) + logger.info(f"Processing file: {file_path}") + error_count += check_blank_line_above_param_section(file_path) + if error_count > 0: + logger.error(f"Found {error_count} docstring errors.") + sys.exit(1) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + logger.warning("Usage: python check_docstrings.py ") + sys.exit(1) + + target_directory = sys.argv[1] + process_directory(target_directory) diff --git a/tox.ini b/tox.ini index 3f6c88b440..c753d9e692 100644 --- a/tox.ini +++ b/tox.ini @@ -55,6 +55,7 @@ commands = flake8 . isort --check-only --diff . black --check --diff . + python docs/check_docstrings.py src/aspire python -m json.tool .zenodo.json /dev/null check-manifest . python -m build From 9c1b756e60450f4521a12819ac1b928e364ef811 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 4 Oct 2023 13:52:07 -0400 Subject: [PATCH 123/294] use glob --- docs/check_docstrings.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py index e085de3451..bad9c7601c 100644 --- a/docs/check_docstrings.py +++ b/docs/check_docstrings.py @@ -3,6 +3,8 @@ import re import sys +from glob import glob + logger = logging.getLogger(__name__) @@ -55,12 +57,8 @@ def process_directory(directory): :param directory: Directory path to walk. """ error_count = 0 - for root, _, files in os.walk(directory): - for file in files: - if file.endswith(".py"): - file_path = os.path.join(root, file) - logger.info(f"Processing file: {file_path}") - error_count += check_blank_line_above_param_section(file_path) + for file in glob(os.path.join(directory, '**/*.py'), recursive=True): + error_count += check_blank_line_above_param_section(file) if error_count > 0: logger.error(f"Found {error_count} docstring errors.") sys.exit(1) From 87c2020a877109780fcb7260d9c373bf03914f97 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 4 Oct 2023 15:12:50 -0400 Subject: [PATCH 124/294] Use re.finditer instead of re.findall to remove double search. --- docs/check_docstrings.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py index bad9c7601c..77bffb7a1a 100644 --- a/docs/check_docstrings.py +++ b/docs/check_docstrings.py @@ -2,7 +2,6 @@ import os import re import sys - from glob import glob logger = logging.getLogger(__name__) @@ -21,31 +20,30 @@ def check_blank_line_above_param_section(file_path): with open(file_path, "r") as file: content = file.read() - docstrings = re.findall(r"\"\"\"(.*?)\"\"\"", content, re.DOTALL) - + docstrings = re.finditer(r"\"\"\"(.*?)\"\"\"", content, re.DOTALL) for docstring in docstrings: - lines = docstring.split("\n") + lines = docstring[0].split("\n") + + # Search for first occurence of either a ':param' or ':return' string. for i, line in enumerate(lines): if line.strip().startswith(r":param") or line.strip().startswith( r":return" ): - body_section = "\n".join(lines[:i]) + # If a body section exists but is not followed by exactly 1 + # new line log an error message and add to the count. + body_section = "\n".join(lines[1:i]) if not body_section: break elif body_section != body_section.rstrip() + "\n": - # Get line number of error. - # Using `re.escape` to deal with non-alphanumeric characters. - match = re.search(re.escape(body_section), content) - docstring_start = match.start() - line_number = content.count("\n", 0, docstring_start) + i + # Get line number. + line_number = content.count("\n", 0, docstring.start()) + i # Log error message. msg = "Must have exactly 1 blank line between docstring body and parameter sections." logger.error(f"{file_path}: {line_number}: {msg}") error_count += 1 - break - else: - break + break + return error_count @@ -57,7 +55,7 @@ def process_directory(directory): :param directory: Directory path to walk. """ error_count = 0 - for file in glob(os.path.join(directory, '**/*.py'), recursive=True): + for file in glob(os.path.join(directory, "**/*.py"), recursive=True): error_count += check_blank_line_above_param_section(file) if error_count > 0: logger.error(f"Found {error_count} docstring errors.") From ca5f0f2adca255699baaa6654874a311afe29ba5 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 4 Oct 2023 15:38:37 -0400 Subject: [PATCH 125/294] Fix found docstring error in basis/steerable.py --- src/aspire/basis/steerable.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 321c2b47bc..ba12d880fc 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -352,8 +352,6 @@ def indices_mask(self, **kwargs): More advanced operations can combine indices attributes. `angular=self.angular_indices>=0, radial=r` selects coefficients with non negative angular indices and some radial index `r`. - - :return: Boolen mask of shape (`count`,). Intended to be broadcast with `Coef` containers. """ From 9fcfa07ab5abcb579b14853136aa90ba0ce21d3c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 5 Oct 2023 09:26:29 -0400 Subject: [PATCH 126/294] Only need one str.startswith(). --- docs/check_docstrings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py index 77bffb7a1a..6480082db6 100644 --- a/docs/check_docstrings.py +++ b/docs/check_docstrings.py @@ -26,9 +26,7 @@ def check_blank_line_above_param_section(file_path): # Search for first occurence of either a ':param' or ':return' string. for i, line in enumerate(lines): - if line.strip().startswith(r":param") or line.strip().startswith( - r":return" - ): + if line.strip().startswith((r":param", r":return")): # If a body section exists but is not followed by exactly 1 # new line log an error message and add to the count. body_section = "\n".join(lines[1:i]) From d69fd35b28d292e74ef8e17ffdac80b8495c6e60 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 5 Oct 2023 09:50:50 -0400 Subject: [PATCH 127/294] Check that target directory is actual directory --- docs/check_docstrings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py index 6480082db6..0d09e80bbb 100644 --- a/docs/check_docstrings.py +++ b/docs/check_docstrings.py @@ -66,4 +66,6 @@ def process_directory(directory): sys.exit(1) target_directory = sys.argv[1] + if not os.path.isdir(target_directory): + raise RuntimeError(f"Invalid target directory path: {target_directory}") process_directory(target_directory) From dc78653dfca882edd68ba3bf5a1b61a70f614fcd Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 09:29:56 -0400 Subject: [PATCH 128/294] Replace logic with regex. --- docs/check_docstrings.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py index 0d09e80bbb..c0e6c7619d 100644 --- a/docs/check_docstrings.py +++ b/docs/check_docstrings.py @@ -20,27 +20,16 @@ def check_blank_line_above_param_section(file_path): with open(file_path, "r") as file: content = file.read() - docstrings = re.finditer(r"\"\"\"(.*?)\"\"\"", content, re.DOTALL) - for docstring in docstrings: - lines = docstring[0].split("\n") + regex = r" {4,}\"\"\"\n(?:^[^:]+?[^\n])+\n {4,}:(?:.*\n)+? {4,}\"\"\"| {4,}\"\"\"\n(?:^.*[^\n]\n)+\n\n+ {4,}:(?:.*\n)+? {4,}\"\"\"" - # Search for first occurence of either a ':param' or ':return' string. - for i, line in enumerate(lines): - if line.strip().startswith((r":param", r":return")): - # If a body section exists but is not followed by exactly 1 - # new line log an error message and add to the count. - body_section = "\n".join(lines[1:i]) - if not body_section: - break - elif body_section != body_section.rstrip() + "\n": - # Get line number. - line_number = content.count("\n", 0, docstring.start()) + i + bad_docstrings = re.finditer(regex, content, re.MULTILINE) + for docstring in bad_docstrings: + line_number = content.count("\n", 0, docstring.start()) + 1 - # Log error message. - msg = "Must have exactly 1 blank line between docstring body and parameter sections." - logger.error(f"{file_path}: {line_number}: {msg}") - error_count += 1 - break + # Log error message. + msg = "Must have exactly 1 blank line between docstring body and parameter sections." + logger.error(f"{file_path}: {line_number}: {msg}") + error_count += 1 return error_count From 1e723abdb897529bbb076210af60e7dfdc12b456 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 09:35:20 -0400 Subject: [PATCH 129/294] refactor docstrings to sphinx format. --- src/aspire/apple/helper.py | 71 ++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/src/aspire/apple/helper.py b/src/aspire/apple/helper.py index 410c3a9b76..25b59a13a3 100644 --- a/src/aspire/apple/helper.py +++ b/src/aspire/apple/helper.py @@ -6,11 +6,11 @@ class PickerHelper: @classmethod def gaussian_filter(cls, size_filter, std): - """Computes low-pass filter. + """ + Computes low-pass filter. - Args: - size_filter: Size of filter (size_filter x size_filter). - std: sigma value in filter. + :param size_filter: Size of filter (size_filter x size_filter). + :param std: sigma value in filter. """ y, x = xp.mgrid[ @@ -27,15 +27,14 @@ def gaussian_filter(cls, size_filter, std): @classmethod def extract_windows(cls, img, block_size): - """Extracts blocks of size (block_size x block_size) from the micrograph. Blocks are + """ + Extracts blocks of size (block_size x block_size) from the micrograph. Blocks are extracted with steps of size (block_size) - Args: - img: Micrograph image. - block_size: required block size. + :param img: Micrograph image. + :param block_size: required block size. - Returns: - 3D Matrix of blocks. For example, img[0] is the first block. + :return: 3D Matrix of blocks. For example, img[0] is the first block. """ # Compute x,y boundary using block_size @@ -57,15 +56,14 @@ def extract_windows(cls, img, block_size): @classmethod def extract_query(cls, img, block_size): - """Extract all query images from the micrograph. windows are + """ + Extract all query images from the micrograph. windows are extracted with steps of size (block_size/2) - Args: - img: Micrograph image. - block_size: Query images must be of size (block_size x block_size). + :param img: Micrograph image. + :param block_size: Query images must be of size (block_size x block_size). - Returns: - 4D Matrix of query images. + :return: 4D Matrix of query images. """ # keep only the portion of the image that can be split into blocks with no remainder @@ -135,16 +133,15 @@ def reference_size(cls, img, container_size): @classmethod def extract_references(cls, img, query_size, container_size): - """Chooses and extracts reference images from the micrograph. + """ + Chooses and extracts reference images from the micrograph. - Args: - img: Micrograph image. - query_size: Reference images must be of the same size of query images, i.e. (query_size x query_size). - container_size: Containers are large regions used to select reference images. The size of each - region is (container_size x container_size) + :param img: Micrograph image. + :param query_size: Reference images must be of the same size of query images, i.e. (query_size x query_size). + :param container_size: Containers are large regions used to select reference images. The size of each + region is (container_size x container_size) - Returns: - 3D Matrix of reference images. windows[0] is the first reference window. + :return: 3D Matrix of reference images. windows[0] is the first reference window. """ img = xp.asarray(img) @@ -220,16 +217,15 @@ def extract_references(cls, img, query_size, container_size): @classmethod def get_training_set(cls, micro_img, bw_mask_p, bw_mask_n, n): - """Gets training set for the SVM classifier. + """ + Gets training set for the SVM classifier. - Args: - micro_img: Micrograph image. - bw_mask_p: Binary image indicating regions from which to extract examples of particles. - bw_mask_n: Binary image indicating regions from which to extract examples of noise. - n: Size of training windows. + :param micro_img: Micrograph image. + :param bw_mask_p: Binary image indicating regions from which to extract examples of particles. + :param bw_mask_n: Binary image indicating regions from which to extract examples of noise. + :param n: Size of training windows. - Returns: - A matrix of features and a vector of labels for the SVM training. + :return: A matrix of features and a vector of labels for the SVM training. """ non_overlap = cls.extract_windows(micro_img, n) @@ -260,15 +256,14 @@ def get_training_set(cls, micro_img, bw_mask_p, bw_mask_n, n): @classmethod def moments(cls, img, query_size): - """Calculates the mean and standard deviation for each window of size (query_size x query_size) + """ + Calculates the mean and standard deviation for each window of size (query_size x query_size) in the micrograph. - Args: - img: Micrograph image. - query_size: Size of windows for which to compute mean and std. + :param img: Micrograph image. + :param query_size: Size of windows for which to compute mean and std. - Returns: - A matrix of mean intensity and a matrix of variance, each containing a single + :return: A matrix of mean intensity and a matrix of variance, each containing a single entry for each possible (query_size x query_size) window in the micrograph. """ From 9be183239ee1a0951be26b1edc3006788d07d964 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 10:21:13 -0400 Subject: [PATCH 130/294] line wrap regex --- docs/check_docstrings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py index c0e6c7619d..e4ec68b6ba 100644 --- a/docs/check_docstrings.py +++ b/docs/check_docstrings.py @@ -20,7 +20,10 @@ def check_blank_line_above_param_section(file_path): with open(file_path, "r") as file: content = file.read() - regex = r" {4,}\"\"\"\n(?:^[^:]+?[^\n])+\n {4,}:(?:.*\n)+? {4,}\"\"\"| {4,}\"\"\"\n(?:^.*[^\n]\n)+\n\n+ {4,}:(?:.*\n)+? {4,}\"\"\"" + regex = ( + r" {4,}\"\"\"\n(?:^[^:]+?[^\n])+\n {4,}(:p|:r)(?:.*\n)+? {4,}\"\"\"" + r"| {4,}\"\"\"\n(?:^[^:]+?[^\n])+\n\n\n+ {4,}(:p|:r)(?:.*\n)+? {4,}\"\"\"" + ) bad_docstrings = re.finditer(regex, content, re.MULTILINE) for docstring in bad_docstrings: From b4afe453abc1dac338087412d93de7c3918fd819 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 11:58:37 -0400 Subject: [PATCH 131/294] Initial test --- tests/saved_test_data/sample_docstrings.py | 39 ++++++++++++++++++++++ tests/test_docstring_checker.py | 18 ++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/saved_test_data/sample_docstrings.py create mode 100644 tests/test_docstring_checker.py diff --git a/tests/saved_test_data/sample_docstrings.py b/tests/saved_test_data/sample_docstrings.py new file mode 100644 index 0000000000..087dd3248c --- /dev/null +++ b/tests/saved_test_data/sample_docstrings.py @@ -0,0 +1,39 @@ +def good_fun1(frog, dog): + """ + This docstring is properly formatted. + + It has a multi-line, multi-section body + followed by exactly one blank line. + + :param frog: This param description is + multiline. + :param dog: Single line description + :return: A frog on a dog + """ + +def good_fun2(): + """ + This function has only a return. + + :return: Just a return. + """ + +def bad_fun1(cat, hat): + """ + This docstring is missing a blank line + between the body and parameter sections. + :param cat: A cat. + :param hat: A hat. + :return: A cat in a hat. + """ + +def bad_fun2(foo): + """ + This docstring has too many blank lines between + the body and parameter sections. + + + :param foo: foo description. + :return: bar + """ + diff --git a/tests/test_docstring_checker.py b/tests/test_docstring_checker.py new file mode 100644 index 0000000000..681e7a2ff9 --- /dev/null +++ b/tests/test_docstring_checker.py @@ -0,0 +1,18 @@ +import sys + +import pytest +import subprocess + +def test_check_docstrings(): + result = subprocess.run( + ["python", "docs/check_docstrings.py", "tests/saved_test_data"], + capture_output=True, + text=True, + ) + + err_1 = "sample_docstrings.py: 22" + err_2 = "sample_docstrings.py: 31" + + assert result.stdout == "" + assert err_1 in result.stderr + assert err_2 in result.stderr From 9fe3393d67c5549472f909dcf206900ad411a1b5 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 12:02:06 -0400 Subject: [PATCH 132/294] tox --- tests/saved_test_data/sample_docstrings.py | 4 +++- tests/test_docstring_checker.py | 8 +++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/saved_test_data/sample_docstrings.py b/tests/saved_test_data/sample_docstrings.py index 087dd3248c..454e3d4dfd 100644 --- a/tests/saved_test_data/sample_docstrings.py +++ b/tests/saved_test_data/sample_docstrings.py @@ -11,6 +11,7 @@ def good_fun1(frog, dog): :return: A frog on a dog """ + def good_fun2(): """ This function has only a return. @@ -18,6 +19,7 @@ def good_fun2(): :return: Just a return. """ + def bad_fun1(cat, hat): """ This docstring is missing a blank line @@ -27,6 +29,7 @@ def bad_fun1(cat, hat): :return: A cat in a hat. """ + def bad_fun2(foo): """ This docstring has too many blank lines between @@ -36,4 +39,3 @@ def bad_fun2(foo): :param foo: foo description. :return: bar """ - diff --git a/tests/test_docstring_checker.py b/tests/test_docstring_checker.py index 681e7a2ff9..16ab712b18 100644 --- a/tests/test_docstring_checker.py +++ b/tests/test_docstring_checker.py @@ -1,8 +1,6 @@ -import sys - -import pytest import subprocess + def test_check_docstrings(): result = subprocess.run( ["python", "docs/check_docstrings.py", "tests/saved_test_data"], @@ -12,7 +10,7 @@ def test_check_docstrings(): err_1 = "sample_docstrings.py: 22" err_2 = "sample_docstrings.py: 31" - + assert result.stdout == "" assert err_1 in result.stderr - assert err_2 in result.stderr + assert err_2 in result.stderr From f0ae1c53562b9d17884d6c7f1bc9c7b01fa4f269 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 12:11:24 -0400 Subject: [PATCH 133/294] update test --- tests/test_docstring_checker.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_docstring_checker.py b/tests/test_docstring_checker.py index 16ab712b18..3924fa6a64 100644 --- a/tests/test_docstring_checker.py +++ b/tests/test_docstring_checker.py @@ -8,9 +8,15 @@ def test_check_docstrings(): text=True, ) - err_1 = "sample_docstrings.py: 22" - err_2 = "sample_docstrings.py: 31" + good_doc_1 = "sample_docstrings.py: 2: Must have exactly 1 blank line" + good_doc_2 = "sample_docstrings.py: 16: Must have exactly 1 blank line" + err_1 = "sample_docstrings.py: 24: Must have exactly 1 blank line" + err_2 = "sample_docstrings.py: 34: Must have exactly 1 blank line" - assert result.stdout == "" + # Check that good docstrings do not log error + assert good_doc_1 not in result.stderr + assert good_doc_2 not in result.stderr + + # Check that bad docstrings log error assert err_1 in result.stderr assert err_2 in result.stderr From 9ea7efa9201b036857588eac778609cf411a7464 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 13:26:00 -0400 Subject: [PATCH 134/294] os.path.join, test cleanup. --- tests/saved_test_data/sample_docstrings.py | 21 ++++++++++++++++ tests/test_docstring_checker.py | 28 +++++++++++++++------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/tests/saved_test_data/sample_docstrings.py b/tests/saved_test_data/sample_docstrings.py index 454e3d4dfd..515bd2375a 100644 --- a/tests/saved_test_data/sample_docstrings.py +++ b/tests/saved_test_data/sample_docstrings.py @@ -20,6 +20,17 @@ def good_fun2(): """ +def good_fun3(): + def nested_fun(bip): + """ + This is a properly formatted docstring + in a nested function. + + :param bip: A small bip + :return: A large bop + """ + + def bad_fun1(cat, hat): """ This docstring is missing a blank line @@ -39,3 +50,13 @@ def bad_fun2(foo): :param foo: foo description. :return: bar """ + + +def bad_fun3(): + def nested_fun(bip): + """ + This is an improperly formatted docstring + in a nested function. + :param bip: A small bip + :return: A large bop + """ diff --git a/tests/test_docstring_checker.py b/tests/test_docstring_checker.py index 3924fa6a64..d335b166e8 100644 --- a/tests/test_docstring_checker.py +++ b/tests/test_docstring_checker.py @@ -1,22 +1,32 @@ +import os import subprocess def test_check_docstrings(): + DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") + DOCS_CHECKER = os.path.join( + os.path.dirname(__file__), "..", "docs", "check_docstrings.py" + ) + result = subprocess.run( - ["python", "docs/check_docstrings.py", "tests/saved_test_data"], + ["python", DOCS_CHECKER, DATA_DIR], capture_output=True, text=True, ) - good_doc_1 = "sample_docstrings.py: 2: Must have exactly 1 blank line" - good_doc_2 = "sample_docstrings.py: 16: Must have exactly 1 blank line" - err_1 = "sample_docstrings.py: 24: Must have exactly 1 blank line" - err_2 = "sample_docstrings.py: 34: Must have exactly 1 blank line" + good_doc_line_nums = [2, 16, 25] + bad_doc_line_nums = [35, 45, 57] # Check that good docstrings do not log error - assert good_doc_1 not in result.stderr - assert good_doc_2 not in result.stderr + for line_num in good_doc_line_nums: + msg = f"sample_docstrings.py: {line_num}: Must have exactly 1 blank line" + assert msg not in result.stderr # Check that bad docstrings log error - assert err_1 in result.stderr - assert err_2 in result.stderr + for line_num in bad_doc_line_nums: + msg = f"sample_docstrings.py: {line_num}: Must have exactly 1 blank line" + assert msg in result.stderr + + # Check total error count log + msg = f"Found {len(bad_doc_line_nums)} docstring errors" + assert msg in result.stderr From 73edd7f0f9d3b564c0e999fd037e7b366a0b4531 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 17 Oct 2023 15:54:06 -0400 Subject: [PATCH 135/294] Better use of OR logic. --- docs/check_docstrings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/check_docstrings.py b/docs/check_docstrings.py index e4ec68b6ba..68d2afe956 100644 --- a/docs/check_docstrings.py +++ b/docs/check_docstrings.py @@ -21,8 +21,7 @@ def check_blank_line_above_param_section(file_path): content = file.read() regex = ( - r" {4,}\"\"\"\n(?:^[^:]+?[^\n])+\n {4,}(:p|:r)(?:.*\n)+? {4,}\"\"\"" - r"| {4,}\"\"\"\n(?:^[^:]+?[^\n])+\n\n\n+ {4,}(:p|:r)(?:.*\n)+? {4,}\"\"\"" + r" {4,}\"\"\"\n(?:^[^:]+?[^\n])+(\n|\n\n\n+) {4,}(:p|:r)(?:.*\n)+? {4,}\"\"\"" ) bad_docstrings = re.finditer(regex, content, re.MULTILINE) From 427d1cf9492b6c1989001c819333c46540310e9f Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 18 Oct 2023 09:36:33 -0400 Subject: [PATCH 136/294] Just test docstring checking function, not entire script. --- tests/test_docstring_checker.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/test_docstring_checker.py b/tests/test_docstring_checker.py index d335b166e8..a077de6f1c 100644 --- a/tests/test_docstring_checker.py +++ b/tests/test_docstring_checker.py @@ -1,32 +1,31 @@ +import logging import os -import subprocess +from docs import check_docstrings -def test_check_docstrings(): - DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") - DOCS_CHECKER = os.path.join( - os.path.dirname(__file__), "..", "docs", "check_docstrings.py" - ) - result = subprocess.run( - ["python", DOCS_CHECKER, DATA_DIR], - capture_output=True, - text=True, +def test_check_docstrings_new(caplog): + test_string = os.path.join( + os.path.dirname(__file__), "saved_test_data", "sample_docstrings.py" ) + caplog.clear() + caplog.set_level(logging.ERROR) + error_count = check_docstrings.check_blank_line_above_param_section(test_string) + + # Line numbers of good and bad docstrings in sample_docstrings.py good_doc_line_nums = [2, 16, 25] bad_doc_line_nums = [35, 45, 57] # Check that good docstrings do not log error for line_num in good_doc_line_nums: msg = f"sample_docstrings.py: {line_num}: Must have exactly 1 blank line" - assert msg not in result.stderr + assert msg not in caplog.text # Check that bad docstrings log error for line_num in bad_doc_line_nums: msg = f"sample_docstrings.py: {line_num}: Must have exactly 1 blank line" - assert msg in result.stderr + assert msg in caplog.text # Check total error count log - msg = f"Found {len(bad_doc_line_nums)} docstring errors" - assert msg in result.stderr + assert error_count == len(bad_doc_line_nums) From d19c87fe279b3e1e45f60f60e10c0c99e2450e97 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 18 Oct 2023 09:47:10 -0400 Subject: [PATCH 137/294] test name --- tests/test_docstring_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docstring_checker.py b/tests/test_docstring_checker.py index a077de6f1c..be899bf0f9 100644 --- a/tests/test_docstring_checker.py +++ b/tests/test_docstring_checker.py @@ -4,7 +4,7 @@ from docs import check_docstrings -def test_check_docstrings_new(caplog): +def test_check_blank_line(caplog): test_string = os.path.join( os.path.dirname(__file__), "saved_test_data", "sample_docstrings.py" ) From b4e1faf152a3b62dd7f2a124a7a5b3edab743e28 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 19 Oct 2023 08:25:25 -0400 Subject: [PATCH 138/294] add missing test case. --- tests/saved_test_data/sample_docstrings.py | 8 ++++++++ tests/test_docstring_checker.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/saved_test_data/sample_docstrings.py b/tests/saved_test_data/sample_docstrings.py index 515bd2375a..dc81b54f43 100644 --- a/tests/saved_test_data/sample_docstrings.py +++ b/tests/saved_test_data/sample_docstrings.py @@ -31,6 +31,14 @@ def nested_fun(bip): """ +def good_fun4(bing, bong): + """ + :param bing: This docstring has no body. + :param bong: Should not error. + :return: Boom. + """ + + def bad_fun1(cat, hat): """ This docstring is missing a blank line diff --git a/tests/test_docstring_checker.py b/tests/test_docstring_checker.py index be899bf0f9..514d06fbbe 100644 --- a/tests/test_docstring_checker.py +++ b/tests/test_docstring_checker.py @@ -14,8 +14,8 @@ def test_check_blank_line(caplog): error_count = check_docstrings.check_blank_line_above_param_section(test_string) # Line numbers of good and bad docstrings in sample_docstrings.py - good_doc_line_nums = [2, 16, 25] - bad_doc_line_nums = [35, 45, 57] + good_doc_line_nums = [2, 16, 25, 35] + bad_doc_line_nums = [43, 53, 65] # Check that good docstrings do not log error for line_num in good_doc_line_nums: From 9d9e1eecef04430f57f53f319f57827bd980150e Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 3 Oct 2023 10:47:08 -0400 Subject: [PATCH 139/294] remove basis check --- src/aspire/denoising/denoiser_cov2d.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/aspire/denoising/denoiser_cov2d.py b/src/aspire/denoising/denoiser_cov2d.py index 126b9d4717..545f6ce642 100644 --- a/src/aspire/denoising/denoiser_cov2d.py +++ b/src/aspire/denoising/denoiser_cov2d.py @@ -142,9 +142,6 @@ def __init__(self, src, basis=None, var_noise=None, batch_size=512, covar_opt=No if basis is None: basis = FFBBasis2D((self.src.L, self.src.L), dtype=src.dtype) - if not isinstance(basis, FFBBasis2D): - raise NotImplementedError("Currently only fast FB method is supported") - self.basis = basis self.cov2d = None self.mean_est = None From 3687e060fe44983f9d0f271b5c6edc4aaf9944ac Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 3 Oct 2023 10:47:30 -0400 Subject: [PATCH 140/294] draft extend denoiser test --- tests/test_covar2d_denoiser.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 07e60a755f..9dc0bd0bc1 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -1,11 +1,11 @@ import numpy as np import pytest -from aspire.basis.ffb_2d import FFBBasis2D +from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D from aspire.denoising import DenoisedSource, DenoiserCov2D from aspire.noise import WhiteNoiseAdder -from aspire.operators.filters import RadialCTFFilter -from aspire.source.simulation import Simulation +from aspire.operators import RadialCTFFilter +from aspire.source import Simulation # TODO, parameterize these further. dtype = np.float32 @@ -17,7 +17,21 @@ RadialCTFFilter(5, 200, defocus=d, Cs=2.0, alpha=0.1) for d in np.linspace(1.5e4, 2.5e4, 7) ] -basis = FFBBasis2D((img_size, img_size), dtype=dtype) +BASIS = [ + FBBasis2D, + FFBBasis2D, + # FLEBasis2D, + # FPSWFBasis2D, +] + + +@pytest.fixture(params=BASIS, scope="module", ids=lambda x: f"basis={x}") +def basis(request): + """ + Construct and return a 2D Basis. + """ + cls = request.param + return cls(img_size, dtype=dtype) @pytest.fixture(scope="module") @@ -34,7 +48,7 @@ def sim(): ) -def test_batched_rotcov2d_MSE(sim): +def test_batched_rotcov2d_MSE(sim, basis): """ Check calling `DenoiserCov2D` via `DenoiserSource` framework yields acceptable error. """ @@ -55,7 +69,7 @@ def test_batched_rotcov2d_MSE(sim): np.testing.assert_allclose(imgs_denoised, src.images[:], rtol=1e-05, atol=1e-08) -def test_source_mismatch(sim): +def test_source_mismatch(sim, basis): """ Assert mismatched sources raises an error. """ From ddf33063ec7692e3b7e4ac7378741b1dfc26bd70 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 3 Oct 2023 10:49:17 -0400 Subject: [PATCH 141/294] Squash stashed work refactoring filter to basis mat tests for FB/FFB. --- src/aspire/basis/fb_2d.py | 77 +++++++++++++------------------- src/aspire/basis/ffb_2d.py | 60 ++++++++++++++++++++++--- src/aspire/covariance/covar2d.py | 2 +- tests/test_covar2d.py | 1 + tests/test_covar2d_denoiser.py | 41 ++++++++++++++++- 5 files changed, 126 insertions(+), 55 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 8deaa8f593..180fc041dd 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -3,10 +3,10 @@ import numpy as np from scipy.special import jv -from aspire.basis import FBBasisMixin, SteerableBasis2D +from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import unique_coords_nd -from aspire.operators import BlkDiagMatrix -from aspire.utils import roll_dim, unroll_dim +from aspire.operators import BlkDiagMatrix, DiagMatrix +from aspire.utils import LogFilterByCount, grid_2d, roll_dim, trange, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape logger = logging.getLogger(__name__) @@ -309,50 +309,33 @@ def calculate_bispectrum( def filter_to_basis_mat(self, f): """ - See SteerableBasis2D.filter_to_basis_mat. - """ - - # These form a circular dependence, import locally until time to clean up. - from aspire.basis.basis_utils import lgwt - - # Get the filter's evaluate function. - h_fun = f.evaluate - - # Set same dimensions as basis object - n_k = self.n_r - n_theta = self.n_theta - radial = self.get_radial() + Convert a filter into a basis representation. - # get 2D grid in polar coordinate - k_vals, wts = lgwt(n_k, 0, 0.5, dtype=self.dtype) - k, theta = np.meshgrid( - k_vals, np.arange(n_theta) * 2 * np.pi / (2 * n_theta), indexing="ij" - ) + :param f: `Filter` object, usually a `CTFFilter`. - # Get function values in polar 2D grid and average out angle contribution - omegax = k * np.cos(theta) - omegay = k * np.sin(theta) - omega = 2 * np.pi * np.vstack((omegax.flatten("C"), omegay.flatten("C"))) - h_vals2d = h_fun(omega).reshape(n_k, n_theta).astype(self.dtype) - h_vals = np.sum(h_vals2d, axis=1) / n_theta + :return: Representation of filter in `basis`. + Return type will be based on the class's `matrix_type`. + """ - # Represent 1D function values in basis - h_basis = BlkDiagMatrix.empty(2 * self.ell_max + 1, dtype=self.dtype) - ind_ell = 0 - for ell in range(0, self.ell_max + 1): - k_max = self.k_max[ell] - rmat = 2 * k_vals.reshape(n_k, 1) * self.r0[ell][0:k_max].T - basis_vals = np.zeros_like(rmat) - ind_radial = np.sum(self.k_max[0:ell]) - basis_vals[:, 0:k_max] = radial[ind_radial : ind_radial + k_max].T - h_basis_vals = basis_vals * h_vals.reshape(n_k, 1) - h_basis_ell = basis_vals.T @ ( - h_basis_vals * k_vals.reshape(n_k, 1) * wts.reshape(n_k, 1) - ) - h_basis[ind_ell] = h_basis_ell - ind_ell += 1 - if ell > 0: - h_basis[ind_ell] = h_basis[ind_ell - 1] - ind_ell += 1 - - return h_basis + coef = Coef(self, np.eye(self.count, dtype=self.dtype)) + img = coef.evaluate() + + # TODO, debug expand has convergence issues, + # there is a note near `cg` hinting this may relate to tolerance + # evaluate_t was not as accurate, but much much faster... + # filt = self.expand(img.filter(f)) + # filt = self.evaluate_t(img.filter(f)) + # return filt.asnumpy().reshape(self.count, self.count) + + # Loop over the expanding the filtered basis vectors one by one + filt = np.zeros((self.count, self.count), self.dtype) + with LogFilterByCount(logger, 1): + for i in trange(self.count): + try: + filt[i] = self.expand(img[i].filter(f)).asnumpy()[0] + except: + logger.warning( + f"Failed to expand basis vector {i} after filter {f}." + ) + + return filt diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index 74d46cc20c..7aaf1dfb88 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -8,6 +8,7 @@ from aspire.basis.basis_utils import lgwt from aspire.nufft import anufft, nufft from aspire.numeric import fft, xp +from aspire.operators import BlkDiagMatrix from aspire.utils import complex_type from aspire.utils.matlab_compat import m_reshape @@ -95,12 +96,6 @@ def _precomp(self): return {"gl_nodes": r, "gl_weights": w, "radial": radial, "freqs": freqs} - def get_radial(self): - """ - Return precomputed radial part - """ - return self._precomp["radial"] - def _evaluate(self, v): """ Evaluate coefficients in standard 2D coordinate basis from those in FB basis @@ -251,3 +246,56 @@ def _evaluate_t(self, x): ind_pos = ind_pos + 2 * self.k_max[ell] return v + + def filter_to_basis_mat(self, f): + """ + See SteerableBasis2D.filter_to_basis_mat. + """ + + # These form a circular dependence, import locally until time to clean up. + from aspire.basis.basis_utils import lgwt + + # Get the filter's evaluate function. + h_fun = f.evaluate + + # Set same dimensions as basis object + n_k = self.n_r + n_theta = self.n_theta + radial = self._precomp["radial"] + + # get 2D grid in polar coordinate + k_vals, wts = lgwt(n_k, 0, 0.5, dtype=self.dtype) + k, theta = np.meshgrid( + k_vals, np.arange(n_theta) * 2 * np.pi / (2 * n_theta), indexing="ij" + ) + + # Get function values in polar 2D grid and average out angle contribution + omegax = k * np.cos(theta) + omegay = k * np.sin(theta) + omega = 2 * np.pi * np.vstack((omegax.flatten("C"), omegay.flatten("C"))) + + h_vals2d = h_fun(omega).reshape(n_k, n_theta).astype(self.dtype) + h_vals = np.sum(h_vals2d, axis=1) / n_theta + + # Represent 1D function values in basis + h_basis = BlkDiagMatrix.empty(2 * self.ell_max + 1, dtype=self.dtype) + ind_ell = 0 + for ell in range(0, self.ell_max + 1): + k_max = self.k_max[ell] + rmat = 2 * k_vals.reshape(n_k, 1) * self.r0[ell][0:k_max].T + basis_vals = np.zeros_like(rmat) + ind_radial = np.sum(self.k_max[0:ell]) + basis_vals[:, 0:k_max] = radial[ind_radial : ind_radial + k_max].T + nrm = self.radial_norms[ind_radial] + # basis_vals /= nrm + h_basis_vals = basis_vals * h_vals.reshape(n_k, 1) + h_basis_ell = basis_vals.T @ ( + h_basis_vals * k_vals.reshape(n_k, 1) * wts.reshape(n_k, 1) + ) + h_basis[ind_ell] = h_basis_ell + ind_ell += 1 + if ell > 0: + h_basis[ind_ell] = h_basis[ind_ell - 1] + ind_ell += 1 + + return h_basis diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index aebbcc1dd0..3ef674981b 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -7,7 +7,7 @@ from aspire.basis import Coef, FFBBasis2D from aspire.operators import BlkDiagMatrix, DiagMatrix from aspire.optimization import conj_grad, fill_struct -from aspire.utils import make_symmat +from aspire.utils import complex_type, make_symmat from aspire.utils.matlab_compat import m_reshape logger = logging.getLogger(__name__) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index b064179efb..2c34337f25 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -20,6 +20,7 @@ DTYPES = [np.float32] # Basis used in FSPCA for class averaging. BASIS = [ + FBBasis2D, FFBBasis2D, ] diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 9dc0bd0bc1..423a31abe6 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -4,8 +4,9 @@ from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D from aspire.denoising import DenoisedSource, DenoiserCov2D from aspire.noise import WhiteNoiseAdder -from aspire.operators import RadialCTFFilter +from aspire.operators import IdentityFilter, RadialCTFFilter from aspire.source import Simulation +from aspire.utils import utest_tolerance # TODO, parameterize these further. dtype = np.float32 @@ -82,3 +83,41 @@ def test_source_mismatch(sim, basis): # Raise because src2 not identical to denoiser.src (sim) with pytest.raises(NotImplementedError, match=r".*must match.*"): _ = DenoisedSource(src2, denoiser) + + +@pytest.skip(reason="Still in development.") +def test_filter_to_basis_mat(sim, basis): + """ + Test that `basis.filter_to_basis_mat` operator is similar to + manual sequence of evaluate->filter->expand. + """ + + # Generate some reference coefficients. + coef = basis.expand(sim.images[:3]) + + # IdentityFilter should produce id + filt = IdentityFilter() + + # Apply the basis filter operator. + # Note transpose because `apply` expects and returns column vectors. + # coef_ftbm = basis.filter_to_basis_mat(filt).apply(coef.asnumpy().T).T + coef_ftbm = (basis.filter_to_basis_mat(filt) @ coef.asnumpy().T).T + + # Apply evaluate->filter->expand manually + imgs = coef.evaluate() + imgs_manual = imgs.filter(filt) + coef_manual = basis.expand(imgs_manual) + + # Sanity check filter_to_basis_mat of IdentityFilter is id + np.testing.assert_allclose(coef_ftbm, coef, atol=utest_tolerance(basis.dtype)) + + # Sanity check manual application of IdentityFilter is id + np.testing.assert_allclose(coef_manual, coef, atol=utest_tolerance(basis.dtype)) + + # Compare coefs from using ftbm operator with coef from eval->filter->exp + np.testing.assert_allclose( + coef_ftbm, + coef_manual, + atol=utest_tolerance(basis.dtype), + err_msg=f"Comparison failed for {filt}", + ) From a5aa404d50e9765b1fb75a26fb161afac7876daa Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 20 Oct 2023 11:36:22 -0400 Subject: [PATCH 142/294] Add from_dense BlkDiagMatrix helper --- src/aspire/operators/blk_diag_matrix.py | 28 +++++++++++++++++++++++++ tests/test_BlkDiagMatrix.py | 8 +++++++ 2 files changed, 36 insertions(+) diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index eeba0a508e..195e95e289 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -941,3 +941,31 @@ def diag(self): diag.extend(list(np.diag(blk))) return DiagMatrix(np.array(diag, dtype=self.dtype)) + + @staticmethod + def from_dense(A, blk_partition): + """ + Create BlkDiagMatrix with `blk_partition` from dense matrix `A`. + + :param A: Dense `Numpy` array. + :param blk_partition: List of block partition shapes. + :return: BlkDiagMatrix with values from A. + """ + + # Instantiate an empty BlkDiagMatrix with `blk_partition` + B = BlkDiagMatrix.zeros(blk_partition, dtype=A.dtype) + + # Set the data + inds = np.array([0, 0]) + for i in range(B.nblocks): + ends = inds + B.partition[i] + B[i][:, :] = A[inds[0] : ends[0], inds[1] : ends[1]] + inds = ends + + # We should reach exactly the end of A when partition was correct + if not np.all(inds == A.shape): + raise RuntimeError( + "Block partition appears to mismatch shape of dense matrix A." + ) + + return B diff --git a/tests/test_BlkDiagMatrix.py b/tests/test_BlkDiagMatrix.py index 3dac18b890..994e77fd3a 100644 --- a/tests/test_BlkDiagMatrix.py +++ b/tests/test_BlkDiagMatrix.py @@ -378,6 +378,14 @@ def test_blk_diag_to_diag(self): """ self.assertTrue(np.allclose(np.diag(self.blk_a.dense()), self.blk_a.diag())) + def test_from_dense(self): + """ + Test truncating dense array returns correct block diagonal entries. + """ + B = BlkDiagMatrix.from_dense(self.dense, self.blk_partition) + + self.allallfunc(B, self.blk_a) + class IrrBlkDiagMatrixTestCase(TestCase): """ From 051471e69807e2c87852e9cdc96bc505af7b9231 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 23 Oct 2023 09:07:57 -0400 Subject: [PATCH 143/294] Fix DiagMatrix.apply --- src/aspire/operators/diag_matrix.py | 4 ++-- tests/test_diag_matrix.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/aspire/operators/diag_matrix.py b/src/aspire/operators/diag_matrix.py index 751d748fbf..49c6fe67f5 100644 --- a/src/aspire/operators/diag_matrix.py +++ b/src/aspire/operators/diag_matrix.py @@ -470,11 +470,11 @@ def apply(self, X): Define the apply option of a diagonal matrix with a matrix of coefficient vectors. - :param X: Coefficient matrix, each column is a coefficient vector. + :param X: Coefficient matrix (ndarray), each column is a coefficient vector. :return: A matrix with new coefficient vectors. """ - return self * DiagMatrix(X) + return (self * DiagMatrix(X.T)).asnumpy().T def rapply(self, X): """ diff --git a/tests/test_diag_matrix.py b/tests/test_diag_matrix.py index e2269f0575..ecce899105 100644 --- a/tests/test_diag_matrix.py +++ b/tests/test_diag_matrix.py @@ -610,7 +610,8 @@ def test_apply(diag_matrix_fixture): """ d1, _, d_np = diag_matrix_fixture - x = d1.apply(d_np) + # Apply is used on column vectors, transpose. + x = d1.apply(d_np.T).T np.testing.assert_allclose(x, d_np[0][None, :] * d_np) @@ -622,9 +623,10 @@ def test_rapply(diag_matrix_fixture): d1, _, d_np = diag_matrix_fixture - x = d1.rapply(d_np) + # Apply is used on column vectors, transpose. + x = d1.rapply(d_np.T) - np.testing.assert_allclose(x, (d_np * d_np[0])) + np.testing.assert_allclose(x.T, (d_np * d_np[0])) def test_solve(diag_matrix_fixture): From 54ee4a4f7021a07d3853ca6bf44e5712580b2a6d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 23 Oct 2023 09:25:07 -0400 Subject: [PATCH 144/294] Minimal FLE change --- src/aspire/covariance/covar2d.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 3ef674981b..9b6df230c2 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -538,6 +538,8 @@ def _build(self): unique_filters = src.unique_filters self.ctf_idx = src.filter_indices self.ctf_basis = [self.basis.filter_to_basis_mat(f) for f in unique_filters] + if isinstance(self.ctf_basis[0], np.ndarray): + self.ctf_basis = [BlkDiagMatrix.from_dense(f, self.basis.blk_diag_cov_shape) for f in self.ctf_basis] def _calc_rhs(self): src = self.src @@ -569,6 +571,10 @@ def _calc_rhs(self): b_mean_k = weight * ctf_basis_k_t.apply(mean_coef_k) + if isinstance(b_mean_k, DiagMatrix): + # Convert to a column vector + b_mean_k = b_mean_k.asnumpy().T + b_mean[k] += b_mean_k covar_coef_k = self._get_covar(coef_k, zero_coef) @@ -664,7 +670,18 @@ def _solve_covar(self, A_covar, b_covar, M, covar_est_opt): return method(A_covar, b_covar, M, covar_est_opt) def _solve_covar_direct(self, A_covar, b_covar, M, covar_est_opt): - raise NotImplementedError("To be implemented in future changeset.") + # A_covar is a list of DiagMatrix, representing each ctf in self.basis. + # b_covar is a BlkDiagMatrix + # M is sum of weighted A squared, only used for cg, ignore here. + A_covar = DiagMatrix(np.concatenate([x.asnumpy() for x in A_covar])) + A2i = A_covar * A_covar + + res = BlkDiagMatrix.empty(b_covar.nblocks, self.dtype) + for b in range(b_covar.nblocks): + res.data[b] = b_covar[b] / A2i[b] + + return res + def _solve_covar_cg(self, A_covar, b_covar, M, covar_est_opt): def precond_fun(S, x): From a4666e27281b18cc6b231bfa96fa29c193f9265d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 23 Oct 2023 11:20:49 -0400 Subject: [PATCH 145/294] Refactor for inheriting PSWF/FPSWF basis mat --- src/aspire/basis/fb_2d.py | 26 ++------------------- src/aspire/basis/fpswf_2d.py | 11 +++++++++ src/aspire/basis/fspca.py | 12 ++++++++++ src/aspire/basis/pswf_2d.py | 11 +++++++++ src/aspire/basis/steerable.py | 39 +++++++++++++++++++++++++++++--- src/aspire/covariance/covar2d.py | 6 +++-- 6 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 180fc041dd..f6f1e86eec 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -6,7 +6,7 @@ from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import unique_coords_nd from aspire.operators import BlkDiagMatrix, DiagMatrix -from aspire.utils import LogFilterByCount, grid_2d, roll_dim, trange, unroll_dim +from aspire.utils import grid_2d, roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape logger = logging.getLogger(__name__) @@ -316,26 +316,4 @@ def filter_to_basis_mat(self, f): :return: Representation of filter in `basis`. Return type will be based on the class's `matrix_type`. """ - - coef = Coef(self, np.eye(self.count, dtype=self.dtype)) - img = coef.evaluate() - - # TODO, debug expand has convergence issues, - # there is a note near `cg` hinting this may relate to tolerance - # evaluate_t was not as accurate, but much much faster... - # filt = self.expand(img.filter(f)) - # filt = self.evaluate_t(img.filter(f)) - # return filt.asnumpy().reshape(self.count, self.count) - - # Loop over the expanding the filtered basis vectors one by one - filt = np.zeros((self.count, self.count), self.dtype) - with LogFilterByCount(logger, 1): - for i in trange(self.count): - try: - filt[i] = self.expand(img[i].filter(f)).asnumpy()[0] - except: - logger.warning( - f"Failed to expand basis vector {i} after filter {f}." - ) - - return filt + return super().filter_to_basis_mat(f) diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index 393a03fddc..3db10562ce 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -366,3 +366,14 @@ def _pswf_integration(self, images_nufft): ] = np.dot(self.blk_r[i], r_n_eval_mat[i * m : (i + 1) * m, :]).T return coef_vec_quad + + def filter_to_basis_mat(self, f): + """ + Convert a filter into a basis representation. + + :param f: `Filter` object, usually a `CTFFilter`. + + :return: Representation of filter in `basis`. + Return type will be based on the class's `matrix_type`. + """ + return super().filter_to_basis_mat(f) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index 172fd22184..e702532a5e 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -613,3 +613,15 @@ def shift(self, coef, shifts): return self.expand_from_image_basis( self.evaluate_to_image_basis(coef).shift(shifts) ) + + def filter_to_basis_mat(self, f): + """ + Convert a filter into a basis representation. + + :param f: `Filter` object, usually a `CTFFilter`. + + :return: Representation of filter in `basis`. + Return type will be based on the class's `matrix_type`. + """ + # This is possible to implement, but there are no current use cases. + raise NotImplementedError("Not currently implemented for compressed basis.") diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index ab396b6da0..25aaf26cee 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -417,3 +417,14 @@ def _pswf_2d_minor_computations(self, big_n, n, bandlimit, phi_approximate_error range_array = np.arange(approx_length, dtype=self.dtype) return d_vec, approx_length, range_array + + def filter_to_basis_mat(self, f): + """ + Convert a filter into a basis representation. + + :param f: `Filter` object, usually a `CTFFilter`. + + :return: Representation of filter in `basis`. + Return type will be based on the class's `matrix_type`. + """ + return super().filter_to_basis_mat(f) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index ba12d880fc..d3540ff673 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -6,12 +6,12 @@ from aspire.basis import Basis, Coef, ComplexCoef from aspire.operators import BlkDiagMatrix -from aspire.utils import complex_type, real_type +from aspire.utils import LogFilterByCount, complex_type, real_type, trange logger = logging.getLogger(__name__) -class SteerableBasis2D(Basis): +class SteerableBasis2D(Basis, abc.ABC): """ SteerableBasis2D is an extension of Basis that is expected to have `rotation` (steerable) and `calculate_bispectrum` methods. @@ -302,7 +302,6 @@ def shift(self, coef, shifts): return self.evaluate_t(self.evaluate(coef).shift(shifts)) - @abc.abstractmethod def filter_to_basis_mat(self, f): """ Convert a filter into a basis representation. @@ -486,3 +485,37 @@ def to_complex(self, coef): ind += np.size(idx) return ComplexCoef(self, complex_coef) + + @abc.abstractmethod + def filter_to_basis_mat(self, f): + """ + Convert a filter into a basis representation. + + :param f: `Filter` object, usually a `CTFFilter`. + + :return: Representation of filter in `basis`. + Return type will be based on the class's `matrix_type`. + """ + + coef = Coef(self, np.eye(self.count, dtype=self.dtype)) + img = coef.evaluate() + + # TODO, debug expand has convergence issues, + # there is a note near `cg` hinting this may relate to tolerance + # evaluate_t was not as accurate, but much much faster... + # filt = self.expand(img.filter(f)) + # filt = self.evaluate_t(img.filter(f)) + # return filt.asnumpy().reshape(self.count, self.count) + + # Loop over the expanding the filtered basis vectors one by one + filt = np.zeros((self.count, self.count), self.dtype) + with LogFilterByCount(logger, 1): + for i in trange(self.count): + try: + filt[i] = self.expand(img[i].filter(f)).asnumpy()[0] + except: + logger.warning( + f"Failed to expand basis vector {i} after filter {f}." + ) + + return filt diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 9b6df230c2..8965e86ec8 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -539,7 +539,10 @@ def _build(self): self.ctf_idx = src.filter_indices self.ctf_basis = [self.basis.filter_to_basis_mat(f) for f in unique_filters] if isinstance(self.ctf_basis[0], np.ndarray): - self.ctf_basis = [BlkDiagMatrix.from_dense(f, self.basis.blk_diag_cov_shape) for f in self.ctf_basis] + self.ctf_basis = [ + BlkDiagMatrix.from_dense(f, self.basis.blk_diag_cov_shape) + for f in self.ctf_basis + ] def _calc_rhs(self): src = self.src @@ -682,7 +685,6 @@ def _solve_covar_direct(self, A_covar, b_covar, M, covar_est_opt): return res - def _solve_covar_cg(self, A_covar, b_covar, M, covar_est_opt): def precond_fun(S, x): p = np.size(S, 0) From 2cbab5a5af913ad3e52bef459b4ab04f4699052f Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 23 Oct 2023 16:49:19 -0400 Subject: [PATCH 146/294] classifier default --- src/aspire/classification/rir_class2d.py | 3 --- src/aspire/denoising/class_avg.py | 21 +++++++++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/aspire/classification/rir_class2d.py b/src/aspire/classification/rir_class2d.py index d989efd84c..6927d19bc5 100644 --- a/src/aspire/classification/rir_class2d.py +++ b/src/aspire/classification/rir_class2d.py @@ -174,9 +174,6 @@ def classify(self, diagnostics=False): self.src, components=self.fspca_components, batch_size=self.batch_size ) - # For convenience, assign the fb_basis used in the pca_basis. - self.fb_basis = self.pca_basis.basis - # Get the expanded coefs in the compressed FSPCA space. self.fspca_coef = self.pca_basis.spca_coef diff --git a/src/aspire/denoising/class_avg.py b/src/aspire/denoising/class_avg.py index d297841156..874d0c5c11 100644 --- a/src/aspire/denoising/class_avg.py +++ b/src/aspire/denoising/class_avg.py @@ -348,6 +348,23 @@ def _images(self, indices): # Finally, apply transforms to resulting Images return self.generation_pipeline.forward(im, indices) + def _get_classifier_basis(self, classifier): + """ + Returns underlying basis of a classifier. + + For classifiers using compressed basis, + returns the underlying uncompressed basis. + + Defaults to `FFBBasis2D` when `pca_basis` is not found. + + :param classifier: Class2D subclass to query. + :return: `classifier` basis + """ + if hasattr(classifier, "pca_basis"): + basis = classifier.pca_basis.basis + else: + basis = FFBBasis2D(self.src.L, dtype=self.src.dtype) + return basis # The following sub classes attempt to pack sensible defaults # into ClassAvgSource so that users don't need to @@ -407,7 +424,7 @@ def __init__( if averager is None: averager = BFRAverager2D( - FFBBasis2D(src.L, dtype=src.dtype), + self._get_classifier_basis(classifier), src, num_procs=num_procs, dtype=dtype, @@ -544,7 +561,7 @@ def __init__( if averager_src is None: averager_src = src - basis_2d = FFBBasis2D(averager_src.L, dtype=dtype) + basis_2d = self._get_classifier_basis(classifier) averager = BFSRAverager2D( composite_basis=basis_2d, From 1fb471d95b63c641dad38d98321a2f8fb668fc4a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 24 Oct 2023 09:43:29 -0400 Subject: [PATCH 147/294] Fixup the FLE coef order returned form filter to basis mat --- src/aspire/basis/fle_2d.py | 6 +++++- src/aspire/basis/steerable.py | 14 ++++++++++++-- src/aspire/denoising/class_avg.py | 3 ++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 034c5a0f33..1f8f9e710e 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -779,12 +779,16 @@ def filter_to_basis_mat(self, f): omegax = k * np.cos(theta) omegay = k * np.sin(theta) omega = 2 * np.pi * np.vstack((omegax.flatten("C"), omegay.flatten("C"))) + h_vals2d = h_fun(omega).reshape(n_k, n_theta).astype(self.dtype) h_vals = np.sum(h_vals2d, axis=1) / n_theta h_basis = np.zeros(self.count, dtype=self.dtype) - # For now we just need handle 1D (stack of one ctf) + # For now we just need to handle 1D (stack of one ctf) for j in range(self.ell_p_max + 1): h_basis[self.idx_list[j]] = self.A3[j] @ h_vals + # Convert from internal FLE ordering to FB convention + h_basis = h_basis[self._fle_to_fb_indices] + return DiagMatrix(h_basis) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index d3540ff673..de1347c080 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -487,15 +487,25 @@ def to_complex(self, coef): return ComplexCoef(self, complex_coef) @abc.abstractmethod - def filter_to_basis_mat(self, f): + def filter_to_basis_mat(self, f, method="evaluate_t"): """ Convert a filter into a basis representation. :param f: `Filter` object, usually a `CTFFilter`. + :param method: `evaluate_t` or `expand`. :return: Representation of filter in `basis`. Return type will be based on the class's `matrix_type`. """ + if method == "evaluate_t": + expand_method = self.evaluate_t + elif method == "expand": + expand_method = self.expand + else: + raise RuntimeError( + "`filter_to_basis_mat` method {method} not supported." + " Try `evaluate_t` or `expand`." + ) coef = Coef(self, np.eye(self.count, dtype=self.dtype)) img = coef.evaluate() @@ -512,7 +522,7 @@ def filter_to_basis_mat(self, f): with LogFilterByCount(logger, 1): for i in trange(self.count): try: - filt[i] = self.expand(img[i].filter(f)).asnumpy()[0] + filt[i] = expand_method(img[i].filter(f)).asnumpy()[0] except: logger.warning( f"Failed to expand basis vector {i} after filter {f}." diff --git a/src/aspire/denoising/class_avg.py b/src/aspire/denoising/class_avg.py index 874d0c5c11..3f232d1737 100644 --- a/src/aspire/denoising/class_avg.py +++ b/src/aspire/denoising/class_avg.py @@ -363,9 +363,10 @@ def _get_classifier_basis(self, classifier): if hasattr(classifier, "pca_basis"): basis = classifier.pca_basis.basis else: - basis = FFBBasis2D(self.src.L, dtype=self.src.dtype) + basis = FFBBasis2D(self.src.L, dtype=self.dtype) return basis + # The following sub classes attempt to pack sensible defaults # into ClassAvgSource so that users don't need to # instantiate every component to get started. From 324812c469f7a0aef9f34d0c832618da17cf4fdc Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 24 Oct 2023 10:59:08 -0400 Subject: [PATCH 148/294] Restore unit tests --- src/aspire/basis/fb_2d.py | 5 ++--- src/aspire/basis/ffb_2d.py | 2 -- src/aspire/basis/steerable.py | 12 +----------- src/aspire/covariance/covar2d.py | 2 +- src/aspire/denoising/class_avg.py | 8 ++++++-- tests/test_class_src.py | 5 ++++- tests/test_covar2d.py | 1 - tests/test_covar2d_denoiser.py | 6 ++---- 8 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index f6f1e86eec..5792457bd0 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -3,10 +3,9 @@ import numpy as np from scipy.special import jv -from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D +from aspire.basis import FBBasisMixin, SteerableBasis2D from aspire.basis.basis_utils import unique_coords_nd -from aspire.operators import BlkDiagMatrix, DiagMatrix -from aspire.utils import grid_2d, roll_dim, unroll_dim +from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape logger = logging.getLogger(__name__) diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index 7aaf1dfb88..254ab6fcc0 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -286,8 +286,6 @@ def filter_to_basis_mat(self, f): basis_vals = np.zeros_like(rmat) ind_radial = np.sum(self.k_max[0:ell]) basis_vals[:, 0:k_max] = radial[ind_radial : ind_radial + k_max].T - nrm = self.radial_norms[ind_radial] - # basis_vals /= nrm h_basis_vals = basis_vals * h_vals.reshape(n_k, 1) h_basis_ell = basis_vals.T @ ( h_basis_vals * k_vals.reshape(n_k, 1) * wts.reshape(n_k, 1) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index de1347c080..2863dede96 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -302,16 +302,6 @@ def shift(self, coef, shifts): return self.evaluate_t(self.evaluate(coef).shift(shifts)) - def filter_to_basis_mat(self, f): - """ - Convert a filter into a basis representation. - - :param f: `Filter` object, usually a `CTFFilter`. - - :return: Representation of filter in `basis`. - Return type will be based on the class's `matrix_type`. - """ - @property def blk_diag_cov_shape(self): """ @@ -523,7 +513,7 @@ def filter_to_basis_mat(self, f, method="evaluate_t"): for i in trange(self.count): try: filt[i] = expand_method(img[i].filter(f)).asnumpy()[0] - except: + except Exception: logger.warning( f"Failed to expand basis vector {i} after filter {f}." ) diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 8965e86ec8..46113579e0 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -7,7 +7,7 @@ from aspire.basis import Coef, FFBBasis2D from aspire.operators import BlkDiagMatrix, DiagMatrix from aspire.optimization import conj_grad, fill_struct -from aspire.utils import complex_type, make_symmat +from aspire.utils import make_symmat from aspire.utils.matlab_compat import m_reshape logger = logging.getLogger(__name__) diff --git a/src/aspire/denoising/class_avg.py b/src/aspire/denoising/class_avg.py index 3f232d1737..c9d3f7dade 100644 --- a/src/aspire/denoising/class_avg.py +++ b/src/aspire/denoising/class_avg.py @@ -360,10 +360,14 @@ def _get_classifier_basis(self, classifier): :param classifier: Class2D subclass to query. :return: `classifier` basis """ - if hasattr(classifier, "pca_basis"): + + if hasattr(classifier, "pca_basis") and classifier.pca_basis is not None: basis = classifier.pca_basis.basis else: - basis = FFBBasis2D(self.src.L, dtype=self.dtype) + # In the cases where a basis is not defined yet, + # construct a FFBBasis2D default. + basis = FFBBasis2D(classifier.src.L, dtype=classifier.dtype) + return basis diff --git a/tests/test_class_src.py b/tests/test_class_src.py index 88be36a013..25f17b3a0e 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from aspire.basis import FBBasis2D, FFBBasis2D +from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D, PSWFBasis2D from aspire.classification import ( BandedSNRImageQualityFunction, BFRAverager2D, @@ -52,6 +52,9 @@ BASIS = [ FFBBasis2D, pytest.param(FBBasis2D, marks=pytest.mark.expensive), + FLEBasis2D, + pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), + pytest.param(FPSWFBasis2D, marks=pytest.mark.skip), ] diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index 2c34337f25..b064179efb 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -20,7 +20,6 @@ DTYPES = [np.float32] # Basis used in FSPCA for class averaging. BASIS = [ - FBBasis2D, FFBBasis2D, ] diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 423a31abe6..369d750728 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D +from aspire.basis import FBBasis2D, FFBBasis2D from aspire.denoising import DenoisedSource, DenoiserCov2D from aspire.noise import WhiteNoiseAdder from aspire.operators import IdentityFilter, RadialCTFFilter @@ -21,8 +21,6 @@ BASIS = [ FBBasis2D, FFBBasis2D, - # FLEBasis2D, - # FPSWFBasis2D, ] @@ -85,7 +83,7 @@ def test_source_mismatch(sim, basis): _ = DenoisedSource(src2, denoiser) -@pytest.skip(reason="Still in development.") +@pytest.mark.skip(reason="Still in development.") def test_filter_to_basis_mat(sim, basis): """ Test that `basis.filter_to_basis_mat` operator is similar to From 613182d97cd5a23d89080dc5b068946a6ab5f921 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 26 Oct 2023 08:07:56 -0400 Subject: [PATCH 149/294] Add optional warning code to BlkDiagMatrix.from_dense --- src/aspire/operators/blk_diag_matrix.py | 14 +++++++++++++- tests/test_BlkDiagMatrix.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index 195e95e289..3843afbd64 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -3,6 +3,8 @@ block diagonal matrices as used by ASPIRE. """ +import warnings + import numpy as np from numpy.linalg import norm, solve from scipy.linalg import block_diag @@ -943,12 +945,14 @@ def diag(self): return DiagMatrix(np.array(diag, dtype=self.dtype)) @staticmethod - def from_dense(A, blk_partition): + def from_dense(A, blk_partition, warn_eps=None): """ Create BlkDiagMatrix with `blk_partition` from dense matrix `A`. :param A: Dense `Numpy` array. :param blk_partition: List of block partition shapes. + :param warn_eps: Optionally warn if off Block values from A + exceed `warn_eps`. Default `None` disables warning. :return: BlkDiagMatrix with values from A. """ @@ -968,4 +972,12 @@ def from_dense(A, blk_partition): "Block partition appears to mismatch shape of dense matrix A." ) + if warn_eps is not None: + max_diff = np.max(np.abs((A - B.dense()))) + if max_diff > warn_eps: + warnings.warn( + f"BlkDiagMatrix.from_dense truncating values exceeding {warn_eps}", + UserWarning, + ) + return B diff --git a/tests/test_BlkDiagMatrix.py b/tests/test_BlkDiagMatrix.py index 994e77fd3a..874cacaee9 100644 --- a/tests/test_BlkDiagMatrix.py +++ b/tests/test_BlkDiagMatrix.py @@ -386,6 +386,18 @@ def test_from_dense(self): self.allallfunc(B, self.blk_a) + def test_from_dense_warns(self): + """ + Test truncating dense array returns correct block diagonal entries, + and that a warning is emitted when values outside the blocks are larger + than some `eps`. + """ + # Add ones to the entire dense matrix, to exceed `warn_eps` below. + dense = self.dense + 1 + + with pytest.warns(UserWarning, match=r".*truncating values.*"): + _ = BlkDiagMatrix.from_dense(dense, self.blk_partition, warn_eps=1e-6) + class IrrBlkDiagMatrixTestCase(TestCase): """ From eb9885cdcc76e8f9a2d5d9bed783a4a406d05a10 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 26 Oct 2023 10:14:31 -0400 Subject: [PATCH 150/294] Extend cov2d_denoiser tests with current performance refs --- tests/test_covar2d_denoiser.py | 99 +++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 19 deletions(-) diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 369d750728..cefd62fe6e 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from aspire.basis import FBBasis2D, FFBBasis2D +from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D, PSWFBasis2D from aspire.denoising import DenoisedSource, DenoiserCov2D from aspire.noise import WhiteNoiseAdder from aspire.operators import IdentityFilter, RadialCTFFilter @@ -19,8 +19,11 @@ for d in np.linspace(1.5e4, 2.5e4, 7) ] BASIS = [ - FBBasis2D, + pytest.param(FBBasis2D, marks=pytest.mark.expensive), FFBBasis2D, + FLEBasis2D, + pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), + FPSWFBasis2D, ] @@ -36,7 +39,7 @@ def basis(request): @pytest.fixture(scope="module") def sim(): """Create a reusable Simulation source.""" - return Simulation( + sim = Simulation( L=img_size, n=num_imgs, unique_filters=filters, @@ -45,12 +48,29 @@ def sim(): dtype=dtype, noise_adder=noise_adder, ) + sim = sim.cache() + return sim + + +@pytest.fixture(scope="module") +def coef(sim, basis): + """Generate small set of reference coefficients.""" + return basis.expand(sim.images[:3]) def test_batched_rotcov2d_MSE(sim, basis): """ Check calling `DenoiserCov2D` via `DenoiserSource` framework yields acceptable error. """ + # Smoke test reference values (chosen by experimentation). + refs = { + "FBBasis2D": 0.25, + "FFBBasis2D": 0.25, + "PSWFBasis2D": 0.75, + "FPSWFBasis2D": 0.75, + "FLEBasis2D": 0.32, + } + # need larger numbers of images and higher resolution for good MSE imgs_clean = sim.projections[:] @@ -60,7 +80,12 @@ def test_batched_rotcov2d_MSE(sim, basis): # Calculate the normalized RMSE of the estimated images. nrmse_ims = (imgs_denoised - imgs_clean).norm() / imgs_clean.norm() - np.testing.assert_array_less(nrmse_ims, 0.25) + ref = refs[basis.__class__.__name__] + np.testing.assert_array_less( + nrmse_ims, + ref, + err_msg=f"Comparison failed for {basis}. Achieved: {nrmse_ims} expected: {ref}.", + ) # Additionally test the `DenoisedSource` and lazy-eval-cache # of the cov2d estimator. @@ -83,15 +108,19 @@ def test_source_mismatch(sim, basis): _ = DenoisedSource(src2, denoiser) -@pytest.mark.skip(reason="Still in development.") -def test_filter_to_basis_mat(sim, basis): +def test_filter_to_basis_mat_id(coef, basis): """ - Test that `basis.filter_to_basis_mat` operator is similar to - manual sequence of evaluate->filter->expand. + Test `basis.filter_to_basis_mat` operator performance against + manual sequence of evaluate->filter->expand for `IdentifyFilter`. """ - # Generate some reference coefficients. - coef = basis.expand(sim.images[:3]) + refs = { + "FBBasis2D": 0.025, + "FFBBasis2D": 8e-7, + "PSWFBasis2D": 0.12, + "FPSWFBasis2D": 0.12, + "FLEBasis2D": 4e-7, + } # IdentityFilter should produce id filt = IdentityFilter() @@ -106,16 +135,48 @@ def test_filter_to_basis_mat(sim, basis): imgs_manual = imgs.filter(filt) coef_manual = basis.expand(imgs_manual) - # Sanity check filter_to_basis_mat of IdentityFilter is id - np.testing.assert_allclose(coef_ftbm, coef, atol=utest_tolerance(basis.dtype)) + # Compare coefs from using ftbm operator with coef from eval->filter->exp + rms = np.sqrt(np.mean(np.square(coef_ftbm - coef_manual))) + ref = refs[basis.__class__.__name__] + np.testing.assert_array_less( + rms, + ref, + err_msg=f"Comparison failed for {basis}. Achieved: {rms} expected: {ref}", + ) + + +def test_filter_to_basis_mat_ctf(coef, basis): + """ + Test `basis.filter_to_basis_mat` operator performance against + manual sequence of evaluate->filter->expand for `RadialCTFFilter`. + """ + + refs = { + "FBBasis2D": 0.11, + "FFBBasis2D": 0.36, + "PSWFBasis2D": 0.07, + "FPSWFBasis2D": 0.07, + "FLEBasis2D": 0.4, + } + + # Create a RadialCTFFilter + filt = RadialCTFFilter(pixel_size=1) - # Sanity check manual application of IdentityFilter is id - np.testing.assert_allclose(coef_manual, coef, atol=utest_tolerance(basis.dtype)) + # Apply the basis filter operator. + # Note transpose because `apply` expects and returns column vectors. + # coef_ftbm = basis.filter_to_basis_mat(filt).apply(coef.asnumpy().T).T + coef_ftbm = (basis.filter_to_basis_mat(filt) @ coef.asnumpy().T).T + + # Apply evaluate->filter->expand manually + imgs = coef.evaluate() + imgs_manual = imgs.filter(filt) + coef_manual = basis.expand(imgs_manual) # Compare coefs from using ftbm operator with coef from eval->filter->exp - np.testing.assert_allclose( - coef_ftbm, - coef_manual, - atol=utest_tolerance(basis.dtype), - err_msg=f"Comparison failed for {filt}", + rms = np.sqrt(np.mean(np.square(coef_ftbm - coef_manual))) + ref = refs[basis.__class__.__name__] + np.testing.assert_array_less( + rms, + ref, + err_msg=f"Comparison failed for {basis}. Achieved: {rms} expected: {ref}", ) From fa00955854f108d465ede0e54bff80c9ce3a397b Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 26 Oct 2023 11:00:16 -0400 Subject: [PATCH 151/294] Cleanup tox (unused import, warning stacklevel) --- src/aspire/operators/blk_diag_matrix.py | 1 + tests/test_covar2d_denoiser.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index 3843afbd64..b15416aa27 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -978,6 +978,7 @@ def from_dense(A, blk_partition, warn_eps=None): warnings.warn( f"BlkDiagMatrix.from_dense truncating values exceeding {warn_eps}", UserWarning, + stacklevel=2, ) return B diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index cefd62fe6e..013d51081a 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -6,7 +6,6 @@ from aspire.noise import WhiteNoiseAdder from aspire.operators import IdentityFilter, RadialCTFFilter from aspire.source import Simulation -from aspire.utils import utest_tolerance # TODO, parameterize these further. dtype = np.float32 From 1f879e1c117f60626207b78a267a1f62f2d96632 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 26 Oct 2023 14:06:52 -0400 Subject: [PATCH 152/294] Address rotation convention change I missed when breaking the code up --- src/aspire/basis/pswf_2d.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 25aaf26cee..6ef03cc4fc 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -428,3 +428,41 @@ def filter_to_basis_mat(self, f): Return type will be based on the class's `matrix_type`. """ return super().filter_to_basis_mat(f) + + def rotate(self, coef, radians, refl=None): + """ + Returns coefs rotated counter-clockwise by `radians`. + + :param coef: Basis coefs. + :param radians: Rotation in radians. + :param refl: Optional reflect image (about y=0) (bool) + :return: rotated coefs. + """ + # {F}PSWF rotation convention is still CW internally. + # This will make things consistent until that is addressed. + return super().rotate(coef, -radians, refl=refl) + + def complex_rotate(self, complex_coef, radians, refl=None): + """ + Returns complex coefs rotated counter-clockwise by `radians`. + + This implementation uses the complex exponential. + It is kept in the code for documentation and + reference purposes. + + To invoke in code: + + self.to_real( + self.complex_rotate( + self.to_complex(coef), radians, refl) + ) + ) + + :param complex_coef: Basis coefs (in complex representation). + :param radians: Rotation in radians. + :param refl: Optional reflect image (about y=0) (bool) + :return: rotated (complex) coefs. + """ + # {F}PSWF rotation convention is still CW internally. + # This will make things consistent until that is addressed. + return super().complex_rotate(complex_coef, -radians, refl=refl) From d001631b2b0f127ddf012a1485daf5e9fa2fe7d2 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 11:09:55 -0400 Subject: [PATCH 153/294] cleanup basis_utils imports --- src/aspire/basis/__init__.py | 18 ++++++++++++++++++ src/aspire/basis/fb.py | 2 +- src/aspire/basis/fb_2d.py | 3 +-- src/aspire/basis/fb_3d.py | 9 +++++++-- src/aspire/basis/ffb_2d.py | 6 +----- src/aspire/basis/ffb_3d.py | 3 +-- src/aspire/basis/fle_2d.py | 5 +---- src/aspire/basis/fpswf_2d.py | 3 +-- src/aspire/basis/pswf_2d.py | 6 ++++-- tests/test_basis_utils.py | 2 +- 10 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/aspire/basis/__init__.py b/src/aspire/basis/__init__.py index 8f292a31bc..beb96c29db 100644 --- a/src/aspire/basis/__init__.py +++ b/src/aspire/basis/__init__.py @@ -2,6 +2,24 @@ # isort: off from .basis import Basis, Coef, ComplexCoef +from .basis_utils import ( + lgwt, + check_besselj_zeros, + besselj_newton, + sph_bessel, + norm_assoc_legendre, + real_sph_harmonic, + besselj_zeros, + all_besselj_zeros, + unique_coords_nd, + d_decay_approx_fun, + p_n, + t_x_mat, + t_x_mat_dot, + t_x_derivative_mat, + t_radial_part_mat, + k_operator, +) from .steerable import SteerableBasis2D from .fb import FBBasisMixin diff --git a/src/aspire/basis/fb.py b/src/aspire/basis/fb.py index 14795ddbd2..7a5d129663 100644 --- a/src/aspire/basis/fb.py +++ b/src/aspire/basis/fb.py @@ -2,7 +2,7 @@ import numpy as np -from aspire.basis.basis_utils import all_besselj_zeros +from aspire.basis import all_besselj_zeros logger = logging.getLogger(__name__) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 5792457bd0..7116d88b86 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -3,8 +3,7 @@ import numpy as np from scipy.special import jv -from aspire.basis import FBBasisMixin, SteerableBasis2D -from aspire.basis.basis_utils import unique_coords_nd +from aspire.basis import FBBasisMixin, SteerableBasis2D, unique_coords_nd from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/fb_3d.py b/src/aspire/basis/fb_3d.py index b787ff618f..6907561cac 100644 --- a/src/aspire/basis/fb_3d.py +++ b/src/aspire/basis/fb_3d.py @@ -2,8 +2,13 @@ import numpy as np -from aspire.basis import Basis, FBBasisMixin -from aspire.basis.basis_utils import real_sph_harmonic, sph_bessel, unique_coords_nd +from aspire.basis import ( + Basis, + FBBasisMixin, + real_sph_harmonic, + sph_bessel, + unique_coords_nd, +) from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index 254ab6fcc0..af97ffe9dc 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -4,8 +4,7 @@ from numpy import pi from scipy.special import jv -from aspire.basis import FBBasis2D -from aspire.basis.basis_utils import lgwt +from aspire.basis import FBBasis2D, lgwt from aspire.nufft import anufft, nufft from aspire.numeric import fft, xp from aspire.operators import BlkDiagMatrix @@ -252,9 +251,6 @@ def filter_to_basis_mat(self, f): See SteerableBasis2D.filter_to_basis_mat. """ - # These form a circular dependence, import locally until time to clean up. - from aspire.basis.basis_utils import lgwt - # Get the filter's evaluate function. h_fun = f.evaluate diff --git a/src/aspire/basis/ffb_3d.py b/src/aspire/basis/ffb_3d.py index 6362a9a703..146c4c0586 100644 --- a/src/aspire/basis/ffb_3d.py +++ b/src/aspire/basis/ffb_3d.py @@ -3,8 +3,7 @@ import numpy as np from numpy import pi -from aspire.basis import FBBasis3D -from aspire.basis.basis_utils import lgwt, norm_assoc_legendre, sph_bessel +from aspire.basis import FBBasis3D, lgwt, norm_assoc_legendre, sph_bessel from aspire.nufft import anufft, nufft from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 1f8f9e710e..2506850965 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -5,8 +5,7 @@ from scipy.fft import dct, idct from scipy.special import jv -from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D -from aspire.basis.basis_utils import besselj_zeros +from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D, besselj_zeros, lgwt from aspire.basis.fle_2d_utils import ( barycentric_interp_sparse, precomp_transform_complex_to_real, @@ -758,8 +757,6 @@ def filter_to_basis_mat(self, f): """ See SteerableBasis2D.filter_to_basis_mat. """ - # These form a circular dependence, import locally until time to clean up. - from aspire.basis.basis_utils import lgwt # Get the filter's evaluate function. h_fun = f.evaluate diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index 3db10562ce..555763d6fe 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -5,8 +5,7 @@ from scipy.optimize import least_squares from scipy.special import jn -from aspire.basis import ComplexCoef -from aspire.basis.basis_utils import lgwt, t_x_mat, t_x_mat_dot +from aspire.basis import ComplexCoef, lgwt, t_x_mat, t_x_mat_dot from aspire.basis.pswf_2d import PSWFBasis2D from aspire.nufft import nufft from aspire.numeric import fft, xp diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 6ef03cc4fc..f1ef38d9fb 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -2,8 +2,10 @@ import numpy as np -from aspire.basis import Coef, ComplexCoef, SteerableBasis2D -from aspire.basis.basis_utils import ( +from aspire.basis import ( + Coef, + ComplexCoef, + SteerableBasis2D, d_decay_approx_fun, k_operator, lgwt, diff --git a/tests/test_basis_utils.py b/tests/test_basis_utils.py index 2d6a3efdb2..035342ccaa 100644 --- a/tests/test_basis_utils.py +++ b/tests/test_basis_utils.py @@ -2,7 +2,7 @@ import numpy as np -from aspire.basis.basis_utils import ( +from aspire.basis import ( all_besselj_zeros, besselj_zeros, lgwt, From d62758fdba4d5a43ab465458fe8175c59fd48a9b Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 11:18:35 -0400 Subject: [PATCH 154/294] Reduce filter_to_basis_mat docstrings. --- src/aspire/basis/fb_2d.py | 11 +++-------- src/aspire/basis/ffb_2d.py | 6 +++++- src/aspire/basis/fle_2d.py | 6 +++++- src/aspire/basis/fpswf_2d.py | 11 +++-------- src/aspire/basis/pswf_2d.py | 11 +++-------- src/aspire/basis/steerable.py | 16 ++++++---------- 6 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 7116d88b86..970fbb0381 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -305,13 +305,8 @@ def calculate_bispectrum( freq_cutoff=freq_cutoff, ) - def filter_to_basis_mat(self, f): + def filter_to_basis_mat(self, f, method="evaluate_t"): """ - Convert a filter into a basis representation. - - :param f: `Filter` object, usually a `CTFFilter`. - - :return: Representation of filter in `basis`. - Return type will be based on the class's `matrix_type`. + See `SteerableBasis2D.filter_to_basis_mat`. """ - return super().filter_to_basis_mat(f) + return super().filter_to_basis_mat(f, method=method) diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index af97ffe9dc..209b105bbf 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -246,10 +246,14 @@ def _evaluate_t(self, x): return v - def filter_to_basis_mat(self, f): + def filter_to_basis_mat(self, f, method=None): """ See SteerableBasis2D.filter_to_basis_mat. """ + if method is not None: + raise NotImplementedError( + "FLEBasis2D.filter_to_basis_mat does not provide alternative `method`s" + ) # Get the filter's evaluate function. h_fun = f.evaluate diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 2506850965..5103807c1c 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -753,10 +753,14 @@ def _radial_convolve_weights(self, b): return a.flatten() - def filter_to_basis_mat(self, f): + def filter_to_basis_mat(self, f, method=None): """ See SteerableBasis2D.filter_to_basis_mat. """ + if method is not None: + raise NotImplementedError( + "FLEBasis2D.filter_to_basis_mat does not provide alternative `method`s" + ) # Get the filter's evaluate function. h_fun = f.evaluate diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index 555763d6fe..b10a238551 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -366,13 +366,8 @@ def _pswf_integration(self, images_nufft): return coef_vec_quad - def filter_to_basis_mat(self, f): + def filter_to_basis_mat(self, f, method="evaluate_t"): """ - Convert a filter into a basis representation. - - :param f: `Filter` object, usually a `CTFFilter`. - - :return: Representation of filter in `basis`. - Return type will be based on the class's `matrix_type`. + See `SteerableBasis2D.filter_to_basis_mat`. """ - return super().filter_to_basis_mat(f) + return super().filter_to_basis_mat(f, method=method) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index f1ef38d9fb..1a6785e37d 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -420,16 +420,11 @@ def _pswf_2d_minor_computations(self, big_n, n, bandlimit, phi_approximate_error range_array = np.arange(approx_length, dtype=self.dtype) return d_vec, approx_length, range_array - def filter_to_basis_mat(self, f): + def filter_to_basis_mat(self, f, method="evaluate_t"): """ - Convert a filter into a basis representation. - - :param f: `Filter` object, usually a `CTFFilter`. - - :return: Representation of filter in `basis`. - Return type will be based on the class's `matrix_type`. + See `SteerableBasis2D.filter_to_basis_mat`. """ - return super().filter_to_basis_mat(f) + return super().filter_to_basis_mat(f, method=method) def rotate(self, coef, radians, refl=None): """ diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 2863dede96..d4a27a4800 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -479,14 +479,15 @@ def to_complex(self, coef): @abc.abstractmethod def filter_to_basis_mat(self, f, method="evaluate_t"): """ - Convert a filter into a basis representation. + Convert a filter into a basis operator representation. :param f: `Filter` object, usually a `CTFFilter`. :param method: `evaluate_t` or `expand`. - :return: Representation of filter in `basis`. + :return: Representation of filter as `basis` operator. Return type will be based on the class's `matrix_type`. """ + # evaluate_t is not as accurate, but much much faster... if method == "evaluate_t": expand_method = self.evaluate_t elif method == "expand": @@ -500,14 +501,9 @@ def filter_to_basis_mat(self, f, method="evaluate_t"): coef = Coef(self, np.eye(self.count, dtype=self.dtype)) img = coef.evaluate() - # TODO, debug expand has convergence issues, - # there is a note near `cg` hinting this may relate to tolerance - # evaluate_t was not as accurate, but much much faster... - # filt = self.expand(img.filter(f)) - # filt = self.evaluate_t(img.filter(f)) - # return filt.asnumpy().reshape(self.count, self.count) - - # Loop over the expanding the filtered basis vectors one by one + # Expansion can fail for some filters on specific basis vectors. + # Loop over the expanding the filtered basis vectors one by one, + # zero-ing failed vectors. filt = np.zeros((self.count, self.count), self.dtype) with LogFilterByCount(logger, 1): for i in trange(self.count): From 996e5dca7a2b336f85ced766bc7820acab9f6e3f Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 11:20:48 -0400 Subject: [PATCH 155/294] Reduce redundate rotate doctrings in pswf2d --- src/aspire/basis/pswf_2d.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 1a6785e37d..11a4660580 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -428,12 +428,7 @@ def filter_to_basis_mat(self, f, method="evaluate_t"): def rotate(self, coef, radians, refl=None): """ - Returns coefs rotated counter-clockwise by `radians`. - - :param coef: Basis coefs. - :param radians: Rotation in radians. - :param refl: Optional reflect image (about y=0) (bool) - :return: rotated coefs. + See `SteerableBasis2D.rotate`. """ # {F}PSWF rotation convention is still CW internally. # This will make things consistent until that is addressed. @@ -441,24 +436,7 @@ def rotate(self, coef, radians, refl=None): def complex_rotate(self, complex_coef, radians, refl=None): """ - Returns complex coefs rotated counter-clockwise by `radians`. - - This implementation uses the complex exponential. - It is kept in the code for documentation and - reference purposes. - - To invoke in code: - - self.to_real( - self.complex_rotate( - self.to_complex(coef), radians, refl) - ) - ) - - :param complex_coef: Basis coefs (in complex representation). - :param radians: Rotation in radians. - :param refl: Optional reflect image (about y=0) (bool) - :return: rotated (complex) coefs. + See `SteerableBasis2D.rotate`. """ # {F}PSWF rotation convention is still CW internally. # This will make things consistent until that is addressed. From 2556f5902924d743bfeedb048bd337ae4012c67c Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 11:26:46 -0400 Subject: [PATCH 156/294] Move blk diag truncation hack from cov2d into basis --- src/aspire/basis/steerable.py | 3 +++ src/aspire/covariance/covar2d.py | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index d4a27a4800..378be3b3af 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -514,4 +514,7 @@ def filter_to_basis_mat(self, f, method="evaluate_t"): f"Failed to expand basis vector {i} after filter {f}." ) + # Truncate off block elements to zero. + filt = BlkDiagMatrix.from_dense(filt, self.blk_diag_cov_shape, warn_eps=1e-6) + return filt diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 46113579e0..68343c418a 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -538,11 +538,6 @@ def _build(self): unique_filters = src.unique_filters self.ctf_idx = src.filter_indices self.ctf_basis = [self.basis.filter_to_basis_mat(f) for f in unique_filters] - if isinstance(self.ctf_basis[0], np.ndarray): - self.ctf_basis = [ - BlkDiagMatrix.from_dense(f, self.basis.blk_diag_cov_shape) - for f in self.ctf_basis - ] def _calc_rhs(self): src = self.src From 862389fedb8fd38cb31b662165e16e02d863686a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 11:42:21 -0400 Subject: [PATCH 157/294] misc string cleanup --- src/aspire/operators/blk_diag_matrix.py | 6 +++--- tests/test_class_src.py | 2 +- tests/test_covar2d_denoiser.py | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index b15416aa27..4c84c8efce 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -951,9 +951,9 @@ def from_dense(A, blk_partition, warn_eps=None): :param A: Dense `Numpy` array. :param blk_partition: List of block partition shapes. - :param warn_eps: Optionally warn if off Block values from A - exceed `warn_eps`. Default `None` disables warning. - :return: BlkDiagMatrix with values from A. + :param warn_eps: Optionally warn if off block values from `A` + exceed `warn_eps`. Default `None` disables warnings. + :return: `BlkDiagMatrix` with values from `A`. """ # Instantiate an empty BlkDiagMatrix with `blk_partition` diff --git a/tests/test_class_src.py b/tests/test_class_src.py index 25f17b3a0e..95052efdcd 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -54,7 +54,7 @@ pytest.param(FBBasis2D, marks=pytest.mark.expensive), FLEBasis2D, pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), - pytest.param(FPSWFBasis2D, marks=pytest.mark.skip), + FPSWFBasis2D, ] diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 013d51081a..f88c4a407e 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -126,7 +126,6 @@ def test_filter_to_basis_mat_id(coef, basis): # Apply the basis filter operator. # Note transpose because `apply` expects and returns column vectors. - # coef_ftbm = basis.filter_to_basis_mat(filt).apply(coef.asnumpy().T).T coef_ftbm = (basis.filter_to_basis_mat(filt) @ coef.asnumpy().T).T # Apply evaluate->filter->expand manually @@ -163,7 +162,6 @@ def test_filter_to_basis_mat_ctf(coef, basis): # Apply the basis filter operator. # Note transpose because `apply` expects and returns column vectors. - # coef_ftbm = basis.filter_to_basis_mat(filt).apply(coef.asnumpy().T).T coef_ftbm = (basis.filter_to_basis_mat(filt) @ coef.asnumpy().T).T # Apply evaluate->filter->expand manually From cc5a4dea6fca82f81a16f2fd94165907f6cf4dca Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 13:14:58 -0400 Subject: [PATCH 158/294] Add test coverage for new ftbm branch cases --- src/aspire/basis/ffb_2d.py | 3 ++- src/aspire/basis/fle_2d.py | 3 ++- src/aspire/basis/steerable.py | 2 +- tests/test_covar2d_denoiser.py | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index 209b105bbf..0369717441 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -252,7 +252,8 @@ def filter_to_basis_mat(self, f, method=None): """ if method is not None: raise NotImplementedError( - "FLEBasis2D.filter_to_basis_mat does not provide alternative `method`s" + "`FFBBasis2D.filter_to_basis_mat` method {method} not supported." + " Use `method=None`." ) # Get the filter's evaluate function. diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 5103807c1c..2b302c3163 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -759,7 +759,8 @@ def filter_to_basis_mat(self, f, method=None): """ if method is not None: raise NotImplementedError( - "FLEBasis2D.filter_to_basis_mat does not provide alternative `method`s" + "`FLEBasis2D.filter_to_basis_mat` method {method} not supported." + " Use `method=None`." ) # Get the filter's evaluate function. diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 378be3b3af..02f82c30d8 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -493,7 +493,7 @@ def filter_to_basis_mat(self, f, method="evaluate_t"): elif method == "expand": expand_method = self.expand else: - raise RuntimeError( + raise NotImplementedError( "`filter_to_basis_mat` method {method} not supported." " Try `evaluate_t` or `expand`." ) diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index f88c4a407e..c3be5c98e5 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -177,3 +177,50 @@ def test_filter_to_basis_mat_ctf(coef, basis): ref, err_msg=f"Comparison failed for {basis}. Achieved: {rms} expected: {ref}", ) + + +def test_filter_to_basis_mat_id_expand(coef, basis): + """ + Test `basis.filter_to_basis_mat` operator performance using slower + `expand` method against manual sequence of + evaluate->filter->expand for `IdentifyFilter`. + """ + + refs = { + "FBBasis2D": 0.025, + "PSWFBasis2D": 0.12, + "FPSWFBasis2D": 0.12, + } + + # IdentityFilter should produce id + filt = IdentityFilter() + + # Some basis do not provide alternative `method`s + if isinstance(basis, FFBBasis2D) or isinstance(basis, FLEBasis2D): + with pytest.raises(NotImplementedError, match=r".*not supported.*"): + _ = basis.filter_to_basis_mat(filt, method="expand") + return + + # Apply the basis filter operator. + # Note transpose because `apply` expects and returns column vectors. + coef_ftbm = (basis.filter_to_basis_mat(filt, method="expand") @ coef.asnumpy().T).T + + # Apply evaluate->filter->expand manually + imgs = coef.evaluate() + imgs_manual = imgs.filter(filt) + coef_manual = basis.expand(imgs_manual) + + # Compare coefs from using ftbm operator with coef from eval->filter->exp + rms = np.sqrt(np.mean(np.square(coef_ftbm - coef_manual))) + ref = refs[basis.__class__.__name__] + np.testing.assert_array_less( + rms, + ref, + err_msg=f"Comparison failed for {basis}. Achieved: {rms} expected: {ref}", + ) + + +def test_filter_to_basis_mat_bad(coef, basis): + filt = IdentityFilter() + with pytest.raises(NotImplementedError, match=r".*not supported.*"): + _ = basis.filter_to_basis_mat(filt, method="bad_method") From 95ef7929cc889a4c6562327018dcda418701eb25 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 13:24:53 -0400 Subject: [PATCH 159/294] Add BlkDiag from dense incorrect shape test --- tests/test_BlkDiagMatrix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_BlkDiagMatrix.py b/tests/test_BlkDiagMatrix.py index 874cacaee9..6a32b09129 100644 --- a/tests/test_BlkDiagMatrix.py +++ b/tests/test_BlkDiagMatrix.py @@ -398,6 +398,18 @@ def test_from_dense_warns(self): with pytest.warns(UserWarning, match=r".*truncating values.*"): _ = BlkDiagMatrix.from_dense(dense, self.blk_partition, warn_eps=1e-6) + def test_from_dense_incorrect_shape(self): + """ + Test truncating dense array returns correct block diagonal entries, + and that a warning is emitted when values outside the blocks are larger + than some `eps`. + """ + # Pad the dense array so there will be a leftover row and column. + dense = np.pad(self.dense, (0, 1)) + + with pytest.raises(RuntimeError, match=r".*mismatch shape.*"): + _ = BlkDiagMatrix.from_dense(dense, self.blk_partition) + class IrrBlkDiagMatrixTestCase(TestCase): """ From 7247e37901bce807b256dfda4e4d1c2c223d4012 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 13:43:26 -0400 Subject: [PATCH 160/294] Extend class_2d tests --- tests/test_class2D.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_class2D.py b/tests/test_class2D.py index db58a914aa..e545b2064f 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -5,7 +5,15 @@ import pytest from sklearn import datasets -from aspire.basis import Coef, FBBasis2D, FFBBasis2D, FSPCABasis +from aspire.basis import ( + Coef, + FBBasis2D, + FFBBasis2D, + FLEBasis2D, + FPSWFBasis2D, + FSPCABasis, + PSWFBasis2D, +) from aspire.classification import RIRClass2D from aspire.classification.legacy_implementations import bispec_2drot_large, pca_y from aspire.noise import WhiteNoiseAdder @@ -28,6 +36,9 @@ BASIS = [ FFBBasis2D, pytest.param(FBBasis2D, marks=pytest.mark.expensive), + FLEBasis2D, + PSWFBasis2D, + FPSWFBasis2D, ] From 8e96f90b43d66da1424c1586b945bb20c33171d9 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 27 Oct 2023 13:52:27 -0400 Subject: [PATCH 161/294] Try to speed up the class_2d tests --- tests/test_class2D.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_class2D.py b/tests/test_class2D.py index e545b2064f..e2255057fc 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -37,22 +37,22 @@ FFBBasis2D, pytest.param(FBBasis2D, marks=pytest.mark.expensive), FLEBasis2D, - PSWFBasis2D, + pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), FPSWFBasis2D, ] -@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}", scope="module") def dtype(request): return request.param -@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}") +@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}", scope="module") def img_size(request): return request.param -@pytest.fixture +@pytest.fixture(scope="module") def volume(dtype, img_size): # Get a volume v = Volume( @@ -62,7 +62,7 @@ def volume(dtype, img_size): return v.downsample(img_size) -@pytest.fixture +@pytest.fixture(scope="module") def sim_fixture(volume, img_size, dtype): """ Provides a clean simulation parameterized by `img_size` and `dtype`. @@ -81,7 +81,7 @@ def sim_fixture(volume, img_size, dtype): return imgs, src, fspca_basis -@pytest.fixture(params=BASIS, ids=lambda x: f"basis={x}") +@pytest.fixture(params=BASIS, ids=lambda x: f"basis={x}", scope="module") def basis(request, img_size, dtype): cls = request.param # Setup a Basis @@ -154,7 +154,7 @@ def test_basis_too_small(sim_fixture, basis): _ = FSPCABasis(src, basis=basis, components=basis.count * 2, noise_var=0) -@pytest.fixture +@pytest.fixture(scope="module") def sim_fixture2(volume, basis, img_size, dtype): """ Provides clean/noisy pair of smaller parameterized simulations, @@ -169,6 +169,7 @@ def sim_fixture2(volume, basis, img_size, dtype): # Clean clean_src = Simulation(L=img_size, n=n_img, vols=volume, dtype=dtype, seed=SEED) + clean_src = clean_src.cache() # With Noise noise_var = 0.01 * np.var(np.sum(volume[0], axis=0)) @@ -181,6 +182,7 @@ def sim_fixture2(volume, basis, img_size, dtype): noise_adder=noise_adder, seed=SEED, ) + noisy_src = noisy_src.cache() # Create Basis, use precomputed Basis clean_fspca_basis = FSPCABasis( From d1b405d604c4db296f44a8aa8a85ee6c8097c1ea Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 30 Oct 2023 11:38:31 -0400 Subject: [PATCH 162/294] Optionally apply truncation to filter_to_basis_mat PSWF and FPSWF have large off blk diag values... --- src/aspire/basis/fb_2d.py | 4 ++-- src/aspire/basis/ffb_2d.py | 5 +++-- src/aspire/basis/fle_2d.py | 5 +++-- src/aspire/basis/fpswf_2d.py | 4 ++-- src/aspire/basis/pswf_2d.py | 4 ++-- src/aspire/basis/steerable.py | 11 ++++++++--- src/aspire/operators/blk_diag_matrix.py | 6 ++++++ tests/test_covar2d_denoiser.py | 2 +- 8 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 970fbb0381..d5e0770de3 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -305,8 +305,8 @@ def calculate_bispectrum( freq_cutoff=freq_cutoff, ) - def filter_to_basis_mat(self, f, method="evaluate_t"): + def filter_to_basis_mat(self, *args, **kwargs): """ See `SteerableBasis2D.filter_to_basis_mat`. """ - return super().filter_to_basis_mat(f, method=method) + return super().filter_to_basis_mat(*args, **kwargs) diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index 0369717441..cf11368d55 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -246,11 +246,12 @@ def _evaluate_t(self, x): return v - def filter_to_basis_mat(self, f, method=None): + def filter_to_basis_mat(self, f, **kwargs): """ See SteerableBasis2D.filter_to_basis_mat. """ - if method is not None: + # Note 'method' and 'truncate' not relevant for this optimized FFB code. + if kwargs.get("method", None) is not None: raise NotImplementedError( "`FFBBasis2D.filter_to_basis_mat` method {method} not supported." " Use `method=None`." diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 2b302c3163..8d845bbe90 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -753,11 +753,12 @@ def _radial_convolve_weights(self, b): return a.flatten() - def filter_to_basis_mat(self, f, method=None): + def filter_to_basis_mat(self, f, **kwargs): """ See SteerableBasis2D.filter_to_basis_mat. """ - if method is not None: + # Note 'method' and 'truncate' not relevant for this optimized FLE code. + if kwargs.get("method", None) is not None: raise NotImplementedError( "`FLEBasis2D.filter_to_basis_mat` method {method} not supported." " Use `method=None`." diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index b10a238551..1f12efd702 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -366,8 +366,8 @@ def _pswf_integration(self, images_nufft): return coef_vec_quad - def filter_to_basis_mat(self, f, method="evaluate_t"): + def filter_to_basis_mat(self, *args, **kwargs): """ See `SteerableBasis2D.filter_to_basis_mat`. """ - return super().filter_to_basis_mat(f, method=method) + return super().filter_to_basis_mat(*args, **kwargs) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 11a4660580..992afeaa5e 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -420,11 +420,11 @@ def _pswf_2d_minor_computations(self, big_n, n, bandlimit, phi_approximate_error range_array = np.arange(approx_length, dtype=self.dtype) return d_vec, approx_length, range_array - def filter_to_basis_mat(self, f, method="evaluate_t"): + def filter_to_basis_mat(self, *args, **kwargs): """ See `SteerableBasis2D.filter_to_basis_mat`. """ - return super().filter_to_basis_mat(f, method=method) + return super().filter_to_basis_mat(*args, **kwargs) def rotate(self, coef, radians, refl=None): """ diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 02f82c30d8..a3ebf93c83 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -477,12 +477,14 @@ def to_complex(self, coef): return ComplexCoef(self, complex_coef) @abc.abstractmethod - def filter_to_basis_mat(self, f, method="evaluate_t"): + def filter_to_basis_mat(self, f, method="evaluate_t", truncate=True): """ Convert a filter into a basis operator representation. :param f: `Filter` object, usually a `CTFFilter`. :param method: `evaluate_t` or `expand`. + :param truncate: Optionally, truncate dense matrix to BlkDiagMatrix. + Defaults to True. :return: Representation of filter as `basis` operator. Return type will be based on the class's `matrix_type`. @@ -514,7 +516,10 @@ def filter_to_basis_mat(self, f, method="evaluate_t"): f"Failed to expand basis vector {i} after filter {f}." ) - # Truncate off block elements to zero. - filt = BlkDiagMatrix.from_dense(filt, self.blk_diag_cov_shape, warn_eps=1e-6) + # Optionally truncate off block elements to zero. + if truncate: + filt = BlkDiagMatrix.from_dense( + filt, self.blk_diag_cov_shape, warn_eps=1e-6 + ) return filt diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index 4c84c8efce..da6fb86533 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -3,6 +3,7 @@ block diagonal matrices as used by ASPIRE. """ +import logging import warnings import numpy as np @@ -12,6 +13,8 @@ from aspire.utils import make_psd from aspire.utils.cell import Cell2D +logger = logging.getLogger(__name__) + def is_scalar_type(x): """ @@ -975,10 +978,13 @@ def from_dense(A, blk_partition, warn_eps=None): if warn_eps is not None: max_diff = np.max(np.abs((A - B.dense()))) if max_diff > warn_eps: + # Warn (once) warnings.warn( f"BlkDiagMatrix.from_dense truncating values exceeding {warn_eps}", UserWarning, stacklevel=2, ) + # Log the specifics for debugging + logger.debug(f"BlkDiagMatrix.from_dense truncated max value {max_diff}") return B diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index c3be5c98e5..7db562fb38 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -162,7 +162,7 @@ def test_filter_to_basis_mat_ctf(coef, basis): # Apply the basis filter operator. # Note transpose because `apply` expects and returns column vectors. - coef_ftbm = (basis.filter_to_basis_mat(filt) @ coef.asnumpy().T).T + coef_ftbm = (basis.filter_to_basis_mat(filt, truncate=False) @ coef.asnumpy().T).T # Apply evaluate->filter->expand manually imgs = coef.evaluate() From 3de3ca95bb5ffc894536a935a385caa3eebed3ee Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 31 Oct 2023 08:45:50 -0400 Subject: [PATCH 163/294] Move tests into expensive --- tests/test_class2D.py | 4 ++-- tests/test_class_src.py | 4 ++-- tests/test_covar2d_denoiser.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_class2D.py b/tests/test_class2D.py index e2255057fc..450c90ba9c 100644 --- a/tests/test_class2D.py +++ b/tests/test_class2D.py @@ -36,9 +36,9 @@ BASIS = [ FFBBasis2D, pytest.param(FBBasis2D, marks=pytest.mark.expensive), - FLEBasis2D, + pytest.param(FLEBasis2D, marks=pytest.mark.expensive), pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), - FPSWFBasis2D, + pytest.param(FPSWFBasis2D, marks=pytest.mark.expensive), ] diff --git a/tests/test_class_src.py b/tests/test_class_src.py index 95052efdcd..d14beeaf61 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -52,9 +52,9 @@ BASIS = [ FFBBasis2D, pytest.param(FBBasis2D, marks=pytest.mark.expensive), - FLEBasis2D, + pytest.param(FLEBasis2D, marks=pytest.mark.expensive), pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), - FPSWFBasis2D, + pytest.param(FPSWFBasis2D, marks=pytest.mark.expensive), ] diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 7db562fb38..37aff4f1fa 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -20,9 +20,9 @@ BASIS = [ pytest.param(FBBasis2D, marks=pytest.mark.expensive), FFBBasis2D, - FLEBasis2D, + pytest.param(FLEBasis2D, marks=pytest.mark.expensive), pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), - FPSWFBasis2D, + pytest.param(FPSWFBasis2D, marks=pytest.mark.expensive), ] From ef14fbb47281b9764c860caea7c45483f26cbacc Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 1 Nov 2023 09:42:39 -0400 Subject: [PATCH 164/294] Fixup some strings --- src/aspire/basis/ffb_2d.py | 2 +- src/aspire/basis/fle_2d.py | 2 +- src/aspire/basis/steerable.py | 2 +- tests/test_BlkDiagMatrix.py | 4 +--- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index cf11368d55..5b337da075 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -248,7 +248,7 @@ def _evaluate_t(self, x): def filter_to_basis_mat(self, f, **kwargs): """ - See SteerableBasis2D.filter_to_basis_mat. + See `SteerableBasis2D.filter_to_basis_mat`. """ # Note 'method' and 'truncate' not relevant for this optimized FFB code. if kwargs.get("method", None) is not None: diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 8d845bbe90..5bc4da1b5d 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -755,7 +755,7 @@ def _radial_convolve_weights(self, b): def filter_to_basis_mat(self, f, **kwargs): """ - See SteerableBasis2D.filter_to_basis_mat. + See `SteerableBasis2D.filter_to_basis_mat`. """ # Note 'method' and 'truncate' not relevant for this optimized FLE code. if kwargs.get("method", None) is not None: diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index a3ebf93c83..c1b4fb04f5 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -13,7 +13,7 @@ class SteerableBasis2D(Basis, abc.ABC): """ - SteerableBasis2D is an extension of Basis that is expected to have + `SteerableBasis2D` is an extension of Basis that is expected to have `rotation` (steerable) and `calculate_bispectrum` methods. """ diff --git a/tests/test_BlkDiagMatrix.py b/tests/test_BlkDiagMatrix.py index 6a32b09129..5bd1b917c1 100644 --- a/tests/test_BlkDiagMatrix.py +++ b/tests/test_BlkDiagMatrix.py @@ -400,9 +400,7 @@ def test_from_dense_warns(self): def test_from_dense_incorrect_shape(self): """ - Test truncating dense array returns correct block diagonal entries, - and that a warning is emitted when values outside the blocks are larger - than some `eps`. + Test truncating dense array returns raises warning on incorrect shape. """ # Pad the dense array so there will be a leftover row and column. dense = np.pad(self.dense, (0, 1)) From fdcec6799a8dd6088ec4de8545f055a24dbaed5d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 2 Nov 2023 07:43:41 -0400 Subject: [PATCH 165/294] reduce cov2d_denoise test size/time and update refs --- tests/test_covar2d_denoiser.py | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/test_covar2d_denoiser.py b/tests/test_covar2d_denoiser.py index 37aff4f1fa..a403a72109 100644 --- a/tests/test_covar2d_denoiser.py +++ b/tests/test_covar2d_denoiser.py @@ -9,7 +9,7 @@ # TODO, parameterize these further. dtype = np.float32 -img_size = 64 +img_size = 32 num_imgs = 1024 noise_var = 0.1848 noise_adder = WhiteNoiseAdder(var=noise_var) @@ -20,9 +20,9 @@ BASIS = [ pytest.param(FBBasis2D, marks=pytest.mark.expensive), FFBBasis2D, - pytest.param(FLEBasis2D, marks=pytest.mark.expensive), + FLEBasis2D, pytest.param(PSWFBasis2D, marks=pytest.mark.expensive), - pytest.param(FPSWFBasis2D, marks=pytest.mark.expensive), + FPSWFBasis2D, ] @@ -63,11 +63,11 @@ def test_batched_rotcov2d_MSE(sim, basis): """ # Smoke test reference values (chosen by experimentation). refs = { - "FBBasis2D": 0.25, - "FFBBasis2D": 0.25, - "PSWFBasis2D": 0.75, - "FPSWFBasis2D": 0.75, - "FLEBasis2D": 0.32, + "FBBasis2D": 0.23, + "FFBBasis2D": 0.23, + "PSWFBasis2D": 0.76, + "FPSWFBasis2D": 0.76, + "FLEBasis2D": 0.52, } # need larger numbers of images and higher resolution for good MSE @@ -115,9 +115,9 @@ def test_filter_to_basis_mat_id(coef, basis): refs = { "FBBasis2D": 0.025, - "FFBBasis2D": 8e-7, - "PSWFBasis2D": 0.12, - "FPSWFBasis2D": 0.12, + "FFBBasis2D": 3e-6, + "PSWFBasis2D": 0.14, + "FPSWFBasis2D": 0.14, "FLEBasis2D": 4e-7, } @@ -150,10 +150,10 @@ def test_filter_to_basis_mat_ctf(coef, basis): """ refs = { - "FBBasis2D": 0.11, - "FFBBasis2D": 0.36, - "PSWFBasis2D": 0.07, - "FPSWFBasis2D": 0.07, + "FBBasis2D": 0.025, + "FFBBasis2D": 0.35, + "PSWFBasis2D": 0.11, + "FPSWFBasis2D": 0.11, "FLEBasis2D": 0.4, } @@ -187,9 +187,9 @@ def test_filter_to_basis_mat_id_expand(coef, basis): """ refs = { - "FBBasis2D": 0.025, - "PSWFBasis2D": 0.12, - "FPSWFBasis2D": 0.12, + "FBBasis2D": 4e-7, + "PSWFBasis2D": 5e-6, + "FPSWFBasis2D": 5e-6, } # IdentityFilter should produce id From f99415150a301750d7eb21070f97b1100081f3dd Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 6 Nov 2023 12:58:00 -0500 Subject: [PATCH 166/294] Update blk diag test docstring to reflect simplified test --- tests/test_BlkDiagMatrix.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_BlkDiagMatrix.py b/tests/test_BlkDiagMatrix.py index 5bd1b917c1..1b92b3b13f 100644 --- a/tests/test_BlkDiagMatrix.py +++ b/tests/test_BlkDiagMatrix.py @@ -388,9 +388,8 @@ def test_from_dense(self): def test_from_dense_warns(self): """ - Test truncating dense array returns correct block diagonal entries, - and that a warning is emitted when values outside the blocks are larger - than some `eps`. + Test that a warning is emitted when values outside the blocks + are larger than some `eps`. """ # Add ones to the entire dense matrix, to exceed `warn_eps` below. dense = self.dense + 1 From c883561f1a322c5b93ab984fadc2df45d39a6dfd Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 6 Nov 2023 13:20:16 -0500 Subject: [PATCH 167/294] Clarify DiagMatrix apply transposes. --- src/aspire/operators/diag_matrix.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/aspire/operators/diag_matrix.py b/src/aspire/operators/diag_matrix.py index 49c6fe67f5..4c94ab83d1 100644 --- a/src/aspire/operators/diag_matrix.py +++ b/src/aspire/operators/diag_matrix.py @@ -468,12 +468,16 @@ def dense(self): def apply(self, X): """ Define the apply option of a diagonal matrix with a matrix of - coefficient vectors. + coefficient column vectors. :param X: Coefficient matrix (ndarray), each column is a coefficient vector. - :return: A matrix with new coefficient vectors. + :return: A matrix with new coefficient column vectors. """ + # Transpose X to become row major because, + # X is a coefficient matrix (ndarray), each column is a coefficient vector. + # Transpose the row major multiplication result back to column major, to + # return a matrix with new coefficient column vectors. return (self * DiagMatrix(X.T)).asnumpy().T def rapply(self, X): From c72b5f57dbbbfe1c8e8a1ad9de7f1943322044ff Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 8 Nov 2023 08:25:56 -0500 Subject: [PATCH 168/294] Revert "cleanup basis_utils imports" This reverts commit 1c3e9d9687deb8e3d7862d7fb1def864cabe639e. --- src/aspire/basis/__init__.py | 18 ------------------ src/aspire/basis/fb.py | 2 +- src/aspire/basis/fb_2d.py | 3 ++- src/aspire/basis/fb_3d.py | 9 ++------- src/aspire/basis/ffb_2d.py | 6 +++++- src/aspire/basis/ffb_3d.py | 3 ++- src/aspire/basis/fle_2d.py | 3 ++- src/aspire/basis/fpswf_2d.py | 3 ++- src/aspire/basis/pswf_2d.py | 6 ++---- tests/test_basis_utils.py | 2 +- 10 files changed, 19 insertions(+), 36 deletions(-) diff --git a/src/aspire/basis/__init__.py b/src/aspire/basis/__init__.py index beb96c29db..8f292a31bc 100644 --- a/src/aspire/basis/__init__.py +++ b/src/aspire/basis/__init__.py @@ -2,24 +2,6 @@ # isort: off from .basis import Basis, Coef, ComplexCoef -from .basis_utils import ( - lgwt, - check_besselj_zeros, - besselj_newton, - sph_bessel, - norm_assoc_legendre, - real_sph_harmonic, - besselj_zeros, - all_besselj_zeros, - unique_coords_nd, - d_decay_approx_fun, - p_n, - t_x_mat, - t_x_mat_dot, - t_x_derivative_mat, - t_radial_part_mat, - k_operator, -) from .steerable import SteerableBasis2D from .fb import FBBasisMixin diff --git a/src/aspire/basis/fb.py b/src/aspire/basis/fb.py index 7a5d129663..14795ddbd2 100644 --- a/src/aspire/basis/fb.py +++ b/src/aspire/basis/fb.py @@ -2,7 +2,7 @@ import numpy as np -from aspire.basis import all_besselj_zeros +from aspire.basis.basis_utils import all_besselj_zeros logger = logging.getLogger(__name__) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index d5e0770de3..7471e08bec 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -3,7 +3,8 @@ import numpy as np from scipy.special import jv -from aspire.basis import FBBasisMixin, SteerableBasis2D, unique_coords_nd +from aspire.basis import FBBasisMixin, SteerableBasis2D +from aspire.basis.basis_utils import unique_coords_nd from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/fb_3d.py b/src/aspire/basis/fb_3d.py index 6907561cac..b787ff618f 100644 --- a/src/aspire/basis/fb_3d.py +++ b/src/aspire/basis/fb_3d.py @@ -2,13 +2,8 @@ import numpy as np -from aspire.basis import ( - Basis, - FBBasisMixin, - real_sph_harmonic, - sph_bessel, - unique_coords_nd, -) +from aspire.basis import Basis, FBBasisMixin +from aspire.basis.basis_utils import real_sph_harmonic, sph_bessel, unique_coords_nd from aspire.utils import roll_dim, unroll_dim from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index 5b337da075..e900c39905 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -4,7 +4,8 @@ from numpy import pi from scipy.special import jv -from aspire.basis import FBBasis2D, lgwt +from aspire.basis import FBBasis2D +from aspire.basis.basis_utils import lgwt from aspire.nufft import anufft, nufft from aspire.numeric import fft, xp from aspire.operators import BlkDiagMatrix @@ -257,6 +258,9 @@ def filter_to_basis_mat(self, f, **kwargs): " Use `method=None`." ) + # These form a circular dependence, import locally until time to clean up. + from aspire.basis.basis_utils import lgwt + # Get the filter's evaluate function. h_fun = f.evaluate diff --git a/src/aspire/basis/ffb_3d.py b/src/aspire/basis/ffb_3d.py index 146c4c0586..6362a9a703 100644 --- a/src/aspire/basis/ffb_3d.py +++ b/src/aspire/basis/ffb_3d.py @@ -3,7 +3,8 @@ import numpy as np from numpy import pi -from aspire.basis import FBBasis3D, lgwt, norm_assoc_legendre, sph_bessel +from aspire.basis import FBBasis3D +from aspire.basis.basis_utils import lgwt, norm_assoc_legendre, sph_bessel from aspire.nufft import anufft, nufft from aspire.utils.matlab_compat import m_flatten, m_reshape diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index 5bc4da1b5d..cad81fcfdd 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -5,7 +5,8 @@ from scipy.fft import dct, idct from scipy.special import jv -from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D, besselj_zeros, lgwt +from aspire.basis import Coef, FBBasisMixin, SteerableBasis2D +from aspire.basis.basis_utils import besselj_zeros, lgwt from aspire.basis.fle_2d_utils import ( barycentric_interp_sparse, precomp_transform_complex_to_real, diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index 1f12efd702..09d67bff5f 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -5,7 +5,8 @@ from scipy.optimize import least_squares from scipy.special import jn -from aspire.basis import ComplexCoef, lgwt, t_x_mat, t_x_mat_dot +from aspire.basis import ComplexCoef +from aspire.basis.basis_utils import lgwt, t_x_mat, t_x_mat_dot from aspire.basis.pswf_2d import PSWFBasis2D from aspire.nufft import nufft from aspire.numeric import fft, xp diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 992afeaa5e..b46bfd60e5 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -2,10 +2,8 @@ import numpy as np -from aspire.basis import ( - Coef, - ComplexCoef, - SteerableBasis2D, +from aspire.basis import Coef, ComplexCoef, SteerableBasis2D +from aspire.basis.basis_utils import ( d_decay_approx_fun, k_operator, lgwt, diff --git a/tests/test_basis_utils.py b/tests/test_basis_utils.py index 035342ccaa..2d6a3efdb2 100644 --- a/tests/test_basis_utils.py +++ b/tests/test_basis_utils.py @@ -2,7 +2,7 @@ import numpy as np -from aspire.basis import ( +from aspire.basis.basis_utils import ( all_besselj_zeros, besselj_zeros, lgwt, From bda169e6d7e9151653cc7518699e8c3900897e2a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 8 Nov 2023 08:27:17 -0500 Subject: [PATCH 169/294] set 1e-3 default for truncation --- src/aspire/basis/steerable.py | 3 ++- src/aspire/operators/blk_diag_matrix.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index c1b4fb04f5..844f56aa11 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -519,7 +519,8 @@ def filter_to_basis_mat(self, f, method="evaluate_t", truncate=True): # Optionally truncate off block elements to zero. if truncate: filt = BlkDiagMatrix.from_dense( - filt, self.blk_diag_cov_shape, warn_eps=1e-6 + filt, + self.blk_diag_cov_shape, ) return filt diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index da6fb86533..7612d38c38 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -948,14 +948,14 @@ def diag(self): return DiagMatrix(np.array(diag, dtype=self.dtype)) @staticmethod - def from_dense(A, blk_partition, warn_eps=None): + def from_dense(A, blk_partition, warn_eps=1e-3): """ Create BlkDiagMatrix with `blk_partition` from dense matrix `A`. :param A: Dense `Numpy` array. :param blk_partition: List of block partition shapes. :param warn_eps: Optionally warn if off block values from `A` - exceed `warn_eps`. Default `None` disables warnings. + exceed `warn_eps`. `None` disables warnings. :return: `BlkDiagMatrix` with values from `A`. """ From e115db6a2c43ac75ebdcdd54089a335bd505c33e Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 8 Nov 2023 08:32:07 -0500 Subject: [PATCH 170/294] Add comment about ftbm abstractmethod --- src/aspire/basis/steerable.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 844f56aa11..698ad37a34 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -476,6 +476,10 @@ def to_complex(self, coef): return ComplexCoef(self, complex_coef) + # `abstractmethod` enforces when a new subclass of + # `SteerableBasis2D` is created that this method is explicitly + # implemented. This is intended to encourage future basis authors + # to consider this method for their application. @abc.abstractmethod def filter_to_basis_mat(self, f, method="evaluate_t", truncate=True): """ From 441bfea9af755b50a0725a43b6cb16445b93d2ff Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 9 Oct 2023 14:52:55 -0400 Subject: [PATCH 171/294] Safeguard dividing by small values in PowerFilter. Temporary change in _rlnOpticGroup to get script to run (possible bug). --- src/aspire/operators/filters.py | 12 +++++++++--- src/aspire/utils/relion_interop.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index d39b475656..50fe0004ba 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -199,11 +199,18 @@ def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): See `Filter.evaluate_grid` for usage. """ - - return ( + filter_vals = ( self._filter.evaluate_grid(L, dtype=dtype, *args, **kwargs) ** self._power ) + # Place safeguard on values below machine epsilon. + if self._power < 0: + eps = np.finfo(filter_vals.dtype).eps + threshold = eps**self._power + filter_vals[filter_vals > threshold] = 0 + + return filter_vals + class LambdaFilter(Filter): """ @@ -340,7 +347,6 @@ def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): See Filter.evaluate_grid for usage. """ - if all(dim == L for dim in self.xfer_fn_array.shape): logger.debug( "Size of transfer function matches evaluate_grid size L exactly," diff --git a/src/aspire/utils/relion_interop.py b/src/aspire/utils/relion_interop.py index 9240e47298..345ac022b8 100644 --- a/src/aspire/utils/relion_interop.py +++ b/src/aspire/utils/relion_interop.py @@ -44,7 +44,7 @@ "_rlnNrOfSignificantSamples": float, "_rlnNrOfFrames": int, "_rlnMaxValueProbDistribution": float, - "_rlnOpticsGroup": int, + "_rlnOpticsGroup": float, "_rlnOpticsGroupName": str, } From b8193e9d86123e0659c54fa2d96cc64fe38a392b Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 10 Oct 2023 14:35:44 -0400 Subject: [PATCH 172/294] Add warning message. --- src/aspire/operators/filters.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index 50fe0004ba..21b2845a3a 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -207,6 +207,11 @@ def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): if self._power < 0: eps = np.finfo(filter_vals.dtype).eps threshold = eps**self._power + ind = np.where(filter_vals > threshold) + if len(ind[0]) > 0: + logger.warning( + f"{self} setting {len(ind[0])} extremal filter value(s) to zero." + ) filter_vals[filter_vals > threshold] = 0 return filter_vals From f3bfedd9cf0a948efeb9fd41fd419751d5b3275a Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 10 Oct 2023 14:56:32 -0400 Subject: [PATCH 173/294] Revert _rlnOpticsGroup type after altering example starfile. --- src/aspire/utils/relion_interop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/utils/relion_interop.py b/src/aspire/utils/relion_interop.py index 345ac022b8..9240e47298 100644 --- a/src/aspire/utils/relion_interop.py +++ b/src/aspire/utils/relion_interop.py @@ -44,7 +44,7 @@ "_rlnNrOfSignificantSamples": float, "_rlnNrOfFrames": int, "_rlnMaxValueProbDistribution": float, - "_rlnOpticsGroup": float, + "_rlnOpticsGroup": int, "_rlnOpticsGroupName": str, } From e4ef77b3d7c0fb2f4a9e667817de6e8a132746bb Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 18 Oct 2023 16:06:08 -0400 Subject: [PATCH 174/294] test for PowerFilter safeguard. --- src/aspire/operators/filters.py | 2 +- tests/test_filters.py | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index 21b2845a3a..049ee4f84c 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -207,7 +207,7 @@ def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): if self._power < 0: eps = np.finfo(filter_vals.dtype).eps threshold = eps**self._power - ind = np.where(filter_vals > threshold) + ind = np.where(abs(filter_vals) > threshold) if len(ind[0]) > 0: logger.warning( f"{self} setting {len(ind[0])} extremal filter value(s) to zero." diff --git a/tests/test_filters.py b/tests/test_filters.py index 40b0fb2a9d..35d7955a9e 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,9 +1,12 @@ +import logging import os.path from unittest import TestCase import numpy as np +import pytest from aspire.operators import ( + ArrayFilter, CTFFilter, FunctionFilter, IdentityFilter, @@ -327,3 +330,34 @@ def testFilterSigns(self): signs = np.sign(ctf_filter.evaluate(self.omega)) sign_filter = ctf_filter.sign self.assertTrue(np.allclose(sign_filter.evaluate(self.omega), signs)) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +def test_power_filter_safeguard(dtype, caplog): + L = 25 + arr = np.ones((L, L), dtype=dtype) + + # Set a few values below machine epsilon. + num_eps = 3 + eps = np.finfo(dtype).eps + arr[L // 2, L // 2 : L // 2 + num_eps] = eps / 2 + + # For negative powers, values below machine eps will be set to zero. + filt = PowerFilter( + filter=ArrayFilter(arr), + power=-0.5, + ) + + caplog.clear() + caplog.set_level(logging.WARN) + filt_vals = filt.evaluate_grid(L, dtype=dtype) + + # Check that extreme values are set to zero. + ref = np.ones((L, L), dtype=dtype) + ref[L // 2, L // 2 : L // 2 + num_eps] = 0 + + np.testing.assert_array_equal(filt_vals, ref) + + # Check caplog for warning. + msg = f"setting {num_eps} extremal filter value(s) to zero." + assert msg in caplog.text From 7e148feda96da1e2666bea735646278c28de4179 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 19 Oct 2023 15:50:12 -0400 Subject: [PATCH 175/294] refactor safeguard. --- src/aspire/operators/filters.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index 049ee4f84c..6841640209 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -199,22 +199,23 @@ def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): See `Filter.evaluate_grid` for usage. """ - filter_vals = ( - self._filter.evaluate_grid(L, dtype=dtype, *args, **kwargs) ** self._power - ) + filter_vals = self._filter.evaluate_grid(L, dtype=dtype, *args, **kwargs) - # Place safeguard on values below machine epsilon. + # Place safeguard on values below machine epsilon for negative powers. if self._power < 0: eps = np.finfo(filter_vals.dtype).eps - threshold = eps**self._power - ind = np.where(abs(filter_vals) > threshold) - if len(ind[0]) > 0: + condition = abs(filter_vals) < eps + num_less_eps = np.count_nonzero(condition) + if num_less_eps > 0: logger.warning( - f"{self} setting {len(ind[0])} extremal filter value(s) to zero." + f"{self} setting {num_less_eps} extremal filter value(s) to zero." ) - filter_vals[filter_vals > threshold] = 0 - return filter_vals + filter_vals = np.where(condition, 0, filter_vals**self._power) + + return filter_vals + + return filter_vals**self._power class LambdaFilter(Filter): From 455f16c90a1862302ddf44a525487f1faa2576b2 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 9 Nov 2023 07:35:50 -0500 Subject: [PATCH 176/294] tmp extend develop timeout to 12 hours --- .github/workflows/long_workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/long_workflow.yml b/.github/workflows/long_workflow.yml index 350e65e133..094e185d29 100644 --- a/.github/workflows/long_workflow.yml +++ b/.github/workflows/long_workflow.yml @@ -9,6 +9,7 @@ on: jobs: expensive_tests: runs-on: self-hosted + timeout-minutes: 720 steps: - uses: actions/checkout@v3 - name: Install dependencies From 3a86325f4b7384828293ba92437bcbe53704ff41 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 9 Nov 2023 10:32:07 -0500 Subject: [PATCH 177/294] reduce class_avg_src test size and compute time --- tests/test_class_src.py | 53 ++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/tests/test_class_src.py b/tests/test_class_src.py index d14beeaf61..b12f7e2987 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -37,8 +37,8 @@ IMG_SIZES = [ - 32, - pytest.param(31, marks=pytest.mark.expensive), + 16, + pytest.param(15, marks=pytest.mark.expensive), ] DTYPES = [ np.float64, @@ -58,7 +58,7 @@ ] -@pytest.fixture(params=BASIS, ids=lambda x: f"basis={x}") +@pytest.fixture(params=BASIS, ids=lambda x: f"basis={x}", scope="module") def basis(request, img_size, dtype): cls = request.param # Setup a Basis @@ -72,17 +72,17 @@ def sim_fixture_id(params): return f"res={res}, dtype={dtype}" -@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}") +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}", scope="module") def dtype(request): return request.param -@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}") +@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}", scope="module") def img_size(request): return request.param -@pytest.fixture +@pytest.fixture(scope="module") def class_sim_fixture(dtype, img_size): """ Construct a Simulation with explicit viewing angles forming @@ -134,12 +134,27 @@ def class_sim_fixture(dtype, img_size): return src -@pytest.fixture(params=CLS_SRCS, ids=lambda param: f"ClassSource={param.__class__}") +@pytest.fixture( + params=CLS_SRCS, ids=lambda param: f"ClassSource={param.__class__}", scope="module" +) def test_src_cls(request): return request.param -def test_basic_averaging(class_sim_fixture, test_src_cls): +@pytest.fixture(scope="module") +def classifier(class_sim_fixture): + return RIRClass2D( + class_sim_fixture, + fspca_components=123, + bispectrum_components=101, # Compressed Features after last PCA stage. + n_nbor=10, + large_pca_implementation="legacy", + nn_implementation="legacy", + bispectrum_implementation="legacy", + ) + + +def test_basic_averaging(class_sim_fixture, test_src_cls, basis, classifier): """ Test that the default `ClassAvgSource` implementations return class averages. @@ -147,8 +162,10 @@ class averages. cmp_n = 5 - # Classify, Select, and compute averaged images. - test_src = test_src_cls(src=class_sim_fixture, num_procs=NUM_PROCS) + test_src = test_src_cls( + src=class_sim_fixture, classifier=classifier, num_procs=NUM_PROCS + ) + test_imgs = test_src.images[:cmp_n] # Fetch reference images from the original source. @@ -194,13 +211,19 @@ def test_heap_helper(): assert popped == a, "Failed to pop min item" -@pytest.fixture() +@pytest.fixture(scope="module") def cls_fixture(class_sim_fixture): """ Classifier fixture. """ # Create the classifier - c2d = RIRClass2D(class_sim_fixture, nn_implementation="sklearn") + c2d = RIRClass2D( + class_sim_fixture, + fspca_components=123, + bispectrum_components=101, # Compressed Features after last PCA stage. + n_nbor=10, + nn_implementation="sklearn", + ) # Compute the classification # (classes, reflections, distances) return c2d.classify() @@ -294,8 +317,10 @@ def test_contrast_selector(dtype): assert np.allclose(selector._quality_scores, ref_scores) -def test_avg_src_starfileio(class_sim_fixture, test_src_cls): - src = test_src_cls(src=class_sim_fixture, num_procs=NUM_PROCS) +def test_avg_src_starfileio(class_sim_fixture, test_src_cls, classifier): + src = test_src_cls( + src=class_sim_fixture, classifier=classifier, num_procs=NUM_PROCS + ) # Save and load the source as a STAR file. # Saving should force classification and selection to occur, From 8f936d454d055e5558e52cc4d766074bda4e556a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 9 Nov 2023 10:37:45 -0500 Subject: [PATCH 178/294] restore 6hour long running time out --- .github/workflows/long_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/long_workflow.yml b/.github/workflows/long_workflow.yml index 094e185d29..4346b93222 100644 --- a/.github/workflows/long_workflow.yml +++ b/.github/workflows/long_workflow.yml @@ -9,7 +9,7 @@ on: jobs: expensive_tests: runs-on: self-hosted - timeout-minutes: 720 + timeout-minutes: 360 steps: - uses: actions/checkout@v3 - name: Install dependencies From 030c89e55404241477da735ffe65f0a1d6de7eb1 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 20 Oct 2023 13:41:58 -0400 Subject: [PATCH 179/294] transform Relion rotations. --- src/aspire/source/relion.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/aspire/source/relion.py b/src/aspire/source/relion.py index 02b8ab9363..9b4e4d7e6c 100644 --- a/src/aspire/source/relion.py +++ b/src/aspire/source/relion.py @@ -155,6 +155,12 @@ def __init__( logger.info(f"Populated {self.n_ctf_filters} CTFFilters from '{filepath}'") + # NOTE: We are currently using a different grid indexing than Relion to produce images. + # If rotations are provided we transform them to align with ASPIRE indexing. + if self._rotations is not None: + flip_xy = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=self.dtype) + self.rotations = self.rotations @ flip_xy + # Any further operations should not mutate this instance. self._mutable = False From 84b55666834431546794b118c7f0bdad70ac2261 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 24 Oct 2023 11:16:08 -0400 Subject: [PATCH 180/294] negate averager rotations. --- src/aspire/classification/averager2d.py | 6 +++--- src/aspire/operators/polar_ft.py | 4 ++-- src/aspire/utils/misc.py | 4 ++-- src/aspire/volume/volume.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/aspire/classification/averager2d.py b/src/aspire/classification/averager2d.py index 3c487070ab..5f9861804a 100644 --- a/src/aspire/classification/averager2d.py +++ b/src/aspire/classification/averager2d.py @@ -404,7 +404,7 @@ def _innerloop(k): for k, result in enumerate(results): rotations[k], correlations[k] = result - return rotations, None, correlations + return -rotations, None, correlations class BFSRAverager2D(BFRAverager2D): @@ -638,7 +638,7 @@ def _innerloop(k): for k, result in enumerate(results): rotations[k], shifts[k], correlations[k] = result - return rotations, shifts, correlations + return -rotations, shifts, correlations def average( self, @@ -836,7 +836,7 @@ def _innerloop(k): for k, result in enumerate(results): rotations[k], shifts[k], correlations[k] = result - return rotations, shifts, correlations + return -rotations, shifts, correlations def average( self, diff --git a/src/aspire/operators/polar_ft.py b/src/aspire/operators/polar_ft.py index 48b08c4c0c..2b90d5cad6 100644 --- a/src/aspire/operators/polar_ft.py +++ b/src/aspire/operators/polar_ft.py @@ -77,8 +77,8 @@ def _precomp(self): # only need half size of ntheta freqs = np.zeros((2, self.ntheta // 2, self.nrad), dtype=self.dtype) for i in range(self.ntheta // 2): - freqs[0, i] = np.cos(i * dtheta) - freqs[1, i] = np.sin(i * dtheta) + freqs[0, i] = np.sin(i * dtheta) + freqs[1, i] = np.cos(i * dtheta) freqs *= omega0 * np.arange(self.nrad) diff --git a/src/aspire/utils/misc.py b/src/aspire/utils/misc.py index f8d3a98c3f..815c7803b0 100644 --- a/src/aspire/utils/misc.py +++ b/src/aspire/utils/misc.py @@ -306,13 +306,13 @@ def fuzzy_mask(L, dtype, r0=None, risetime=None): if not (L[0] == L[1]): raise ValueError(f"A 2D fuzzy_mask must be square, found L={L}.") grid = grid_2d(**grid_kwargs) - axes.append("y") + axes.insert(0, "y") elif dim == 3: if not (L[0] == L[1] == L[2]): raise ValueError(f"A 3D fuzzy_mask must be cubic, found L={L}.") grid = grid_3d(**grid_kwargs) - axes.extend(["y", "z"]) + axes.insert(0, ["z", "y"]) else: raise RuntimeError( diff --git a/src/aspire/volume/volume.py b/src/aspire/volume/volume.py index c7738db9f6..365cca803e 100644 --- a/src/aspire/volume/volume.py +++ b/src/aspire/volume/volume.py @@ -658,7 +658,7 @@ def rotated_grids(L, rot_matrices): Frequencies are in the range [-pi, pi]. """ - grid2d = grid_2d(L, indexing="xy", dtype=rot_matrices.dtype) + grid2d = grid_2d(L, indexing="yx", dtype=rot_matrices.dtype) num_pts = L**2 num_rots = rot_matrices.shape[0] pts = np.pi * np.vstack( @@ -672,7 +672,7 @@ def rotated_grids(L, rot_matrices): for i in range(num_rots): pts_rot[:, i, :] = rot_matrices[i, :, :] @ pts - pts_rot = pts_rot.reshape((3, num_rots, L, L)) + pts_rot = pts_rot.reshape((3, num_rots, L, L))[::-1] return pts_rot From 11435f2752bfecb177d5b90e77715a4f224c1b4c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 24 Oct 2023 14:43:30 -0400 Subject: [PATCH 181/294] fix grid axes insert in fuzzy_mask --- src/aspire/utils/misc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aspire/utils/misc.py b/src/aspire/utils/misc.py index 815c7803b0..b113bcf85b 100644 --- a/src/aspire/utils/misc.py +++ b/src/aspire/utils/misc.py @@ -312,7 +312,8 @@ def fuzzy_mask(L, dtype, r0=None, risetime=None): if not (L[0] == L[1] == L[2]): raise ValueError(f"A 3D fuzzy_mask must be cubic, found L={L}.") grid = grid_3d(**grid_kwargs) - axes.insert(0, ["z", "y"]) + axes.insert(0, "y") + axes.insert(0, "z") else: raise RuntimeError( From 44760b462c5dce82368e7a5678dd13ea051e27df Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 25 Oct 2023 08:48:31 -0400 Subject: [PATCH 182/294] resolve FLE test --- tests/test_FLEbasis2D.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_FLEbasis2D.py b/tests/test_FLEbasis2D.py index 14eb8faf5b..dd37303f2c 100644 --- a/tests/test_FLEbasis2D.py +++ b/tests/test_FLEbasis2D.py @@ -264,7 +264,7 @@ def testRotate(): ims_fle_pi = basis.evaluate(coefs_fle_pi) # test reflection - assert np.allclose(np.flipud(ims.asnumpy()[0]), ims_fle_pi[0], atol=1e-4) + assert np.allclose(np.fliplr(np.flipud(ims.asnumpy()[0])), ims_fle_pi[0], atol=1e-4) # make sure you can pass in a 1-D array if you want _ = basis.lowpass(Coef(basis, np.zeros((basis.count,))), np.pi) From f8bacd8029fde877467d7defc6ee7d5d56a92cca Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 25 Oct 2023 08:49:56 -0400 Subject: [PATCH 183/294] Switch synthetic volume grid indexing. Resolves failing orient_symmetric tests. --- src/aspire/volume/volume_synthesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/volume/volume_synthesis.py b/src/aspire/volume/volume_synthesis.py index 579eb05919..693d94ca8d 100644 --- a/src/aspire/volume/volume_synthesis.py +++ b/src/aspire/volume/volume_synthesis.py @@ -141,7 +141,7 @@ def _eval_gaussians(self, Q, D, mu): :return: An L x L x L array. """ - g = grid_3d(self.L, indexing="xyz", dtype=self.dtype) + g = grid_3d(self.L, indexing="zyx", dtype=self.dtype) coords = np.array( [g["x"].flatten(), g["y"].flatten(), g["z"].flatten()], dtype=self.dtype ) From 3e6cfd23b5a21cefac2c523d6e1b068bf7eb273f Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 25 Oct 2023 09:56:18 -0400 Subject: [PATCH 184/294] rotated_grid_3d xyz ~~> zyx. Resolved failing synthetic vol test. --- src/aspire/volume/volume.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aspire/volume/volume.py b/src/aspire/volume/volume.py index 365cca803e..22420b9e97 100644 --- a/src/aspire/volume/volume.py +++ b/src/aspire/volume/volume.py @@ -687,7 +687,7 @@ def rotated_grids_3d(L, rot_matrices): Frequencies are in the range [-pi, pi]. """ - grid3d = grid_3d(L, indexing="xyz", dtype=rot_matrices.dtype) + grid3d = grid_3d(L, indexing="zyx", dtype=rot_matrices.dtype) num_pts = L**3 num_rots = rot_matrices.shape[0] pts = np.pi * np.vstack( @@ -702,4 +702,4 @@ def rotated_grids_3d(L, rot_matrices): pts_rot[:, i, :] = rot_matrices[i, :, :] @ pts # Note we return grids as (Z,Y,X) - return pts_rot.reshape(3, -1) + return pts_rot.reshape(3, -1)[::-1] From 0c0dd2a4eaff657d8aaf82b561b0731232f3bf78 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 25 Oct 2023 10:03:58 -0400 Subject: [PATCH 185/294] Resolve failing volume rotate test. --- tests/test_volume.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index 7ad409a891..ca191ab401 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -320,17 +320,17 @@ def test_rotate(L, dtype): # Create a dict with map from axis and angle of rotation to new location of nonzero voxel. ref_pts = { ("x", 0): (1, 1, 1), - ("x", pi / 2): (1, 1, -1), - ("x", pi): (1, -1, -1), + ("x", pi / 2): (-1, 1, 1), + ("x", pi): (-1, -1, 1), ("x", 3 * pi / 2): (1, -1, 1), ("y", 0): (1, 1, 1), - ("y", pi / 2): (-1, 1, 1), + ("y", pi / 2): (1, 1, -1), ("y", pi): (-1, 1, -1), - ("y", 3 * pi / 2): (1, 1, -1), + ("y", 3 * pi / 2): (-1, 1, 1), ("z", 0): (1, 1, 1), ("z", pi / 2): (1, -1, 1), - ("z", pi): (-1, -1, 1), - ("z", 3 * pi / 2): (-1, 1, 1), + ("z", pi): (1, -1, -1), + ("z", 3 * pi / 2): (1, 1, -1), } center = np.array([L // 2] * 3) From 902dca87429ed36d4b5ca790677619f0a034665c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 25 Oct 2023 11:36:43 -0400 Subject: [PATCH 186/294] swap axes of volume in test_simulation. --- tests/test_simulation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 11c7779245..16e489f6b4 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -108,11 +108,14 @@ def setUp(self): self.L = 8 self.dtype = np.float32 - self.vols = LegacyVolume( + vols = LegacyVolume( L=self.L, dtype=self.dtype, ).generate() + # Note: swapping x and z axes to account for new Volume "zyx" grid convention. + self.vols = Volume(np.swapaxes(vols.asnumpy(), 1, 3)) + self.sim = Simulation( n=self.n, L=self.L, From f1d6e09b24143a5e085b32f76c29a676b2cd2a8b Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 26 Oct 2023 11:43:21 -0400 Subject: [PATCH 187/294] Initial rotation swap. --- tests/test_simulation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 16e489f6b4..36650b2ab8 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -10,7 +10,7 @@ from aspire.operators import RadialCTFFilter from aspire.source.relion import RelionSource from aspire.source.simulation import Simulation -from aspire.utils.types import utest_tolerance +from aspire.utils import J_conjugate, utest_tolerance from aspire.volume import LegacyVolume, SymmetryGroup, Volume from .test_utils import matplotlib_dry_run @@ -115,8 +115,8 @@ def setUp(self): # Note: swapping x and z axes to account for new Volume "zyx" grid convention. self.vols = Volume(np.swapaxes(vols.asnumpy(), 1, 3)) - - self.sim = Simulation( + + sim = Simulation( n=self.n, L=self.L, vols=self.vols, @@ -127,6 +127,9 @@ def setUp(self): dtype=self.dtype, ) + flip_xy = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=self.dtype) + self.sim = sim.update(rotations = sim.rotations[:, ::-1] @ flip_xy) + def tearDown(self): pass From f960cc0f51a68368ad8404e6e5bc8ecff11c0601 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 26 Oct 2023 14:22:30 -0400 Subject: [PATCH 188/294] Rotation transformation method. Fix test_simulation. --- src/aspire/utils/__init__.py | 6 +++++- src/aspire/utils/relion_interop.py | 27 +++++++++++++++++++++++++++ tests/test_simulation.py | 21 ++++++++++++++------- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/aspire/utils/__init__.py b/src/aspire/utils/__init__.py index e691e0ba5e..dede0dcb6a 100644 --- a/src/aspire/utils/__init__.py +++ b/src/aspire/utils/__init__.py @@ -67,7 +67,11 @@ virtual_core_cpu_suggestion, ) from .random import Random, choice, rand, randi, randn, random -from .relion_interop import RelionStarFile, relion_metadata_fields +from .relion_interop import ( + RelionStarFile, + relion_metadata_fields, + rots_zyx_to_legacy_aspire, +) from .resolution_estimation import FourierRingCorrelation, FourierShellCorrelation from .rotation import Rotation from .units import ratio_to_decibel, voltage_to_wavelength, wavelength_to_voltage diff --git a/src/aspire/utils/relion_interop.py b/src/aspire/utils/relion_interop.py index 9240e47298..392bf52ec5 100644 --- a/src/aspire/utils/relion_interop.py +++ b/src/aspire/utils/relion_interop.py @@ -70,6 +70,33 @@ def dict_to_relion_types(d): return retval +def rots_zyx_to_legacy_aspire(rots): + """ + Helper function to transform rotations to mimic original aspire python + grid indexing. Now that we are enforcing "zyx" grid indexing across the + code base, in particular for the rotated_grids used for volume projection, + we must transform rotation matrices to allow for existing hardcoded tests + to remain valid. + + Note, this transformation is it's own inverse. + + :param rots: n_rot x 3 x 3 array of rotation matrices. + :return: Transformed rotations. + """ + dtype = rots.dtype + + # Handle singletons + og_shape = rots.shape + if len(og_shape) == 2: + rots = np.expand_dims(rots, axis=0) + + # Transform rots + flip_xy = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=dtype) + new_rots = rots[:, ::-1] @ flip_xy + + return new_rots.reshape(og_shape) + + class RelionStarFile(StarFile): """ A star file generated by RELION representing particles, micrographs, or movies. diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 36650b2ab8..2e519e3bd4 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -10,7 +10,7 @@ from aspire.operators import RadialCTFFilter from aspire.source.relion import RelionSource from aspire.source.simulation import Simulation -from aspire.utils import J_conjugate, utest_tolerance +from aspire.utils import rots_zyx_to_legacy_aspire, utest_tolerance from aspire.volume import LegacyVolume, SymmetryGroup, Volume from .test_utils import matplotlib_dry_run @@ -113,9 +113,10 @@ def setUp(self): dtype=self.dtype, ).generate() - # Note: swapping x and z axes to account for new Volume "zyx" grid convention. + # Note: swapping x and z axes to compensate for new Volume "zyx" grid convention. + # This leaves the volume unchanged for hardcoded tests. self.vols = Volume(np.swapaxes(vols.asnumpy(), 1, 3)) - + sim = Simulation( n=self.n, L=self.L, @@ -127,9 +128,9 @@ def setUp(self): dtype=self.dtype, ) - flip_xy = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=self.dtype) - self.sim = sim.update(rotations = sim.rotations[:, ::-1] @ flip_xy) - + # Transform rotations so hardcoded tests pass under new "zyx" grid convention. + self.sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) + def tearDown(self): pass @@ -141,7 +142,7 @@ def testGaussianBlob(self): def testSimulationRots(self): self.assertTrue( np.allclose( - self.sim.rotations[0, :, :], + rots_zyx_to_legacy_aspire(self.sim.rotations[0, :, :]), np.array( [ [0.91675498, 0.2587233, 0.30433956], @@ -174,6 +175,12 @@ def testSimulationCached(self): noise_adder=WhiteNoiseAdder(var=1), dtype=self.dtype, ) + + # Transform rotations so hardcoded tests pass under new "zyx" grid convention. + sim_cached = sim_cached.update( + rotations=rots_zyx_to_legacy_aspire(sim_cached.rotations) + ) + sim_cached = sim_cached.cache() self.assertTrue( np.array_equal(sim_cached.images[:].asnumpy(), self.sim.images[:].asnumpy()) From 90b2cac17415800edf44ad58a328eac5ecfbfaed Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 26 Oct 2023 14:30:59 -0400 Subject: [PATCH 189/294] Resolve failing hardcoded volume test. --- tests/test_volume.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index ca191ab401..bdbd872fbe 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -8,9 +8,14 @@ from numpy import pi from pytest import raises, skip -from aspire.utils import Rotation, grid_2d, powerset -from aspire.utils.matrix import anorm -from aspire.utils.types import utest_tolerance +from aspire.utils import ( + Rotation, + anorm, + grid_2d, + powerset, + rots_zyx_to_legacy_aspire, + utest_tolerance, +) from aspire.volume import ( AsymmetricVolume, CnSymmetryGroup, @@ -300,6 +305,10 @@ def test_project(vols_1, dtype): vols = Volume(np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol_down8.npy"))) rots = np.load(os.path.join(DATA_DIR, "rand_rot_matrices32.npy")) rots = np.moveaxis(rots, 2, 0) + + # Note, transforming rotations to compensate for "zyx" grid convention. + rots = rots_zyx_to_legacy_aspire(rots) + imgs_clean = vols.project(rots).asnumpy() assert np.allclose(results, imgs_clean, atol=1e-7) From ab67f658df8029e7576af5c730beef44471316ab Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 26 Oct 2023 14:47:56 -0400 Subject: [PATCH 190/294] Resolve failing hardcoded anisotropic_noise test. --- tests/test_anisotropic_noise.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_anisotropic_noise.py b/tests/test_anisotropic_noise.py index dc4732732e..dbaea7eda2 100644 --- a/tests/test_anisotropic_noise.py +++ b/tests/test_anisotropic_noise.py @@ -6,8 +6,8 @@ from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator from aspire.operators import RadialCTFFilter from aspire.source import ArrayImageSource, Simulation -from aspire.utils.types import utest_tolerance -from aspire.volume import LegacyVolume +from aspire.utils import rots_zyx_to_legacy_aspire, utest_tolerance +from aspire.volume import LegacyVolume, Volume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -15,8 +15,13 @@ class SimTestCase(TestCase): def setUp(self): self.dtype = np.float32 - self.vol = LegacyVolume(L=8, dtype=self.dtype).generate() - self.sim = Simulation( + + # Note: swapping x and z axes to compensate for new Volume "zyx" grid convention. + # This leaves the volume unchanged for hardcoded tests. + vol = LegacyVolume(L=8, dtype=self.dtype).generate() + self.vol = Volume(np.swapaxes(vol.asnumpy(), 1, 3)) + + sim = Simulation( n=1024, vols=self.vol, unique_filters=[ @@ -25,6 +30,9 @@ def setUp(self): dtype=self.dtype, ) + # Transform rotations so hardcoded tests pass under new "zyx" grid convention. + self.sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) + def tearDown(self): pass From c9e9f65ed6af94ab7c1c54695f61bd3edc364b15 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 26 Oct 2023 14:53:48 -0400 Subject: [PATCH 191/294] Resolve failing hardcoded covar2d test. --- tests/test_covar2d.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index b064179efb..ba281d9ae6 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -10,7 +10,7 @@ from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter from aspire.source.simulation import Simulation -from aspire.utils import randi, utest_tolerance +from aspire.utils import randi, rots_zyx_to_legacy_aspire, utest_tolerance from aspire.volume import Volume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -107,6 +107,10 @@ def cov2d_fixture(volume, basis, ctf_enabled): dtype=volume.dtype, noise_adder=noise_adder, ) + + # Transform rotations so hardcoded tests pass under new "zyx" grid convention. + sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) + sim.cache() cov2d = RotCov2D(basis) From 7797bc733ff54cbe6837f6afe1935df77b204d35 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 26 Oct 2023 15:01:58 -0400 Subject: [PATCH 192/294] Resolve failing hardcoded covar3d test. --- tests/test_covar3d.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/test_covar3d.py b/tests/test_covar3d.py index 1d3dadbc71..465a7fcaed 100644 --- a/tests/test_covar3d.py +++ b/tests/test_covar3d.py @@ -13,7 +13,7 @@ from aspire.operators import RadialCTFFilter from aspire.reconstruction import MeanEstimator from aspire.source.simulation import Simulation -from aspire.utils import eigs +from aspire.utils import eigs, rots_zyx_to_legacy_aspire from aspire.utils.random import Random from aspire.volume import LegacyVolume, Volume @@ -24,8 +24,13 @@ class Covar3DTestCase(TestCase): @classmethod def setUpClass(cls): cls.dtype = np.float32 - cls.vols = LegacyVolume(L=8, dtype=cls.dtype).generate() - cls.sim = Simulation( + vols = LegacyVolume(L=8, dtype=cls.dtype).generate() + + # Note: swapping x and z axes to compensate for new Volume "zyx" grid convention. + # This leaves the volume unchanged for hardcoded tests. + cls.vols = Volume(np.swapaxes(vols.asnumpy(), 1, 3)) + + sim = Simulation( n=1024, vols=cls.vols, unique_filters=[ @@ -33,6 +38,10 @@ def setUpClass(cls): ], dtype=cls.dtype, ) + + # Transform rotations so hardcoded tests pass under new "zyx" grid convention. + cls.sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) + basis = FBBasis3D((8, 8, 8), dtype=cls.dtype) cls.noise_variance = 0.0030762743633643615 From 9db24fae3e7f72de3c0e0f75b5abcdda086f1faf Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 26 Oct 2023 15:59:49 -0400 Subject: [PATCH 193/294] legacy_simulation helper function. Resolve mean_estimator failing tests. --- src/aspire/utils/__init__.py | 1 + src/aspire/utils/relion_interop.py | 32 ++++++++++++++++++++++++++++++ tests/test_mean_estimator.py | 11 +++++++--- tests/test_simulation.py | 20 +++++++------------ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/aspire/utils/__init__.py b/src/aspire/utils/__init__.py index dede0dcb6a..57793502ae 100644 --- a/src/aspire/utils/__init__.py +++ b/src/aspire/utils/__init__.py @@ -69,6 +69,7 @@ from .random import Random, choice, rand, randi, randn, random from .relion_interop import ( RelionStarFile, + legacy_simulation, relion_metadata_fields, rots_zyx_to_legacy_aspire, ) diff --git a/src/aspire/utils/relion_interop.py b/src/aspire/utils/relion_interop.py index 392bf52ec5..39da3a406c 100644 --- a/src/aspire/utils/relion_interop.py +++ b/src/aspire/utils/relion_interop.py @@ -97,6 +97,38 @@ def rots_zyx_to_legacy_aspire(rots): return new_rots.reshape(og_shape) +def legacy_simulation(sim): + """ + This converts a `Simulation` generated using the zyx grid convention into + a legacy Simulation generated with the legacy grid convention. + + :param sim: Simulation object + :return: Legacy Simulation + """ + # Transform original volume. + vols = sim.vols.__class__(np.swapaxes(sim.vols.asnumpy(), 1, 3)) + + legacy_sim = sim.__class__( + n=sim.n, + L=sim.L, + vols=vols, + angles=sim.angles, + offsets=sim.offsets, + amplitudes=sim.amplitudes, + unique_filters=sim.unique_filters, + noise_adder=sim.noise_adder, + symmetry_group=sim.symmetry_group, + dtype=sim.dtype, + ) + + # Transform rotations so hardcoded tests pass under new "zyx" grid convention. + legacy_sim = legacy_sim.update( + rotations=rots_zyx_to_legacy_aspire(legacy_sim.rotations) + ) + + return legacy_sim + + class RelionStarFile(StarFile): """ A star file generated by RELION representing particles, micrographs, or movies. diff --git a/tests/test_mean_estimator.py b/tests/test_mean_estimator.py index a3bb9a3a0f..e561a760d4 100644 --- a/tests/test_mean_estimator.py +++ b/tests/test_mean_estimator.py @@ -8,6 +8,7 @@ from aspire.operators import RadialCTFFilter from aspire.reconstruction import MeanEstimator from aspire.source.simulation import Simulation +from aspire.utils import legacy_simulation from aspire.volume import LegacyVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -18,7 +19,7 @@ def setUp(self): self.dtype = np.float32 self.resolution = 8 self.vols = LegacyVolume(L=self.resolution, dtype=self.dtype).generate() - self.sim = sim = Simulation( + self.sim = Simulation( n=1024, vols=self.vols, unique_filters=[ @@ -26,12 +27,16 @@ def setUp(self): ], dtype=self.dtype, ) + + # Transform Simulation to Legacy Simulation for hardcoded tests. + self.sim = legacy_simulation(self.sim) + basis = FBBasis3D((self.resolution,) * 3, dtype=self.dtype) - self.estimator = MeanEstimator(sim, basis, preconditioner="none") + self.estimator = MeanEstimator(self.sim, basis, preconditioner="none") self.estimator_with_preconditioner = MeanEstimator( - sim, basis, preconditioner="circulant" + self.sim, basis, preconditioner="circulant" ) def tearDown(self): diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 2e519e3bd4..6c1c5aaab0 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -10,7 +10,7 @@ from aspire.operators import RadialCTFFilter from aspire.source.relion import RelionSource from aspire.source.simulation import Simulation -from aspire.utils import rots_zyx_to_legacy_aspire, utest_tolerance +from aspire.utils import legacy_simulation, rots_zyx_to_legacy_aspire, utest_tolerance from aspire.volume import LegacyVolume, SymmetryGroup, Volume from .test_utils import matplotlib_dry_run @@ -108,16 +108,12 @@ def setUp(self): self.L = 8 self.dtype = np.float32 - vols = LegacyVolume( + self.vols = LegacyVolume( L=self.L, dtype=self.dtype, ).generate() - # Note: swapping x and z axes to compensate for new Volume "zyx" grid convention. - # This leaves the volume unchanged for hardcoded tests. - self.vols = Volume(np.swapaxes(vols.asnumpy(), 1, 3)) - - sim = Simulation( + self.sim = Simulation( n=self.n, L=self.L, vols=self.vols, @@ -128,8 +124,8 @@ def setUp(self): dtype=self.dtype, ) - # Transform rotations so hardcoded tests pass under new "zyx" grid convention. - self.sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) + # Transform to legacy Simulation to pass hardcoded tests. + self.sim = legacy_simulation(self.sim) def tearDown(self): pass @@ -176,10 +172,8 @@ def testSimulationCached(self): dtype=self.dtype, ) - # Transform rotations so hardcoded tests pass under new "zyx" grid convention. - sim_cached = sim_cached.update( - rotations=rots_zyx_to_legacy_aspire(sim_cached.rotations) - ) + # Transform to leagacy Simulation so hardcoded tests pass under new "zyx" grid convention. + sim_cached = legacy_simulation(sim_cached) sim_cached = sim_cached.cache() self.assertTrue( From e0b2c86f341abf2a7c1f69c6e28ea9c6a109f998 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 27 Oct 2023 09:46:01 -0400 Subject: [PATCH 194/294] resolve failing hardcoded weighted_mean_estimator test. --- tests/test_weighted_mean_estimator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_weighted_mean_estimator.py b/tests/test_weighted_mean_estimator.py index a66440a330..02460ee9e1 100644 --- a/tests/test_weighted_mean_estimator.py +++ b/tests/test_weighted_mean_estimator.py @@ -8,6 +8,7 @@ from aspire.operators import RadialCTFFilter from aspire.reconstruction import WeightedVolumesEstimator from aspire.source.simulation import Simulation +from aspire.utils import legacy_simulation from aspire.volume import LegacyVolume logger = logging.getLogger(__name__) @@ -29,6 +30,10 @@ def setUp(self): ], dtype=self.dtype, ) + + # Convert Simulation to legacy Simulation for hardcoded tests. + self.sim = legacy_simulation(self.sim) + self.basis = FBBasis3D((L, L, L), dtype=self.dtype) self.weights = np.ones((self.n, self.r)) / np.sqrt(self.n) self.estimator = WeightedVolumesEstimator( From 53b20f67e19a052039dcb7d12d28ea80c079adee Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 27 Oct 2023 10:03:50 -0400 Subject: [PATCH 195/294] remove relion rotation transformation --- src/aspire/source/relion.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/aspire/source/relion.py b/src/aspire/source/relion.py index 9b4e4d7e6c..02b8ab9363 100644 --- a/src/aspire/source/relion.py +++ b/src/aspire/source/relion.py @@ -155,12 +155,6 @@ def __init__( logger.info(f"Populated {self.n_ctf_filters} CTFFilters from '{filepath}'") - # NOTE: We are currently using a different grid indexing than Relion to produce images. - # If rotations are provided we transform them to align with ASPIRE indexing. - if self._rotations is not None: - flip_xy = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=self.dtype) - self.rotations = self.rotations @ flip_xy - # Any further operations should not mutate this instance. self._mutable = False From 4cc38065d0968a06853763618def7cb44eb71173 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 27 Oct 2023 10:24:20 -0400 Subject: [PATCH 196/294] negate rotations in averager2d test. Revert rots in averager methods. --- src/aspire/classification/averager2d.py | 6 +++--- tests/test_averager2d.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/aspire/classification/averager2d.py b/src/aspire/classification/averager2d.py index 5f9861804a..3c487070ab 100644 --- a/src/aspire/classification/averager2d.py +++ b/src/aspire/classification/averager2d.py @@ -404,7 +404,7 @@ def _innerloop(k): for k, result in enumerate(results): rotations[k], correlations[k] = result - return -rotations, None, correlations + return rotations, None, correlations class BFSRAverager2D(BFRAverager2D): @@ -638,7 +638,7 @@ def _innerloop(k): for k, result in enumerate(results): rotations[k], shifts[k], correlations[k] = result - return -rotations, shifts, correlations + return rotations, shifts, correlations def average( self, @@ -836,7 +836,7 @@ def _innerloop(k): for k, result in enumerate(results): rotations[k], shifts[k], correlations[k] = result - return -rotations, shifts, correlations + return rotations, shifts, correlations def average( self, diff --git a/tests/test_averager2d.py b/tests/test_averager2d.py index c09da2c42e..702a47f2fd 100644 --- a/tests/test_averager2d.py +++ b/tests/test_averager2d.py @@ -234,11 +234,11 @@ def testAverager(self): self.assertIsNone(_shifts) # Crude check that we are closer to known angle than the next rotation - self.assertTrue(check_angle_diff(_rotations, self.thetas, self.step / 2)) + self.assertTrue(check_angle_diff(-_rotations, self.thetas, self.step / 2)) # Fine check that we are within n_angles. self.assertTrue( - check_angle_diff(_rotations, self.thetas, 2 * np.pi / self.n_search_angles) + check_angle_diff(-_rotations, self.thetas, 2 * np.pi / self.n_search_angles) ) @@ -275,11 +275,11 @@ def testAverager(self): _rotations, _shifts, _ = avgr.align(self.classes, self.reflections, self.coefs) # Crude check that we are closer to known angle than the next rotation - self.assertTrue(check_angle_diff(_rotations, self.thetas, self.step / 2)) + self.assertTrue(check_angle_diff(-_rotations, self.thetas, self.step / 2)) # Fine check that we are within n_angles. self.assertTrue( - check_angle_diff(_rotations, self.thetas, 2 * np.pi / self.n_search_angles) + check_angle_diff(-_rotations, self.thetas, 2 * np.pi / self.n_search_angles) ) # Check that we are _not_ shifting the base image @@ -313,10 +313,10 @@ def testAverager(self): _rotations, _shifts, _ = avgr.align(self.classes, self.reflections, self.coefs) # Crude check that we are closer to known angle than the next rotation - self.assertTrue(check_angle_diff(_rotations, self.thetas, self.step / 2)) + self.assertTrue(check_angle_diff(-_rotations, self.thetas, self.step / 2)) # Fine check that we are within 4 degrees. - self.assertTrue(check_angle_diff(_rotations, self.thetas, np.pi / 45)) + self.assertTrue(check_angle_diff(-_rotations, self.thetas, np.pi / 45)) # Check that we are _not_ shifting the base image self.assertTrue(np.all(_shifts[0][0] == 0)) From 0cec119c9798a1c230daafea232f3b0e0e823c45 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 27 Oct 2023 10:37:48 -0400 Subject: [PATCH 197/294] update pipeline demo section title. --- gallery/tutorials/pipeline_demo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gallery/tutorials/pipeline_demo.py b/gallery/tutorials/pipeline_demo.py index 77b34af9b9..f2547d2c00 100644 --- a/gallery/tutorials/pipeline_demo.py +++ b/gallery/tutorials/pipeline_demo.py @@ -9,7 +9,7 @@ # %% # Download an Example Volume -# ----------------- +# -------------------------- # We begin by downloading a high resolution volume map of the 80S # Ribosome, sourced from EMDB: https://www.ebi.ac.uk/emdb/EMD-2660. # This is one of several volume maps that can be downloaded with @@ -213,8 +213,8 @@ # %% -# Mean Squared Error -# ------------------ +# Mean Error of Estimated Rotations +# --------------------------------- # ASPIRE has the built-in utility function, ``mean_aligned_angular_distance``, which globally # aligns the estimated rotations to the true rotations and computes the mean # angular distance (in degrees). From af3d550c9584cc435e7d98aefa798cc2f5015c83 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 27 Oct 2023 11:20:21 -0400 Subject: [PATCH 198/294] use legacy_simulation everywhere. --- tests/test_anisotropic_noise.py | 17 ++++++----------- tests/test_covar3d.py | 15 +++++---------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/tests/test_anisotropic_noise.py b/tests/test_anisotropic_noise.py index dbaea7eda2..98626e39b8 100644 --- a/tests/test_anisotropic_noise.py +++ b/tests/test_anisotropic_noise.py @@ -6,8 +6,8 @@ from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator from aspire.operators import RadialCTFFilter from aspire.source import ArrayImageSource, Simulation -from aspire.utils import rots_zyx_to_legacy_aspire, utest_tolerance -from aspire.volume import LegacyVolume, Volume +from aspire.utils import legacy_simulation, utest_tolerance +from aspire.volume import LegacyVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -15,13 +15,8 @@ class SimTestCase(TestCase): def setUp(self): self.dtype = np.float32 - - # Note: swapping x and z axes to compensate for new Volume "zyx" grid convention. - # This leaves the volume unchanged for hardcoded tests. - vol = LegacyVolume(L=8, dtype=self.dtype).generate() - self.vol = Volume(np.swapaxes(vol.asnumpy(), 1, 3)) - - sim = Simulation( + self.vol = LegacyVolume(L=8, dtype=self.dtype).generate() + self.sim = Simulation( n=1024, vols=self.vol, unique_filters=[ @@ -30,8 +25,8 @@ def setUp(self): dtype=self.dtype, ) - # Transform rotations so hardcoded tests pass under new "zyx" grid convention. - self.sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) + # Transform Simulation to legacy Simulation for hardcoded tests. + self.sim = legacy_simulation(self.sim) def tearDown(self): pass diff --git a/tests/test_covar3d.py b/tests/test_covar3d.py index 465a7fcaed..bfc0e05fbb 100644 --- a/tests/test_covar3d.py +++ b/tests/test_covar3d.py @@ -13,7 +13,7 @@ from aspire.operators import RadialCTFFilter from aspire.reconstruction import MeanEstimator from aspire.source.simulation import Simulation -from aspire.utils import eigs, rots_zyx_to_legacy_aspire +from aspire.utils import eigs, legacy_simulation from aspire.utils.random import Random from aspire.volume import LegacyVolume, Volume @@ -24,13 +24,8 @@ class Covar3DTestCase(TestCase): @classmethod def setUpClass(cls): cls.dtype = np.float32 - vols = LegacyVolume(L=8, dtype=cls.dtype).generate() - - # Note: swapping x and z axes to compensate for new Volume "zyx" grid convention. - # This leaves the volume unchanged for hardcoded tests. - cls.vols = Volume(np.swapaxes(vols.asnumpy(), 1, 3)) - - sim = Simulation( + cls.vols = LegacyVolume(L=8, dtype=cls.dtype).generate() + cls.sim = Simulation( n=1024, vols=cls.vols, unique_filters=[ @@ -39,8 +34,8 @@ def setUpClass(cls): dtype=cls.dtype, ) - # Transform rotations so hardcoded tests pass under new "zyx" grid convention. - cls.sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) + # Transform Simulation to legacy Simulation for hardcoded tests. + cls.sim = legacy_simulation(cls.sim) basis = FBBasis3D((8, 8, 8), dtype=cls.dtype) cls.noise_variance = 0.0030762743633643615 From c270fa8dca87fd64230e427acce74dc10d6b134b Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 27 Oct 2023 11:24:39 -0400 Subject: [PATCH 199/294] add commment about grid conversion in rotated_grids. --- src/aspire/volume/volume.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/aspire/volume/volume.py b/src/aspire/volume/volume.py index 22420b9e97..c8d539974f 100644 --- a/src/aspire/volume/volume.py +++ b/src/aspire/volume/volume.py @@ -661,6 +661,8 @@ def rotated_grids(L, rot_matrices): grid2d = grid_2d(L, indexing="yx", dtype=rot_matrices.dtype) num_pts = L**2 num_rots = rot_matrices.shape[0] + + # Frequency points flattened and placed in xyz order to apply rotations. pts = np.pi * np.vstack( [ grid2d["x"].flatten(), @@ -672,6 +674,7 @@ def rotated_grids(L, rot_matrices): for i in range(num_rots): pts_rot[:, i, :] = rot_matrices[i, :, :] @ pts + # Reshape rotated frequency points and convert back into zyx convention. pts_rot = pts_rot.reshape((3, num_rots, L, L))[::-1] return pts_rot @@ -690,6 +693,8 @@ def rotated_grids_3d(L, rot_matrices): grid3d = grid_3d(L, indexing="zyx", dtype=rot_matrices.dtype) num_pts = L**3 num_rots = rot_matrices.shape[0] + + # Frequency points flattened and placed in xyz order to apply rotations. pts = np.pi * np.vstack( [ grid3d["x"].flatten(), From f30a961fd1bcc4ceee832ee384bf61ffe20e9b8c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 27 Oct 2023 11:32:07 -0400 Subject: [PATCH 200/294] update relion interop comments. --- src/aspire/utils/relion_interop.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aspire/utils/relion_interop.py b/src/aspire/utils/relion_interop.py index 39da3a406c..fd50172d12 100644 --- a/src/aspire/utils/relion_interop.py +++ b/src/aspire/utils/relion_interop.py @@ -105,7 +105,7 @@ def legacy_simulation(sim): :param sim: Simulation object :return: Legacy Simulation """ - # Transform original volume. + # Transform Volume in zyx convention to Volume in xyz convention. vols = sim.vols.__class__(np.swapaxes(sim.vols.asnumpy(), 1, 3)) legacy_sim = sim.__class__( @@ -121,7 +121,7 @@ def legacy_simulation(sim): dtype=sim.dtype, ) - # Transform rotations so hardcoded tests pass under new "zyx" grid convention. + # Transform rotations to legacy rotations. legacy_sim = legacy_sim.update( rotations=rots_zyx_to_legacy_aspire(legacy_sim.rotations) ) From b32e036004ac6c4a06f0a622381bac9b3a96a753 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 09:09:47 -0400 Subject: [PATCH 201/294] enforce that Legacy_Volume is still that same volume. Subclass Legacy_Simulation. Update test_simulation.py. --- src/aspire/source/simulation.py | 56 ++++++++++++++++++++++++++- src/aspire/volume/volume_synthesis.py | 11 ++++-- tests/test_simulation.py | 17 +++----- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index 46a01fdfe9..f924df7dbf 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -10,6 +10,7 @@ from aspire.source import ImageSource from aspire.source.image import _ImageAccessor from aspire.utils import ( + Rotation, acorr, ainner, anorm, @@ -148,7 +149,7 @@ def __init__( self.states = states if angles is None: - angles = uniform_random_angles(n, seed=seed, dtype=self.dtype) + angles = self._init_angles() self.angles = angles if unique_filters is None: @@ -192,6 +193,9 @@ def __init__( # Any further operations should not mutate this instance. self._mutable = False + def _init_angles(self): + return uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) + def _populate_ctf_metadata(self, filter_indices): # Since we are not reading from a starfile, we must construct # metadata based on the CTF filters by hand and set the values @@ -532,3 +536,53 @@ def true_snr(self, *args, **kwargs): noise_power = self.noise_adder.noise_var signal_power = self.true_signal_power(*args, **kwargs) return signal_power / noise_power + + +class Legacy_Simulation(Simulation): + """ + Legacy Simulation enforces the legacy grid convention for generating projection + images. + + Note, that `angles`, and thus `rotations`, are altered upon initialization. + To recover the rotations associated with the input angles use the staticmethod + `rots_zyx_to_legacy_aspire()`. + """ + + def _init_angles(self): + angles = uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) + + # Convert to rotations. + rots = Rotation.from_euler(angles).matrices + + # Transform rotations to replicate legacy grid convention. + legacy_rots = Rotation(self.rots_zyx_to_legacy_aspire(rots)) + + # Convert back to angles. + return legacy_rots.angles.astype(self.dtype) + + @staticmethod + def rots_zyx_to_legacy_aspire(rots): + """ + Helper function to transform rotations to mimic original aspire python + grid indexing. Now that we are enforcing "zyx" grid indexing across the + code base, in particular for the rotated_grids used for volume projection, + we must transform rotation matrices to allow for existing hardcoded tests + to remain valid. + + Note, this transformation is it's own inverse. + + :param rots: n_rot x 3 x 3 array of rotation matrices. + :return: Transformed rotations. + """ + dtype = rots.dtype + + # Handle singletons + og_shape = rots.shape + if len(og_shape) == 2: + rots = np.expand_dims(rots, axis=0) + + # Transform rots + flip_xy = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=dtype) + new_rots = rots[:, ::-1] @ flip_xy + + return new_rots.reshape(og_shape) diff --git a/src/aspire/volume/volume_synthesis.py b/src/aspire/volume/volume_synthesis.py index 693d94ca8d..31dd0c38af 100644 --- a/src/aspire/volume/volume_synthesis.py +++ b/src/aspire/volume/volume_synthesis.py @@ -80,7 +80,7 @@ def _gaussian_blob_vols(self): """ Generates a Volume object composed of Gaussian blobs. - :return: A Volume instance containing C Gaussian blob volumes. + :return: An ndarray containing C Gaussian blob volumes. """ vols = np.zeros(shape=((self.C,) + (self.L,) * 3)).astype(self.dtype) with Random(self.seed): @@ -88,7 +88,7 @@ def _gaussian_blob_vols(self): Q, D, mu = self._gen_gaussians() Q_rot, D_sym, mu_rot = self._symmetrize_gaussians(Q, D, mu) vols[c] = self._eval_gaussians(Q_rot, D_sym, mu_rot) - return Volume(vols) + return vols def _gen_gaussians(self): """ @@ -263,4 +263,9 @@ def generate(self): """ Generates an asymmetric volume composed of random 3D Gaussian blobs. """ - return self._gaussian_blob_vols() + vols = self._gaussian_blob_vols() + + # Swap axes to retain Legacy xyz-indexing. + vols = np.swapaxes(vols, 1, 3) + + return Volume(vols) diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 6c1c5aaab0..3b9c72ed12 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -9,8 +9,8 @@ from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter from aspire.source.relion import RelionSource -from aspire.source.simulation import Simulation -from aspire.utils import legacy_simulation, rots_zyx_to_legacy_aspire, utest_tolerance +from aspire.source.simulation import Legacy_Simulation, Simulation +from aspire.utils import utest_tolerance from aspire.volume import LegacyVolume, SymmetryGroup, Volume from .test_utils import matplotlib_dry_run @@ -113,7 +113,7 @@ def setUp(self): dtype=self.dtype, ).generate() - self.sim = Simulation( + self.sim = Legacy_Simulation( n=self.n, L=self.L, vols=self.vols, @@ -124,9 +124,6 @@ def setUp(self): dtype=self.dtype, ) - # Transform to legacy Simulation to pass hardcoded tests. - self.sim = legacy_simulation(self.sim) - def tearDown(self): pass @@ -138,7 +135,7 @@ def testGaussianBlob(self): def testSimulationRots(self): self.assertTrue( np.allclose( - rots_zyx_to_legacy_aspire(self.sim.rotations[0, :, :]), + self.sim.rots_zyx_to_legacy_aspire(self.sim.rotations[0, :, :]), np.array( [ [0.91675498, 0.2587233, 0.30433956], @@ -146,6 +143,7 @@ def testSimulationRots(self): [-0.00507853, 0.76938412, -0.63876622], ] ), + atol=utest_tolerance(self.dtype), ) ) @@ -161,7 +159,7 @@ def testSimulationImages(self): ) def testSimulationCached(self): - sim_cached = Simulation( + sim_cached = Legacy_Simulation( n=self.n, L=self.L, vols=self.vols, @@ -172,9 +170,6 @@ def testSimulationCached(self): dtype=self.dtype, ) - # Transform to leagacy Simulation so hardcoded tests pass under new "zyx" grid convention. - sim_cached = legacy_simulation(sim_cached) - sim_cached = sim_cached.cache() self.assertTrue( np.array_equal(sim_cached.images[:].asnumpy(), self.sim.images[:].asnumpy()) From fb8a2436f2b9533190e4c5c59f7d90e04d111b80 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 09:45:18 -0400 Subject: [PATCH 202/294] Use LegacySimulation in anisotropic noise test. --- src/aspire/source/__init__.py | 2 +- src/aspire/source/simulation.py | 2 +- tests/test_anisotropic_noise.py | 9 +++------ tests/test_simulation.py | 7 +++---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/aspire/source/__init__.py b/src/aspire/source/__init__.py index 7fd7691a3b..f05352d974 100644 --- a/src/aspire/source/__init__.py +++ b/src/aspire/source/__init__.py @@ -8,7 +8,7 @@ OrientedSource, ) from aspire.source.relion import RelionSource -from aspire.source.simulation import Simulation +from aspire.source.simulation import LegacySimulation, Simulation # isort: off from aspire.source.micrograph import ( diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index f924df7dbf..e7a79cc044 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -538,7 +538,7 @@ def true_snr(self, *args, **kwargs): return signal_power / noise_power -class Legacy_Simulation(Simulation): +class LegacySimulation(Simulation): """ Legacy Simulation enforces the legacy grid convention for generating projection images. diff --git a/tests/test_anisotropic_noise.py b/tests/test_anisotropic_noise.py index 98626e39b8..37bc2ece56 100644 --- a/tests/test_anisotropic_noise.py +++ b/tests/test_anisotropic_noise.py @@ -5,8 +5,8 @@ from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator from aspire.operators import RadialCTFFilter -from aspire.source import ArrayImageSource, Simulation -from aspire.utils import legacy_simulation, utest_tolerance +from aspire.source import ArrayImageSource, LegacySimulation +from aspire.utils import utest_tolerance from aspire.volume import LegacyVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -16,7 +16,7 @@ class SimTestCase(TestCase): def setUp(self): self.dtype = np.float32 self.vol = LegacyVolume(L=8, dtype=self.dtype).generate() - self.sim = Simulation( + self.sim = LegacySimulation( n=1024, vols=self.vol, unique_filters=[ @@ -25,9 +25,6 @@ def setUp(self): dtype=self.dtype, ) - # Transform Simulation to legacy Simulation for hardcoded tests. - self.sim = legacy_simulation(self.sim) - def tearDown(self): pass diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 3b9c72ed12..115e4aa20f 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -8,8 +8,7 @@ from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter -from aspire.source.relion import RelionSource -from aspire.source.simulation import Legacy_Simulation, Simulation +from aspire.source import LegacySimulation, RelionSource, Simulation from aspire.utils import utest_tolerance from aspire.volume import LegacyVolume, SymmetryGroup, Volume @@ -113,7 +112,7 @@ def setUp(self): dtype=self.dtype, ).generate() - self.sim = Legacy_Simulation( + self.sim = LegacySimulation( n=self.n, L=self.L, vols=self.vols, @@ -159,7 +158,7 @@ def testSimulationImages(self): ) def testSimulationCached(self): - sim_cached = Legacy_Simulation( + sim_cached = LegacySimulation( n=self.n, L=self.L, vols=self.vols, From 4a44ecee9348846328f7448c262c2cf603de5264 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 09:50:03 -0400 Subject: [PATCH 203/294] Use LegacySimulation in test_covar2d.py. --- tests/test_covar2d.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index ba281d9ae6..83715ccd3b 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -9,8 +9,8 @@ from aspire.covariance import RotCov2D from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter -from aspire.source.simulation import Simulation -from aspire.utils import randi, rots_zyx_to_legacy_aspire, utest_tolerance +from aspire.source.simulation import LegacySimulation +from aspire.utils import randi, utest_tolerance from aspire.volume import Volume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -97,7 +97,7 @@ def cov2d_fixture(volume, basis, ctf_enabled): noise_adder = WhiteNoiseAdder(var=NOISE_VAR) - sim = Simulation( + sim = LegacySimulation( n=n, vols=volume, unique_filters=unique_filters, @@ -108,9 +108,6 @@ def cov2d_fixture(volume, basis, ctf_enabled): noise_adder=noise_adder, ) - # Transform rotations so hardcoded tests pass under new "zyx" grid convention. - sim = sim.update(rotations=rots_zyx_to_legacy_aspire(sim.rotations)) - sim.cache() cov2d = RotCov2D(basis) From 7f7e699483804c747f009a06782cc3b18ed06207 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 09:53:58 -0400 Subject: [PATCH 204/294] Use LegacySimulation in test_covar3d.py. --- tests/test_covar3d.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_covar3d.py b/tests/test_covar3d.py index bfc0e05fbb..d2d1e12aad 100644 --- a/tests/test_covar3d.py +++ b/tests/test_covar3d.py @@ -12,8 +12,8 @@ from aspire.denoising import src_wiener_coords from aspire.operators import RadialCTFFilter from aspire.reconstruction import MeanEstimator -from aspire.source.simulation import Simulation -from aspire.utils import eigs, legacy_simulation +from aspire.source.simulation import LegacySimulation +from aspire.utils import eigs from aspire.utils.random import Random from aspire.volume import LegacyVolume, Volume @@ -25,7 +25,7 @@ class Covar3DTestCase(TestCase): def setUpClass(cls): cls.dtype = np.float32 cls.vols = LegacyVolume(L=8, dtype=cls.dtype).generate() - cls.sim = Simulation( + cls.sim = LegacySimulation( n=1024, vols=cls.vols, unique_filters=[ @@ -34,9 +34,6 @@ def setUpClass(cls): dtype=cls.dtype, ) - # Transform Simulation to legacy Simulation for hardcoded tests. - cls.sim = legacy_simulation(cls.sim) - basis = FBBasis3D((8, 8, 8), dtype=cls.dtype) cls.noise_variance = 0.0030762743633643615 From 1a2ea651ad6a82eec94402d94c05aab52273a612 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 09:55:31 -0400 Subject: [PATCH 205/294] Use LegacySimulation in test_mean_estimator.py. --- tests/test_mean_estimator.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_mean_estimator.py b/tests/test_mean_estimator.py index e561a760d4..121a72ad76 100644 --- a/tests/test_mean_estimator.py +++ b/tests/test_mean_estimator.py @@ -7,8 +7,7 @@ from aspire.basis import FBBasis3D from aspire.operators import RadialCTFFilter from aspire.reconstruction import MeanEstimator -from aspire.source.simulation import Simulation -from aspire.utils import legacy_simulation +from aspire.source.simulation import LegacySimulation from aspire.volume import LegacyVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -19,7 +18,7 @@ def setUp(self): self.dtype = np.float32 self.resolution = 8 self.vols = LegacyVolume(L=self.resolution, dtype=self.dtype).generate() - self.sim = Simulation( + self.sim = LegacySimulation( n=1024, vols=self.vols, unique_filters=[ @@ -28,9 +27,6 @@ def setUp(self): dtype=self.dtype, ) - # Transform Simulation to Legacy Simulation for hardcoded tests. - self.sim = legacy_simulation(self.sim) - basis = FBBasis3D((self.resolution,) * 3, dtype=self.dtype) self.estimator = MeanEstimator(self.sim, basis, preconditioner="none") From c3e5bd773139c0591f753b0b3b6e1b537f36fa58 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 10:00:48 -0400 Subject: [PATCH 206/294] Use LegacySimulation in test_weighted_mean_estimator.py. --- tests/test_weighted_mean_estimator.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_weighted_mean_estimator.py b/tests/test_weighted_mean_estimator.py index 02460ee9e1..f96a504791 100644 --- a/tests/test_weighted_mean_estimator.py +++ b/tests/test_weighted_mean_estimator.py @@ -7,8 +7,7 @@ from aspire.basis import FBBasis3D from aspire.operators import RadialCTFFilter from aspire.reconstruction import WeightedVolumesEstimator -from aspire.source.simulation import Simulation -from aspire.utils import legacy_simulation +from aspire.source import LegacySimulation from aspire.volume import LegacyVolume logger = logging.getLogger(__name__) @@ -22,7 +21,7 @@ def setUp(self): self.n = 1024 self.r = 2 self.L = L = 8 - self.sim = Simulation( + self.sim = LegacySimulation( vols=LegacyVolume(L, dtype=self.dtype).generate(), n=self.n, unique_filters=[ @@ -31,9 +30,6 @@ def setUp(self): dtype=self.dtype, ) - # Convert Simulation to legacy Simulation for hardcoded tests. - self.sim = legacy_simulation(self.sim) - self.basis = FBBasis3D((L, L, L), dtype=self.dtype) self.weights = np.ones((self.n, self.r)) / np.sqrt(self.n) self.estimator = WeightedVolumesEstimator( From 5e9451e8b684aa7d84f692e2ecb9d7a4151b2c8e Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 10:23:40 -0400 Subject: [PATCH 207/294] Remove helper functions from relion_interop --- src/aspire/utils/__init__.py | 7 +--- src/aspire/utils/relion_interop.py | 59 ------------------------------ 2 files changed, 1 insertion(+), 65 deletions(-) diff --git a/src/aspire/utils/__init__.py b/src/aspire/utils/__init__.py index 57793502ae..e691e0ba5e 100644 --- a/src/aspire/utils/__init__.py +++ b/src/aspire/utils/__init__.py @@ -67,12 +67,7 @@ virtual_core_cpu_suggestion, ) from .random import Random, choice, rand, randi, randn, random -from .relion_interop import ( - RelionStarFile, - legacy_simulation, - relion_metadata_fields, - rots_zyx_to_legacy_aspire, -) +from .relion_interop import RelionStarFile, relion_metadata_fields from .resolution_estimation import FourierRingCorrelation, FourierShellCorrelation from .rotation import Rotation from .units import ratio_to_decibel, voltage_to_wavelength, wavelength_to_voltage diff --git a/src/aspire/utils/relion_interop.py b/src/aspire/utils/relion_interop.py index fd50172d12..9240e47298 100644 --- a/src/aspire/utils/relion_interop.py +++ b/src/aspire/utils/relion_interop.py @@ -70,65 +70,6 @@ def dict_to_relion_types(d): return retval -def rots_zyx_to_legacy_aspire(rots): - """ - Helper function to transform rotations to mimic original aspire python - grid indexing. Now that we are enforcing "zyx" grid indexing across the - code base, in particular for the rotated_grids used for volume projection, - we must transform rotation matrices to allow for existing hardcoded tests - to remain valid. - - Note, this transformation is it's own inverse. - - :param rots: n_rot x 3 x 3 array of rotation matrices. - :return: Transformed rotations. - """ - dtype = rots.dtype - - # Handle singletons - og_shape = rots.shape - if len(og_shape) == 2: - rots = np.expand_dims(rots, axis=0) - - # Transform rots - flip_xy = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=dtype) - new_rots = rots[:, ::-1] @ flip_xy - - return new_rots.reshape(og_shape) - - -def legacy_simulation(sim): - """ - This converts a `Simulation` generated using the zyx grid convention into - a legacy Simulation generated with the legacy grid convention. - - :param sim: Simulation object - :return: Legacy Simulation - """ - # Transform Volume in zyx convention to Volume in xyz convention. - vols = sim.vols.__class__(np.swapaxes(sim.vols.asnumpy(), 1, 3)) - - legacy_sim = sim.__class__( - n=sim.n, - L=sim.L, - vols=vols, - angles=sim.angles, - offsets=sim.offsets, - amplitudes=sim.amplitudes, - unique_filters=sim.unique_filters, - noise_adder=sim.noise_adder, - symmetry_group=sim.symmetry_group, - dtype=sim.dtype, - ) - - # Transform rotations to legacy rotations. - legacy_sim = legacy_sim.update( - rotations=rots_zyx_to_legacy_aspire(legacy_sim.rotations) - ) - - return legacy_sim - - class RelionStarFile(StarFile): """ A star file generated by RELION representing particles, micrographs, or movies. From ad67945c6bf4bd1254688ee6a8140e117c1a5d62 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 10:42:15 -0400 Subject: [PATCH 208/294] Fix import of moved function. --- tests/test_volume.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index bdbd872fbe..9dabff8396 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -8,14 +8,8 @@ from numpy import pi from pytest import raises, skip -from aspire.utils import ( - Rotation, - anorm, - grid_2d, - powerset, - rots_zyx_to_legacy_aspire, - utest_tolerance, -) +from aspire.source import LegacySimulation +from aspire.utils import Rotation, anorm, grid_2d, powerset, utest_tolerance from aspire.volume import ( AsymmetricVolume, CnSymmetryGroup, @@ -306,8 +300,8 @@ def test_project(vols_1, dtype): rots = np.load(os.path.join(DATA_DIR, "rand_rot_matrices32.npy")) rots = np.moveaxis(rots, 2, 0) - # Note, transforming rotations to compensate for "zyx" grid convention. - rots = rots_zyx_to_legacy_aspire(rots) + # Note, transforming rotations to compensate for legacy grid convention used in saved data. + rots = LegacySimulation.rots_zyx_to_legacy_aspire(rots) imgs_clean = vols.project(rots).asnumpy() assert np.allclose(results, imgs_clean, atol=1e-7) From 7ff2e7dc22c63ef29153ab4067a9d4d7e1969a63 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 11:03:18 -0400 Subject: [PATCH 209/294] refactor _init_angles. --- src/aspire/source/simulation.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index e7a79cc044..e3592eaf96 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -148,9 +148,7 @@ def __init__( states = randi(self.C, n, seed=seed) self.states = states - if angles is None: - angles = self._init_angles() - self.angles = angles + self._init_angles(angles) if unique_filters is None: unique_filters = [] @@ -193,8 +191,10 @@ def __init__( # Any further operations should not mutate this instance. self._mutable = False - def _init_angles(self): - return uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) + def _init_angles(self, angles): + if angles is None: + angles = uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) + self.angles = angles def _populate_ctf_metadata(self, filter_indices): # Since we are not reading from a starfile, we must construct @@ -548,8 +548,9 @@ class LegacySimulation(Simulation): `rots_zyx_to_legacy_aspire()`. """ - def _init_angles(self): - angles = uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) + def _init_angles(self, angles): + if angles is None: + angles = uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) # Convert to rotations. rots = Rotation.from_euler(angles).matrices @@ -558,7 +559,7 @@ def _init_angles(self): legacy_rots = Rotation(self.rots_zyx_to_legacy_aspire(rots)) # Convert back to angles. - return legacy_rots.angles.astype(self.dtype) + self.angles = legacy_rots.angles.astype(self.dtype) @staticmethod def rots_zyx_to_legacy_aspire(rots): From 054acbb7b85c5ffaee70eb58c360d9120094fcba Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 11:34:48 -0400 Subject: [PATCH 210/294] one more refactor of _init_angles --- src/aspire/source/simulation.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index e3592eaf96..fb1b5338b5 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -148,7 +148,7 @@ def __init__( states = randi(self.C, n, seed=seed) self.states = states - self._init_angles(angles) + self.angles = self._init_angles(angles) if unique_filters is None: unique_filters = [] @@ -194,7 +194,7 @@ def __init__( def _init_angles(self, angles): if angles is None: angles = uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) - self.angles = angles + return angles def _populate_ctf_metadata(self, filter_indices): # Since we are not reading from a starfile, we must construct @@ -549,8 +549,7 @@ class LegacySimulation(Simulation): """ def _init_angles(self, angles): - if angles is None: - angles = uniform_random_angles(self.n, seed=self.seed, dtype=self.dtype) + angles = super()._init_angles(angles) # Convert to rotations. rots = Rotation.from_euler(angles).matrices @@ -559,7 +558,7 @@ def _init_angles(self, angles): legacy_rots = Rotation(self.rots_zyx_to_legacy_aspire(rots)) # Convert back to angles. - self.angles = legacy_rots.angles.astype(self.dtype) + return legacy_rots.angles.astype(self.dtype) @staticmethod def rots_zyx_to_legacy_aspire(rots): From bfd54d3eadaa59a6459adcbd19e67b6b802c5725 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 2 Nov 2023 13:17:25 -0400 Subject: [PATCH 211/294] negate thetas instead of rotations. --- tests/test_averager2d.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_averager2d.py b/tests/test_averager2d.py index 702a47f2fd..12bf4d4cdf 100644 --- a/tests/test_averager2d.py +++ b/tests/test_averager2d.py @@ -129,7 +129,7 @@ def _construct_rotations(self): # Generate rotations to be used by `Simulation` self.rotations = Rotation.about_axis( - "z", self.thetas, dtype=self.dtype, gimble_lock_warnings=False + "z", -self.thetas, dtype=self.dtype, gimble_lock_warnings=False ) @@ -234,11 +234,11 @@ def testAverager(self): self.assertIsNone(_shifts) # Crude check that we are closer to known angle than the next rotation - self.assertTrue(check_angle_diff(-_rotations, self.thetas, self.step / 2)) + self.assertTrue(check_angle_diff(_rotations, self.thetas, self.step / 2)) # Fine check that we are within n_angles. self.assertTrue( - check_angle_diff(-_rotations, self.thetas, 2 * np.pi / self.n_search_angles) + check_angle_diff(_rotations, self.thetas, 2 * np.pi / self.n_search_angles) ) @@ -275,11 +275,11 @@ def testAverager(self): _rotations, _shifts, _ = avgr.align(self.classes, self.reflections, self.coefs) # Crude check that we are closer to known angle than the next rotation - self.assertTrue(check_angle_diff(-_rotations, self.thetas, self.step / 2)) + self.assertTrue(check_angle_diff(_rotations, self.thetas, self.step / 2)) # Fine check that we are within n_angles. self.assertTrue( - check_angle_diff(-_rotations, self.thetas, 2 * np.pi / self.n_search_angles) + check_angle_diff(_rotations, self.thetas, 2 * np.pi / self.n_search_angles) ) # Check that we are _not_ shifting the base image @@ -313,10 +313,10 @@ def testAverager(self): _rotations, _shifts, _ = avgr.align(self.classes, self.reflections, self.coefs) # Crude check that we are closer to known angle than the next rotation - self.assertTrue(check_angle_diff(-_rotations, self.thetas, self.step / 2)) + self.assertTrue(check_angle_diff(_rotations, self.thetas, self.step / 2)) # Fine check that we are within 4 degrees. - self.assertTrue(check_angle_diff(-_rotations, self.thetas, np.pi / 45)) + self.assertTrue(check_angle_diff(_rotations, self.thetas, np.pi / 45)) # Check that we are _not_ shifting the base image self.assertTrue(np.all(_shifts[0][0] == 0)) From c97d18a8a568ce8b2fb6f4cf580ab3d7413f7ae1 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 3 Nov 2023 13:29:42 -0400 Subject: [PATCH 212/294] Remove added blank lines. --- tests/test_covar2d.py | 1 - tests/test_covar3d.py | 1 - tests/test_mean_estimator.py | 1 - tests/test_simulation.py | 1 - tests/test_weighted_mean_estimator.py | 1 - 5 files changed, 5 deletions(-) diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index 83715ccd3b..1167957a7a 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -107,7 +107,6 @@ def cov2d_fixture(volume, basis, ctf_enabled): dtype=volume.dtype, noise_adder=noise_adder, ) - sim.cache() cov2d = RotCov2D(basis) diff --git a/tests/test_covar3d.py b/tests/test_covar3d.py index d2d1e12aad..fb4ae38b35 100644 --- a/tests/test_covar3d.py +++ b/tests/test_covar3d.py @@ -33,7 +33,6 @@ def setUpClass(cls): ], dtype=cls.dtype, ) - basis = FBBasis3D((8, 8, 8), dtype=cls.dtype) cls.noise_variance = 0.0030762743633643615 diff --git a/tests/test_mean_estimator.py b/tests/test_mean_estimator.py index 121a72ad76..ec4e711288 100644 --- a/tests/test_mean_estimator.py +++ b/tests/test_mean_estimator.py @@ -26,7 +26,6 @@ def setUp(self): ], dtype=self.dtype, ) - basis = FBBasis3D((self.resolution,) * 3, dtype=self.dtype) self.estimator = MeanEstimator(self.sim, basis, preconditioner="none") diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 115e4aa20f..7634d957ad 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -168,7 +168,6 @@ def testSimulationCached(self): noise_adder=WhiteNoiseAdder(var=1), dtype=self.dtype, ) - sim_cached = sim_cached.cache() self.assertTrue( np.array_equal(sim_cached.images[:].asnumpy(), self.sim.images[:].asnumpy()) diff --git a/tests/test_weighted_mean_estimator.py b/tests/test_weighted_mean_estimator.py index f96a504791..797d22202b 100644 --- a/tests/test_weighted_mean_estimator.py +++ b/tests/test_weighted_mean_estimator.py @@ -29,7 +29,6 @@ def setUp(self): ], dtype=self.dtype, ) - self.basis = FBBasis3D((L, L, L), dtype=self.dtype) self.weights = np.ones((self.n, self.r)) / np.sqrt(self.n) self.estimator = WeightedVolumesEstimator( From 67def1f4208ae05fb67d767bd50bfc61eebc8c7e Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 6 Nov 2023 14:51:15 -0500 Subject: [PATCH 213/294] update docstring. --- src/aspire/volume/volume_synthesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/volume/volume_synthesis.py b/src/aspire/volume/volume_synthesis.py index 31dd0c38af..c10c466081 100644 --- a/src/aspire/volume/volume_synthesis.py +++ b/src/aspire/volume/volume_synthesis.py @@ -78,7 +78,7 @@ def generate(self): def _gaussian_blob_vols(self): """ - Generates a Volume object composed of Gaussian blobs. + Generates a 4D array representing a stack of volumes composed of Gaussian blobs. :return: An ndarray containing C Gaussian blob volumes. """ From 3ec0523a6a5bfbf1ba123ef0b84cfc22818cd8bf Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 8 Nov 2023 11:21:49 -0500 Subject: [PATCH 214/294] LegacySimulation ~~> _LegacySimulation. --- src/aspire/source/__init__.py | 2 +- src/aspire/source/simulation.py | 2 +- tests/test_anisotropic_noise.py | 4 ++-- tests/test_covar2d.py | 4 ++-- tests/test_covar3d.py | 4 ++-- tests/test_mean_estimator.py | 4 ++-- tests/test_simulation.py | 6 +++--- tests/test_volume.py | 4 ++-- tests/test_weighted_mean_estimator.py | 4 ++-- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/aspire/source/__init__.py b/src/aspire/source/__init__.py index f05352d974..c3703db8bc 100644 --- a/src/aspire/source/__init__.py +++ b/src/aspire/source/__init__.py @@ -8,7 +8,7 @@ OrientedSource, ) from aspire.source.relion import RelionSource -from aspire.source.simulation import LegacySimulation, Simulation +from aspire.source.simulation import Simulation, _LegacySimulation # isort: off from aspire.source.micrograph import ( diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index fb1b5338b5..f27597b21d 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -538,7 +538,7 @@ def true_snr(self, *args, **kwargs): return signal_power / noise_power -class LegacySimulation(Simulation): +class _LegacySimulation(Simulation): """ Legacy Simulation enforces the legacy grid convention for generating projection images. diff --git a/tests/test_anisotropic_noise.py b/tests/test_anisotropic_noise.py index 37bc2ece56..12b52064ea 100644 --- a/tests/test_anisotropic_noise.py +++ b/tests/test_anisotropic_noise.py @@ -5,7 +5,7 @@ from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator from aspire.operators import RadialCTFFilter -from aspire.source import ArrayImageSource, LegacySimulation +from aspire.source import ArrayImageSource, _LegacySimulation from aspire.utils import utest_tolerance from aspire.volume import LegacyVolume @@ -16,7 +16,7 @@ class SimTestCase(TestCase): def setUp(self): self.dtype = np.float32 self.vol = LegacyVolume(L=8, dtype=self.dtype).generate() - self.sim = LegacySimulation( + self.sim = _LegacySimulation( n=1024, vols=self.vol, unique_filters=[ diff --git a/tests/test_covar2d.py b/tests/test_covar2d.py index 1167957a7a..05e0eda509 100644 --- a/tests/test_covar2d.py +++ b/tests/test_covar2d.py @@ -9,7 +9,7 @@ from aspire.covariance import RotCov2D from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter -from aspire.source.simulation import LegacySimulation +from aspire.source.simulation import _LegacySimulation from aspire.utils import randi, utest_tolerance from aspire.volume import Volume @@ -97,7 +97,7 @@ def cov2d_fixture(volume, basis, ctf_enabled): noise_adder = WhiteNoiseAdder(var=NOISE_VAR) - sim = LegacySimulation( + sim = _LegacySimulation( n=n, vols=volume, unique_filters=unique_filters, diff --git a/tests/test_covar3d.py b/tests/test_covar3d.py index fb4ae38b35..9bf04b4594 100644 --- a/tests/test_covar3d.py +++ b/tests/test_covar3d.py @@ -12,7 +12,7 @@ from aspire.denoising import src_wiener_coords from aspire.operators import RadialCTFFilter from aspire.reconstruction import MeanEstimator -from aspire.source.simulation import LegacySimulation +from aspire.source.simulation import _LegacySimulation from aspire.utils import eigs from aspire.utils.random import Random from aspire.volume import LegacyVolume, Volume @@ -25,7 +25,7 @@ class Covar3DTestCase(TestCase): def setUpClass(cls): cls.dtype = np.float32 cls.vols = LegacyVolume(L=8, dtype=cls.dtype).generate() - cls.sim = LegacySimulation( + cls.sim = _LegacySimulation( n=1024, vols=cls.vols, unique_filters=[ diff --git a/tests/test_mean_estimator.py b/tests/test_mean_estimator.py index ec4e711288..2c7563dbb8 100644 --- a/tests/test_mean_estimator.py +++ b/tests/test_mean_estimator.py @@ -7,7 +7,7 @@ from aspire.basis import FBBasis3D from aspire.operators import RadialCTFFilter from aspire.reconstruction import MeanEstimator -from aspire.source.simulation import LegacySimulation +from aspire.source.simulation import _LegacySimulation from aspire.volume import LegacyVolume DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") @@ -18,7 +18,7 @@ def setUp(self): self.dtype = np.float32 self.resolution = 8 self.vols = LegacyVolume(L=self.resolution, dtype=self.dtype).generate() - self.sim = LegacySimulation( + self.sim = _LegacySimulation( n=1024, vols=self.vols, unique_filters=[ diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 7634d957ad..c3144ae598 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -8,7 +8,7 @@ from aspire.noise import WhiteNoiseAdder from aspire.operators import RadialCTFFilter -from aspire.source import LegacySimulation, RelionSource, Simulation +from aspire.source import RelionSource, Simulation, _LegacySimulation from aspire.utils import utest_tolerance from aspire.volume import LegacyVolume, SymmetryGroup, Volume @@ -112,7 +112,7 @@ def setUp(self): dtype=self.dtype, ).generate() - self.sim = LegacySimulation( + self.sim = _LegacySimulation( n=self.n, L=self.L, vols=self.vols, @@ -158,7 +158,7 @@ def testSimulationImages(self): ) def testSimulationCached(self): - sim_cached = LegacySimulation( + sim_cached = _LegacySimulation( n=self.n, L=self.L, vols=self.vols, diff --git a/tests/test_volume.py b/tests/test_volume.py index 9dabff8396..26c49a7d80 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -8,7 +8,7 @@ from numpy import pi from pytest import raises, skip -from aspire.source import LegacySimulation +from aspire.source import _LegacySimulation from aspire.utils import Rotation, anorm, grid_2d, powerset, utest_tolerance from aspire.volume import ( AsymmetricVolume, @@ -301,7 +301,7 @@ def test_project(vols_1, dtype): rots = np.moveaxis(rots, 2, 0) # Note, transforming rotations to compensate for legacy grid convention used in saved data. - rots = LegacySimulation.rots_zyx_to_legacy_aspire(rots) + rots = _LegacySimulation.rots_zyx_to_legacy_aspire(rots) imgs_clean = vols.project(rots).asnumpy() assert np.allclose(results, imgs_clean, atol=1e-7) diff --git a/tests/test_weighted_mean_estimator.py b/tests/test_weighted_mean_estimator.py index 797d22202b..4d185ce142 100644 --- a/tests/test_weighted_mean_estimator.py +++ b/tests/test_weighted_mean_estimator.py @@ -7,7 +7,7 @@ from aspire.basis import FBBasis3D from aspire.operators import RadialCTFFilter from aspire.reconstruction import WeightedVolumesEstimator -from aspire.source import LegacySimulation +from aspire.source import _LegacySimulation from aspire.volume import LegacyVolume logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def setUp(self): self.n = 1024 self.r = 2 self.L = L = 8 - self.sim = LegacySimulation( + self.sim = _LegacySimulation( vols=LegacyVolume(L, dtype=self.dtype).generate(), n=self.n, unique_filters=[ From 9a645de1b43c385e2f82497000c77559c3e1166b Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 8 Nov 2023 12:22:15 -0500 Subject: [PATCH 215/294] Add comment about negative thetas. --- tests/test_averager2d.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_averager2d.py b/tests/test_averager2d.py index 12bf4d4cdf..efa3a683ac 100644 --- a/tests/test_averager2d.py +++ b/tests/test_averager2d.py @@ -127,7 +127,9 @@ def _construct_rotations(self): 0, 2 * np.pi, num=self.n_img, endpoint=False, retstep=True, dtype=self.dtype ) - # Generate rotations to be used by `Simulation` + # Generate rotations to be used by `Simulation`. Since `Simulation` rotates + # the coordinate grid and the averager aligns by rotating the projection images, + # we negate the angles fed into `Simulation` for direct comparison later. self.rotations = Rotation.about_axis( "z", -self.thetas, dtype=self.dtype, gimble_lock_warnings=False ) From 9d5d9b4eddb349320bd6245cc258c2e53741ad27 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 8 Nov 2023 07:55:19 -0500 Subject: [PATCH 216/294] add author note and paper to ctf module --- src/aspire/ctf/ctf_estimator.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/aspire/ctf/ctf_estimator.py b/src/aspire/ctf/ctf_estimator.py index 98d0a2dc37..b9c2307c04 100644 --- a/src/aspire/ctf/ctf_estimator.py +++ b/src/aspire/ctf/ctf_estimator.py @@ -1,8 +1,26 @@ """ +Contains code supporting CTF parameter estimation. +Generally, this is a port of ASPIRE-CTF from MATLAB. + +See paper: + + | "Reducing bias and variance for CTF estimation in single particle cryo-EM" + | Ayelet Heimowitz, Joakim Andén, Amit Singer + | Ultramicroscopy, Volume 212, 2020 + | https://doi.org/10.1016/j.ultramic.2020.112950. + +Note: +`CtfEstimator` computes the background as a monotonically decreasing +function of spatial frequency. This practice may lead to an inaccurate +background estimation for experimental images produced using a K2 +camera in counting mode, as the background in this case is not +monotonically decreasing. Despite this, CTF parameters are captured +successfully in such situations. + Created on Sep 10, 2019 @author: Ayelet Heimowitz, Amit Moscovich -Integrated into ASPIRE by Garrett Wright Feb 2021. +Integrated into ASPIRE-Python by Garrett Wright Feb 2021. """ import logging From 6b2f7a786e39aff28dc5fde70602c944e3010c90 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 9 Nov 2023 11:09:16 -0500 Subject: [PATCH 217/294] add extra backticks for sphinx markup --- src/aspire/ctf/ctf_estimator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/ctf/ctf_estimator.py b/src/aspire/ctf/ctf_estimator.py index b9c2307c04..0288c0ae6f 100644 --- a/src/aspire/ctf/ctf_estimator.py +++ b/src/aspire/ctf/ctf_estimator.py @@ -10,7 +10,7 @@ | https://doi.org/10.1016/j.ultramic.2020.112950. Note: -`CtfEstimator` computes the background as a monotonically decreasing +``CtfEstimator`` computes the background as a monotonically decreasing function of spatial frequency. This practice may lead to an inaccurate background estimation for experimental images produced using a K2 camera in counting mode, as the background in this case is not From fba69be51bf73afef7200c099d396f2f2ed8c0dc Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 10 Nov 2023 09:24:43 -0500 Subject: [PATCH 218/294] Add small array handling hack used in MATLAB code --- src/aspire/classification/legacy_implementations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/classification/legacy_implementations.py b/src/aspire/classification/legacy_implementations.py index 5fb94649bf..2809a175ce 100644 --- a/src/aspire/classification/legacy_implementations.py +++ b/src/aspire/classification/legacy_implementations.py @@ -154,7 +154,7 @@ def bispec_2drot_large(coef, freqs, eigval, alpha, sample_n, seed=None): m = np.exp(o1 * coef_norm + 1j * o2 * phase) # svd of the reduced bispectrum - u, s, v = pca_y(m, 300, seed=seed) + u, s, v = pca_y(m, min(300, len(m)), seed=seed) coef_b = np.einsum("i, ij -> ij", s, np.conjugate(v)) coef_b_r = np.conjugate(u.T).dot(np.conjugate(m)) From 4cc4d61aa02e430875fe3890f57ade7db5fb4a25 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 10 Nov 2023 09:53:46 -0500 Subject: [PATCH 219/294] fix seed for class avg src classifier --- tests/test_class_src.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_class_src.py b/tests/test_class_src.py index b12f7e2987..5251c98037 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -35,6 +35,8 @@ DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") +# RNG SEED, should help small class average tests be deterministic. +SEED = 5552368 IMG_SIZES = [ 16, @@ -151,6 +153,7 @@ def classifier(class_sim_fixture): large_pca_implementation="legacy", nn_implementation="legacy", bispectrum_implementation="legacy", + seed=SEED, ) @@ -223,6 +226,7 @@ def cls_fixture(class_sim_fixture): bispectrum_components=101, # Compressed Features after last PCA stage. n_nbor=10, nn_implementation="sklearn", + seed=SEED, ) # Compute the classification # (classes, reflections, distances) From c9e224ad4efde4fcbdbdd2a35232c8124693349c Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 14 Nov 2023 13:40:57 -0500 Subject: [PATCH 220/294] remove unused pswf indices --- src/aspire/basis/pswf_2d.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index b46bfd60e5..7e2f75901e 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -94,14 +94,6 @@ def _precomp(self): """ self._generate_samples() - self.non_neg_freq_inds = slice(0, len(self.complex_angular_indices)) - - tmp = np.nonzero(self.complex_angular_indices == 0)[0] - self.zero_freq_inds = slice(tmp[0], tmp[-1] + 1) - - tmp = np.nonzero(self.complex_angular_indices > 0)[0] - self.pos_freq_inds = slice(tmp[0], tmp[-1] + 1) - def _generate_samples(self): """ Generate sample points for PSWF functions From ba46bcace3a9e4534d22f797bff245e155296362 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 14 Nov 2023 13:59:16 -0500 Subject: [PATCH 221/294] remove deprecated 2d `indices` and `_indices` --- src/aspire/basis/fb_2d.py | 11 ----------- src/aspire/basis/ffb_2d.py | 7 ++----- src/aspire/basis/fle_2d.py | 10 ---------- src/aspire/basis/pswf_2d.py | 12 ------------ src/aspire/basis/steerable.py | 1 - 5 files changed, 2 insertions(+), 39 deletions(-) diff --git a/src/aspire/basis/fb_2d.py b/src/aspire/basis/fb_2d.py index 7471e08bec..2698e41ed6 100644 --- a/src/aspire/basis/fb_2d.py +++ b/src/aspire/basis/fb_2d.py @@ -63,7 +63,6 @@ def _build(self): # generate 1D indices for basis functions self._compute_indices() - self._indices = self.indices() # get normalized factors self.radial_norms, self.angular_norms = self.norms() @@ -110,16 +109,6 @@ def _compute_indices(self): self.radial_indices = indices_ks self.signs_indices = indices_sgns - def indices(self): - """ - Return the precomputed indices for each basis function. - """ - return { - "ells": self.angular_indices, - "ks": self.radial_indices, - "sgns": self.signs_indices, - } - def _precomp(self): """ Precompute the basis functions at defined sample points diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index e900c39905..27748d2132 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -51,7 +51,6 @@ def _build(self): # generate 1D indices for basis functions self._compute_indices() - self._indices = self.indices() # get normalized factors self.radial_norms, self.angular_norms = self.norms() @@ -118,7 +117,6 @@ def _evaluate(self, v): # go through each basis function and find corresponding coefficient pf = np.zeros((n_data, 2 * n_theta, n_r), dtype=complex_type(self.dtype)) - mask = self._indices["ells"] == 0 ind = 0 @@ -126,7 +124,7 @@ def _evaluate(self, v): # include the normalization factor of angular part into radial part radial_norm = self._precomp["radial"] / np.expand_dims(self.angular_norms, 1) - pf[:, 0, :] = v[:, mask] @ radial_norm[idx] + pf[:, 0, :] = v[:, self._zero_angular_indices] @ radial_norm[idx] ind = ind + np.size(idx) ind_pos = ind @@ -216,11 +214,10 @@ def _evaluate_t(self, x): # go through each basis function and find the corresponding coefficient ind = 0 idx = ind + np.arange(self.k_max[0]) - mask = self._indices["ells"] == 0 # include the normalization factor of angular part into radial part radial_norm = self._precomp["radial"] / np.expand_dims(self.angular_norms, 1) - v[:, mask] = pf[:, :, 0].real @ radial_norm[idx].T + v[:, self._zero_angular_indices] = pf[:, :, 0].real @ radial_norm[idx].T ind = ind + np.size(idx) ind_pos = ind diff --git a/src/aspire/basis/fle_2d.py b/src/aspire/basis/fle_2d.py index cad81fcfdd..423d37c093 100644 --- a/src/aspire/basis/fle_2d.py +++ b/src/aspire/basis/fle_2d.py @@ -143,16 +143,6 @@ def _build_indices(self): ci += len(ks) - def indices(self): - """ - Return the precomputed indices for each basis function. - """ - return { - "ells": self.angular_indices, - "ks": self.radial_indices, - "sgns": self.signs_indices, - } - def _precomp(self): """ Precompute the basis functions and other objects used in the evaluation of diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 7e2f75901e..3fc0638e4a 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -181,18 +181,6 @@ def _generate_samples(self): ci += len(ks) - # Added for compatibility. - # Probably can remove `indices` dict wholesale later (MATLAB holdover). - def indices(self): - """ - Return the precomputed indices for each basis function. - """ - return { - "ells": self.angular_indices, - "ks": self.radial_indices, - "sgns": self.signs_indices, - } - def _evaluate_t(self, images): """ Evaluate coefficient vectors in PSWF basis using the direct method diff --git a/src/aspire/basis/steerable.py b/src/aspire/basis/steerable.py index 698ad37a34..a2b9872886 100644 --- a/src/aspire/basis/steerable.py +++ b/src/aspire/basis/steerable.py @@ -33,7 +33,6 @@ def __init__(self, *args, **kwargs): self._blk_diag_cov_shape = None # Centralize indices attributes between FB/PSWF/FLE in SteerableBasis2D - self._indices = self.indices() self.complex_count = self.count - sum(self._neg_angular_inds) self.complex_angular_indices = self.angular_indices[self._non_neg_angular_inds] self.complex_radial_indices = self.radial_indices[self._non_neg_angular_inds] From 1e7648c31fafc90423d60b3d14b44f36af7a2941 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 14 Nov 2023 14:55:08 -0500 Subject: [PATCH 222/294] update deprecated 2d `indices` and `_indices` test references to class vars --- src/aspire/basis/basis.py | 6 ------ src/aspire/basis/ffb_2d.py | 4 ++-- tests/_basis_util.py | 12 +++++------- tests/test_FBbasis2D.py | 7 +++---- tests/test_FFBbasis2D.py | 7 +++---- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 5988ec37ca..873e16080c 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -409,12 +409,6 @@ def _build(self): """ raise NotImplementedError("subclasses must implement this") - def indices(self): - """ - Create the indices for each basis function - """ - raise NotImplementedError("subclasses must implement this") - def _precomp(self): """ Precompute the basis functions at defined sample points diff --git a/src/aspire/basis/ffb_2d.py b/src/aspire/basis/ffb_2d.py index 27748d2132..5a5c7c3f27 100644 --- a/src/aspire/basis/ffb_2d.py +++ b/src/aspire/basis/ffb_2d.py @@ -124,7 +124,7 @@ def _evaluate(self, v): # include the normalization factor of angular part into radial part radial_norm = self._precomp["radial"] / np.expand_dims(self.angular_norms, 1) - pf[:, 0, :] = v[:, self._zero_angular_indices] @ radial_norm[idx] + pf[:, 0, :] = v[:, self._zero_angular_inds] @ radial_norm[idx] ind = ind + np.size(idx) ind_pos = ind @@ -217,7 +217,7 @@ def _evaluate_t(self, x): # include the normalization factor of angular part into radial part radial_norm = self._precomp["radial"] / np.expand_dims(self.angular_norms, 1) - v[:, self._zero_angular_indices] = pf[:, :, 0].real @ radial_norm[idx].T + v[:, self._zero_angular_inds] = pf[:, :, 0].real @ radial_norm[idx].T ind = ind + np.size(idx) ind_pos = ind diff --git a/tests/_basis_util.py b/tests/_basis_util.py index 5cc0bb5237..5138887496 100644 --- a/tests/_basis_util.py +++ b/tests/_basis_util.py @@ -45,8 +45,6 @@ def testIndices(self, basis): ell_max = basis.ell_max k_max = basis.k_max - indices = basis.indices() - i = 0 for ell in range(ell_max + 1): @@ -57,9 +55,9 @@ def testIndices(self, basis): for sgn in sgns: for k in range(k_max[ell]): - assert indices["ells"][i] == ell - assert indices["sgns"][i] == sgn - assert indices["ks"][i] == k + assert basis.angular_indices[i] == ell + assert basis.signs_indices[i] == sgn + assert basis.radial_indices[i] == k i += 1 @@ -99,7 +97,7 @@ def testIsotropic(self, basis): coef_np = basis.expand(im).asnumpy() - ells = basis.indices()["ells"] + ells = basis.angular_indices energy_outside = np.sum(np.abs(coef_np[..., ells != 0]) ** 2) energy_total = np.sum(np.abs(coef_np) ** 2) @@ -125,7 +123,7 @@ def testModulated(self, basis): coef_np = basis.expand(im1).asnumpy() - ells = basis.indices()["ells"] + ells = basis.angular_indices energy_outside = np.sum(np.abs(coef_np[..., ells != ell]) ** 2) energy_total = np.sum(np.abs(coef_np) ** 2) diff --git a/tests/test_FBbasis2D.py b/tests/test_FBbasis2D.py index b24c14568d..2682747ba1 100644 --- a/tests/test_FBbasis2D.py +++ b/tests/test_FBbasis2D.py @@ -33,10 +33,9 @@ def _testElement(self, basis, ell, k, sgn): # This is covered by the isotropic test. assert ell > 0 - indices = basis.indices() - ells = indices["ells"] - sgns = indices["sgns"] - ks = indices["ks"] + ells = basis.angular_indices + sgns = basis.signs_indices + ks = basis.radial_indices g2d = grid_2d(basis.nres, dtype=basis.dtype) mask = g2d["r"] < 1 diff --git a/tests/test_FFBbasis2D.py b/tests/test_FFBbasis2D.py index d2977f6e4e..04b7a9b4c7 100644 --- a/tests/test_FFBbasis2D.py +++ b/tests/test_FFBbasis2D.py @@ -31,10 +31,9 @@ class TestFFBBasis2D(Steerable2DMixin, UniversalBasisMixin): seed = 9161341 def _testElement(self, basis, ell, k, sgn): - indices = basis.indices() - ells = indices["ells"] - sgns = indices["sgns"] - ks = indices["ks"] + ells = basis.angular_indices + sgns = basis.signs_indices + ks = basis.radial_indices g2d = grid_2d(basis.nres, dtype=basis.dtype) mask = g2d["r"] < 1 From dc8df29f9bd7cc50b73df6cd80ff01f797a0e81d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 15 Nov 2023 09:37:18 -0500 Subject: [PATCH 223/294] continue updating tests, removing old indice refs --- src/aspire/basis/fspca.py | 14 +++++++------- src/aspire/covariance/covar2d.py | 13 +++++++------ src/aspire/ctf/ctf_estimator.py | 8 ++++---- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index e702532a5e..d7cc4c96a6 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -71,7 +71,7 @@ def __init__( self.complex_count = self.basis.complex_count self.angular_indices = self.basis.angular_indices self.radial_indices = self.basis.radial_indices - self.signs_indices = self.basis._indices["sgns"] + self.signs_indices = self.basis.signs_indices self.complex_angular_indices = self.basis.complex_angular_indices self.complex_radial_indices = self.basis.complex_radial_indices @@ -243,7 +243,7 @@ def _compute_spca(self): self.mean_coef_zero = self.mean_coef_est.asnumpy()[0][self.angular_indices == 0] # Define mask for zero angular mode, used in loop below - zero_ell_mask = self.basis._indices["ells"] == 0 + zero_ell_mask = self.basis.angular_indices == 0 # Apply Data matrix batchwise num_batches = (self.src.n + self.batch_size - 1) // self.batch_size @@ -275,9 +275,9 @@ def _compute_spca(self): for ell in range( 1, self.basis.ell_max + 1 ): # `ell` in this code is `k` from paper - mask_ell = self.basis._indices["ells"] == ell - mask_pos = mask_ell & (self.basis._indices["sgns"] == +1) - mask_neg = mask_ell & (self.basis._indices["sgns"] == -1) + mask_ell = self.basis.angular_indices == ell + mask_pos = mask_ell & (self.basis.signs_indices == +1) + mask_neg = mask_ell & (self.basis.signs_indices == -1) A.append(batch_coef[:, mask_pos]) A.append(batch_coef[:, mask_neg]) @@ -405,8 +405,8 @@ def _get_compressed_indices(self): top_components = list(ordered_components)[: self.components] # Now we need to find the locations of both the + and - sgns. - pos_mask = self.basis._indices["sgns"] == 1 - neg_mask = self.basis._indices["sgns"] == -1 + pos_mask = self.basis.signs_indices == 1 + neg_mask = self.basis.signs_indices == -1 compressed_indices = [] for k, q in top_components: # Compute the locations of coefs we're interested in. diff --git a/src/aspire/covariance/covar2d.py b/src/aspire/covariance/covar2d.py index 68343c418a..a4f1971b78 100644 --- a/src/aspire/covariance/covar2d.py +++ b/src/aspire/covariance/covar2d.py @@ -123,9 +123,10 @@ def _get_mean(self, coefs): if coefs.size == 0: raise RuntimeError("The coefficients need to be calculated first!") - mask = self.basis._indices["ells"] == 0 mean_coef = np.zeros(self.basis.count, dtype=coefs.dtype) - mean_coef[mask] = np.mean(coefs[..., mask], axis=0) + mean_coef[self.basis._zero_angular_inds] = np.mean( + coefs[..., self.basis._zero_angular_inds], axis=0 + ) return mean_coef @@ -147,16 +148,16 @@ def _get_covar(self, coefs, mean_coef=None, do_refl=True): covar_coef = BlkDiagMatrix.empty(0, dtype=coefs.dtype) ell = 0 - mask = self.basis._indices["ells"] == ell + mask = self.basis.angular_indices == ell coef_ell = coefs[..., mask] - mean_coef[mask] covar_ell = np.array(coef_ell.T @ coef_ell / coefs.shape[0]) covar_coef.append(covar_ell) for ell in range(1, self.basis.ell_max + 1): - mask_ell = self.basis._indices["ells"] == ell - mask_pos = mask_ell & (self.basis._indices["sgns"] == +1) - mask_neg = mask_ell & (self.basis._indices["sgns"] == -1) + mask_ell = self.basis.angular_indices == ell + mask_pos = mask_ell & (self.basis.signs_indices == +1) + mask_neg = mask_ell & (self.basis.signs_indices == -1) covar_ell_diag = np.array( coefs[:, mask_pos].T @ coefs[:, mask_pos] diff --git a/src/aspire/ctf/ctf_estimator.py b/src/aspire/ctf/ctf_estimator.py index 0288c0ae6f..b6983cb071 100644 --- a/src/aspire/ctf/ctf_estimator.py +++ b/src/aspire/ctf/ctf_estimator.py @@ -287,13 +287,13 @@ def elliptical_average(self, ffbbasis, amplitude_spectrum, circular): coefs_s = ffbbasis.evaluate_t(amplitude_spectrum).asnumpy().copy().T coefs_n = coefs_s.copy() - coefs_s[np.argwhere(ffbbasis._indices["ells"] == 1)] = 0 + coefs_s[np.argwhere(ffbbasis.angular_indices == 1)] = 0 if circular: - coefs_s[np.argwhere(ffbbasis._indices["ells"] == 2)] = 0 + coefs_s[np.argwhere(ffbbasis.angular_indices == 2)] = 0 noise = amplitude_spectrum else: - coefs_n[np.argwhere(ffbbasis._indices["ells"] == 0)] = 0 - coefs_n[np.argwhere(ffbbasis._indices["ells"] == 2)] = 0 + coefs_n[np.argwhere(ffbbasis.angular_indices == 0)] = 0 + coefs_n[np.argwhere(ffbbasis.angular_indices == 2)] = 0 noise = Coef(ffbbasis, coefs_n.T).evaluate() psd = Coef(ffbbasis, coefs_s.T).evaluate() From b8b46c049b5407995e00115a4a519efd8bdfb0c4 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 8 Nov 2023 11:22:38 -0500 Subject: [PATCH 224/294] update internal 2d prolate grid orientation to yx --- src/aspire/basis/fpswf_2d.py | 4 ++-- src/aspire/basis/pswf_2d.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index 09d67bff5f..4143171c53 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -177,8 +177,8 @@ def _generate_pswf_quad( ] ) - pts_x = quad_rule_pts_r * np.cos(quad_rule_pts_theta) - pts_y = quad_rule_pts_r * np.sin(quad_rule_pts_theta) + pts_x = quad_rule_pts_r * np.sin(quad_rule_pts_theta) + pts_y = quad_rule_pts_r * np.cos(quad_rule_pts_theta) return ( pts_x, diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 3fc0638e4a..4546fb48a7 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -83,7 +83,7 @@ def _generate_grid(self): """ Generate the 2D sampling grid """ - grid = grid_2d(self.nres, normalized=False, indexing="xy") + grid = grid_2d(self.nres, normalized=False, indexing="yx") self._disk_mask = grid["r"] <= self.rcut self._r_disk = grid["r"][self._disk_mask] / self.rcut self._theta_disk = grid["phi"][self._disk_mask] From 3ae2a1c0f08303cfb3ddd968d456613ea9d41cef Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 8 Nov 2023 11:23:01 -0500 Subject: [PATCH 225/294] remove rotation orientation hack --- src/aspire/basis/pswf_2d.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/aspire/basis/pswf_2d.py b/src/aspire/basis/pswf_2d.py index 4546fb48a7..c9795ec1bc 100644 --- a/src/aspire/basis/pswf_2d.py +++ b/src/aspire/basis/pswf_2d.py @@ -403,19 +403,3 @@ def filter_to_basis_mat(self, *args, **kwargs): See `SteerableBasis2D.filter_to_basis_mat`. """ return super().filter_to_basis_mat(*args, **kwargs) - - def rotate(self, coef, radians, refl=None): - """ - See `SteerableBasis2D.rotate`. - """ - # {F}PSWF rotation convention is still CW internally. - # This will make things consistent until that is addressed. - return super().rotate(coef, -radians, refl=refl) - - def complex_rotate(self, complex_coef, radians, refl=None): - """ - See `SteerableBasis2D.rotate`. - """ - # {F}PSWF rotation convention is still CW internally. - # This will make things consistent until that is addressed. - return super().complex_rotate(complex_coef, -radians, refl=refl) From 4e4ba413353acc71900252d4c9ff6ffa21b46241 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 8 Nov 2023 11:38:00 -0500 Subject: [PATCH 226/294] update prolate unit tests after 'yx' internal grid change --- tests/test_FPSWFbasis2D.py | 13 +++++++------ tests/test_PSWFbasis2D.py | 13 ++++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/test_FPSWFbasis2D.py b/tests/test_FPSWFbasis2D.py index 7c9a7133fb..b53c043a2c 100644 --- a/tests/test_FPSWFbasis2D.py +++ b/tests/test_FPSWFbasis2D.py @@ -17,12 +17,11 @@ @pytest.mark.parametrize("basis", test_bases, ids=show_basis_params) class TestFPSWFBasis2D(UniversalBasisMixin): def testFPSWFBasis2DEvaluate_t(self, basis): - img_ary = np.load( - os.path.join(DATA_DIR, "ffbbasis2d_xcoef_in_8_8.npy") - ).T # RCOPT + img_ary = np.load(os.path.join(DATA_DIR, "ffbbasis2d_xcoef_in_8_8.npy")) images = Image(img_ary) result = basis.evaluate_t(images) + # Historically, FPSWF returned complex values. # Load and convert them for this hard coded test. ccoefs = np.load(os.path.join(DATA_DIR, "pswf2d_vcoefs_out_8_8.npy")).T # RCOPT @@ -37,7 +36,9 @@ def testFPSWFBasis2DEvaluate(self, basis): coefs = ComplexCoef(basis, ccoefs).to_real() result = coefs.evaluate() - result = basis.evaluate(coefs) - images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoef_out_8_8.npy")).T # RCOPT + # This hardcoded reference result requires transposing the stack axis. + images = np.transpose( + np.load(os.path.join(DATA_DIR, "pswf2d_xcoef_out_8_8.npy")), (2, 0, 1) + ) - np.testing.assert_allclose(result, images, rtol=1e-05, atol=1e-08) + np.testing.assert_allclose(result.asnumpy(), images, rtol=1e-05, atol=1e-08) diff --git a/tests/test_PSWFbasis2D.py b/tests/test_PSWFbasis2D.py index 21c182d6da..3660843ccd 100644 --- a/tests/test_PSWFbasis2D.py +++ b/tests/test_PSWFbasis2D.py @@ -16,9 +16,7 @@ @pytest.mark.parametrize("basis", test_bases, ids=show_basis_params) class TestPSWFBasis2D(UniversalBasisMixin): def testPSWFBasis2DEvaluate_t(self, basis): - img_ary = np.load( - os.path.join(DATA_DIR, "ffbbasis2d_xcoef_in_8_8.npy") - ).T # RCOPT + img_ary = np.load(os.path.join(DATA_DIR, "ffbbasis2d_xcoef_in_8_8.npy")) images = Image(img_ary) result = basis.evaluate_t(images) @@ -37,5 +35,10 @@ def testPSWFBasis2DEvaluate(self, basis): coefs = ComplexCoef(basis, ccoefs).to_real() result = coefs.evaluate() - images = np.load(os.path.join(DATA_DIR, "pswf2d_xcoef_out_8_8.npy")).T # RCOPT - assert np.allclose(result.asnumpy(), images) + + # This hardcoded reference result requires transposing the stack axis. + images = np.transpose( + np.load(os.path.join(DATA_DIR, "pswf2d_xcoef_out_8_8.npy")), (2, 0, 1) + ) + + np.testing.assert_allclose(result.asnumpy(), images, rtol=1e-05, atol=1e-08) From 84450f600d40c53adc74baaf032af535a79feebb Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 16 Nov 2023 07:57:16 -0500 Subject: [PATCH 227/294] restore ptx_{x,y} labels, swap in returned param order --- src/aspire/basis/fpswf_2d.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aspire/basis/fpswf_2d.py b/src/aspire/basis/fpswf_2d.py index 4143171c53..5476eed61d 100644 --- a/src/aspire/basis/fpswf_2d.py +++ b/src/aspire/basis/fpswf_2d.py @@ -177,12 +177,12 @@ def _generate_pswf_quad( ] ) - pts_x = quad_rule_pts_r * np.sin(quad_rule_pts_theta) - pts_y = quad_rule_pts_r * np.cos(quad_rule_pts_theta) + pts_x = quad_rule_pts_r * np.cos(quad_rule_pts_theta) + pts_y = quad_rule_pts_r * np.sin(quad_rule_pts_theta) return ( - pts_x, pts_y, + pts_x, quad_rule_weights, radial_quad_points, quad_rule_radial_weights, From 750517daf17d0b3b8e246a08663100eee11dc9b9 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 17 Nov 2023 09:34:58 -0500 Subject: [PATCH 228/294] increase sample_n for small class averaging problems --- tests/test_class_src.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_class_src.py b/tests/test_class_src.py index 5251c98037..a1cc9cbc6b 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -150,6 +150,7 @@ def classifier(class_sim_fixture): fspca_components=123, bispectrum_components=101, # Compressed Features after last PCA stage. n_nbor=10, + sample_n=50000, large_pca_implementation="legacy", nn_implementation="legacy", bispectrum_implementation="legacy", @@ -225,6 +226,7 @@ def cls_fixture(class_sim_fixture): fspca_components=123, bispectrum_components=101, # Compressed Features after last PCA stage. n_nbor=10, + sample_n=50000, nn_implementation="sklearn", seed=SEED, ) From f3b43195b0f793bdb38153d4d6eddf5ed4ba97c6 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Fri, 17 Nov 2023 14:14:04 -0500 Subject: [PATCH 229/294] Loosen test tolerance for nonzero shifts. --- tests/test_orient_sync_voting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_orient_sync_voting.py b/tests/test_orient_sync_voting.py index bd91e98e71..31d6b20e94 100644 --- a/tests/test_orient_sync_voting.py +++ b/tests/test_orient_sync_voting.py @@ -87,6 +87,9 @@ def test_build_clmatrix(source_orientation_objs): # Check that at least 98% of estimates are within 5 degrees. tol = 0.98 + if src.offsets.all() != 0: + # Set tolerance to 95% when using nonzero offsets. + tol = 0.95 assert within_5 / angle_diffs.size > tol From ff14f5a1ceaf32b6c6d13169b2a5643bc9db3513 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 10 Nov 2023 13:59:10 -0500 Subject: [PATCH 230/294] Add some details to the cg call under test. Add `tol` and `atol` args. Keeps defaults for now. [skip ci] --- src/aspire/basis/basis.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 873e16080c..1af960851c 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -523,7 +523,7 @@ def mat_evaluate_t(self, X): """ return mdim_mat_fun_conj(X, len(self.sz), 1, self._evaluate_t) - def expand(self, x): + def expand(self, x, tol=None, atol=0): """ Obtain coefficients in the basis from those in standard coordinate basis @@ -565,9 +565,10 @@ def expand(self, x): dtype=self.dtype, ) - # TODO: (from MATLAB implementation) - Check that this tolerance make sense for multiple columns in v - tol = 10 * np.finfo(x.dtype).eps - logger.info("Expanding array in basis") + if tol is None: + # TODO: (from MATLAB implementation) - Check that this tolerance make sense for multiple columns in v + tol = 10 * np.finfo(x.dtype).eps + logger.info(f"Expanding array in basis with tol={tol} atol={atol}") # number of image samples n_data = x.shape[0] @@ -576,9 +577,9 @@ def expand(self, x): for isample in range(0, n_data): b = self.evaluate_t(self._cls(x[isample])).asnumpy().T # TODO: need check the initial condition x0 can improve the results or not. - v[isample], info = cg(operator, b, tol=tol, atol=0) + v[isample], info = cg(operator, b, tol=tol, atol=atol) if info != 0: - raise RuntimeError("Unable to converge!") + raise RuntimeError(f"Unable to converge! cg info={info}") # return v coefficients with the last dimension of self.count v = v.reshape((*sz_roll, self.count)) From da3f6e7543462d5e3fbf8fd64289df6e5af80d9a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 13 Nov 2023 08:08:04 -0500 Subject: [PATCH 231/294] set constant amplitudes and offsets for FLE basis test --- tests/test_FLEbasis2D.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_FLEbasis2D.py b/tests/test_FLEbasis2D.py index dd37303f2c..9fb5504202 100644 --- a/tests/test_FLEbasis2D.py +++ b/tests/test_FLEbasis2D.py @@ -55,7 +55,9 @@ def create_images(L, n): np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype(np.float64) ) v = v.downsample(L) - sim = Simulation(L=L, n=n, vols=v, dtype=v.dtype, seed=1103) + sim = Simulation( + L=L, n=n, vols=v, dtype=v.dtype, offsets=0, amplitudes=1, seed=1103 + ) img = sim.clean_images[:] return img From 7a1a1651d7a167e1cf0dac4c74980025aa1f36c5 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 13 Nov 2023 10:22:56 -0500 Subject: [PATCH 232/294] add docstring for optional params --- src/aspire/basis/basis.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 1af960851c..173bf28900 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -532,6 +532,9 @@ def expand(self, x, tol=None, atol=0): :param x: An array whose last two or three dimensions are to be expanded the desired basis. These dimensions must equal `self.sz`. + :param tol: Tolerances for convergence, `norm(residual) <= max(tol*norm(b), atol)`. + Deafult `None` sets to dtype's `eps`*10. + :param atol: Tolerances for convergence, `norm(residual) <= max(tol*norm(b), atol)`. :return: The coefficients of `v` expanded in the desired basis. The last dimension of `v` is with size of `count` and the first dimensions of the return value correspond to From 326347a67c872b4cabbdca56f5729c7be89cf451 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 16 Nov 2023 07:42:32 -0500 Subject: [PATCH 233/294] improve basis.expand docstring --- src/aspire/basis/basis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aspire/basis/basis.py b/src/aspire/basis/basis.py index 173bf28900..3dd768a788 100644 --- a/src/aspire/basis/basis.py +++ b/src/aspire/basis/basis.py @@ -532,9 +532,9 @@ def expand(self, x, tol=None, atol=0): :param x: An array whose last two or three dimensions are to be expanded the desired basis. These dimensions must equal `self.sz`. - :param tol: Tolerances for convergence, `norm(residual) <= max(tol*norm(b), atol)`. + :param tol: Relative tolerance for convergence, `norm(residual) <= max(tol*norm(b), atol)`. Deafult `None` sets to dtype's `eps`*10. - :param atol: Tolerances for convergence, `norm(residual) <= max(tol*norm(b), atol)`. + :param atol: Absolute tolerance for convergence, `norm(residual) <= max(tol*norm(b), atol)`. :return: The coefficients of `v` expanded in the desired basis. The last dimension of `v` is with size of `count` and the first dimensions of the return value correspond to From 8d2275a5f3a95917db5c0d1133db07ba0f9e0b58 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 20 Nov 2023 11:19:56 -0500 Subject: [PATCH 234/294] use magnitude of eigvals in bispect random vector selection --- src/aspire/classification/legacy_implementations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/classification/legacy_implementations.py b/src/aspire/classification/legacy_implementations.py index 2809a175ce..58671735c5 100644 --- a/src/aspire/classification/legacy_implementations.py +++ b/src/aspire/classification/legacy_implementations.py @@ -138,7 +138,7 @@ def bispec_2drot_large(coef, freqs, eigval, alpha, sample_n, seed=None): phase = coef[freqs_not_zero] / np.absolute(coef[freqs_not_zero]) phase = np.arctan2(np.imag(phase), np.real(phase)) - eigval = eigval[freqs_not_zero] + eigval = np.abs(eigval[freqs_not_zero]) o1, o2 = bispec_operator_1(freqs[freqs_not_zero]) # GBW, naively handle vanishing eigvals. From a676e0f684436f39ac83f9d2739977383f7b839f Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 21 Nov 2023 07:01:17 -0500 Subject: [PATCH 235/294] Revert "use magnitude of eigvals in bispect random vector selection" This reverts commit 4a42a42c91ecebe18638da7ac9094ae703f1bc77. --- src/aspire/classification/legacy_implementations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/classification/legacy_implementations.py b/src/aspire/classification/legacy_implementations.py index 58671735c5..2809a175ce 100644 --- a/src/aspire/classification/legacy_implementations.py +++ b/src/aspire/classification/legacy_implementations.py @@ -138,7 +138,7 @@ def bispec_2drot_large(coef, freqs, eigval, alpha, sample_n, seed=None): phase = coef[freqs_not_zero] / np.absolute(coef[freqs_not_zero]) phase = np.arctan2(np.imag(phase), np.real(phase)) - eigval = np.abs(eigval[freqs_not_zero]) + eigval = eigval[freqs_not_zero] o1, o2 = bispec_operator_1(freqs[freqs_not_zero]) # GBW, naively handle vanishing eigvals. From 81abef9e532601692cfc4938a570cd40423ac75b Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 27 Nov 2023 08:58:38 -0500 Subject: [PATCH 236/294] Update arg order. --- src/aspire/operators/filters.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aspire/operators/filters.py b/src/aspire/operators/filters.py index 6841640209..696482d649 100644 --- a/src/aspire/operators/filters.py +++ b/src/aspire/operators/filters.py @@ -109,7 +109,7 @@ def scale(self, c=1): """ return ScaledFilter(self, c) - def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): + def evaluate_grid(self, L, *args, dtype=np.float32, **kwargs): """ Generates a two dimensional grid with prescribed dtype, yielding the values (omega) which are then evaluated by @@ -191,7 +191,7 @@ def __init__(self, filter, power=1): def _evaluate(self, omega): return self._filter.evaluate(omega) ** self._power - def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): + def evaluate_grid(self, L, *args, dtype=np.float32, **kwargs): """ Calls the provided filter's evaluate_grid method in case there is an optimization. @@ -199,7 +199,7 @@ def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): See `Filter.evaluate_grid` for usage. """ - filter_vals = self._filter.evaluate_grid(L, dtype=dtype, *args, **kwargs) + filter_vals = self._filter.evaluate_grid(L, *args, dtype=dtype, **kwargs) # Place safeguard on values below machine epsilon for negative powers. if self._power < 0: @@ -338,7 +338,7 @@ def _evaluate(self, omega): return result - def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): + def evaluate_grid(self, L, *args, dtype=np.float32, **kwargs): """ Optimized evaluate_grid method for ArrayFilter. @@ -361,7 +361,7 @@ def evaluate_grid(self, L, dtype=np.float32, *args, **kwargs): res = self.xfer_fn_array else: # Otherwise call parent code to generate a grid then evaluate. - res = super().evaluate_grid(L, dtype=dtype, *args, **kwargs) + res = super().evaluate_grid(L, *args, dtype=dtype, **kwargs) return res From 08312985c8a60be7baae9620e81fbf33197b72a7 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 21 Nov 2023 14:59:32 -0500 Subject: [PATCH 237/294] Fix mistake in class name param log --- tests/test_class_src.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_class_src.py b/tests/test_class_src.py index a1cc9cbc6b..ede7cb397e 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -137,7 +137,7 @@ def class_sim_fixture(dtype, img_size): @pytest.fixture( - params=CLS_SRCS, ids=lambda param: f"ClassSource={param.__class__}", scope="module" + params=CLS_SRCS, ids=lambda param: f"ClassSource={param.__class__.__name__}", scope="module" ) def test_src_cls(request): return request.param From 8947c5536cf07a2cb6e336349a3b9ea028528051 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 22 Nov 2023 14:12:42 -0500 Subject: [PATCH 238/294] reduce bispect settings for smaller image size (32~>16) --- tests/test_class_src.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_class_src.py b/tests/test_class_src.py index ede7cb397e..0c169621cd 100644 --- a/tests/test_class_src.py +++ b/tests/test_class_src.py @@ -137,7 +137,9 @@ def class_sim_fixture(dtype, img_size): @pytest.fixture( - params=CLS_SRCS, ids=lambda param: f"ClassSource={param.__class__.__name__}", scope="module" + params=CLS_SRCS, + ids=lambda param: f"ClassSource={param.__class__.__name__}", + scope="module", ) def test_src_cls(request): return request.param @@ -147,8 +149,8 @@ def test_src_cls(request): def classifier(class_sim_fixture): return RIRClass2D( class_sim_fixture, - fspca_components=123, - bispectrum_components=101, # Compressed Features after last PCA stage. + fspca_components=63, + bispectrum_components=51, # Compressed Features after last PCA stage. n_nbor=10, sample_n=50000, large_pca_implementation="legacy", @@ -223,8 +225,8 @@ def cls_fixture(class_sim_fixture): # Create the classifier c2d = RIRClass2D( class_sim_fixture, - fspca_components=123, - bispectrum_components=101, # Compressed Features after last PCA stage. + fspca_components=63, + bispectrum_components=51, # Compressed Features after last PCA stage. n_nbor=10, sample_n=50000, nn_implementation="sklearn", From e64841a536e30f350b07f40e1182d8047eab0642 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 22 Nov 2023 14:13:46 -0500 Subject: [PATCH 239/294] force workflow run --- .github/workflows/long_workflow.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/long_workflow.yml b/.github/workflows/long_workflow.yml index 4346b93222..51a5f36f2b 100644 --- a/.github/workflows/long_workflow.yml +++ b/.github/workflows/long_workflow.yml @@ -2,9 +2,6 @@ name: ASPIRE Python Long Running Test Suite on: push: - branches: - - 'main' - - 'develop' jobs: expensive_tests: From 8d664b290f45451e950087c2b8c126a0a4ba62c5 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 27 Nov 2023 08:09:20 -0500 Subject: [PATCH 240/294] Revert "force workflow run" This reverts commit ed0119bf79b6f233dbbb6cd803ea4251f6e1688f. --- .github/workflows/long_workflow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/long_workflow.yml b/.github/workflows/long_workflow.yml index 51a5f36f2b..4346b93222 100644 --- a/.github/workflows/long_workflow.yml +++ b/.github/workflows/long_workflow.yml @@ -2,6 +2,9 @@ name: ASPIRE Python Long Running Test Suite on: push: + branches: + - 'main' + - 'develop' jobs: expensive_tests: From 711f4bb5802026aa4964f4ee6713561884e635a6 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 13 Nov 2023 11:41:21 -0500 Subject: [PATCH 241/294] Refactor Volume.rotate to rotate grid by inverse. Arbitrary rotations test. --- src/aspire/volume/volume.py | 22 ++++++++-------- tests/test_volume.py | 50 ++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/aspire/volume/volume.py b/src/aspire/volume/volume.py index c8d539974f..b271d17c20 100644 --- a/src/aspire/volume/volume.py +++ b/src/aspire/volume/volume.py @@ -434,10 +434,9 @@ def shift(self): def rotate(self, rot_matrices, zero_nyquist=True): """ - Rotate volumes using a `Rotation` object. If the `Rotation` object - is a single rotation, each volume will be rotated by that rotation. - If the `Rotation` object is a stack of rotations of length n_vols, - the ith volume is rotated by the ith rotation. + Rotate volumes, within a fixed grid, by `rot_matrices`. If `rot_matrices` is a single + rotation, each volume will be rotated by that rotation. If `rot_matrices` is a stack of + rotations of length n_vols, the ith volume will be rotated by the ith rotation. :param rot_matrices: `Rotation` object of length 1 or n_vols. :param zero_nyquist: Option to keep or remove Nyquist frequency for even resolution. @@ -454,13 +453,14 @@ def rotate(self, rot_matrices, zero_nyquist=True): rot_matrices, Rotation ), f"Argument must be an instance of the Rotation class. {type(rot_matrices)} was supplied." - # Get numpy representation of Rotation object. - rot_matrices = rot_matrices.matrices + # Invert the rotations passed to `rotated_grids_3d` and get numpy representation of Rotation object. + rots_inverted = rot_matrices.invert() + rots_inverted = rots_inverted.matrices - K = len(rot_matrices) # Rotation stack size + K = len(rots_inverted) # Rotation stack size assert K == self.n_vols or K == 1, "Rotation object must be length 1 or n_vols." - if rot_matrices.dtype != self.dtype: + if rots_inverted.dtype != self.dtype: logger.warning( f"{self.__class__.__name__}" f" rot_matrices.dtype {rot_matrices.dtype}" @@ -470,19 +470,19 @@ def rotate(self, rot_matrices, zero_nyquist=True): # If K = 1 we broadcast the single Rotation object across each volume. if K == 1: - pts_rot = rotated_grids_3d(self.resolution, rot_matrices) + pts_rot = rotated_grids_3d(self.resolution, rots_inverted) vol_f = nufft(self.asnumpy(), pts_rot) vol_f = vol_f.reshape(-1, self.resolution, self.resolution, self.resolution) # If K = n_vols, we apply the ith rotation to ith volume. else: - rot_matrices = rot_matrices.reshape((K, 1, 3, 3)) + rots_inverted = rots_inverted.reshape((K, 1, 3, 3)) pts_rot = np.zeros((K, 3, self.resolution**3), dtype=self.dtype) vol_f = np.empty( (self.n_vols, self.resolution**3), dtype=complex_type(self.dtype) ) for i in range(K): - pts_rot[i] = rotated_grids_3d(self.resolution, rot_matrices[i]) + pts_rot[i] = rotated_grids_3d(self.resolution, rots_inverted[i]) vol_f[i] = nufft(self[i].asnumpy(), pts_rot[i]) diff --git a/tests/test_volume.py b/tests/test_volume.py index 26c49a7d80..2c36a114c7 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -309,7 +309,7 @@ def test_project(vols_1, dtype): # Parameterize over even and odd resolutions @pytest.mark.parametrize("L", RES) -def test_rotate(L, dtype): +def test_rotate_axes(L, dtype): # In this test we instantiate Volume instance `vol`, containing a single nonzero # voxel in the first octant, and rotate it by multiples of pi/2 about each axis. # We then compare to reference volumes containing appropriately located nonzero voxel. @@ -358,6 +358,54 @@ def test_rotate(L, dtype): assert np.allclose(ref_vol, rot_vol, atol=utest_tolerance(dtype)) +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("L", [32, 33]) +def test_rotate(L, dtype): + """ + We rotate Volumes containing random hot/cold spots by random rotations and check that + hot/cold spots in the rotated Volumes are in the expected locations. + """ + n_vols = 5 + + # Generate random locations for hot/cold spots, each at a distance of approximately + # L // 4 from (0, 0, 0). Note, these points are considered to be in (z, y, x) order. + hot_cold_locs = np.random.uniform(low=-1, high=1, size=(n_vols, 2, 3)) + hot_cold_locs = np.round( + (hot_cold_locs / np.linalg.norm(hot_cold_locs, axis=-1)[:, :, None]) * (L // 4) + ).astype("int") + + # Generate Volumes, each with one hot and one cold spot. + vols = np.zeros((n_vols, L, L, L), dtype=dtype) + vol_center = np.array((L // 2, L // 2, L // 2), dtype="int") + for i in range(n_vols): + vols[i][tuple(vol_center + hot_cold_locs[i, 0])] = 1 + vols[i][tuple(vol_center + hot_cold_locs[i, 1])] = -1 + vols = Volume(vols) + + # Generate random rotations. + rots = Rotation.generate_random_rotations(n=n_vols, dtype=dtype) + rots_mat = rots.matrices + + # Expected location of hot/cold spots relative to (0, 0, 0) origin in (x, y, z) order. + expected_hot_cold = np.transpose( + rots_mat @ np.transpose(hot_cold_locs[..., ::-1], axes=(0, 2, 1)), + axes=(0, 2, 1), + ) + + # Expected location of hot/cold spots relative to Volume center (L/2, L/2, L/2) in (z, y, x) order. + expected_locs = np.round(expected_hot_cold[..., ::-1] + vol_center) + + # Rotate Volumes. + rotated_vols = vols.rotate(rots) + + # Check that new hot/cold spots are in expectecd locations. + for i in range(n_vols): + new_hot_loc = np.unravel_index(np.argmax(rotated_vols.asnumpy()[i]), (L, L, L)) + new_cold_loc = np.unravel_index(np.argmin(rotated_vols.asnumpy()[i]), (L, L, L)) + np.testing.assert_allclose(new_hot_spot, expected_locs[i, 0]) + np.testing.assert_allclose(new_cold_spot, expected_locs[i, 1]) + + def test_rotate_broadcast_unicast(asym_vols): # Build `Rotation` objects. A singleton for broadcasting and a stack for unicasting. # The stack consists of copies of the singleton. From 33409cec0b918a8c0a8b11982f20390f85f54402 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 13 Nov 2023 12:02:23 -0500 Subject: [PATCH 242/294] Adjust test_rotate_axes reference points. --- tests/test_volume.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index 2c36a114c7..69be718793 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -320,20 +320,20 @@ def test_rotate_axes(L, dtype): data[L // 2 + 1, L // 2 + 1, L // 2 + 1] = 1 vol = Volume(data) - # Create a dict with map from axis and angle of rotation to new location of nonzero voxel. + # Create a dict with map from axis and angle of rotation to new location (z, y, x) of nonzero voxel. ref_pts = { ("x", 0): (1, 1, 1), - ("x", pi / 2): (-1, 1, 1), + ("x", pi / 2): (1, -1, 1), ("x", pi): (-1, -1, 1), - ("x", 3 * pi / 2): (1, -1, 1), + ("x", 3 * pi / 2): (-1, 1, 1), ("y", 0): (1, 1, 1), - ("y", pi / 2): (1, 1, -1), + ("y", pi / 2): (-1, 1, 1), ("y", pi): (-1, 1, -1), - ("y", 3 * pi / 2): (-1, 1, 1), + ("y", 3 * pi / 2): (1, 1, -1), ("z", 0): (1, 1, 1), - ("z", pi / 2): (1, -1, 1), + ("z", pi / 2): (1, 1, -1), ("z", pi): (1, -1, -1), - ("z", 3 * pi / 2): (1, 1, -1), + ("z", 3 * pi / 2): (1, -1, 1), } center = np.array([L // 2] * 3) @@ -402,8 +402,8 @@ def test_rotate(L, dtype): for i in range(n_vols): new_hot_loc = np.unravel_index(np.argmax(rotated_vols.asnumpy()[i]), (L, L, L)) new_cold_loc = np.unravel_index(np.argmin(rotated_vols.asnumpy()[i]), (L, L, L)) - np.testing.assert_allclose(new_hot_spot, expected_locs[i, 0]) - np.testing.assert_allclose(new_cold_spot, expected_locs[i, 1]) + np.testing.assert_allclose(new_hot_loc, expected_locs[i, 0]) + np.testing.assert_allclose(new_cold_loc, expected_locs[i, 1]) def test_rotate_broadcast_unicast(asym_vols): From 4b0bba89eb5e72ed8e18f6ee971f5868a75c59a3 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 15 Nov 2023 15:52:55 -0500 Subject: [PATCH 243/294] use fixtures instead of parametrize in rotate test. --- tests/test_volume.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index 69be718793..f903c70901 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -307,15 +307,14 @@ def test_project(vols_1, dtype): assert np.allclose(results, imgs_clean, atol=1e-7) -# Parameterize over even and odd resolutions -@pytest.mark.parametrize("L", RES) -def test_rotate_axes(L, dtype): +def test_rotate_axes(res, dtype): # In this test we instantiate Volume instance `vol`, containing a single nonzero # voxel in the first octant, and rotate it by multiples of pi/2 about each axis. # We then compare to reference volumes containing appropriately located nonzero voxel. # Create a Volume instance to rotate. # This volume has a value of 1 in the first octant at (1, 1, 1) and zeros elsewhere. + L = res data = np.zeros((L, L, L), dtype=dtype) data[L // 2 + 1, L // 2 + 1, L // 2 + 1] = 1 vol = Volume(data) @@ -358,13 +357,12 @@ def test_rotate_axes(L, dtype): assert np.allclose(ref_vol, rot_vol, atol=utest_tolerance(dtype)) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("L", [32, 33]) -def test_rotate(L, dtype): +def test_rotate(res, dtype): """ We rotate Volumes containing random hot/cold spots by random rotations and check that hot/cold spots in the rotated Volumes are in the expected locations. """ + L = res n_vols = 5 # Generate random locations for hot/cold spots, each at a distance of approximately From d6fad155f11629ac18cac4fb73e0e075ee072da2 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 16 Nov 2023 13:57:45 -0500 Subject: [PATCH 244/294] hot cold volume fixture. test_project with arbitrary rotations. --- tests/test_volume.py | 93 +++++++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index f903c70901..897ade3244 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -30,7 +30,7 @@ def res_id(params): RES = [42, 43] -@pytest.fixture(params=RES, ids=res_id) +@pytest.fixture(params=RES, ids=res_id, scope="module") def res(request): return request.param @@ -42,7 +42,7 @@ def dtype_id(params): DTYPES = [np.float32, np.float64] -@pytest.fixture(params=DTYPES, ids=dtype_id) +@pytest.fixture(params=DTYPES, ids=dtype_id, scope="module") def dtype(request): return request.param @@ -87,6 +87,29 @@ def asym_vols(res, dtype): return vols +@pytest.fixture(scope="module") +def vols_hot_cold(res, dtype): + L = res + n_vols = 5 + + # Generate random locations for hot/cold spots, each at a distance of approximately + # L // 4 from (0, 0, 0). Note, these points are considered to be in (z, y, x) order. + hot_cold_locs = np.random.uniform(low=-1, high=1, size=(n_vols, 2, 3)) + hot_cold_locs = np.round( + (hot_cold_locs / np.linalg.norm(hot_cold_locs, axis=-1)[:, :, None]) * (L // 4) + ).astype("int") + + # Generate Volumes, each with one hot and one cold spot. + vols = np.zeros((n_vols, L, L, L), dtype=dtype) + vol_center = np.array((L // 2, L // 2, L // 2), dtype="int") + for i in range(n_vols): + vols[i][tuple(vol_center + hot_cold_locs[i, 0])] = 1 + vols[i][tuple(vol_center + hot_cold_locs[i, 1])] = -1 + vols = Volume(vols) + + return vols, hot_cold_locs, vol_center + + @pytest.fixture def random_data(res, dtype): return np.random.randn(res, res, res).astype(dtype) @@ -261,7 +284,45 @@ def test_save_load(vols_1): assert np.allclose(vols_1, vols_loaded_double) -def test_project(vols_1, dtype): +def test_project(vols_hot_cold): + """ + We project Volumes containing random hot/cold spots using random rotations and check that + hot/cold spots in the projections are in the expected locations. + """ + vols, hot_cold_locs, vol_center = vols_hot_cold + dtype = vols.dtype + L = vols.resolution + + # Generate random rotations. + rots = Rotation.generate_random_rotations(n=vols.n_vols, dtype=dtype) + rots_mat = rots.matrices + + # To find the expected location of hot/cold spots in the projections we rotate the 3D + # vector of locations by the transpose, ie. rots.invert(), (since our projections are + # produced by rotating the underlying grid) and then project along the z-axis. + + # Expected location of hot/cold spots relative to (0, 0, 0) origin in (x, y, z) order. + expected_hot_cold = np.transpose( + rots.invert().matrices @ np.transpose(hot_cold_locs[..., ::-1], axes=(0, 2, 1)), + axes=(0, 2, 1), + ) + + # Expected location of hot/cold spots relative to center (L/2, L/2, L/2) in (z, y, x) order. + # Then projected along z-axis by dropping the z component. + expected_locs = np.round(expected_hot_cold[..., ::-1] + vol_center)[..., 1:] + + # Generate projection images. + projections = vols.project(rots) + + # Check that new hot/cold spots are within 1 pixel of expectecd locations. + for i in range(vols.n_vols): + new_hot_loc = np.unravel_index(np.argmax(projections.asnumpy()[i]), (L, L)) + new_cold_loc = np.unravel_index(np.argmin(projections.asnumpy()[i]), (L, L)) + np.testing.assert_allclose(new_hot_loc, expected_locs[i, 0], atol=1) + np.testing.assert_allclose(new_cold_loc, expected_locs[i, 1], atol=1) + + +def test_project_axes(vols_1, dtype): L = vols_1.resolution # first test with synthetic data # Create a stack of rotations to test. @@ -357,31 +418,17 @@ def test_rotate_axes(res, dtype): assert np.allclose(ref_vol, rot_vol, atol=utest_tolerance(dtype)) -def test_rotate(res, dtype): +def test_rotate(vols_hot_cold): """ We rotate Volumes containing random hot/cold spots by random rotations and check that hot/cold spots in the rotated Volumes are in the expected locations. """ - L = res - n_vols = 5 - - # Generate random locations for hot/cold spots, each at a distance of approximately - # L // 4 from (0, 0, 0). Note, these points are considered to be in (z, y, x) order. - hot_cold_locs = np.random.uniform(low=-1, high=1, size=(n_vols, 2, 3)) - hot_cold_locs = np.round( - (hot_cold_locs / np.linalg.norm(hot_cold_locs, axis=-1)[:, :, None]) * (L // 4) - ).astype("int") - - # Generate Volumes, each with one hot and one cold spot. - vols = np.zeros((n_vols, L, L, L), dtype=dtype) - vol_center = np.array((L // 2, L // 2, L // 2), dtype="int") - for i in range(n_vols): - vols[i][tuple(vol_center + hot_cold_locs[i, 0])] = 1 - vols[i][tuple(vol_center + hot_cold_locs[i, 1])] = -1 - vols = Volume(vols) + vols, hot_cold_locs, vol_center = vols_hot_cold + dtype = vols.dtype + L = vols.resolution # Generate random rotations. - rots = Rotation.generate_random_rotations(n=n_vols, dtype=dtype) + rots = Rotation.generate_random_rotations(n=vols.n_vols, dtype=dtype) rots_mat = rots.matrices # Expected location of hot/cold spots relative to (0, 0, 0) origin in (x, y, z) order. @@ -397,7 +444,7 @@ def test_rotate(res, dtype): rotated_vols = vols.rotate(rots) # Check that new hot/cold spots are in expectecd locations. - for i in range(n_vols): + for i in range(vols.n_vols): new_hot_loc = np.unravel_index(np.argmax(rotated_vols.asnumpy()[i]), (L, L, L)) new_cold_loc = np.unravel_index(np.argmin(rotated_vols.asnumpy()[i]), (L, L, L)) np.testing.assert_allclose(new_hot_loc, expected_locs[i, 0]) From 6f2757c561603190b074d7c669a23426f1c563b7 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 16 Nov 2023 13:59:07 -0500 Subject: [PATCH 245/294] remove unused variable --- tests/test_volume.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index 897ade3244..bc4f3b31a6 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -295,7 +295,6 @@ def test_project(vols_hot_cold): # Generate random rotations. rots = Rotation.generate_random_rotations(n=vols.n_vols, dtype=dtype) - rots_mat = rots.matrices # To find the expected location of hot/cold spots in the projections we rotate the 3D # vector of locations by the transpose, ie. rots.invert(), (since our projections are From 2468a292cff3fc58ee17bd2d09aa3cbbefd5c9fd Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 21 Nov 2023 11:02:20 -0500 Subject: [PATCH 246/294] one-line call to matrices --- src/aspire/volume/volume.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/aspire/volume/volume.py b/src/aspire/volume/volume.py index b271d17c20..5993a1dd4e 100644 --- a/src/aspire/volume/volume.py +++ b/src/aspire/volume/volume.py @@ -454,8 +454,7 @@ def rotate(self, rot_matrices, zero_nyquist=True): ), f"Argument must be an instance of the Rotation class. {type(rot_matrices)} was supplied." # Invert the rotations passed to `rotated_grids_3d` and get numpy representation of Rotation object. - rots_inverted = rot_matrices.invert() - rots_inverted = rots_inverted.matrices + rots_inverted = rot_matrices.invert().matrices K = len(rots_inverted) # Rotation stack size assert K == self.n_vols or K == 1, "Rotation object must be length 1 or n_vols." From 76a7325b3d24dbd46faa6cd2ebdcc69527737847 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 21 Nov 2023 11:31:48 -0500 Subject: [PATCH 247/294] cleaner matrix multiplication --- tests/test_volume.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index bc4f3b31a6..1a030b0426 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -301,10 +301,8 @@ def test_project(vols_hot_cold): # produced by rotating the underlying grid) and then project along the z-axis. # Expected location of hot/cold spots relative to (0, 0, 0) origin in (x, y, z) order. - expected_hot_cold = np.transpose( - rots.invert().matrices @ np.transpose(hot_cold_locs[..., ::-1], axes=(0, 2, 1)), - axes=(0, 2, 1), - ) + # Note, we write the simpler `(x, y, z) @ rots` in place of `(rots.T @ (x, y, z).T).T` + expected_hot_cold = hot_cold_locs[..., ::-1] @ rots.matrices # Expected location of hot/cold spots relative to center (L/2, L/2, L/2) in (z, y, x) order. # Then projected along z-axis by dropping the z component. @@ -428,13 +426,10 @@ def test_rotate(vols_hot_cold): # Generate random rotations. rots = Rotation.generate_random_rotations(n=vols.n_vols, dtype=dtype) - rots_mat = rots.matrices # Expected location of hot/cold spots relative to (0, 0, 0) origin in (x, y, z) order. - expected_hot_cold = np.transpose( - rots_mat @ np.transpose(hot_cold_locs[..., ::-1], axes=(0, 2, 1)), - axes=(0, 2, 1), - ) + # Note, we write the simpler `(x, y, z) @ rots.T` in place of `(rots @ (x, y, z).T).T` + expected_hot_cold = hot_cold_locs[..., ::-1] @ rots.invert().matrices # Expected location of hot/cold spots relative to Volume center (L/2, L/2, L/2) in (z, y, x) order. expected_locs = np.round(expected_hot_cold[..., ::-1] + vol_center) From d10508745cb3f6290ef82ac8587def4d7fc0cbb6 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 21 Nov 2023 11:36:48 -0500 Subject: [PATCH 248/294] more cleanup --- tests/test_volume.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index 1a030b0426..480667ddef 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -313,8 +313,9 @@ def test_project(vols_hot_cold): # Check that new hot/cold spots are within 1 pixel of expectecd locations. for i in range(vols.n_vols): - new_hot_loc = np.unravel_index(np.argmax(projections.asnumpy()[i]), (L, L)) - new_cold_loc = np.unravel_index(np.argmin(projections.asnumpy()[i]), (L, L)) + p = projections.asnumpy()[i] + new_hot_loc = np.unravel_index(np.argmax(p), (L, L)) + new_cold_loc = np.unravel_index(np.argmin(p), (L, L)) np.testing.assert_allclose(new_hot_loc, expected_locs[i, 0], atol=1) np.testing.assert_allclose(new_cold_loc, expected_locs[i, 1], atol=1) @@ -439,8 +440,9 @@ def test_rotate(vols_hot_cold): # Check that new hot/cold spots are in expectecd locations. for i in range(vols.n_vols): - new_hot_loc = np.unravel_index(np.argmax(rotated_vols.asnumpy()[i]), (L, L, L)) - new_cold_loc = np.unravel_index(np.argmin(rotated_vols.asnumpy()[i]), (L, L, L)) + v = rotated_vols.asnumpy()[i] + new_hot_loc = np.unravel_index(np.argmax(v), (L, L, L)) + new_cold_loc = np.unravel_index(np.argmin(v), (L, L, L)) np.testing.assert_allclose(new_hot_loc, expected_locs[i, 0]) np.testing.assert_allclose(new_cold_loc, expected_locs[i, 1]) From 865f84ae154dad5a9ad6412285aa8656456ccf20 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 28 Nov 2023 11:35:32 -0500 Subject: [PATCH 249/294] loosen tolerance for 1hot rotate test. --- tests/test_volume.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_volume.py b/tests/test_volume.py index 480667ddef..9d039c4c9f 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -438,13 +438,13 @@ def test_rotate(vols_hot_cold): # Rotate Volumes. rotated_vols = vols.rotate(rots) - # Check that new hot/cold spots are in expectecd locations. + # Check that new hot/cold spots are within 1 pixel of expectecd locations. for i in range(vols.n_vols): v = rotated_vols.asnumpy()[i] new_hot_loc = np.unravel_index(np.argmax(v), (L, L, L)) new_cold_loc = np.unravel_index(np.argmin(v), (L, L, L)) - np.testing.assert_allclose(new_hot_loc, expected_locs[i, 0]) - np.testing.assert_allclose(new_cold_loc, expected_locs[i, 1]) + np.testing.assert_allclose(new_hot_loc, expected_locs[i, 0], atol=1) + np.testing.assert_allclose(new_cold_loc, expected_locs[i, 1], atol=1) def test_rotate_broadcast_unicast(asym_vols): From b31bb566dd28866c48f67241d4f3ca1e9e52831f Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 15 Nov 2023 15:48:31 -0500 Subject: [PATCH 250/294] add simulated_channelspin dataset to registry --- src/aspire/downloader/data_fetcher.py | 22 ++++++++++++++++++++++ src/aspire/downloader/registry.py | 3 +++ 2 files changed, 25 insertions(+) diff --git a/src/aspire/downloader/data_fetcher.py b/src/aspire/downloader/data_fetcher.py index 44b11dbd9d..6a5bed6104 100644 --- a/src/aspire/downloader/data_fetcher.py +++ b/src/aspire/downloader/data_fetcher.py @@ -1,5 +1,6 @@ import shutil +import numpy as np import pooch from aspire import config @@ -263,3 +264,24 @@ def emdb_6458(): vol = Volume.load(file_path, symmetry_group="C11") return vol + + +def simulated_channelspin(): + """ + Downloads the Simulated ChannelSpin dataset and returns the file path. + + This dataset includes a stack of 54 volumes sized (54,54,54) + and a corresponding stack of 10000 projection images (54,54). + + :return: Dictionary containing Volume and Image instances, + along with associated metadata fields in Numpy arrays. + """ + file_path = fetch_data("simulated_channelspin.npz") + data = np.load(file_path) + ["vols", "angles", "shifts", "images", "amplitudes", "rots", "states", "weights"] + + # Instantiate ASPIRE objects for the main entries. + data["vols"] = Volume(data["vols"]) + data["images"] = Images(data["images"]) + + return data diff --git a/src/aspire/downloader/registry.py b/src/aspire/downloader/registry.py index 8c82b4701e..467ad3c772 100644 --- a/src/aspire/downloader/registry.py +++ b/src/aspire/downloader/registry.py @@ -13,6 +13,7 @@ "emdb_14621.map": "b45774245c2bd5e1a44e801b8fb1705a44d5850631838d060294be42e34a6900", "emdb_2484.map": "6a324e23352bea101c191d5e854026162a5a9b0b8fc73ac5a085cc22038e1999", "emdb_6458.map": "645208af6d36bbd3d172c549e58d387b81142fd320e064bc66105be0eae540d1", + "simulated_channelspin.npz": "c0752674acb85417f6a77a28ac55280c1926c73fda9e25ce0a9940728b1dfcc8", } registry_urls = { @@ -29,6 +30,7 @@ "emdb_14621.map": "https://ftp.ebi.ac.uk/pub/databases/emdb/structures/EMD-14621/map/emd_14621.map.gz", "emdb_2484.map": "https://ftp.ebi.ac.uk/pub/databases/emdb/structures/EMD-2484/map/emd_2484.map.gz", "emdb_6458.map": "https://ftp.ebi.ac.uk/pub/databases/emdb/structures/EMD-6458/map/emd_6458.map.gz", + "simulated_channelspin.npz": "https://zenodo.org/records/8186548/files/example_FakeKV_dataset.npz", } file_to_method_map = { @@ -45,4 +47,5 @@ "emdb_14621.map": "emdb_14621", "emdb_2484.map": "emdb_2484", "emdb_6458.map": "emdb_6458", + "simulated_channelspin.npz": "simulated_channelspin", } From 0cf2d9c6cb993bf23c9bbdd749f664c538c7f1f1 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 16 Nov 2023 10:42:12 -0500 Subject: [PATCH 251/294] stash initial wt vol nb [skip ci] --- .../experiments/weighted_volume_estimation.py | 87 +++++++++++++++++++ src/aspire/downloader/__init__.py | 1 + src/aspire/downloader/data_fetcher.py | 13 ++- 3 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 gallery/experiments/weighted_volume_estimation.py diff --git a/gallery/experiments/weighted_volume_estimation.py b/gallery/experiments/weighted_volume_estimation.py new file mode 100644 index 0000000000..dd25d2ad00 --- /dev/null +++ b/gallery/experiments/weighted_volume_estimation.py @@ -0,0 +1,87 @@ +""" +Weighted Volume Reconstruction +============================== + +This tutorial demonstrates using weighted volume reconstruction, +using a reference dataset. +""" + +# %% +# Download an Example Dataset +# --------------------------- +from aspire import downloader +from aspire.source import ArrayImageSource + +sim_data = downloader.simulated_channelspin() + +# This data contains a Volume stack, an Image stack, weights and +# corresponding parameters that were used to derive the image stack +# from the volumes. For example. the rotations below are the known +# true simulation projections. In practice these would be derived from +# an orientation estimation component. + +imgs = sim_data["images"] # Simulated image stack. +rots = sim_data["rots"] # True projection rotations +weights = sim_data["weights"] # Volume weights +vols = sim_data["vols"] # True reference volumes + +# %% +# Create a ImageSource +# ---------------- +# lorem ipsum +src = ArrayImageSource(imgs, angles=rots.angles) + + +# %% +# Volume Reconstruction +# --------------------- +# Now that we have our class averages and rotation estimates, we can +# estimate the mean volume by supplying the class averages and basis +# for back projection. + +from aspire.basis import FFBBasis3D +from aspire.reconstruction import WeightedVolumesEstimator + +# Create a reasonable Basis +basis = FFBBasis3D(src.L, dtype=src.dtype) + +# Setup an estimator to perform the back projection. +estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") + +# XXX +import os + +import numpy as np + +from aspire.volume import Volume + +fn = "est_vol.npy" +if not os.path.exists(fn): + # Perform the estimation. + estimated_volume = estimator.estimate() + np.save(fn, estimated_volume.asnumpy()) + +estimated_volume = Volume(np.load(fn)) + + +# .. note: +# The ``estimate`` requires a fair amount of compute time, +# but there should be regularly logged progress towards convergence. + +# %% +# Comparison of Estimated Volume with Source Volume +# ------------------------------------------------- +# Generate and compare several random projections between the estimated volumes and the known volumes. + +from aspire.utils import uniform_random_angles + +v = 0 # Volume under comparison +m = 3 # Number of projections + +angles = uniform_random_angles(m, dtype=src.dtype) + +# Estimated volume projections +estimated_volume[v].project(angles).show() + +# Source volume projections +vols[v].project(angles).show() diff --git a/src/aspire/downloader/__init__.py b/src/aspire/downloader/__init__.py index e29b9a0983..be0d375878 100644 --- a/src/aspire/downloader/__init__.py +++ b/src/aspire/downloader/__init__.py @@ -18,4 +18,5 @@ emdb_10835, emdb_14621, remove_downloads, + simulated_channelspin, ) diff --git a/src/aspire/downloader/data_fetcher.py b/src/aspire/downloader/data_fetcher.py index 6a5bed6104..26ce85d0f8 100644 --- a/src/aspire/downloader/data_fetcher.py +++ b/src/aspire/downloader/data_fetcher.py @@ -5,7 +5,9 @@ from aspire import config from aspire.downloader import file_to_method_map, registry, registry_urls +from aspire.image import Image from aspire.volume import Volume +from aspire.utils import Rotation # Initialize pooch data fetcher instance. _data_fetcher = pooch.create( @@ -277,11 +279,14 @@ def simulated_channelspin(): along with associated metadata fields in Numpy arrays. """ file_path = fetch_data("simulated_channelspin.npz") - data = np.load(file_path) - ["vols", "angles", "shifts", "images", "amplitudes", "rots", "states", "weights"] + # Use context manager so the file handle closes. + with np.load(file_path) as data: + # Convert to dict so that the entries can be modified + data = dict(data) - # Instantiate ASPIRE objects for the main entries. + # Instantiate ASPIRE objects where appropriate data["vols"] = Volume(data["vols"]) - data["images"] = Images(data["images"]) + data["images"] = Image(data["images"]) + data['rots'] = Rotation(data['rots']) return data From b103b15a854de93cf7ed2b1687c6e9ae466dfa35 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 16 Nov 2023 11:19:06 -0500 Subject: [PATCH 252/294] stashing --- gallery/experiments/weighted_volume_estimation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gallery/experiments/weighted_volume_estimation.py b/gallery/experiments/weighted_volume_estimation.py index dd25d2ad00..8494c3473d 100644 --- a/gallery/experiments/weighted_volume_estimation.py +++ b/gallery/experiments/weighted_volume_estimation.py @@ -46,6 +46,7 @@ basis = FFBBasis3D(src.L, dtype=src.dtype) # Setup an estimator to perform the back projection. +breakpoint() estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") # XXX @@ -73,15 +74,15 @@ # ------------------------------------------------- # Generate and compare several random projections between the estimated volumes and the known volumes. -from aspire.utils import uniform_random_angles +from aspire.utils import Rotation, uniform_random_angles v = 0 # Volume under comparison m = 3 # Number of projections -angles = uniform_random_angles(m, dtype=src.dtype) +random_rotations = Rotation.from_euler(uniform_random_angles(m, dtype=src.dtype)) # Estimated volume projections -estimated_volume[v].project(angles).show() +estimated_volume[v].project(random_rotations).show() # Source volume projections -vols[v].project(angles).show() +vols[v].project(random_rotations).show() From 3bd85c227d9eb122efd8c47a696dee69bead8348 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 16 Nov 2023 15:41:23 -0500 Subject: [PATCH 253/294] convert legacy rots in reference simulated_channelspin dataset loader --- gallery/experiments/weighted_volume_estimation.py | 1 - src/aspire/downloader/data_fetcher.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gallery/experiments/weighted_volume_estimation.py b/gallery/experiments/weighted_volume_estimation.py index 8494c3473d..3e03132831 100644 --- a/gallery/experiments/weighted_volume_estimation.py +++ b/gallery/experiments/weighted_volume_estimation.py @@ -46,7 +46,6 @@ basis = FFBBasis3D(src.L, dtype=src.dtype) # Setup an estimator to perform the back projection. -breakpoint() estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") # XXX diff --git a/src/aspire/downloader/data_fetcher.py b/src/aspire/downloader/data_fetcher.py index 26ce85d0f8..745e7f7767 100644 --- a/src/aspire/downloader/data_fetcher.py +++ b/src/aspire/downloader/data_fetcher.py @@ -6,6 +6,7 @@ from aspire import config from aspire.downloader import file_to_method_map, registry, registry_urls from aspire.image import Image +from aspire.source import _LegacySimulation from aspire.volume import Volume from aspire.utils import Rotation @@ -287,6 +288,7 @@ def simulated_channelspin(): # Instantiate ASPIRE objects where appropriate data["vols"] = Volume(data["vols"]) data["images"] = Image(data["images"]) - data['rots'] = Rotation(data['rots']) + data['rots'] = Rotation(_LegacySimulation.rots_zyx_to_legacy_aspire(data['rots'])) return data + From 238f003d445ad55043e4ba4a5c890ad9c229786d Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 17 Nov 2023 08:09:24 -0500 Subject: [PATCH 254/294] add initial text to weighted volume tutorial --- .../experiments/weighted_volume_estimation.py | 87 ----------------- .../tutorials/weighted_volume_estimation.py | 94 +++++++++++++++++++ 2 files changed, 94 insertions(+), 87 deletions(-) delete mode 100644 gallery/experiments/weighted_volume_estimation.py create mode 100644 gallery/tutorials/tutorials/weighted_volume_estimation.py diff --git a/gallery/experiments/weighted_volume_estimation.py b/gallery/experiments/weighted_volume_estimation.py deleted file mode 100644 index 3e03132831..0000000000 --- a/gallery/experiments/weighted_volume_estimation.py +++ /dev/null @@ -1,87 +0,0 @@ -""" -Weighted Volume Reconstruction -============================== - -This tutorial demonstrates using weighted volume reconstruction, -using a reference dataset. -""" - -# %% -# Download an Example Dataset -# --------------------------- -from aspire import downloader -from aspire.source import ArrayImageSource - -sim_data = downloader.simulated_channelspin() - -# This data contains a Volume stack, an Image stack, weights and -# corresponding parameters that were used to derive the image stack -# from the volumes. For example. the rotations below are the known -# true simulation projections. In practice these would be derived from -# an orientation estimation component. - -imgs = sim_data["images"] # Simulated image stack. -rots = sim_data["rots"] # True projection rotations -weights = sim_data["weights"] # Volume weights -vols = sim_data["vols"] # True reference volumes - -# %% -# Create a ImageSource -# ---------------- -# lorem ipsum -src = ArrayImageSource(imgs, angles=rots.angles) - - -# %% -# Volume Reconstruction -# --------------------- -# Now that we have our class averages and rotation estimates, we can -# estimate the mean volume by supplying the class averages and basis -# for back projection. - -from aspire.basis import FFBBasis3D -from aspire.reconstruction import WeightedVolumesEstimator - -# Create a reasonable Basis -basis = FFBBasis3D(src.L, dtype=src.dtype) - -# Setup an estimator to perform the back projection. -estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") - -# XXX -import os - -import numpy as np - -from aspire.volume import Volume - -fn = "est_vol.npy" -if not os.path.exists(fn): - # Perform the estimation. - estimated_volume = estimator.estimate() - np.save(fn, estimated_volume.asnumpy()) - -estimated_volume = Volume(np.load(fn)) - - -# .. note: -# The ``estimate`` requires a fair amount of compute time, -# but there should be regularly logged progress towards convergence. - -# %% -# Comparison of Estimated Volume with Source Volume -# ------------------------------------------------- -# Generate and compare several random projections between the estimated volumes and the known volumes. - -from aspire.utils import Rotation, uniform_random_angles - -v = 0 # Volume under comparison -m = 3 # Number of projections - -random_rotations = Rotation.from_euler(uniform_random_angles(m, dtype=src.dtype)) - -# Estimated volume projections -estimated_volume[v].project(random_rotations).show() - -# Source volume projections -vols[v].project(random_rotations).show() diff --git a/gallery/tutorials/tutorials/weighted_volume_estimation.py b/gallery/tutorials/tutorials/weighted_volume_estimation.py new file mode 100644 index 0000000000..204e83669e --- /dev/null +++ b/gallery/tutorials/tutorials/weighted_volume_estimation.py @@ -0,0 +1,94 @@ +""" +Weighted Volume Reconstruction +============================== + +This tutorial demonstrates a weighted volume reconstruction, +using a published reference dataset. +""" + +# %% +# Download an Example Dataset +# --------------------------- +# ASPIRE's downloader will download, cache, +# and unpack the reference dataset. +# More information about the dataset can be found here: +# https://zenodo.org/records/8186548 + +from aspire import downloader + +sim_data = downloader.simulated_channelspin() + +# This data contains a Volume stack, an Image stack, weights and +# corresponding parameters that were used to derive the image stack +# from the volumes. For example. the rotations below are the known +# true simulation projections. In practice these would be derived from +# an orientation estimation component. + +imgs = sim_data["images"] # Simulated image stack (``Image`` object) +rots = sim_data["rots"] # True projection rotations (``Rotation`` object) +weights = sim_data["weights"] # Volume weights (``Numpy`` array) +vols = sim_data["vols"] # True reference volumes (``Volume`` object) + +# %% +# Create a ``ImageSource`` +# ------------------------ +# The image stack and projection rotation (Euler) angles can be +# associated together during instantiation of an ``ImageSource``. + +from aspire.source import ArrayImageSource + +src = ArrayImageSource(imgs, angles=rots.angles) + +# The images are downsampled for the sake of a quicker tutorial. +# This line can be commented out to achieve the reference size (54 pixels). + +src = src.downsample(24) + +# .. note: +# This tutorial demonstrates bringing data reference data. +# It is also possible to just create a ``Simulation`` or use other +# ``ImageSource`` objects here, so long as the rotations required +# for backprojecting are assigned. + + +# %% +# Volume Reconstruction +# --------------------- +# Performing a weighted volume reconstruction requires defining an +# appropriate 3D basis and supplying an associated image to volume +# weight mapping as a ``Numpy`` array. + +from aspire.basis import FFBBasis3D +from aspire.reconstruction import WeightedVolumesEstimator + +# Create a reasonable Basis +basis = FFBBasis3D(src.L, dtype=src.dtype) + +# Setup an estimator to perform the back projections and volume estimation. +estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") + +# Perform the estimation, returning a ``Volume`` stack. +estimated_volume = estimator.estimate() + +# .. note: +# The ``estimate()`` method requires a fair amount of compute time, +# but there should be regularly logged progress towards convergence. + +# %% +# Comparison of Estimated Volume with Source Volume +# ------------------------------------------------- +# Generate several random projections rotations, then compare these +# projections between the estimated volumes and the known volumes. + +from aspire.utils import Rotation, uniform_random_angles + +v = 0 # Volume under comparison +m = 3 # Number of projections + +random_rotations = Rotation.from_euler(uniform_random_angles(m, dtype=src.dtype)) + +# Estimated volume projections +estimated_volume[v].project(random_rotations).show() + +# Source volume projections +vols[v].project(random_rotations).show() From a34889e02876ca0ce6396b4de12390b58f55bbc1 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 17 Nov 2023 08:09:50 -0500 Subject: [PATCH 255/294] lint data_fetcher --- src/aspire/downloader/data_fetcher.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/aspire/downloader/data_fetcher.py b/src/aspire/downloader/data_fetcher.py index 745e7f7767..3fe163521b 100644 --- a/src/aspire/downloader/data_fetcher.py +++ b/src/aspire/downloader/data_fetcher.py @@ -7,8 +7,8 @@ from aspire.downloader import file_to_method_map, registry, registry_urls from aspire.image import Image from aspire.source import _LegacySimulation -from aspire.volume import Volume from aspire.utils import Rotation +from aspire.volume import Volume # Initialize pooch data fetcher instance. _data_fetcher = pooch.create( @@ -288,7 +288,6 @@ def simulated_channelspin(): # Instantiate ASPIRE objects where appropriate data["vols"] = Volume(data["vols"]) data["images"] = Image(data["images"]) - data['rots'] = Rotation(_LegacySimulation.rots_zyx_to_legacy_aspire(data['rots'])) + data["rots"] = Rotation(_LegacySimulation.rots_zyx_to_legacy_aspire(data["rots"])) return data - From 7a557d45b732390bc443ed710da2fe599d389cae Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 17 Nov 2023 09:34:00 -0500 Subject: [PATCH 256/294] update hyperlinks etc --- .../tutorials/tutorials/weighted_volume_estimation.py | 11 +++++++---- tox.ini | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gallery/tutorials/tutorials/weighted_volume_estimation.py b/gallery/tutorials/tutorials/weighted_volume_estimation.py index 204e83669e..dde1b28b60 100644 --- a/gallery/tutorials/tutorials/weighted_volume_estimation.py +++ b/gallery/tutorials/tutorials/weighted_volume_estimation.py @@ -11,8 +11,9 @@ # --------------------------- # ASPIRE's downloader will download, cache, # and unpack the reference dataset. -# More information about the dataset can be found here: -# https://zenodo.org/records/8186548 +# More information about the dataset can be found on +# `Zenodo `_ +# and in this `paper `_ from aspire import downloader @@ -41,16 +42,15 @@ # The images are downsampled for the sake of a quicker tutorial. # This line can be commented out to achieve the reference size (54 pixels). - src = src.downsample(24) +# %% # .. note: # This tutorial demonstrates bringing data reference data. # It is also possible to just create a ``Simulation`` or use other # ``ImageSource`` objects here, so long as the rotations required # for backprojecting are assigned. - # %% # Volume Reconstruction # --------------------- @@ -70,6 +70,7 @@ # Perform the estimation, returning a ``Volume`` stack. estimated_volume = estimator.estimate() +# %% # .. note: # The ``estimate()`` method requires a fair amount of compute time, # but there should be regularly logged progress towards convergence. @@ -79,6 +80,8 @@ # ------------------------------------------------- # Generate several random projections rotations, then compare these # projections between the estimated volumes and the known volumes. +# If ``src`` was downsampled above, the resulting estimated volumes +# and projections will be similarly downsampled. from aspire.utils import Rotation, uniform_random_angles diff --git a/tox.ini b/tox.ini index c753d9e692..2157c65745 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,7 @@ per-file-ignores = gallery/tutorials/turorials/data_downloader.py: E402 gallery/tutorials/tutorials/ctf.py: T201, E402 gallery/tutorials/tutorials/micrograph_source.py: T201, E402 + gallery/tutorials/tutorials/weighted_volume_estimation.py: E402 # Ignore Sphinx gallery builds docs/build/html/_downloads/*/*.py: T201, E402, F401, E265 docs/source/auto*/*.py: T201, E402, F401, E265 From b3504227ede4f07b5c520a4570340c20ee19c422 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 20 Nov 2023 11:43:35 -0500 Subject: [PATCH 257/294] Gallery string updates --- .../tutorials/tutorials/weighted_volume_estimation.py | 9 ++++++--- tox.ini | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/gallery/tutorials/tutorials/weighted_volume_estimation.py b/gallery/tutorials/tutorials/weighted_volume_estimation.py index dde1b28b60..0e8db1e711 100644 --- a/gallery/tutorials/tutorials/weighted_volume_estimation.py +++ b/gallery/tutorials/tutorials/weighted_volume_estimation.py @@ -46,9 +46,9 @@ # %% # .. note: -# This tutorial demonstrates bringing data reference data. +# This tutorial demonstrates bringing reference data. # It is also possible to just create a ``Simulation`` or use other -# ``ImageSource`` objects here, so long as the rotations required +# ``ImageSource`` objects, so long as the rotations required # for backprojecting are assigned. # %% @@ -65,6 +65,9 @@ basis = FFBBasis3D(src.L, dtype=src.dtype) # Setup an estimator to perform the back projections and volume estimation. +# In this case, the ``weights`` array comes from the reference data set, +# and is shaped to map images to volumes. +print("`weights shape:`", weights.shape) estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") # Perform the estimation, returning a ``Volume`` stack. @@ -81,7 +84,7 @@ # Generate several random projections rotations, then compare these # projections between the estimated volumes and the known volumes. # If ``src`` was downsampled above, the resulting estimated volumes -# and projections will be similarly downsampled. +# and projections will be of similar downsampled quality. from aspire.utils import Rotation, uniform_random_angles diff --git a/tox.ini b/tox.ini index 2157c65745..723d15daf5 100644 --- a/tox.ini +++ b/tox.ini @@ -73,7 +73,7 @@ per-file-ignores = gallery/tutorials/turorials/data_downloader.py: E402 gallery/tutorials/tutorials/ctf.py: T201, E402 gallery/tutorials/tutorials/micrograph_source.py: T201, E402 - gallery/tutorials/tutorials/weighted_volume_estimation.py: E402 + gallery/tutorials/tutorials/weighted_volume_estimation.py: T201, E402 # Ignore Sphinx gallery builds docs/build/html/_downloads/*/*.py: T201, E402, F401, E265 docs/source/auto*/*.py: T201, E402, F401, E265 From 95da13babe354fd4525c3c4c35bc465067bdff56 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 20 Nov 2023 13:31:48 -0500 Subject: [PATCH 258/294] More wt vol gallery cleanup --- .../tutorials/weighted_volume_estimation.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/gallery/tutorials/tutorials/weighted_volume_estimation.py b/gallery/tutorials/tutorials/weighted_volume_estimation.py index 0e8db1e711..d443f4691e 100644 --- a/gallery/tutorials/tutorials/weighted_volume_estimation.py +++ b/gallery/tutorials/tutorials/weighted_volume_estimation.py @@ -19,22 +19,24 @@ sim_data = downloader.simulated_channelspin() -# This data contains a Volume stack, an Image stack, weights and -# corresponding parameters that were used to derive the image stack +# This data contains a `Volume` stack, an `Image` stack, weights and +# corresponding parameters that were used to derive images # from the volumes. For example. the rotations below are the known # true simulation projections. In practice these would be derived from # an orientation estimation component. -imgs = sim_data["images"] # Simulated image stack (``Image`` object) -rots = sim_data["rots"] # True projection rotations (``Rotation`` object) -weights = sim_data["weights"] # Volume weights (``Numpy`` array) -vols = sim_data["vols"] # True reference volumes (``Volume`` object) +imgs = sim_data["images"] # Simulated image stack (`Image` object) +rots = sim_data["rots"] # True projection rotations (`Rotation` object) +weights = sim_data["weights"] # Volume weights (`Numpy` array) +vols = sim_data["vols"] # True reference volumes (`Volume` object) # %% # Create a ``ImageSource`` # ------------------------ # The image stack and projection rotation (Euler) angles can be # associated together during instantiation of an ``ImageSource``. +# Because this example starts with a dense array of images, +# an ``ArrayImageSource`` is used. from aspire.source import ArrayImageSource @@ -45,7 +47,7 @@ src = src.downsample(24) # %% -# .. note: +# .. note:: # This tutorial demonstrates bringing reference data. # It is also possible to just create a ``Simulation`` or use other # ``ImageSource`` objects, so long as the rotations required @@ -56,7 +58,7 @@ # --------------------- # Performing a weighted volume reconstruction requires defining an # appropriate 3D basis and supplying an associated image to volume -# weight mapping as a ``Numpy`` array. +# weight mapping as an array. from aspire.basis import FFBBasis3D from aspire.reconstruction import WeightedVolumesEstimator @@ -65,16 +67,16 @@ basis = FFBBasis3D(src.L, dtype=src.dtype) # Setup an estimator to perform the back projections and volume estimation. -# In this case, the ``weights`` array comes from the reference data set, +# In this case, the `weights` array comes from the reference data set, # and is shaped to map images to volumes. print("`weights shape:`", weights.shape) estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") -# Perform the estimation, returning a ``Volume`` stack. +# Perform the estimation, returning a `Volume` stack. estimated_volume = estimator.estimate() # %% -# .. note: +# .. note:: # The ``estimate()`` method requires a fair amount of compute time, # but there should be regularly logged progress towards convergence. From ec9f7f9109420f04ca85d9f861182502287f24e9 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 21 Nov 2023 11:38:55 -0500 Subject: [PATCH 259/294] Review suggestions round 1 --- .../tutorials/weighted_volume_estimation.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/gallery/tutorials/tutorials/weighted_volume_estimation.py b/gallery/tutorials/tutorials/weighted_volume_estimation.py index d443f4691e..c44abf77de 100644 --- a/gallery/tutorials/tutorials/weighted_volume_estimation.py +++ b/gallery/tutorials/tutorials/weighted_volume_estimation.py @@ -21,9 +21,9 @@ # This data contains a `Volume` stack, an `Image` stack, weights and # corresponding parameters that were used to derive images -# from the volumes. For example. the rotations below are the known -# true simulation projections. In practice these would be derived from -# an orientation estimation component. +# from the volumes. For example, the rotations below are the known +# true simulation projection rotations. In practice these would be +# derived from an orientation estimation component. imgs = sim_data["images"] # Simulated image stack (`Image` object) rots = sim_data["rots"] # True projection rotations (`Rotation` object) @@ -68,7 +68,11 @@ # Setup an estimator to perform the back projections and volume estimation. # In this case, the `weights` array comes from the reference data set, -# and is shaped to map images to volumes. +# and is shaped to map images to spectral volumes. +# Note that we can have many more actual/reference volumes generating +# the image stack than spectral volumes. In this case the input +# images were generated from 54 volumes, but are described by 16 +# spectral volumes. print("`weights shape:`", weights.shape) estimator = WeightedVolumesEstimator(weights, src, basis, preconditioner="none") @@ -84,19 +88,28 @@ # Comparison of Estimated Volume with Source Volume # ------------------------------------------------- # Generate several random projections rotations, then compare these -# projections between the estimated volumes and the known volumes. +# projections between the estimated spectral volumes and a known volume. # If ``src`` was downsampled above, the resulting estimated volumes # and projections will be of similar downsampled quality. +# +# Note that the estimated spectral volumes are treated as `Volume` +# objects purely for convienience and are not expected to correspond +# exactly to any particular reference volume. The spectral volumes +# collectively describe motion features derived from the input data. +# However, basic visual comparison is useful as a sanity check to +# demonstrate that we are in fact generating spectral volumes that +# appear reasonably similar to the input volumes. from aspire.utils import Rotation, uniform_random_angles -v = 0 # Volume under comparison +reference_v = 0 # Actual volume under comparison +spectral_v = 0 # Estimated spectral volume m = 3 # Number of projections random_rotations = Rotation.from_euler(uniform_random_angles(m, dtype=src.dtype)) # Estimated volume projections -estimated_volume[v].project(random_rotations).show() +estimated_volume[spectral_v].project(random_rotations).show() # Source volume projections -vols[v].project(random_rotations).show() +vols[reference_v].project(random_rotations).show() From 7eee2408f522d1d5732dca2828dd6cc91f181123 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 29 Nov 2023 14:45:03 -0500 Subject: [PATCH 260/294] Fixup some words in wt vol gallery doc strings --- gallery/tutorials/tutorials/weighted_volume_estimation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/tutorials/tutorials/weighted_volume_estimation.py b/gallery/tutorials/tutorials/weighted_volume_estimation.py index c44abf77de..a8e7893292 100644 --- a/gallery/tutorials/tutorials/weighted_volume_estimation.py +++ b/gallery/tutorials/tutorials/weighted_volume_estimation.py @@ -66,7 +66,7 @@ # Create a reasonable Basis basis = FFBBasis3D(src.L, dtype=src.dtype) -# Setup an estimator to perform the back projections and volume estimation. +# Set up an estimator to perform the backprojections and volume estimation. # In this case, the `weights` array comes from the reference data set, # and is shaped to map images to spectral volumes. # Note that we can have many more actual/reference volumes generating From b4849103e2c652ce62dcf791da445c397be1dd2b Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 20 Nov 2023 14:52:05 -0500 Subject: [PATCH 261/294] Add test_relion_interop.py. --- tests/test_relion_interop.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/test_relion_interop.py diff --git a/tests/test_relion_interop.py b/tests/test_relion_interop.py new file mode 100644 index 0000000000..37d6606324 --- /dev/null +++ b/tests/test_relion_interop.py @@ -0,0 +1,36 @@ +import logging +import os + +import numpy as np +import pytest + +from aspire.source import RelionSource, Simulation +from aspire.volume import Volume + +DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") + + +def test_projections(): + # Create RelionSource from Relion generated projection images. + starfile = os.path.join(DATA_DIR, "rln_proj.star") + rln_src = RelionSource(starfile) + + # Create Simulation source using same volume and angles. + # Note, Relion projections are shifted by 1 pixel cmopared to ASPIRE. + dtype = rln_src.dtype + vol_path = os.path.join(DATA_DIR, "clean70SRibosome_vol.npy") + vol = Volume(np.load(vol_path), dtype=dtype) + sim_src = Simulation( + n=rln_src.n, + vols=vol, + offsets=-np.ones((rln_src.n, 2), dtype=dtype), + angles=rln_src.angles, + dtype=dtype, + ) + + # Compute the Fourier Ring Correlation. + res, corr = rln_src.images[:].frc(sim_src.images[:], cutoff=0.143) + + # Check that res is small and corr is close to 1. + np.testing.assert_array_less(res, 2.5) + np.testing.assert_array_less(1 - corr[:, -2], 0.0015) From 71494be6923b05bd88c76a9a10d3beea502909d6 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 20 Nov 2023 14:53:50 -0500 Subject: [PATCH 262/294] remove unused imports --- tests/test_relion_interop.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_relion_interop.py b/tests/test_relion_interop.py index 37d6606324..ff86f73592 100644 --- a/tests/test_relion_interop.py +++ b/tests/test_relion_interop.py @@ -1,8 +1,6 @@ -import logging import os import numpy as np -import pytest from aspire.source import RelionSource, Simulation from aspire.volume import Volume From a07965efad28d4018f6a3e4fbc4e91cefa13872d Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 20 Nov 2023 15:15:00 -0500 Subject: [PATCH 263/294] typo --- tests/test_relion_interop.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_relion_interop.py b/tests/test_relion_interop.py index ff86f73592..9f90df0676 100644 --- a/tests/test_relion_interop.py +++ b/tests/test_relion_interop.py @@ -14,7 +14,7 @@ def test_projections(): rln_src = RelionSource(starfile) # Create Simulation source using same volume and angles. - # Note, Relion projections are shifted by 1 pixel cmopared to ASPIRE. + # Note, Relion projections are shifted by 1 pixel compared to ASPIRE. dtype = rln_src.dtype vol_path = os.path.join(DATA_DIR, "clean70SRibosome_vol.npy") vol = Volume(np.load(vol_path), dtype=dtype) @@ -29,6 +29,6 @@ def test_projections(): # Compute the Fourier Ring Correlation. res, corr = rln_src.images[:].frc(sim_src.images[:], cutoff=0.143) - # Check that res is small and corr is close to 1. + # Check that estimated resolution is small and correlation is close to 1. np.testing.assert_array_less(res, 2.5) np.testing.assert_array_less(1 - corr[:, -2], 0.0015) From 69b9c4b7252b5a9c13c432f90dc824522ffb27c7 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 20 Nov 2023 15:17:25 -0500 Subject: [PATCH 264/294] add test files --- tests/saved_test_data/rln_proj.mrcs | Bin 0 -> 85524 bytes tests/saved_test_data/rln_proj.star | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 tests/saved_test_data/rln_proj.mrcs create mode 100644 tests/saved_test_data/rln_proj.star diff --git a/tests/saved_test_data/rln_proj.mrcs b/tests/saved_test_data/rln_proj.mrcs new file mode 100644 index 0000000000000000000000000000000000000000..58ae2769e4b01b1212de8e89ca8f1a945a042e6c GIT binary patch literal 85524 zcmeFYX*8E#^f!#mnWrdI5h7FgUgvuqm5{kPDw0$}nha?U8PZ@Z$_2~laDm1Yg}_`kTU5|DW;yUxEMcR^VFPW~SfzDl_=}CZk#)!3;LZabR|sW8{*!gmH&C7q2Al zs^KI!(B%aMKbAt&V+lxFaGoffRCuc~ zjwJlI559jjM^e^T(Wc#ELYCVRi2%wLAWi|-tT@i*E~-m=qh zHo%a)?p_Arii*g(={NJsk3nswe;Grj7JcGug@v`!!cPz5QC`a`=4xILGfH_U>PRj` zCrp}9E>nd*Z_GmrqjsU*+2ifQKTLybWHso9ABCREe3&4$8`8z+!0W*#uJqh!Zq*-g z23=8Rw(b6pd9Y0nsdOXMA0&zR<}J*mX)4U}b=jiepSfJh-z+ZIQWokZ9&iu)YPc`* z@m#FV5H}-1214fMF#hG@XiM)M1k)!9`}Ic)bzCG-$$$CKu(AXG)4I)BZT3YVHCi|v zA}fqqx)Lh?DUn;v;$+y&l&PMfA*_m-Bs{eh31bJc(c|!Y;Ji_eXsE0Ju@k$w@X7*a zgK`=Y&s0ICQL9m+k-Ts~a{yUy_`|ea=;n@Y*vFYz?PaRoePd=kT8#u>&CuZs<5B-P z1$1&rQ{90YX$VYT372J$z+s0h2#iaHJrUs$BcTdS8<%sX6IB`WojdC4c8o$nFNA1@ z&tjCK!J;UMv537<#3ZWBb2vT9l5_6z;clP1$0Z$X=EQe7aI5!ei|j|~bH(jtTz#81 zcj}2fV|4EcdUjxpP;IA@ko#tiQ0Ox7nB@ots`Hq*6=Fh%bwZ)v?R#j}tE(W8J2#?PQxehkoDE2!$O*am zWHP(Tp9^kFjpa_?6LC5TmpM-11lKhwiJPD`m0K;}>|plwvS9B8DW7=^P7ExOgr63id{Axi^sa zock!l*AFcbE`XTuZm3hRhM`P^(l;=u`-L}Btdd0f|GnqNrRk!F{m)TMLIlc)G2o>3 zc)_7nA}+iZF++hB%7VH2yjVBr$#a7fds?_tTg0Jpa6F)N zb+F6=u56V8S9CZ_An71t?*Ev85&}Yy$C?1tm^K@&ZRZ#ZN!vPyN^kCz_dCwwj2(E~ zS^>Yn1O8yBM`X;HZ|v+<)4IEDSA$HldP2v)B&q>!va=4ciC9 z6=RWEcz|%Cs+&-_Kt^a>UIo@)9mp271Him#72N&hfxg0CrZF)Jc3x_RP2VJm!A3h! z`?wsv^UXskUAK{0r91QXz&zM-rdD(vC!p<_4(P6EAs7BRUv${5o4IIoiZSbZ!l*T| zXd)AYLWDnneEMXor z4l(ilQHaS(LXT{m5N|jJ9nvZh{0(TUTc-Vzv+bM&hd(cXMb|T6D?jp(WMbBzLdfJ)xQRN=H7#s*G@6hQY?k<=ObaXk+jg; z_Z$=qc0%)#-&~xA3=AJp;tX${hsXb9$RvL$Vxyu?wk>5G<~S9gtFz9bI~R?G+RqoD z4=P#EQSuB9%#();mG`-%8h_A_xyyKuHbUMRTN(WW)6lq#3+Q!hAG+$l9~p_yL2VEI zG5M}M!G4mY-s7){)?KfY93=5=D zh0g`X*VTwac^UjFZi20F z1lH|7#dO~^VB)lOk?3P7N=xiT%TIqtf1h7Lf}~DVJh2=t4H$zqyF?+uet&d$-9hH} z*#yD!vT=ehd%8q8&72$ZoWN~8(Zbv?{LJ+4Ge%pcH!x9FDooME5bll2JFau79$a@b zg~8Vo;nmnCu49Hgl>AD7#|taLao`Pf+x&!+n?A$r%#*OHeteye<7O0TKgcNz7J%V( z9WwUL7Z~5v%;=k!pbF1)G|{gHIadZlZ7je&$bgll(&YYz@ucN;7V&OPBE|>Ph);U} zxt;J3T+Q{t=3g^$BCAQM zNOO82vX*Z`en(%S*AGW*?#W&hnR5#bM14kh$wjn|+lBD-@hEn$6!R}^3s*KVgR`GI zD9Vb=WwaD?8Q00ug6p5uYNKo?a}MVlI9jL-mgO@+{_|9*VI`olm*FnB=ZQS;xk0tq zey9tJ1*MwDAUE#^c(EKb&kzHt@+wCC*gPmHD}<|o1m^Zv)@hIa${fD3gmE5~$<3B~ zSr>Sq8G-|QK_mDQ7{!&4v~7)Ko9RCwg<@ zJ36Q^O6cZ1Ryg(R5V}fgQ2TsibfZR7v^A-i6F9eX4G(^aK67!*m+$)>s+PKO%3c3N zp|UAlw|hJnJVpYB?Z!dS0(EHIwS{|dwp~=c=a@(@Wem`Rdq5bx29n%3$cycOIUjF9 zZAvMe(%l34?rpHAw+H-c8sSBK3QRQUg=c#>P+pZow%_X}Dm7J8PV=)ZQ8H-ifG9BlND?h8T@#bm{p!8OVixRQh!|{!hYoO z^9jUeSqeN4E9V?-_K6lY^>9_a4T5T~UgpR;Hx!)Nj0_hKBY9&rp%(8V)J)b99DNATs;7HIv9nJ);HQ*(r{c?1%*@V{OSq?`5Q_F^nwY-;lJ&@-$Yzk!U66k-cVHN#k$_k+wFWx_WbI$bJXv)wP7a z$zM-r9SWvyts%5YS4jUJm`J5-MdY1VE_vK_m3VAAOX$vY5_jK>SZ_>#>fJX5OSVKi zv@$8M#%w(I=9z|I`-}?a*t0TJ*z+Bkme~qxreWdpD^Jkf@7`#8d?j=8PrHaNlYo+z zsc>w?a_;tqYn;=%bnc>WBj?n?S1b39 zD|;ycN+sh!!(cKj*eVO2(WcBM^<7Z7(U^Q2r%G;(n?trf@+H-VdX^QF&pape;C&7k2hsYW1lk(7joL)d zp?ZJy>6D|BXtmdDYV&gibzK-n2M;IF%FhKfE3tr<*dC-Gv(jnonlc*amO+2>^J(OA zJ$mDsAsrM6=_O4a8q-)$EcXwA`}Jrv{;9lhwpuY#2@kj+299hNrkM z@oik*hs)gfLowWG{-5alQ%$Z~wup1kmw|`dR>8Ii1ZqX~jKa|wC@ifT{?*JU$tBsO z^W+CI-G-7jJsX%L-vJiqC23G!Ae~^HO^?q#Nu_NM(WAvX>4Wq0>0j%$)cw{$nri~| zp5J3?_^XeuH|(V)qFyR-u8+zr@1PG$+ow|y%!`}-2%xdtV)ZnUvz;?5}adgpb7B0i&J+JZbjEkd_9J9BYzhah)(C+!b@L!QZvoQ);6yh*{`fOtA97g)y)DmWe0-e714QaSuNw)8Gpetsj z(f;%MsYBQ?I`(rVov&O?TLy~hyx)7N!jUQ(`MI0w&HGDV`6}}XUW$CzBq{z+kTf53 zUzv~nEzchc66fVF-J_>v>*?Hx8ah$uC{;AyOGjM|px)S)dX9cgc1V~I?=wn54>Jwn zhv$z`XxmL>e*Od6-qD6y7A`}NeFRJ*vlYzFUV(1Sh49-+8+JB2gTy>%C^fYO_hoia z`p6HSZXO4nZys|-{777!9|Skog~M9+L!k9E6I^uDVO>`)d^+L*O9mFhks~h9{aO)a zxEi58>&>!WwFB`5mLC5PUX?WXq1?euFx2lbJ?OAXXoY15Sk8c=+i4&P6pxPBpZ z{g6h#7+s~Kk3FX8%YRbO{?UAd!Z`jwg$dvPh2h&e9e9t!ru_MSHQxI7dumM|(yyJr z>8EH_eo*fdbzGfJU(QP=msRHowKR%^j}{LIH-EY&T=`ufJiagib)8$vbt=CDr`V$~ zeA0o#Wy#z&`!ddCw-Ri8wGe94RH0>@Bb<8}4nt`fus$LODyMbB=~-3qen|@Getip> zo9&3k>RS;0TbmQ4IWhOLGr4lnZ&*+7k>X-cTC;EqT^OHE-wu`0RVR+oGe0-dqc45v zOo>W5=qJfXJ(lKc&&%~t7oz#^Mvrm^P5QQZ-Y>Gt_Aygy*Ud@j)PZM z3qAJMgg;bl!{<6#@F!Nu@?$+7&}A!M(Le9x_;Xvd`8(BGyl2=0`qpy>HK-Va)-o}| z)+x~_@aZgMa#t5lME62%%~9yJ&aW#Dy2;51=fQ@yBsgq*2{t_IgFAORAxZir^jL{t ziqmZzU36em3|ZkinPmrD*j-hntm|W0e6q^|JLyir5G;#N;fd_cjPFG2(*e3+W)(G_ zX-f0ejL3?0m&pFPNwi6CkUn~3!l&`ty!o#dDq8MA3;#&c1^J`s z%-?C~i@9+k#RK4i0t zHH%4xeFy#7q0E=+jOMp#cF^kV<5X<+KDu^+fXd50ATE!5h-LZ?NYrTNEvec}OZ()a?`YdH989SCnKo^r2OyEEor+)%xZI&+V6L}o?b(XKVB=$y-bD1Nn+ zJh3V#3!?6mL#fWRw7Zz@8tS9t9Hefo9J^x(5@5`>hUh95JFVZ%T? zG>-A$UfAAX&KW47B+ovk<@z0_)M!4sEJ{MtX9uCb@vC4o+D-0j-X$`{6X~qa`>11m zBejYhr2SuH_)E7q`qp3>=#D?iX7K%N(Os93TA=uYdLm&%AR9 z_d6Hi4`NyPbjAUk{VWfcX`R3uv=88|;`6b6%{_Khn59smERxP133=Caou;L+Mbx8k zGo81`fxghIA)8VkLtsrMEc4k7Le~;#SauU4J1)Vx=lkH=hd9V-ScZ%TjSvZ3E_kw6 zf-6$iVs17UGy5vNm>|yyf>|lQ!Fv87(xCpHD0sM18TDMMQ*?)pTQ0`;&-*~_16I=M zN5ohU>nE)76m`5|nk+W(y~r*|{ExM`Ad8=I?pS?pB+khW!_w1S@mDuFd||kjRSxWD=nK!?%zj?CM3~@!&~WOyM^?i=}*!F>&c7? zY2uZ;7hG)mK>Ftgm}_?iysTzH#ZCby)l!MfpSvL5)|)x|5pyo%(zzL1PB3qchM1|9 z^Wnr1Yx3}TEO{@wM;<))r@tSar8^e(((JJN^t${Z`k&tmawJWIofq27hOBta?w@g$ zb(opX8t=ukLB#}?)@w9y<3iW4L`(gpWVO)v{H7qj2sT`^u|XwZo}JWZpV8n z-0_Yy9qi|9gwYNc>|47D=gi2&7S~enmXYrc?+6v2rIN;|U%DC$M zb+&wKH0zToMv7b4QqQCa>Nhc-%G^$*`EKF#S&S9!5A7sb;7dYUf5W;*3dC()2lT#< zgWC_sahHwbP_&>6#e26QQyhV~mTip5ry#C9_8KQ{5do=V8^HYNEf8yJgv@2FL?`_y zo#6M0{_PawHy`h%TN6`gt#b{za#o$4)s@A@qz1C%H~kg*GEzdlaxwJvnI>yx)5o?; zjm49MZ&+2qcJ?3d$T~JVv$y;1vVG1oaOsppob8i?r|_}3+G`AcZ}F5pwfqHpQC|^1 zinYX%(fXKN(8VuDDc~B}m#oE!5A0p%A=Wxy92b>$u({h0vYon*gt^nkQK#!A^h{zI zbsfy5?|*EfPgA_<2`h&7$%|9oY(Hr*!=(J@7;-7>BG^?Yb3KQqBh3SsQ4#J%hFh*9 z?P*I;Kxmi1vHK8c`V4^F6$kQcl7Ps5SW9|Emx*bZkOr>2Nvog8^52ii^Y^B|r)S+_ zD7z~OJcApBzn6FjW90S{o827I*)@qMwN?w+ulv{`@fWQ7D+w(B=?ZJ7FrP)ya_r&0 zQS6C_JiElv469Uz;kQo?VR~f`#%JttxW-#HRrfv{5haE<87bo0|9aWr>7Utcg$DRx zfeAhoXphB*=V5orP^|Oa6$hw)W@`@&36~w)Opm*W@s$A*{4SeIG_Eg--f}djsnmof zxDFBjQ%S^ZwIXRWy#Sr%#c}}_V*NV?&v$u+<-+07(c?kJ&NQ#%0ETy&*4)k!56#eCUlDs@Wg?L1~1^cIq;b(s% zC(`d^7Ps7DRB9uU>%0QylGj!w;`32K?`HHZyIT;obP=Z#5CGF0o49o+g1N1if5IaR zHR{#fN>i%G@Za>5_(R%>bi(l=Vf&(Tws#A|ZkJm}jp%K9G~o?-Ejvbt?v;Vrwsm3B7 zUg`5vey*AUU+nN7O^hBxeVHQiH(8OCL}i0nR0RBVddDeLiZO!45h(Fq5-NyVf+`;@ zK|LAvA15-61j5~w5QjbfOjYR;i@C9so99*kEdW0*HoOoXE$aK?ZT(D z*5R&Gs3E-I(k6N^_qJ<0B)uAiw%t07;uhUO=E}wBQ`Ix{XyJU6>amH5y4t`L z;Ck-)!BTEcO98Vycn&ggehAJ@Yw)JZm4kQ`)eQhAN zcF$#F!m3C~fQs;i-ffoOHwo__xdZz;=i=iH^Ktm&dAOizHcpx1g3F$KVpXQjX5Y8Z zW&6@YSn192?6fI&SmisaI9bmeYu4CcugD4b*IrfJHe-ODA63jAo7uol_w&SAt(Dki z{s2CrpzQeFU&e9g*dDyIDG|$$TEw~sms6+EKXmz_CVD(+2~}A%kIa1EjJyx-6{1n| zgk@wm5_U_Y|B5Mcx%&x4hrUJg%f6y@3OCT^xcx|OaW{&c){54yT+F<4xW*0o%WyT7 z9dNQujxOz=MU_u+v}C3hzq(3||L@;SYMv*Jr!j?CxfQSnIf7k>%L#Wv%_ru}8DI*_AGK*s5w5b}cxF$NC+{Jr7r7< zXte=7=ITf@$0$+0!+dPcrSEu=GZPSQV5x@p-Dz@+@tBw2jV7JxqhdF463$OB4%E(SQLFt(~IGANuOd8+uyu&C_4dIc0mP z@O?AcmU^13{AEnv1*W5?(Z|_M%jEH+9TvEDvI^Gd^k)yu`%T{dilEO%{G+Ota9Vio zF_DPOAgT+M$oJy~P!MnjWPY81%^Mp>qnDV9y5p_LW1`E@6~5>QI7rM97$A78TU1s0z-@xLzmNh9X(} z;3@>}SpH2K4BxD!q zetUylGZLp+(JcLHoJ+%ppV9ekf9Tr$$|Tw$c@EZ}7oY6ZP@?jg_T*zk3J$FI$2h_uEX4uVsmNT#55A)b`(i!+{!W97y}Y>lb{8~vi5L`Ejiu=)a8 z*8Y%+nGR6b-22of{V}~<^o(+ul6<7P6`y@ipMQM3nT8iy(!{xPBe}Q9)YbSTJvT#~ zKhgGq=FhxH=ZxD$UqARm#^h`y-&B3cuPvHn+-Dn6*yR(b^wV6%x2uQS`6PqOl@I0? zyu889>{>Qc4w7n9L*%w-(2yZ-+WloO?eKj^e_gTU zwNKjdr3MrE^Z;r8>Z-H!>nt1kKDLHj4Neq3X=mB2ol8=v zTNwMu13J}A;BKQFIh+?k!mgAOMWa6?xYLdPjV_=~1^?*IXnX$WTf|SXx8|p}%kx`f za%sbj`=BT@hW)5n#0I`^V`JK@*$RsiHq3Su-s|XYpbA{K zs}SE`y&hXCO~W~s)3B44BkrWZIOI?gR?tkrp6?Uz?%~Dw_02!5x#mTo+?^a+{&6(l z7-!D!%d+HO@qZ~VRY`5%mC%SQS7_>=8ai<22(^2bO9kc&sj0CH-KBAk9PF4t-2Yj@ z-%UQCI!O(t`^ds((`>;PK2zlKeGW7CWF#jcm@8;;9|w;_+ra!sI#lNgA*DzJEqEF! z^iLwzVd7M=A%a>eo~8XiNAXc7t$EGG3_mr`fxnFg>B5fpBtL#VJGb&PyZNg*-abWu z6K0s;o@Hb4^-Mi1F6oJT&uzvz@_X>{>-(^R?0#IlF%GwE3B}u1vv}_*J52rDu&-k< zc2mv4hZ<_|ScYP~F@SRePU8!1k$9`gYu54DUD3Ae#)D7ug?lr5xR`X`vg&%$r2&az(n*VD4m6ZkEo-S{1|Z1_VL{!-7=RW!NhAPt^VM5lV4p!-eZ zXm*hVow%@!#JZc2tOW_^gGN4*aSY*P+>GH{rwb_Rn?cQ%AXsW-3%|#F=kzjHhyper zV9G8BIUHUj58)4nxToFqu=K@p;@NtLtZaQi&bNuvlc8&9iS~W^Xrm54bCx4txoa~2 z+^&!QXFHir;HI&w2A{IyqIK|j|LOSO-4%FIULf9*$YAJl!iiy<@WQH$k$89?7N=1- z_@yh(9z7OIWV~l<#QRu{AqCue(I3y9pNro_p27@Yh7;9K;x$p_INd)R#}zNeSL(mB z>7iA`C{u~Yr#$%WlQ2JPkveZ-(o0u8K2E>gE2OC&2Pk{Mfy%jDB1s-01o3;(=>Rw3 zuis)qS3?`*H(d_AMOkn)>nM!ZOa)I*AJFP41rfD`R~o0eZ_yrbiuHu5b`~a$-VUeD z<;c|H%_PVDHPO8|g?>`Wr-ydFpr?N7@P8y(K54QwemTI#O+|jrff+p0WWnw_{gOf-K`VO9hD{WF_{y|TYc_W)F**ri}Dg|`% zxdhrNb(&r|^McMFsgv{;YVp?X8hrTPL7MV3j#ea(VI9XmV{2u;vw<#>`1LAT%&5F$ z3)w$x(<=j9)Q<7R1b=)nF&tYiXYo+`Q+8%*B0JV*Kbt-JJbP5|lO4TK75_QZ&WdIk zu%BH&lRYmZs9cH`9o(l+Cf`^=j2FbxYv*KnE9J@j-d6_v(W#33)bJFApzHErSa`0PbiglBs7b$(lzM#5h5jYTG!|&Hb@-)wC0I+^!Z{ zsoOylN9rlf^+)K0E+Z;#`dDaXaE!fR+sIBoG0gT`7~$q`=6K?3ckIWl!If$wxgU*{ zxY6Gm*XQVCrQvRtY>H%0wa2p+W3I9gDUSzBB=MFcSGGZHHC<&jhS&5J@YV-(d2f|o z+WGn@ovWQglXag^mjVrb(=H{x@xXn$X4+PI0xQxmt%<~(IRFxei%IH%JW}24HWKr1 zKtb)t7=hY-E?03KY&vxkwk0({sYM<{pV?OHzcRDSRj;1NO`E>uc96Co}L#G^@NB*5WgAQjk z3Rk4MuvXuJy;-h>7bE@h9`p)bf|-9^y-ayMuCAeM_MHiQC+r(Dymr_4L|bTlp` zl39H4E#sT#PhO7TsF*z~=_R!kdL}-G%8L-~8dXS2ofJ{DRk!e*SSY(N>mC~tlyPB)~}CN3QD z@k7vKUs<$t&j15gySV2=_n8-Adhln(Vkkde4xXhi;IjOC(6_w>qw?AyJwFS+m2QBi z7mUEe#sjjqyyMoDzjhF=;p)Z(EMNj=siDP(bg;%cK9h2$((9&@O|_SWdLEf<>k0!b{csaDA?f(ypM!Yfl7skT z&VHPBd>hW+?2Tg-$6?j+z#cg|o7Eh5O^8pgC&I1Q$=e4kT_fH>KP=Yf)k|&pML$@6 zPv~UcfRp8SIk(dGKe<$Iat8HRI7oG-%%xkUK9bO5^~AHxhUDBE13NPRq8Bw^QKw}p z7hl?H?>9pR^56N`vmLIWpce_xFDJs6o&zvpa2sS*Uf`G=ecblmu^{C?1ukvz<0NJz zFn?^nFq%*8P}8Pq$STDH{AjaT~i`ziCE95 zo7o`E%1P+Gl^Jt+@(xfT5kx(poqSWDL%&2ErL#7-Q2zu8erlsWANSpyC-;^3%~M~~ zoYZu>k6R_wE9qpf-BH3?lfSZz=@wR5u?{`_aE4eH7}3tgbMWx~ec>0YTf+MEt8k=j zEB$!uF1f z1b2FVG9_cIU4)(LhlP#z-Gyx~7-fD-uRHrj9$pH{VeXnJ$nTyGDa+?Tp>Zt4G)Ruv z1~ZVoxd=vO#l!mw3yA!&3jEbJfm^62^nXj?oaUr+x5_3m=sjXuZdNdFzGREG7w18I zt_4|lay5y0SxzM4zmhw{m4C`pa%Mb;?Dd!%2TSWI1xr{bl%! zU-Ep?vqma*bOYThe~AR2`~enk8_DzqtLVMshpAX`0aevHMOQwJre$Hjh~yF_$`~8b z+4Vn3TH|B#YDGVpT=A6H`InNo!SiIO{t9vb-UE{4H(FiLkIvinGFJxTz@)PPL}7gp zC;k*n+x5Wy$_D`-FV3jW%H(3Qr6IM9f#MTW;A)U7Wc-kaZL6(e{}NAlTfQ5tEq-uI z{f>&%Kfh%*xGh3O!}qxW!zPfv*aiX7kxcG)YvGUDBf@$wec^>`h2;7{JG%1M0UG^A zMBzTAtFG*)=>c+7dCC}6^x=}w^X3zn`XQXk#N4KPI5~d*L1lh>iWG0!(?!D**3;JG zo#g(9YBE1Zh4%Y8(TP35l#7}}HNRXVMj5w((-Ohm~3*%*@D57z@|Al8kvpZ48u<_T{X2@@6`M14CnX_fc_kd1SpdNm_VA!UL*$=&k}2|) z6I@$(oeS(6184p!f^v-!s0Hf5wWQ@BGFOKk%Kx}2XKDl!q)nN9_IuIwo4t&!cQ6{bs@Eybnyekv6gli$Y&B^EpXfcbKgg1)CiDxZ30OXu~QwA(s7# zo{pL)P_dCGVf)vSlYJM6(m6@0(c(n2w;iI9H(Tk#vwie?^;l+9Vbao@-c8fvx%s= z^^hL^3|VsTGEBN`iavD|!(gBRF?k_L66`O7y4q>rKXt<9@ylS%rgdDtbvC!ctC#zI z%pJ^p2*(J{PM&F!I#Cra@KiHu6ZZa``U4a7P)BU;^XM0#s|iIXau`>%ZLnbL&Rsm zpB$GsMlv>DBa{CKsO^SQ`szyutvr5_zEjMjPohfbdAkZKwjiGNjqH2c$bGqM7(?Cu zOQ3Q(9@J}^9=*8SpE}HspzOSs_#LJz7DiHr!S+?@hnwcJ5*W*AweG?^|N zoknxgMzB-f0W?i9o_0?Oqx+o{ss2(4x+7PDdR+ZLv^Q(eg?DO*MNAk`x}iib51LY~ zZcSRmbPy}NkBoAxgkDcq)IIMZNZha_5e>JYRe3GEYZt<-@jR$rkteI1T0u$B1dk2A zfnRq4G?o40yt6lPj`ol0WY>J=9N&)z->q*rot+l2^yg#-B4DIG=)hI!0N zK?w{WlqcD}cI5p03h0g7#9UNWMl*6n%z`!VxWb$nBy`{*A4Z= z@nIYhloye9*%IQkUWNGPC=>RzCV6k7LZY`&Vjc3CY)|_^O#Uk-4o4@DOIMG;v{(nw zi!6bqB}eMQy;9Nd+uxYwD1&>;6hO~US+bWe1>^cWsA&NRdQb$9Vt0VWtORaApvZj~ zmW23ofzaA-0%;L4aKrN>ry>~SQo`PG=NgTx<&v(3 zAvQJ&Epu^4PDc62D8(D1t|Sr}aW|5lP!5kq`k3n8FCbZ`SCIH=&ZKrh5b@GmNSf+o ziJ`|Jz$jZ{X?2q9mUu!oPJc6uFPSnekG1sSm6@5#X0zU`( zVBXSE#H2uttia9iC8-zYj{N~QlTUz)W-@G~hHx|`fU8!;+|$#lu;}e+E=79=*Zy@a zXJ4_~A$9jfL1SDFb9Y{%;7!XNd!vA5-1X#Y?pgR-Zpp-7+_SZ=oT}Ow2Yt;eOyl8j z#$mU_2H{td_NSu_mSKAMnKKMct;GzitN zCCG{j1EME=4|4zQhpYGE;pW?HuKJF(NWD0PlMeqW^4vIEpj6)=Xuh+NIoDUiH1tRz zN4H!iD(DcGE2|8Z2k&xc%+7I=^clDNjuA|or~nGnq&Y?*pF1o5jB{>4+=Fei>KclQ z1S>C#809JJ8Otq_(B>ckXFWp%k7l?a-P#WHWBPSutS&%1m+Xcjr*_bL{Q$hK)d7Fy z7)gZ_a1&yODl5Rnjx?1SBsGz>q}+R9y~-g@;q&o*IE$ZsW*fDZ3FHpi7cN z*C@&h?y)--Q|9nHOUN8FYiC?^KQStT(MbGJKhu=)S0MYbnrm~a;p*>J za1n+Q(3v|4jt@+S+7Lg8Z3zeK9M9Wp4lY3TCsJ2TI+2WyFV8sEh3v z=fo960`D1-XtL1*@oFJ#S1yF1*<$2zu_oz})FNqb#o(8MI#QCw{$ z`q_9~(5ChVsvgZ8!4~}ntvwYGQ`7k zBWfxlrl9F@$lex|s5oYY4tQM-~}+AjtRu>X97qDXRo;LIW5p6*U22ANpqob#y*Hwu~e$-c#uQ`YeF3X^E?Z=_PPnyiU1klkU z0kKilT=2@(!05NZoIwghd&|K0c%Gr*9 z3vA!FLiVg=AS>xmhD-aBptms{_GH=Osm8IaoV|u{)Z|Duc<74w?5jdH{E33B$Z@qFoH=->xWc016xjPG0;;E+gSYmBBEPHv%mAJ+Z$fh+OlAd46s;qvF$nwF151@MjY-LvIgK(_CTPB?iLh&481p zLs%e_B)YTOUi87gNbodN$VhssF$z1|nU1mY?8tE)jOVd@fl0v%+-&;_&kD=IU1c|M z(-?WkZwv-$ohTS>%Rs391(B-SYG#kP36s*}R4KeUi+y2mg)Q9lnKjpX!v3uPE9h#> zg^fMq$@h~Du;G=pNP!J!ua6rgJTiR?^ZbQ0Nja=Y+9aECRgtbR{?jDkj$KdKJ$;I7 z#)2%6zupGvF+#u_PT-x3{NQ7AsKD&n1a?!~S+>_)O}KrXy6}Q=GuxCR!FafYFr|l1 zut!Ql*!d^3*@m;VZ1Ce0cDSLJ!NDs<1;gTNu6Kij>raCJlSC*RHh?i7v+-JyF}|JZ zYhT_g#i%x(X9i^#K=_n3@U+$+^f%3eC$)~SHgW>ko7Ca(rkdlB5@H%XUXC}EYA4?xGy+19O$*V78j}M=5 zyW~kc$x{MmUOXp?`mR!WOI8iflkvve3eI7-*H^Lk3LS{9G=#3N&M-7!2CTT=9(~&y zy2`}8#$Tqe@P`=~{5OcVrf$OvX%6;nxs1O|)c^%C%e3-QFV;7mhdpA>R+cUaV7Bb} z#f)z1Wu$NGR|bD}0riY%@QaLxl`k^EO(Fqub&WxDu|3#rtcGHrepqeP2o>MQ0;*ff zj`Kdx2KsMjUt8odGuE)onp9PGpCFlSZP65JZeAm#hR20nH{FHJTLeOhS5t*mH?!D6 zEn{$B{}o^JFPFDqBU5GAB*Q@R7Bh%X9>%wOZsS_zk3vkNkV*F~7_#M2vh*=`n!YmSwW^=ht=8w}iQP8nbaA3$4 zbnFOB-6hU7Rs_Kc&r|R<*c%)!?0|W3ao~Nt5ujYWA4)}F@0$#}tK%-Ss(&YBSD3?I zLB62hD2Ub2{mAZXVTE$KdxQ^F%!C{Jf3bPP8SI{Y@uFFcBAB)6FPs*y`zxYnKtOnb z$j@#h8+&Fl%Y3b1dXEoa>w}tbkw1k0U7aB~G8>CV!ADWmXK)5O=Kjt*D>XLZZH9g+r?iWCi)bSfa}H2kzHa7%$@xV^o4(*DO-tX z+J{123xfKmdazS=GEk{>5US}z!mjd4!%5+cpYB446aR;L!`3mP{!&(JMGxz;{1a=H zBr81pJe950G+^zI4~P_w8-ZD1BFLxb!#BgVP;4a)=*CRZnVb@)=|nqY^#w6f6;JUo z{bD@vvk+gg+b!DOJx&l#OUn+Q#UoCR|@1f|&>ID0@96fYY<;<+*Kd*C9@%5lP9=ijeC3au7PpvS-%o-WnvK}E${D<;nSt`DC2-^DK4@z@4H_YjU|GX?=((!|_YS5rRSRda zv6Ukji4{Tkd9NdkIxNEJ<*JY~966quR| zgIM{?1~^^7;urb5@TuRv5SzFQRySXUvbR~VTY8(w^lA#b)I(W#g$%Gt1OBYSh9c2& z?Vb2}5-{ti9ILypn7u=!ga_I!g%2JZ2)6|7VEy7o2<*?z5Of-jXXMkq;o6&tu#vZh z2NzF^(h|JwdTk;FzI9`9w}B&URa^9pD9zCV0S8O&ti` zw-Mh?_*$8>)QEZiOq1$?vo;~|qVTy_;7$BWC|%pgxN_6k zweyPEk#8l0aw}umbInFL>OW<;IPxp@Z}$LupLlpT=^J*wlExfgzn=Nnl7J=b9Py4! zO$fT|1clee!@F8f*gNBS>>OMrNI2Xqk`*Qka&N~n2m6*Y$DCquO|330 z40(pbum<>#7l8GrN$?~8jhJ_Qiets~_YM7c!(0b^*oq1QU9}nIvsTQk zvQm*(wGuW?ZWf)|^;a;V@;ojrXn^Ndy>Q}_G#UG70#Q-=24}m@gSA36Ofz}{1Fu?O zxHJu9tApUF>2)}3s6y&`JjlkugGAT=JPH3VpZKocMi#jWNl%hKc_yhs*82#E=a?xZ zAy|`i9#AIhFKLrm#Z6#ctp}y;GB9yQ4OW>W4`QAkPmh*{$RHlwN-pBXq*KuYV*LqQUQ+2h6-}JLl}n_mzmSc)0sOGM;PPEAV%B&HIuN|i2d2? z!?;@egVpS6s5h4&i7Hcw@S`eObEOmRH#I?3S`)N*TZwB0$BAC&d2-M69nlEUqq%|h z^rgQgEna0oFE3}s^)zwD{+}N0-Y%d+x4mdk*K8VAHkQ`p+#$sS2Z`g^B}C_~5eYPG zgb^(wsJ>tWo2v+Rtnq<=>(;`<)1PtVjH~#{%;$ppml5;2RFhHe&J;YFybwzyHdT6l zy&)Q6s*htLGt#W6#aHxgC-v7Air-Z5xH^c`1;XQk=>+= z;MvhYuE^Wdty>P#Nh89ke3UmmsHsTD<>nDTom_HXyMVkNv4;3Q{tYWKqhZ0|axju| zhCag!IPldDL1%Fvvp(IFZMfRZq+<`}`pW=Oky8*}I3pEr$@?yT_ix3Kcbf6B=|)f< zXbF#Zj(`B86ny6AkSM?AByK7Fh{u=pV>3xj7|OSWjWK%AwLA|idF~Va*nEe1zU>zy z33<%Ehad6n&}1<$9SRQ?Zia;~rhvbbBy3;j2>QPYAtpkbguk>STDG&u!|z8(>cavu z^lJn?CY(g$9e)uGnOC51`%gI3UBKz||Kt+Vl+odj+GvB5JX&m4z!{vJ#;xjz722yy zldz;!WL-c80lxbUbNry9GLq7qNwe4m%}G#7uM0WM|pD zWc+@w6C7Bdge~_*;(B#W!5{xv(X5Anzqs7Q-X)*$&U+Nchg`sn|UUjRN@hhRci8_v^xz{F3Bd-eSo?WKYASjf?ju^wu zl^^0xY0gJJmk*(smSD6ZH42>$3q~TV`Do%DQ*{5IEV9wL!3mv0xK6Fd!kt@cVBfFz zWX&ugl`giUej^p>>_dg*=8zn5nViUEUvg%ps3f~)XdWB2TY=g2R~LUje;>~^4Z_<* zt1(lc1Mk8NK>mD}XkNy1fh?#n<4bcYU1rr|&p#Tl!_@);YBTUmnOHp5@(Z55(F2bC zvViv`tMH$LZlb9Use(5NW~^p8V(Z#c*u0SKY^hp2*8V0zHuvk1pYkKfdZqvlgpCoL zP>aAvLWW>)>^f3q`j(Ve+t6L9{&XNVg5K6Oqf*+ZS&foXZuD4J^e-*}wYI0A|E|TL zVw)Ir@?{j-UV0qu*t`klmM=ueh(R)+^N z?n?%d`8xA)ZRZ1f=b8en$1no3U~5qLxd6VdI|_RT>|o|sU+6WO3@xX-1*M@9Y}H?L zrW717O_7DT!IiKlBM^+LrozOHkMPK+a_}{_1@HE)#X%ENvBbs)c%roh)a$$BbH*2$ z#~&Bs!UhFqsoWekf3zWEwy^?^nNB6Yr!OFuuO^d^E)~$UTaKtTRuT359`Ym7mS!}D zQ5VBfiVxJ#sqa?OnHhJ5EBZ#D_bt0o*B=NyBE zqmf{C`U4!iPz||_iJ%oY5_Xe^xc#po^C9Ui^SWb|{q9Ha@zEQW;PYx7)Qz(Si(zGW z;GhDAA9%dAJ_ZLWd*EWRmn`Cp8x&;AQ0Z;lwon_NdQ#7L z*@xlFwsPd?Cjk*f%oV#R){#%iSID95rc|kVHf=JCqiyCqJ!)Wpt?NT^TC5|oqd(Wl}3`T`YdvC=UH-pT`cmpw{>O~Ym0V35v~%g7tB@VDYw_V(4J#X5Z zJS6V3Rne0l1MJIM?{mrl}uSW40J7q#glc}4oz;|8fbxPcs!xCXBEdC+r;1%rSOqA=%u%wZ2kkp3|m z_xzLrC9&6w`i_7No2P@yv>g1}eigIkoHuq^--#z4UJK8HFG6mAKJ0%t0`_ZG;mD~L zupVy0ZJp=f8+09dx*oye8?waHqZqs!bigNI9SOFoB^z#>A}Xcj#970Ib{~wRqxM~; z`SpM3j0_Wg|L>`M+1o!9YYS*!u7}`i{TL$iwU9dPvF3+Oo%z#MkEv;v8HYb;qg!`o zBRwfcbWHs_=OlAi_)Do-I#LI>^%#ek`pX zYf20D{v}&d%E-!l6Nr^>8Au;@gV$M^IOe7!zP_#>AAalvEA};Db17rIedQmK#a}1J zNTp05YS|^Ci_+NTqr%y}--8%o+ZwoH{sI03l)~!=akyiI2D9P(0{D4m2;3~>$nc*J zpqrpgBs}#U%W(`em*BZ>h2KvCDZ7+aPc$S+D@OI9Ld{@84Tv{Og=@! zp64=0@^kOMrZ+8q(H1fP&L^(sXWla5ANJ-FnGOS_&4;5Mb0X2q2}_aaObgefcY-wK z6w<2AKd5Ty82-=j7=BmE5Y-%?N9zu)p^ar~biyfJ+TAsu?iagvhPFH)lQmjMVP7;6 zmZ_32H|jxPb{v+ET@BOn9YAWwV|>Jm;%v2ctogwPOQEGgZcPfW84iPHxg zQtuN^vdbjtFZa9R-(X13ZaGVn#=fPeg=+l2NfY?F?>OE;&WZmnKa+2JKATrj^5l21 zzWnKr7QFdVHJY_f8UZAt@CJ;Ej%A~*``4iDwUo0*#?<5eZ<@Pg0&mr1$y+Qk<}DV> z@uSMG(E6@$TG6qHzB)OMe${9rEp__1`>iTF`2Hl>646W)>(7vNb=IU%FBabA&xcwa z@p-2}6Hi+;5saiYVL;m$Kk0sq>7>WFT}qLC=~x9i0bTgN=?RS2U>9Q(zRiBPAREe^ zhhX-UOlY-^7f3FUWBu0U!{Xicle@n#X3a&rXz+hR*)E#m3j zyG<0?{h~W-W%!$8hw19}@6`AB2)<~64Zp$Lo%ee`lh3bJ;~!|~&>F!5E^YcYwDJ1| z^h%-{O(E&1IMWF|JG+DP7v~uyUykS1&N}jg0}lMzUNb)HfhxZ_vW;GVBpRtVMOhYUMlA*e@nj=kywCQdK}BBHvI4ODTTZ#ZEe+DTL-~?Izr z@pWm!&wJ*Q^jq1a?@tK1w9%M6RA_{f>+!&M4q~(Ozp>ht`Oq||368O`us>g!1g`4^ z`Xmmv1htFSuUOAU?0?6q&uM4JCj_(ddIE^}*9Pj_hJm>_nzUKwiMbdns{LpIojkCF z%6wl++mtX>cM&2YL&B(}%W+#8Q+V$TTP5S#NYjShVK z79;+{v#+$yyNv#OH-Q#ehX{S1UE#(CYM`3)#;7fA3_6tahU=<2#x0P)AnbD63#(10 z5<8ViB=z(y__6;d{yJwX9uoJYueH>P4hDJQ26P2lw#pJ)n;LK!^Z{inCHBmSwKV-IyN))I3M+US%+6f)P$L@~<8QHkRybgT6#*R44LJynfBXIx5REXjemq;M*e*+b24Nbr&gpJ>iQX@2Cg@qBpfSl;!HB>z|80gWFYMsHk1 zM7PI;^YOmP-P1HcfA2HMVVVt+S+0pAC#oxuG~u%ba?HK9wi zSmxG0ikEFx`kA1&;mUzhKp^ULP4wp$l-o5$61Cl8E6&hKZS zBc3zR?jPo8U~ri0&pXNKTvOw=ayX9qGxL$fibym~eI&B% zZxOyTs1o|0e#>1MyARcb55asbWguJzh|`} zpZ&&=-+j!K|2^79010gR|u|2`*!WC!(tP-&m;%uOzd`+SC%LGhMcp0?>3Z?He0xbBz34rO ze|C2jzql-dU+o&r{|sHgZ(5p5=f92NhKz;iy`&eiu4&^u&c7zN#0=vF-k*+e-Nwn< zIHNx$QAoJ%AiDN#GLno;;I>?tNZu`5MOBVOQX@iW_CNI{eI zedhJItL%8?X7-_G9d2EcM!F{msH|uQ_2p8iQ)~w<{yC0+v}F-b>!qG@m^7^z+1~(($yr9lWP#7sEr3uc*IPU@6*63 zC@vRf$bTg%t6k{cSFtp1LpNoD75HyzALwl7D!Q+;f)*Qor$1+n<@0Nd_|e_+y!Ys1 zbo_9fFxlW3_xo2M_omp58)4_ez5X_j3wWr@&AApX9GYk(T;dlA?VtUKcKjbWA0uWL zuld2PXI2p6cNIT9Aj96fevBnqli6toschqqm+V&87B;Lcm^H~i%(kg$2$cuIg(o~c zg&J9&0QHspT4?D_1ynamk2}tfa5~mT@=W_3)#+hogBks_BQxt%cAp2EaHNt z5%tOt%E^|}?gKxlc770n`P7fNr^XV$ zI!T&3;ti3oizGK^i2L^QWXZ~;2Ii-<6*1L6Kn^iQaBo98Oq}lk!MmN{Nld!v$CL(U zLPImtJ^TO{zdnWY@LWOU%I&OMb1~}_eTDtgFE5l?ts`7eJjJf}@DsBByZ9Q=Ra#}% zNu#EIrH6R^_vol+cchQ2IR~jvgFUPFH)}q8VKUv|`~( zdQoWvjYHg05Mrmqv&PBO$xE{%orUnT$_76YBs1$-Zj z25NJR(aO$XLx#Ip(Nj6$v;GpcUGpP^HA~Yfao%#^HcvPGxlMH@Hq$CpOJ_>P&`YC- zNL)M1MGRUaua2|GCZ-V?&u>I)lv2_9QhoF|2DA-DA*!DJ`60 zt2oQIcMQKnN{jy<_*~33U!fOP*3v*tDPHF4Xufs0la`H3pgoJ+=;38e@}yhf7TC{0FI!b@P2({>`p@*$1NMp`iRP`zf9UPI0Zs|p!vPqNC-fNq=^8OC;qFRA2 zZumnwa;MPc$KG z_WUHAD|Z_w?VbnR+WBDDt^%VA$1-Zp9gL3ABW7E_Bs*_!5$rD(8_CU9;H-IKukTPP z*?iK8Ef-JM zFF?^NMM&e4E3)XKuEx7kb2ZL}PRK1UZ zF%zrd&?6p>_~ig^bRA~ONs>>AcErIU2QvPu$%|jYadQy~iDL z?Ng{w|GPJru=FBl?Ap)W-8}|r$J(O7v$K#%g%_GqJ{#TYvP1_D$fGr7;`hUKrs$c> zQgmnMIwT#SftKHF5JszAq>I%g_<`UD^l0WW_+-|{#Wb!&)9;-_?vL-I+3%&MiIk+L zU9Dg5X z^yQ4(B%o$F*}5PBJbj0-i`F(c(4z|1F<@n$3Vb;BD znqch68}Q^*h)`GW9Cx^V6moGAqTegrQDL$JstBBnR&-dQ#WsskWzZ^QwciB2(v9J4 zBeZFNem?Cmv!@nn30%w=gzg=PLH%(Tk<<69n!6$qwIxfT*Ig%sBV|qL zKEE*f%in=skG7@G9~aO+%4=xm#%SuVc!^GVevwMtjG=QHO=;A$bmAnhOY#R_!Hmh_ zaHQQ>%yuQ?zCSYTgkvk1=|?*7T!(uY``SbQ#Vv57HV*hF4e%+U1AK8m%LD&J0!o?Bsv{4h{3W6Iu zzto6dJ7B|~@X+Re&V5A5*=D-V^)D@R9M7kAkKns@gwWRIFNKTNDN zxRqPCJD_>sa`<_ zEp*GFq1lPlNb@h*Z`ui`%+J8e@6TY7N)lw%RYIp-8my!8V7Vq8U8}}_sI6lx{sCTEH-Hd!i z&Z^Rssh&@394FDNWuHmMm_ovAF<`TFJ%xQc44JtF6R1o=JXJedMD-k#sbYpag|*`P zw@8M}+h<7*OeqFWqY$VO`_i{1rGV=i3?H3D%r=W-?6%BMHmqh8yJxN|ykn%ulat=$ zoy&EyO+|qg-51jR8$#&ZGu1T9VvxSRqrj`ql;zikR@1B9d#PEJ2W>5Jq}63Cby_kH z{$s*9#(kJmpJalbowq>0T1TPO2tA}W)dB4dutw%v?{ZaZ!r0h1n<)A!N%y;pJ))ES zP~4ha%!AJ2hdLn&HYxRkQj=2L@~ z`LuFwCrQ&{*=2u=SgjkyWarEI)Z%;)y=Hcp%B?k~{SV!V#rI@<_HQ&un>Sa+>xGK0 z`JTfEwJl*;N)+xpd!_PFb&SCA$4$ZLkG0I(;AePYOTXB?yo#7jy-wCH(Wco|0%{WJ zOTEt((Z&Z)=}+yq)YO8|En`;G{r5&tsVV8?q{?ivDe8vMI{6o;Gi5KDrImsbePWSB zfG3LFDTzcSAGqlkUUTP-qq(x<8(?~JHVq0YqFie^ylxrf%#Zzt9?m$2cEv{_H@Ok0 zQhhZyKSPo`C$GxI{y5InN)>P(GBMnLV-z@ir=1PBqVj*ArNrnQYH(nkxVEN1FXy+4 zXIb1O-*W9}#S)HIJIc^`uk*>=#g$-7LYaO0TLjYu`m8{|mpK|@#Xd-W&3uvNnI8*U zVEOeNFkG+|N-KV0u67-~5wD+z&d8AmuR@8^jl0CXM~+VMv!#yP&(N!G59lJ>8`Ls) z3yqc-O?L=K&`smoNL9ExguIRA_7?e|=^@p~p!Nm2r{9291jeAE#&M`DqK^BteB`5D|S=#?yvdS7<47F+ zQj=c{x=SuX_+D{sb>Tl|*$K>6t3P2^`PH)jZrZWm$4P-hD-Ukld!WbW8WgS5f5ea>b^d>$*T7~)OqpLGAvpmOrmDwn2UJ*_8*1w=>4)^Hme`grdUVL@AkAw|p)(Fv(8<%Q=!(}F^y*wY+P=Mp{A*oIhxh$Q_n%TA8kgU4t`j5C zRxJ_I{#=6&eab~C4ztnHW;d?2$$;KC2XwOKYnok9OkI~d63;WPoLS2Sj$Ky6opoZl z=TUP>^k6jETA{(UC|pG{!Di0c=kwy<+{6eZ$#&9yP zjgaJ!01axXqD{-caefl_xi?*en||#ox9eUqXYwplc<;($`s>ecdgzTMAJlEmhs~Ab z*J)ook4*N=U?scyt`0k)PM2v@kYf%% z>J$C*+k)M`B!Ry8Ui5nRK#*@7nBN_bTf{7Y^Wm!`z|Mxg7#Bx(Ub{=97yYF_e!Qg? z`*P@$QEt@wVi3_m7Q*L^i#hZ654oKqg=qJI^O z8Y&9NdG>5nwATgg*dd3`XeDrIiDjVn_Y4gax0mw_wCR?t{oJm~uiXC97u=rb>D)xG zM9zQZWlm+mAMW2mbyPe_0>$r$<(xO1CikRn(ETZ@yqT&dzcl#={o;LvLWC79{O>3+ zFuerwV;|uVafYdVev|!wHDmC;>y+`{7QyO%ILBUDHB>BzdnFcj=tYC_0eZNB)U( zzNUJe!fut*Tzr)oTC*Y=72YmInY&7n&&~++PH`H#%_yUpv#pWD*bwx!F%MlR&qISW z1dYC7j~uiTxzc|}h)cCKEz(><#`s(n#&2>Gwg`hsyYYRZex{m@+?U3Ajvb3mESrG> zXIY6|b+@>5sT$V!V+8%}{+5>Bmg39F9U4TB&`G6|bWF)A@;$XuG-~)DYc^Y+)j4pO z{XSv?dwlygwy@Keb=yQ&Q!4*9ujBTY(H8UeHFBeihMc$|>}!bv#{}{gTX#6Z_WMUS1Yqx#GwblYw|-Iycoz zfqS)oJ)87LjViu3-$a)WgMqV{Z*|pWj*_|1o-K(Vr4-BR5aDWZ_iW zIZ#Q*?|DJp#ZLdXH)hh?6W@|A+x*G7q>=D`ve*%9FV9wFC9}eWX7-WW3--m;V{9VL zXLD8!vXOiAg(kPh3T0-<2{Q~T*@;3GHY4yB4jK6vJA10(=~vbRy*H5id@3J=(lB?#*Jj(kaf)AJ6YwE*>`Z|)*3Q)_%~5pbDU-$E~cTc9#Y!_ zAE<-4A21@@g0H`^fZt)Ti{EzJlP}!TMVGGb5f1pW$bWJG+GFF30_z;m0cio!Fj$Tx z7oJ3OwqHbT5d_)$P&DY2kKUY%K!#D%(eF()+!p!oLha4TkeH@N+l}|p71lm<>6_cc z!e&so);@>J?@8gzR7P+-oa8_GS%l|sjMSs{&qxyasX4Jx%= zi%jC(k=-T>lpQ0DZWP_%uI_2(Jl@Knix;%SH8~CRWT7G&=>N-o8QsFYv0KNz2weqd zzKo$8)|t^FfjQlwwvs9hoTn*D=jp3oiLxBD006bfkiq!~Z;vDkx2+mYMN%vDj17lrfrLIm4B|ePkPd_}E(B z{-ix`o;6G-74q~(%_%Xr>reL>DN+lCPB^z&gH23-D9qWuj%y9f=lT!Vai?q=Io-kM z+@%}R=;#D{(@bZhsVgx}k#B9+KzZG*0Drj&a~+5}f$NNdjK!l{Fv0 z*zra?EO-g~k^EU@2|l{=A3fAsKwV8G==1t}!mLe~IMc{K+=wc17Ngw_{euH2R+x;k z~>cONZ+F1Rbb;Yl^9Oi(JIeS85KRa#@X4XFb!OYw#V%pde z!Qx6;{L3{9Uw>yMQd)bCnfXPX9k+iUW9(HY=)EAohr&+bbDE<;zHkpncn!i5^*EC5 ze1k;(YZ2cI3)-%KoSu@oL?vf;(VXvJ=*4U5{MQO=zJ+()sU5@+?LsFUdsTZG%bsH*7~K=WoN#KtTPha z$OvRSwt6u>hi2kH)F={OogzeS& z*N$sIWyc)&@O&b~)r}GSlhj=SgLWwwsAg%^9-nb{bha`4m~VvI|~)oJYEiO3AB?Q^W&}r9>`-9_ucl z$%oshoRQz~ebA zZ*hG(Z=@yU=XIFzKcuDj#WiJg;6XGkp~u8N$))tbQE{DUYBq^js7Fr!9fbSBF~seF zD*5zEo1niva9ccQ;DPCLn4J9)7F>#kPH9hY8W{<3MG+8qell!%`4NZbT44WR4aVfr z6G67DI@2$4l6j)vAULQMg_AVb;^p6EuwRge=)uo*_-=oJ$V28nZr&%JdzU;xUu z5xFF=d1nTHDk?=YRZp{VPxnK+nlp)RTu6kHuSoefWtz*mQ&gEn6Sv%=U-!JDvK!U- zOb-+Oke5E+xLAX~eq@MV*?67$MTyUaUB77jQL%@P=XfhgH{LwMlJ8ie$y@(Y;#aQd zpg%ItP#Vpk?)1s23dgr&V%OJRd~4bGDI||fwW@<+<*2F z!o~BHr>}ei%Y6D^w&M%f{i7UC)bE5li@xBZy;sFEtt0TtJC3-n$czbpSizVc+$%x{ z>hNOp1V7figuVNHu*Siu*kDCDUio1hJZ??K$&aJ3oddzG1=8YNsFFaacZ&5JN@kVi zEpVNA6AZc95=ZsRhsq%N@ ze^C{ICNC74@>lPV;|0-Uc%zf*{1fBP^mSc5CF?6_vV530qdSA{-8ou(_P9r`uDL+A zZ<|Km9eEFDCxn3e2n%Qkoy=@UZo)^4RIz&c`LbW-gCafa6rB3I9N*g>jkjmE+MkiEz^!_^ zV0y>~Ldzawt=z-R=n`+HYleVLRQ$>0#XGVy-}JD{ME|i>g3HVz2?>y#twa*|IFfpV z5}E2k(lz}T(civ~dK#wD{Eru@dEGr)RM1V=54O`=yDO>M{dAfTQA>+8UQx06gg)rF zPRnQK)7^(p&;xiqbyT;aDz?u^`^RK*U$BrYcBz9E*}$A%4zez>u-Y#ITu+~d_`+D2v3w(FCrt;r`fqrn z+dUkAG6|o^j>4H97IH|g(&U7skOQedaSL8OVmWo+E+C=&~ISW7*ZEI?UJ)u1r)`s-X8<6BcW@1Wf5JJfa{Fwy3#) zVcQ0HAok)FDMrKZF`Hn#^FFXy=m$k}9Kk!*9@OpBz&kGwzt~`k-_A(Gz8x>Iai%8} zU0MNFcQioao;Qt1u+F3BvzgbAvUL)2tWC5&PWbo+kF;6^I$A3r zvUwCVu5yE#^ABNEZ7rObUJc(Ac*uw@gt~)vB()`tJoh+HrhF8Uy|3<)lR@W*qTcZ zHFKx4j~myr=4mNRVT=X#;a_8kPY3X!lVd`~ETWj#lt$uL0<}w(%o`+{qkzigF3ME6~aCUkQM0siw zKUYVxa{PSK`r;Ikn36_}d`FOZN))^6W#hi8EP=~*mTig|A@sGMDQwZZ!;T8&ak6r! z*i~dh0-yE6P|yU}*>8dIYysHc`2?z@M?70N9ahho3bRVCGIlCCY(Y~zTP$bF3cD=X z#YgQ}<%6@CpP9!+A%=-qCH^)ZsQ7_*`zwP@mkB7I8Uy?0Ys0b_Tey406c&UlgZX|T zB*)By=m~1D>kb1ly3$b7$zsBhlkZrQk-_X5V{O(rH%s7LYXb!lcSUO#?qo|l zG4uR=Cg}egfYF9pq|f;`Onp5Q4!1Qh z^nIJ4*fJMy>Ar?D7Y^gFooY~IWdR`*7|@Uy2aClSRVOF0Pp;Vp95m$Mt&B5VcaaBc ziE2Dqas?iKy-*NG8W_RcGfazjcjbx*ZK(J%20|_zz~L2{jA`X0CjCwfzEo3%2c^zp z>;74o&ziy<+iYFo^go8qJ0Qm{4C4)n($r9zR-35abDrmwe1^`7xWq@-jP*-q|%*^_ZW4R;1=I~>9e(ZS|GPV?Ej!1=H&DHIa_hgGl z)|o&jkpr|j{{_F4CmUJ*{3WGAXp|BT&F>5wNb+GHqxux_?IrGH)?6hz;NM2BL>(b#%uC6MOncIIXb@Zpd?wnw zcQ3hCR6~3#CAVUilR)r{I_Rm}1#O4*!0tXdZfkHQ>`Y4rMyIpI6J)CZtEv-EY|kd0 z2~SD-1xzB+7n8|xr$vmkEQ`yx5=w(%!!*l$iUyu8S#w_C}sRSU>biRZJp6%c#*KiG4wW;v z9tdPca|sIO9Jgf(ccMg|Ygr!yW&2;_<%2HddjD_|o|!@>?I|Qm`zDb)Qqysf_yv%; z9u0z(CgYzM-r>bq3hTJa;^$p6@vJL7l`~pLi-FH=%vYNbl~;=7;(1$QF?u|C^VEv$ zo(4(KnjfMD_c36(Z=C4ml;JqwRTj2v8$s0bN07l?gZOIiZCw7h4jVUJ!xNPciq2>R8T#UL#LLl%lpLC=!x41|RC! z586Lm!QTB0Ki%j>cJ9w4*6vrpsl6X6S3kEQrIT)wE0TS1iQEwKyV@0l^^;)GMP;J+ z^D9wYx|hJ2hGf^s-FRP@7JTIKTBO|WCwdmYNqW8LQaNxNl zGJKGbCi!fVdT}Lgi+BtwF24Zeb|Q!^+$@?T*C8J0P!~)6s}pUkvIKfi11JQgh>reH z7iCO5DBco%S$tz;4Ibb><3;l^e*4~#BpteoH$FDOHs2Xu zcd|L+*^%$@1hW>bQTz~VPpH8u*@yAGj7Or=TaDq~N*Op*frvI|CjrNjkNEhP7?RtN zhOISfE8hq9+g)`Ui5m(lv9z-~9@Q)lgUSM+SYZm}77d7tURXftwhW3sHDiln16XXK ziZ>bG~)bR(T5i zYTaYEH^M{Q!H$c^Ya~>r9^X>A;*}Jz5-$KhV_%8({-*#+w=0N3{r`w)dbhaty)&M4 zatFS=@f@z)Fb22N2R73E=3vsAq43)41tP!Vx!^~%2|k+QNQ%Dg!&jyT;>A~dK>5}E z;=PZ?;Bnq_ah}o$@L4VkPEA-1$H~gT>AxD_ld*MhM$A31$T}Y0+1dm)MF-o-zrBa$ zU#r{3ROCb4eL(ytAx3Oa{1_j%?&s~4IAPLXLt?c zp5Gk`)ux)_G5wx6yda-+SZ0xN$1Tav_cz27Z4U#aGy44gh@BvgCf4K0ElVOakctlDrv z{Jz(bNWa)dLXx}jg0vsvfJy0~`;|8g{XPoro;n;B`2~aA-P7>)o5{GbxfqxGTZ1o= zL7+A{1=!W8fz{tnh?!{-p6Gr7|M|5BH?7MB=R5y_fuZ)mFWC{d@3|~;o}^v*$YmJR zy|4kUI1>d{_e{_tLj$6_lH ze=&fkr%M$6J};R3!=4n~3L>m~By0;egCG7nKvzv|s1-k!411_d$of8PZiI;VX%em| zykw`&r+~kqbK$3^*|7J~6u8UbGMG2*kof*;g!}ey!opZbaPt-h3yv!Qr8X7(Giw>n zOUuD83|er|A$gK{PezO;G=ljv2Eb4g9y}~L0!Y|BkbBk#p4hk%y3Ozbwr1nk)u)D%k`0m`I9v+v4k!Xz!}Orfo>g#H zfd!Nb(180{JWzZ#Ppq0+SovR!Es$R;(My(wi4O4Pl|DE0@izY;oQS7m5Oxwr50%uo zXI*wCpL#*%!dU~1N5ScBxv;at2b!ZeAYGJ$`(-5=!15Q#%H_G_g{~Ee^gKXr zO*n~X%6|g+KW#}sq#54PUWohV^^xx<21tbaP`dlS{nY+k3XMKyM8nk&zzs5!xz|P$ zIi7B$bct3IS z*N4{s=|Jz>oAHKvO{jBy3^2H6M3$ZSi$BeX75TypFg02iI?Wgh7idYtm%p!rq0SV|@FIopb8!i>eJq z5WYVSlOG;|(cem7LQ^&wX#!?m0RWh-4D%obV3!adE0N*s3!Xb|>@TP0^ zk{)0K*yp@%^EFtcbrsG! z@CHn=o=<#3ZjrF|JLIhLcnY>1pw`wI)b7|{k`ZD}oBLkT7sp@HQKvd7O4ekitBl!) zQ;=O%wq`?Hx~R0zRr+mp2K{nZpJIo0thNYnro9}ew9f;2^M9a&;}&k3^-*q5X(~7E zZ68b-5=u<#6Un$WX~13k8`$$^1b4F92sY;H6U_^!@Sma2D~rujvHITzZ2QE8cnlH_fi&j{Xp9s^+f6uZX3*CO-qPOyESA^f(^UKDcr zlX!1U9@hMH3C}Dy1h+;`f_tJn;IrS^PzQX%J5xW9fl=NxrKp-N{V|OFm}tj#9b3-^ zXf6vjxWcyVInEsZ<+F-smzihr9d@R(npv#QX1nLFV#-GXSZ~Hb20}2y%{$m^t1g=P zoKY!P3%2Fr6ed=zqM7C@H0WI#DSs(pFJzc->B9rKWo}_y-7OvNVoVDR`JDmFFAZSt zm$yjC?w@24$so6JJN6XWlBr>#Q) zqY>Pz&;ibORe;2^yTucFZerC3s^oe0W<1vPIvDV`0zGz8;6Irvam3fNcy+t8==SZy z;J@ciu-Ebwc(go{G#N|g;bJFhGv_4zc1??&l}TfBPF!G^Hv268D#}gcadTv40C&9~H}@-KVnH zBUk9Vu1GpzrWN%s-ODlE4z9qhj2q*r!9_Q2gjq{Q!}!=@a=${E+8;Vf=CrOCi(Q*> zLG>zPZSWQUn>vr!e2XSQO>;=yobjYJp#u-&i?P*qS^R36gor*2sr~X`Qw798Ml)4Rvmae*FgoF$2nR^-AC?u0I_gHe| zMZAQQ5k-pj>67zo`f!PsEcW+_65n2zh@U(#70bM{2>EdwrG&gzg1z&!gDaRIzt?FMS<2yd@M(PuVTS(O4yg|Y8KvbjoE)^ z?1&tjr16GGCo+D&VE3px5|-qp0hdYy zUVj|V)i^HaeAZY)m-`9SHfs$#^}US^ni&a~mrN37D=7(=Ev~cY){;B6oDW!TnHt|# zxSjvPF7Y|h*ZIW_llij5^|bZ3bME5gp*!F z1QnlQf__jx`!D4x3#m?LGgLN9fK^kNqqM|7uH%ydzvM*a$feOx!X@DhjqAAkt;0Co z6aQ$!kUFL_&RR%yUn&UDQ3!TuV_%nBv2XVZxdT@(@d-8R=qW!H-H35R71d@a{KR>F za8nj6eyYJ5BFflS`FpHK{~@cXmJ*g$%Luydui0Hmlyc;&D{O3DGdmFdj$I0P#X|lt z7Fu?Jtz4JH=$QHJ%=|@cx#v7)`vkIy&DW`8*CcY=sS(;ny@HO4y_NGu>?7r!k;Hr6 zKBA#JfJ>LHgr~>V!}!}*;Id7TaG_*Yi=F%m&uPAhw-!3!VROci?O(c{uaAzw5RV2HG!TDsAssorQ{U(B&^1S<}jDW}IUYi_@9?r~S-*?`n24#h6{>JE^Vc zE{)uBo9>ucLODfs`p|-h=G!x2etsN0_sk!vRvZ$2s(*ox=qBN%nLXm>IZv^w&11>I zSS#^&FPG@tHl%#=WK5#w*}d+;SZT;r@pW}S@Mc~zcJO{g)aOgnWsd12LtUN@DJ`aF z<2bf>RvF_ye_{G(M+oMr$S7kJRknmn5<9O>l1sWDy!p2k$ z7q*yd3%Xf~0z+?D*yCf&D}!f!_S$TkyFMFnN1tgMnXs?jnrxQzZ^<11q42*|^wHUg z#7)78g!-Qy1C?amgDvD~V;i>G5Kl`qCo|{fOxCdJ30pW?LrDFhCj>3O z%%&{5&q;rmL3Vn+=)3BE6umeV$-jz3iqn=NuNRY1lHzqf_?r%YFsg-9Kk$^>>pO#Y zcacInDpsKTv!l@PBRi1jqZ^v-+sePQi{UoCb!Tf93>Er-si3Q`BHWlQBmAwu$L0p5 zF?D5q7PwJ|ebEbNFDB=(O_DkrU=Yr_?yE4-O*#5UveR(4nMm6=w2@@RL1L-xo0_ zt|qCZ<3%z5NyY)G-bqIBrX@(HCLh_G zu13EiI{8?iUTB`4!S??!6z0WE7rx%o7bexdXRAt6Sk!}QY(b+AYdWpYf=10|9&W4I zlXiDj+O5K%bP_$XIv1*S7Kz#|9chzxAnoh+rT%OJoiVbBydRfOE`Ki}XU*SBvSn_P z*7kX1T7I(lOTQRwUVH@HHqHm`XN$2^qA5|S^B^Ar9mr`#S!z}^ln!}$lfHejkolx! zGZ}+BO#6q5p!dpLF!-b^tWQ7Bs&*ZbxEM6i-gRLpYSlgz>KTtztD?~chc&1t-3nc1 zwq2UsE2k;&*1%?x@!>kTQ&2&E>C7XGT{jR}`3t0n+$C(A8F>VVrCv zarapZ%@xm*z=kVSr&o&|CAVH{CbuJD z6lc>O2tDLtfcGbP{P3g;j`yAi{EbG!koD7`kaJMvYfW+drcCk}6_VXkElKz5JC)bD zP`u1hhMtfsr!U5;v#U;C?Bn@THddpR-CjLhxVDfNvW;B@g`qma^h2ra!F2=vioX^r z=m|pYCnC|{sNG216o*3Q#Uqsxf3(+6A62OlKDs8Co8L5+6`ZhQ9v+4CBw5EP_@C$V zS9+mKXG+csIT9%<7 zZr(^7#G_@lR;XI;7yomw2cLB^f!1a(WId~$S?QN4)GNW0M?NMfK{EoSxJILORZ(c( z7Ed&!ekl5@T);D;&0lPB=5Mwh;y+?X-Z4Fkx&_2CU4s+Mr}GjkyLgUi=T|Z#vqI(? zv5bX!S~2}E?UMa{6uGizJm;eo&FPdxao@Fm!EFZnL3~Xpk%>D&o=BY_+ZMW$>c4@+ zsoIHXLX5rh<;kNl(}|zkVWhGNxphg6Nng!pQC)Sc-0=w;?lhN!T15h`QL(c{pa=-eKxGR@Cmatl@pkoqHxG~h`@d9V}|b6SzhBI_G$KZ)@?Y9P>>Z<`V7AY#gze#ghx2*Kz03YRr2~C9M%Mq&RvzQPCyGbB4W??9%k1XuCn#@?F>kIc*}LBg2V46Lbu0yAr?;(jIWzACDoy* zwJ#F!Hp%G7A%tF!Ealb9cJON-b@PK-rfBDup(r_jBVSs3hQW~V)}XkSV*aFbrlZQRugN6!z2Yt@I5&w&Ej z=(mZq{z)adIxFz;BVR@R`yGIyrx(F+201759;Z&-hxImX0Y=vb!JB^)z5V<@62C#0 zZ9AI6CTzIOHb3lTyPgae@)mHy%cdQ|{Gq#q!TY8{=IqaK>$gRc|3E1kR+WwZ)0=`8 zKRwDf7iRI6L%R8bQC4W8`eYO~>>W>^Me;w~7W2*A0Y1b2Ij^^!N6*&9AtR}Dl96!AZ?f>bS6i4k>puH0 zXd7$tGGk?Tq}bVxBh<75(uFpG)PK$dx)e^NACiKo=eC7(XO|BBS<^>mU(F}0PyDNF zo1QNoQT9fBj%x$W>E_V;yA&jfjd=cjEm9tCL5y==5~n>bv}Sx44cWVr?OavK_I>%o zqjvZMkB$T8I5j&HFDFfO^$P#$^jUtavN2k8 zX)P*!xE1{o?9lkAGd!sr!zV9D=IdmJqxV5UC}(Us%B;;oP2=;?X#W`0Icoyi-FSl+ zEHCn(tfkTDYbNMShAR3NX3YQoBFXTy1cH~Zxv+lz3pV`hQkJcKlMakMMRjZsP>r{R zwCz|8ZKs2@>X8l$chq4E#|&kM+OT90TR|uO)1gmmB1nhK7jgcU0=w;o%fa$Jao~K< z1d(xZC-{u*p^sfS48GPUVn3JAfOdhdv~Q%tn$6hmL&Xy2gtRbs^ekb;oH*g?yh8#% zKU$bPvxKc2?|?Kk8qocj?P!GuMgKXhK!&@U_#5*XKj{TRn_BmvmChT{Q?rqhtWv}` zE-Q!8jfc4YjuU*tJ8v}7B@dMjD@27O51_Z+f#`PEJ^nZH;QniP3>Oro@C}c;_)VTw z{DTiF+N{82{YPMxh=CtWPIs7czz=WZfJWc$^Uu*Yf8Vu zZvPHw?VJsUO7dVnvEzjA5a?m^y(~(?s*>qgC2U-jE9C6S6-Gykgs-n&awTp1(1T-- zQOLVT=%!B|npSL$9RlZ$TvD9P&w=i`<4^;UDI2 z;CZieysFhCRHHl<86}_Mw=6nI?`|()GvjcBxRqp+$b zlbY8UpkLJ?sM*^e?JWJtuT9y_wL09V^#}KJ_e@{$JLWi`at{NPsS?AF{g}qh%*o{P zLZ|L)r|{LyW_Hu$9_zN2Xt+K{B@w`f`7~!m z{(8;`w&}7@e!&)#0O}_{zOlwJVBDGs?#kOg?~_ZxZ3vzb9av zk{d)0qu|3uKVXQ=LXo12D|M{uq|I+JYcQE6-00XO`1INcZ(|zhxkG*YN-!Huo9l{d zrZw{~G-5b(YXymY8N#>i)JI#rW+P=Z8ohU5Jb(T?KYf2IzbbubXg z#?3}5ccvi8E6R@;Gn(DYlo9NvQI>kIhTGlrod1Km_{P(&d~f&yHmplm7~$_Kc-a~Y zSA}{8&mUlklAd#Ok|ejypTZl}??fpJqqqlU*_^+3Bn=!iU}dq^tky)D@%ue!lkG$j z`G?|(Q}xJ|@nK}ZGMVVT2^3k69}cfaM1gEmDbkQ~oXj(ROLnTxr4bf!l;3Pg{lX8C z{?l^I@!cC{G}cF05)&t^JnSvZlM7<6+OG09{@J47-KGfK75KlG75FpN#oWB=e7;m# z4L$D}j~-Skqv-Rkd|%TK{$2Sb)Y$EV&VSmC+^)r=RhnDSto6ajBi0keRJHRKV{L!_wIj*fW&26Gi~^A= zdn(?rNeRAmcns#ZREgzwn@e=bU8HEQ8HqjGOJ+q`(UX(aX~iyyPIY@f1Md}u%V~ka z`U^?I7bOp2?Z6-UH?Nm(I|q=jvmU}9E=Ukz%6!-zZ|)*Syh9Y@a}@d>+Zmq-L7-Yz<4!l+K|lcSQ*D#S~T#w&qksq z7hME42z*Ju9B(&A!oyX%O{ESd(S~{I)bEBbvG%P5F%NuUY4{`}=c}bG|La>Z} zJ}ggVj*Tbf>k+spr3go5!~pF$OS~=OIC(kEfqZoG!|IPq!L`0C@IsCTH({GGcg|rs zF$a>G+;79)&aP)~j!h8Q;l+aS4G+O;xrAqTIG#VE?~XAT*;}8Wo_UWY~hI zOwsERJ6W1RpPIDtwv7m_yKIMMstoWu6OZvTW0Lq@*+^cctCg?!)7?7r1%*BGfba7oT)rllKbpAfFXS(RHmhWd2NdsIq<{toP6Z ztE-eqsNw-~>GNu_rEEW#l(PzsEog*>)#lu+OL|;ih%8he456uJ&*&fdNo;P{DR%Jp zD~6mE1-}9np2i&RkV($9>cs<3rcJi9bv8~^R1 zD#~6fVHvnt^Ria&>26xUxV96lqTQdQ&pE;0U1yHkKhH#C;d6dj!d-6ngJf=bVmW_x zh~)PPNTNqbp+8Djc=h1#y!AqD)FEq%)|7VheyN|iPp&EyZJf@XzJHJ3^S}j-Sr~$T zxd)(#=Mv6ba1uX3vbSz7m_m0XSktDpsRU{0fj?tLzzlOKm@sxO-qz8B*L=zasoN(* z-nt5EC_IPzY!1MbBdjub=V{Wl%#;p_OX(xIK{~tsY`~xGc@iYbMb-DhK%0)|1inMS&=7>3sC)>SVON z$qlVn+KxVC$0C>TWoX9c<3EiytaZ? z4c|l0{VXOSk)`0nwZFjPX`0=(i>GmfSw1#wN)Yco^aI>Jy&CqURlq{Y-O|Ay)wqB5 zRsv2wBb}G*sFtXhx?jCXSGP%6G!s@b=a!4?(ApMuLw+YKJZ8v0ey@&Dy+3N77>!m$ zuSBYLKX@72eVokUcB&p7QvQm|M&UA)`=9xZA#uP56j^bdU2HLDs!ndgz z@S9JmLGz>{>N#M?=J%zs^uzPn;<8-&tf__j^H$79O&pG;}K(PkXkYl6d-6>d##?4Dc+@u2J*O!a-)bEw((yl1#uN!i= zU4-OQr=iSyDu}JO;(gzFvxCc$S!}Q&i^-WtyC)3=n?ISuul-_ZIG8G7DOkeZ9~}6R zTxT~qI0Ognc!+1Oo<)?ELy2?p2{Lo30^J>GO@~jjlW?M*5f{}wntE&!(>-yR%}o2j z{%EKQV>XQz%xei0xf%J^N_eWCJCXi2gpSF!@msYUIlXsXbdtvjs=wm__n{z{5A=J+ z>vc+Y7WrdPkS0Xl;}P<)3qolllF;Fp45WHB4!u|0gS_lx(98)N(R5D>WR!Z0|L|WT z)qcL8d5&Jksy02LH?tE7A3heE?b8R9ZJR)CK|8q7>nob4JCyW{s=!6=nWBp?9)PS| zE4WKM1=xb|hl{G=~yxbPpza;Q;mC^|cQ2=aFv zif%}>mCP_tG-6xiT^tJH9_j!-SZpSxQ~08>1bWHa?Nsj z)V+#0Mb=>H6MsZsyKS(~Y$v?5ehATcC%NapS1nn~JOFclZiC_T1Zb5|0{8#)hlw(_ zpwuxF*L=+&gQ*^L=hDCQRYx#88Z2S4oxa1sn^x9x?K`_#qApyxG)f3`P!)D&4j0O# zYFN$(7kV{QpYP6{%JVtKd`;y@o;pSHu?L5sJgHUa=AvTM%ZrijN{WiV97X%i??mT* zOh;3;8lX*U^iaszF~~|n-zeVng^w8`iw*{U zo6Jnmm^HR&h13Mp`{XVE==Lja<;CSJ$G(6C<;-CvJ+(AAzlivo6Ioyp&U{iUbi)~L}iy6PZY>fQ=%-bwC=z1qbIu?p03#~PY; z<}vNIoyaQgtzaZ4nyG}PFu$KC*`Q1}yIwt1ICxS?*!@CN(Chll%70E{PimvdhfnLc z6AP^PjnT2Zjw<6P+70mQoD9*@kIu;K-*gn@?TDcuLJ12#oy}I)Ww|jil3R^*vTycC^85ZuvRP^!nZk`DHUsDIqV|*G>h9a( zXP?A&M&A;F_UtcUO7vf_^71@5!zvCMMyvsUK?FYd?-?0iXHJXzifBRUQ~KgEp|e#n zb@C~otE-ynn1CpDq_K*1sorH`<+tqXhu7@AOc{GQWI4N-EzP|BFF?n@3A_T^%kK&c z;ZFrm;eXy5&8I~N;qPbO(X5FDbcx~;?vRuUpEh@Z+Y^hp{|-trZJYjWj|JTDu!v*jc4JBdQff0D=zkOCkGp|xp6Cw(CmYeOfNo?S*6cn z4(>IyHRda^3qMcn($Wrkl6;s&! zWhqRlHj!<7eViHI?PNhVZ7jt35*sx;j&+S5&oq`6(D3eb(v+SbAGhO=%A7;V}hvr=*(R#`EdPxFoXllQ-G+ zXdQ{GgydC_2~mGJgp}9b!`shRVP3i)ci(@3k53#<#*KW9XJ$6y*%^ZEZL^`EHlz$s z7x4r|TNAUTWjJy5Mc^h^0lGRb;Ujx2@JNqQFhlt_NqbgKQ@($v)m884W&=y+ak`8x zF{oi1+%L2FNAlRst?OBaO$ZC%)-jKg18l3yefBm>Q*fT^C;X_05|nMW32V>$32V2{ z6*Nj6giAzImHoTP!8=8G=I8f#<17WTEKQ22C01j1 z9g0sCsFMSd9pks#F671Gb9j8KEPgn>81uII_=tB7wkj_YxsK@uv)1f}_s12$%EuSr zV(-^5cbPfo;AzU;S~!%m5H|t85J`5rvIctn#wS}@;ZA*^IiBJ<5i zW0!pMSpGl>)BJvusXdbtB9lx6ba8YcJyGCkODWfpIv=Sy3D_D*-Q8yNp}U?z8o)NyfJ3$^;iJAq7`_ zNZaiNWY0=hd~EVlae>cXEaMO*@jjo%_N_VCLwE>OXIz3$zdCct&zEy@6FJUl=so!I zZvjl|D}Yo(7Vh2g2u67ra*aQ3!%Igmk&O64>h5mF%zy1ovQX#>#4T;${M?E=y#!Z&KOB5`UKHBhOxpPoj$(t%-wIB7AzM1iExb0ImIUWa=z= zqJK`2B;22h>1idHd}jo#Q}hrIzc3EuoEl>*{bm-vlWRzVKX{UfX^v!rzY+Q8)QSyO z&Bm$zhs5a{2gQ3aW`vIZj*fTgiU>!!T#OwWV<{z zv#!^XEPOK0MB5(IxkeFm?rkk9zhDFjHnCO{Xg1CQO3hGTICFj`t8_FcUeAKjgTQNI}xhuM=K`Csr}CxZVAj>fkx#^8mM zci`_cT=2^3QqYkR16{N>!k@8T5H$4wyOH+r(Zw6^=(kRoBH44Fl^=wu8_>HJ;o9m3)za~7*-WFlC9G*V0w3_vBX$8mgAL1&s!MK zYRkJMc*AJY78{GdHFt?@`Wx(q&fAVPt=dI(E5bm6V+gPSW8kWLQz6oYu+40iL|2Az zeOD839`#Bz*LkVf!^j1Pc0Iv~XUxgQ?=SG|Eyu8%>@9rt;W_ModI6p-RN$uusle-? zJoHOaCUGuviPPF~Wap!FAU*aN^nUOe3ax%v#q}h${9H?CKA|+9o~AwL6qv{@o^^g> z?A+EHEGIvWRoXtG_J=8)>0HKr_-Doq&7VeRT+O7X|ERE@OLpvhjtTQQr^V8L|D>^F z4A~5s8SKTuWz52DCVL7S>Ar)6F5UHv-WeHAJ^S7g=iq5%So3-Dmrl#d7}Ae_I`)Yd z4_$&!ezyW^)*S#Una9Dy2S-5OH)W_P(GA>7zJUEt85j?nL4E9OFiMg1~)wx>LD7t(GPIc=Du1unVW`}9_;|nzD^&Z;tw3;Toox(142C@|zDXhq49{cacZ{jDM z<#e^Q_`yg?U){twMde5E1(c;x3T@Q#`!Cwio=XqMThLEUlc~4nYT97Bmp+pBroCG2 zWM{iIU0S?^GXIz4-MCPqin_(l8%@Csdru%=s6zfCJyP_}7RwCm2Opx&iNyX4G|a6R z$;hUIH(@_PpGpT+l4QBl;=CH;yZrhbnd}7*g-YPPe zySpa@Zs~EME?ei&M`#F9nf*j;e!C04cx=ao55ED8zE6U#+%quN?>Nzj)S&8$k)$nN zgg;r|0jXm`LG_*CfE1s_kM{n+?92hXT?5Uc2y1CvzibrD`cndK%}D@5K8ygzGG~LU zlDv(H+NVX*NmE506=wLo^HaO%@^zqML9Sg^V_)T!^Pj}nd>nqJk%;T+R$=WaEAVX? zkF~Wp(V~t_ak|PmaOJWjv+et6ajAn3EZuPjEXh|4sfGQd;DJ63GM1rN=sJDhI+Z@$B=IJ?e1Q!wQn+hNUUUB{8@aBxvpJKcw_r(@I#i6Y zCDuMYB=_Y6T6zG$SHWYs^14e<`_%{VyXz#FQ>;s#WDSyIZl2_UdZxq|coxLRJORJ; zXM<@6C*hqBLhyIByh`U)(&9)9eWIt@Eh-mxfZg)bV33?Ee4>~K&bE}`h4Xgdme1|t zyTJ>^-R-l**OtbEU#AX$;rt5l98bbJ+L!RzbG3N$mP2^ERsyzq+JN_l-WKa7=;Nsk zjv)EoSW&)9E4Vaw37ofA38vgvBd$S(WRv_|@@6oZ-0xmW#F3-nPQ52k@t`MM6KIN4 z4TjPdhj*kzC5dE={DA#77;^2i#BsXWU0@8gycn&wZ}t%)=_16{7VJwRx#xEWah)$@A84)N>}k@y>$F?R}${n zbq|lv%7iv!b75brJeHmt4fS)ym_PcH-xQ7GCxR`H+QhO&FTqq%tjOlY5}fV6AHSDU1Fo+g;QKGW z;DsY9>|Urwf^eq-@JmM@*6wHpdb2izh%5U=n=XH@Tz6Xu^xaeg+c&g`_DM_Q7xT)o zy_aORJ}E~ke!j)$yEVxM+fjrI9ZocrFpirUi(^uKfYSU3uvPH_@O^zseBr({QStGu zR9RwF`2rV1K~IJo^pNJxPf38I%VcrD*E+2D%MI5(sl^Z5Yw=CIn;f>?gQuShgY$N* zuGH!ffRV~Z;(OeQ-rS@{-#U#V;FuBF?rT_iquvy**D4fm2s9O&s$`0k=VTG95pQta zrCB)X&Yw#s7FU6c90#%C0S`b1As+cofsFbxh5S#^nYdH+wPDyir4UJq3?-U`oINZJ zQV}Jo@S~EVNs5w65-E~QrBH+vQOI=myVjPXL6Sx)Lq&s%28BY!x4(bkT<6+rt@nMN z`~H)iAl~2Q$vs^<4wlY|PEjs%%j)^{|eiL?iEuq*S~`GAi4tsov!x zy?zyO{F_wHB(H;u9NWv;wrRk0+fm?QD+x#Qy173$G~spmXKvllNzOj=hv@yJ3d*Ca zgDSe#R=&1Kzz*-|6J1UXr_-l>qX&&%(^_v*=q+nbh_7{y2CoUb;Bwerc-vzHzZDK~ z_hz|4M2$DMDC;S`!(bI7|%zqDa$vFI{aF)f2T zSyj%(6vc6J8PU{uU1jQ};#*PNyA|STy)?IO^S-iwItJ8-xXsk;<+&mW&(Ykf z$Bo?8-8#bguLC+qKXS`R8P_@RfLk9`$R)mu<8rfXxk9y7)KmX%YTx-}>Y(;~kz(}+ z4x9?Ocb;45oHNhqmaePx2lGaX_nrV+0S++lkKmDh=KywXt6)jTAg8j=6E;SU-hxV^_z-pbngw-dRi)TFR-TiXU&QHc(-7Q;ZQS2K zSGk>^PI1PD3Xngwhm&l+AE(-Oy&&|Cu19~Sv;i?WvQp10} z=)TG@`qnfV$XJ^UQ0)qKW$U?FS;gFiJ4+zfAb?xHZSMbjLZECkj<%?|K}|#T+G161rM0){~Xq!H(W(%61Qpo*D`NCY1+H}DtGHpjMyfC7n^*Y z#aYVC6Rn`ei^nG_K=s;nQ0eId$?sht(!&U3_jz-Fbfl==e@;?+;*zQDMMk2bf)m{I zvAZ~%*;C8C)tsmxxdSjp5+I^T7Mq!9;yGD5c+vYb=vO8kzjT;(iGM_& zojQ)Ddiy!2@z0>`)F8Z>&<%T~FM(HYA*7q`<2GzKBhGnpk{cV8!yToBsMn)pZlEVt z{PIr)_0#{a=;`S9+>cKbT(3JJI(c_C?RV`3HQMSRowz)PmNj*!Yh@J0Z+H5^uEJKX zy>S&?-*Js*nQ8R=MlqF*GQ^f;JHYit1GF9q=bo$H7BAU;m=pO8QRz*qDfWa4w;?tW zuI7J-h?zSfcx@Mlo_ygZtQ_JRt(9Tl9a}gwtDM_$grh=g{)+wf>cg>oft$2pB?NW; zg0gTG+)rELRN4i9JGclZ)Q`d8pLm#dSB72}Uq`QKQDo^}@UFGmi9(TdBzQm@CV!=WxmhMnS)=?Ap#zYF5n!|>;! zCoGXKx6v4M+W zh0qcI@i&*-s%|G9|54yp&RYnd5<9qvVJA?SHlAz88$~sVL*mIEP2!nW=^Rzx$KCkm z%_V;0sfs0g=vf===@C;0sf6mcTwAa=?6qy;*6FrU>!tPRqjTPH7e7~X!vd>k!!1uZ z5L(BHUzt((vJw6E&uNi&jz2ea-5wTe$AOT&fw1jYVfo6Fuyd9_Sp2i)d{rN_a#BN3*lJ%H>@1K@d53okxofs1}v;8;adTu?X@zqv9JU%exZS;;cs zzHOymDHqT+by7@msU(wIA4#|CPJ!%>vk-G76LvJMhP*3o@RCdiJLmhHOJaEWq0|aa z=XE&n&-Q|-;|SDP`%xqPrqcV?7K)R+vZ!f;i@78ECn0KNJy+1>C%!p)4t@Q~7CN*~ zot`<{3*wyt^q7T$M^qc$ET2jJ$#><%UIuWZsuF%?)qpeW#!bCP(;J$M=&3_j#I-#e zD7JVamo#IIz|A*6tbC!y3bg>ne0r3Xrz>1$p= z^kM7q93P<%%G^>IY#$3(ES?Cy*EBBh0pUiWH{$!-U8rkyqoLvODX3|^2*K}?%S}Uq z>GhJ6MBT45MRiNF#plM0p?dFOC@)yX?PjxR{+t(mS?z%+-PQ|ESv&&g#AJB0C>JD* zKk?o9r9IHFPckW054>eBlDo z&)#u⋘pi{>s9YEMM>rmd1K{ose+i4Ji0NgMY@sFg>;uh8liA%H&=!^i>c#sxtVh z`+c}l{t?EeN@33eY5eoo7<{x%9gn?o8iq|Q>33ht>B4uX=)$H!s_yg%?$<>bVBTuN ztfpOJy(C%gW%+OJmCI{x&W(ifPa$61WZC85lyVHD*1K>4OSaLC_bRFHj}o~42M4*- zub1G6N);%dWq`<9&=2oFpcbww3<78L5g!@TQafJs*%4~n@G0We+qS_w4%mT>(= zpSk1hS(Z;$#nQ^tJ!rYJGwAXlNvcuf8@I=0E96_2!&c8bz!xK!cfvp{XS*6guZDx~ z7+uVW4e&lZ5}OxE;N*gKU`PLe!ga0ivGOfgY5#(MFCRf|_W>BKcLI*1Yj8Dd1eR-M zaP0vG$4u403FFk@R>(wpxq>f!dT1Oy$LW-~bi+pKar!KXaWe(kQyaOt)4qr!=Pee= zZ8;^*(buJdJzt956jX)t5SL%ET}i(rBHF5U3_RMA0U!6a!M?GbP$_E-nFn~P)psqm z%1~dNJEfY7*>Vf|jh91pw;ISOtl>PyUo3x7kS?Bb%24Y$?tGAemtJ}X8@kwY=t?k zy#mKZ0at(d4SGdgFgN!bSl+(}L8rXIaE=tXSB-#gwN-){0f-3q}8YM^hu-KjdyC zHF8ZG7H}tA=Wt(*HDUat>ySBWHh%WV0XrC!gMXDgb-QgAJ-+)k^+0&*eR(H~`?WuT zPumF~Of+YGa1YmEmdx!gTgkQMOot@}ZZIQg1KbIYhiknYjMn)G8}vuw&!^>Z&bjZf zBI^YFQ<(;nlZ&Y|xf-e{GuHC%3Mcrh{|gR{m&XOhQaJI)YsgJ}09$0=LF0}nIPv5Z zsCrz0$4hrZie(*l_%O$P&XR)6a0To?dnq2gBf^VDd;nWm4;cf9)3fYe3HkNymrVcxelZEf<3;40_F}#{Ai>GHx;_@gf zY<^VmGU;5!Hj?{sYEc%Pnj&QKzC_YYuo8AuJ}Y!}(y>-%AP#EP$J_fmVOd}c+*n!) zvg3Y0=0aoa^JXUAGV~i{a$kdj-&NSZcR3VvRg2xR4`=&q6799AfxfnKnEsyBNc;YC zp@(~_C@NhQNdH|Jx$Y16GxZR=KNB{-@Q3?}tGG3<;>23%x46MTEr{mpInCxh+?xsB z+|Bq@SeC1R?ewPN3u*HBzNHTsPWi;;-&dmb*Thpd!V)O|sB!S)O&2uDMMG=*K6onj zho&h@!D>nkNal^e7lp36^L9I2dPxequF%J7_suZ>#2(KnxP@E(NE79+tvFu15W96R zq3<3X4-dYj;d(zs5^b+bwCBqb!x^=BWXlnJOTr1ix@wQ@qSxd5Xe-wLVTq&ogAmLp zi|1zCrWKGXGc{=(Gn6=*>6N!&K7N?VEEeVtAB>LEc5CK~jGZRH^ymt1UA#Yx9sdFp zCF{XjuMCvC#z5SaWKJe+j;N$2Thw|wT9nS0aUTQEab;ae99v=v;{>-m^WYPV)q4-J zJ*&WCvL57q(-W1ndWpw%EZ~-%)CE}|9lZRRJl1o#2~zV-LCZT1G`0XNa5sVCfvep3 zvq@Ca0)Hl=yMmc^{s}{?RWT{MU10g}0ABy&H@5L9z!6R=c+kTJx3?=1>+`F~noHZr zcWMEVKCVSRz5R%_WQuTy`bB);%1d15_zYJb+=wseKM{Q?>87Xlx-c?oUX1(=mYJ|! zlkr{ng1&P+j~;0t%$DT5Kw|hW7}+Xfv*<85cYixKud0u8JhKCCyOzN9y}sb-F$+xj zE1Z4>Bi=J9lRB~_TC~69lxTsSJ?F7@9C#=vfy1C5XwNH$IR(COUr!3ECJWrm#YS*2 z;SfBSxCcs<#^IO%SA6rUJx*A+7&_LLS{j8YC!o4n{Tb9X@0bNkikL=`o9?*t!CAwL7eYog$~ z!!j=K))d-z#5>C1Up8em^oV041a6>OIHW(3hd(+!;u+&8>df#Baer(c7kzFDR~wr^ z-NnJ88vQ|Pm;DG@!%r}t_AU{=Db{fP>J2V$K^8m~E8~gPj<{2!6i<2=gXIbZzNhm= z#zr!X-H=$#mR|bKhI^=>ZM5L5oI;_#N3+oUTVqhnsx)^0s3VZ^+nzLeC6aeb#N^?^ zr=(}@7jo=oBiZum7}q61X-|AB1=M_Y+(un?`wExJvmsX>%JDd$FM+?>3m+#Nkdo-_ETR-cw7T`*XXrcDWZRI(dMYge<;XMD{I~=J62|{+ZQWK4H*|S2B4=);;RN$L{&E;?U73 zZPP+@?A9_=yKx12cFGZ{teJ+keEH6d+wNrjDOZxFI)oLb8(|4u87|J! zMXa)WWx401bl~<}hOV@H@I5YuSW^R#I{AgOUun**q~)lJ6K}*$OXi9nH+j)+qG2kn zzJc4yieLeWfS{P?pqA7GKD*w+-{%dWu5t_9{`&}Dl|Mo5F-aU=9*ARepWrvT{rFM! zSYmX`j&%RqK>EbiWK`>9k|uD-M_O-XT_;{*pVT$7kIeenC0UQzhQy<6Q|c4Ofz zBv(d6X5FN2`WXJ#77O0x7~yLHUMGsB5_o+WmYRT4WiD zo{o%0m#>AQXKF%@b2y2;Hg5#{&37c>`HRVr+ha`6{tX*@Qy^FHa#oa#z-P@g@K0k2 zOvnhlw7V37KDTlIes_vgYzL|O>08Cr_qE_~*&f_lS~(5R7+Cb94HEb-5Wa63o~vny zBQELU()sFGZ_Z6vw6zOVsy;(E3;^R5hm$;1NVXdykC$v9kL%LNk7AC@y#mDCsF>^& zxY{#TkD}67S+Py7f>+2fiJfR?!*2A?XBO0EKvvIG;xW0F%vMq0Lj^yy^9Kw5M&xXM zf1x^m>)Txta_%!OJ-eCxxJv?U(lJ8LhKtd~AInifzu+)Ant-hAv(Sxo(df4RTr^QH zm)&oghRs&36EZL}2|ub7zwnrinPVz=yOIpvKpsMwram+sPvh#AmT{^6@;AaMG6xhxYI=|qPvMhFgD36VYvcc_77*yiZ z@J^=>&W1+gHN}eLT$vxK(@7%ljueuT?hX@h5JTN-P`J#}>Ua zLUkSEkXL~cx)}45J#W>=b_6d$XDUykjJw%r`rTtl&3+7epz@qix!!~0Cru(z_IwI~f~!ZM zV>J!j8e=GV(I6UhJVvXyrqFZr9cdn1AvRqMk5W`QHYJX;ccJmk8AW)%i@>;vI!r#R zJgTKkw5832b=1Qj5OcL$i>W(hugrm9bOHq@`O?JwH6^weT z5^gvi$(EdSK)1JMpmit}b>vy1+q1W_OGgXuv7`Zf`_Uk_J+KQ;v$Tc~*Z%TR>zhTA zk2|RAfrq%a7oB0ZTrs%2R6uP*KB#!c(GLx>XzlHX=_e9XXdSmRw1;#PJ!-g--u|P5 z)>dqxUu+4a1{f6kEy&DG?m(>VU8?;L*e{JH$DWJ}&7-;kdgq{{zt)#UZ}tMawA zJ>*714Jq9{h-2c;vhB9^2yP~!TNSy;yDA)wpOM2(-xf>u84rytrvBwLRH%H-Y`Q z*&EqCyn?cFF&b+fjpDkGu&F5mKgVw+@6;^C7u7QCiUAg>Fa8hB8liyZWy`THr=Eil zO~Fpb%CJotg_jy+LH&{+mV4~;MB=H*v#Xg@ZmZFLPNN98#P1@yd1e!>)9A;@q0P)b zXNtjgO>{%85pCS<%w5n8=bFMD=@EO+($>$Hit`lzgPAMdLtd32&U?3jB=3(WZ=Xhz z`S%=&e-b75GLPc>^KeHet7NLtJL~ z6YTO*;I6&EGzjez&HA%}K3W|{Z_iO==1+EHe)Gyq=kI^?_T|@UQ&6V6gQK}?0vBoA zJ~eRIyqkKix0JTbct{nxN=*oJ+$a3|&IutmF660pT_bt6@wiOFl|5g5n9WSmr+?F1$dLho7dF3@ z%>Ve3oHqs{6_`Uj{Ew3h?Qdw)*UCuC=PW9YeT>>>6(Rd$a_C-l9Z7cG&Rh2^;vdAu zlZ_T1n8c}J?1gm(Oo>ATSuQRk+lG#lW0U5RD{se;>(@(gK&TI{`=p68@)7jNP2eoA zSI~PtO=6g$(4y+(X% z^du58E1Zm*y_+DpvMBCi$W7}H?vGWQ$gubam4fdP>VGI%oJ6 zu1$|5)6btL3crHLh3%*D{)1<^_2J7HgIycS=f3%ci(Zc*Qq9ZBgNet<^MXLqUMRtI z59*-UfKv3Kb^x`Ob))g4;t(BT&AL^8CU0Lbyzy=YK3q48jNBSZ#{Zc~wg&_;H+)pl zp1mi9BFh~VDpP}GOb(z#ITyw;Qg*Ef!Ui zg)dH$)lPHC>_1ib%-|ZVdOHF(Hgi-lN*1aAx+tC&@MYNGV#i0CxEY0!Bhf;%6! zn!BZOkCU0~2a9jqfq&UQ;DpCbusSgw{#dW11A2Anl4KY!%c%7vJ|;Wmy^g+J}bxU|0--0gY~?$+QqIJ|T<#NXQq z(Y~cHTEhtfty3U>O)I<)Qx>>BjrfGNANehKGv9djlgJ|)yoF5<8KHEIM08}5BZmWt zX{-gg)~Z3WGpq@7KY-jAXG?-!XtNV8`k|hn*N7TB!peJsq}6=gGsx_%0ekb841ZEI zhi6wR@|m*xNOp-L2~A~5b;3>DM2cCPW*4-;G8yf+%tSLvy-{RMID7Z%YH~ZPhU^i4 zC1aG%ZQ zhO5(E$<<>eMDMH|A0MbIOtOsloh=IdiqL0dzQ;47C7n%_uIrJ%*N@`ingl$%na6F0 zS8+O1%`|!n-btHls9o|c3iPeC-)bZGwI&mOXcumN@*cks z-d{JqI4%bNMR8JyRuQPF&L>MC%+{Ao&bDg=(CD1YHN0{{yjcjo5dbD6# z5_+X{32}1A(C6P>Y~_kbJZ?h_c|?Vi&B6?zkJi91dwXfQpiU+@SK_9-3!u-X8iLOLf#4~Nz-oN0*f3_0 z3O}Yq)h=GkEz(iImmQzrkNYjj>cyLcoZ(f{-1LELULemWjaB0B)whzhhx15jbsibJ z{{*R!Fe9#MUny73dF;8}iLArrbT-(;mc6rCot4f$$Ig2$hbEfQXc8BS&iaHSA8d%a z<@O(bZp!j;BeXhr+Y_de<-VdQg$qb-Y1fnFm=L{R+2SFUH-iZP=`_0T1PG#l0gs zIeELgRIGCmeJArZ%{uO(r#U2wUq@Vp9`*rr9#{cC|qv(whSYK z3L|k-Z2_w|5h2DV2;J9qMblkAv;LBbVxQp4BuV=cJ~VXCt@b<2It^vxAH^lOdCQ%CUq?0{%378ao#RLyXI!5z-Go|_dpJ`a@ivltzi$q|IRosW*Nv{1*;{S?rVd}pl5H;NnJnBlI(_;plJ26!LZ(jiw zcu19ow}f6)>rA@{vzGybbvQzBRc_CAA+I|VNZ6u8@_YG4qWMgURM0mWc6%p#awUb{ z|1(Eh3!k&bJ(}!v=RW3AOClTcrJr5Be>6HEe!#jeuxH~97qLa3U$SL49nq`Tk?68| z6zb?TM+=TcGhHWNlY{04e5mO+GX2aXCZ~5I^65@Q|5;X|xm_hF(PKN>eIS}0EgwiM z&P$P1HP6`2{Eg_{6c4mDqlE2wa+uh;kK1lEb=_NyDdqcvt&# z?5=SUC;BhL6B4xXPCNvKYwy4zvrFQZMW)oal*RPt>`nB|V>}&ozk-Sp-=YQ(*^()K!;Sk><*KNe__q?Xk<{i%pG($Yc&E}rOoxHpQ(pNIyZrm*uw z(^=%}$7W@8vVj)LXptVUl!h_uW3rr`x^)yfTN8*>Bf^noyEDp(!fds(Hd#FD5h+^P zLL!G&k<0UaSd|h(G_GzVnyYvUeVm(#qFN84-4ZX@Gwdn+;D#}?@LwOh>dOi=X{jYz zZcVYz)iEjdHs^()4FA6T0g0C1Ky2dH5}ySd$w}W8JEESG=C${oy6MMh8ldWtF zWdk0*WDcEO%XWMd3W~c9BdeL)&{MBb$mZS=rd}q7NNrj}-fe#hPiEM$NALEqFUTY` za9j(ixf-CD?@LhULMQa-p(cuvJiJYCCV%z=Y2+!la9f7LBbu#d3y;KHvR{-mBCP;unUfEj-!H7 za^aQp1DIsc3XyNq;iAf0IM$_zt4|ET=ELE*N>_{gRtX@NgMj#se@HgvWRRuZ=ET3v zkPh-Eto!xR$V){LjZeD7&Nwlh_55ScMnMX@yGIrcxcp*ux6WWQ*Q(-oX}6j8c_UGu z%MMg#zY<9YHnZeENIMfQj|A40qRd=O__&vxltCe7=-`Rx-U`R4&^_{T(>-!4@|oYgJK*&;(y z_}Gqok=7-n^YXCQ-&J_)12z20`MSUh=Aba1f<_rPXtRC<8G<)pzxs1<>W%_Bb_NW@ zj)oAw-H@<&5)SZ~B5)UyNyWW8B%z9b5@byX>BHBu1WIAN1qeDYDDTZW!ZoD80!$y#&m_q6PpY11Yh|rZI>AiAI`y*hZnPRlzLgc{P751kU`&vOxV3EwaE5i1@h$C8P;pS0X_7IM&1XH zqJk+)kh0TQ*#yH6!%6AYtK^WiEIHVUnIE55P=8W4l4>hEG90JE zw&r|e*ER@u-?3lV^hR}7k(x>dBktkUy?N~4$ffA3d^Ae^7m0=p*P*EeLKif15?ZgP zgmxV7Wv8mDqTWzv^nHFJ%F!-B)1IXwI?xzZ{guZ-`;2&a6~LQ*UdOBU81YNJtI5L? zuEgH`26lZ}0*!kLxG5gHAb@)S_2(~vM{pSE+AIN`ia~)@}Y7ZId|NOt;p+Ob@zW@lhiJgFB<;z5@0o{`ofDCsdJAdg&tlp^I)gg9N|n1#S99akPJq9T3LY)B z8%KKo#`a&G$i%t($?s#1Bw(c?*%okvOTBoKy|#}+0|h6L!~1Bosfa>y@{ifdq*_LA z+$xBAI+dMNP{~ftEoW=Y&#^We|FWNrG|<$ zNc#A3bm(C+@}7Gd>8(43dLHaYM!kk8oZMnJ8>Fymlg_ab*FLfryyptL7Jrn+TcMwe zs@censZ3i;6bYL8nVkOqm$(UT*kple6?(;;eE6nFG(%+Z^Bh%L`db9`q5T4_<7~uq zJG(OSdp#JZTl1MPBL_>z>x!_Qm%x9eXX0sFH1Oa%0E6orK;paS64= zpU}S597E4^{3yQxsOZx$_uF_OHyjR!Dh}h8jYD|he=fwL?i6Wsy-uDg3SGmu3(4&y zE4JBuD*AZ#5c(<|iIz5RMBV4)kci4;8!Y-*9Iy%Ja@f)3p$y%HD5hD-mE-&3C94zp;g z_*wlXj_sr6%p2+U#m{NUGEX{Z#|e6hjXM1Dn1P>^3_{u2OelR34rc4WaFc>o zfkT)Dtj)9m^&ua*Ffqt?w;K99yPXk*ZiaGA zZMLIeh`nfh6m4~@M3W1jp_fw|k>k^BbX;Bw$=kEcd6g()E1XM*ZoVTGk(hK&3?`da zw_-(|HMrJ8l0-jtB9G%DNWimXQsB0m+uv~W{=PI+4I`XcSnpxMx%{w3o_q=X#g&m(jTo{d) zPgf@&TcSw#_73t(Vm!ZCbsE1;G?F)d^8@>R>|wV}+A4624x)w@OQbA+k*)Fh#>~N^ zSo2ps>}>7bDExj73hNZ3ui|`Elaz=I%om|aj<4A2-RW%RiRY}NuOX6_+=V{O%RxoW zVkG~x0PXb+MwRQz*@mw_@T=Xaq_pumiGSNnoXz`)=6MBP+V?NHj_#7~dC}x%gBNj8 z^&k)X=8%oT4pjPV8rE=QaM0=*0{=xmFP;zc$1!-?i}X)|q(LxhdFl zNFK8FU14O|LCz!GjmypL7C%tC%h@H5gdcIH;LE-S4Y_%^YN#21s&^-QEeeUWsE-61 z81Ypmvw3sl2SirRisc1vZ1Ivq$mHNkWLYh+;;zZE2a*jjUK`59$5pU{sg~$wMljl~ zb`XWfc%rIMOO*0{JZe9sgSKyGQCiGWq~d3bZVBwUj_KQwm-zyedF>9HATkHHYhI+` z!vS)TNhD?ggxvIz;Cs4F`8EBs`Pn98c#YM!iT6MR>FiujnCwZ!N52qT4JUyn>f*-E zt>#`2jS!PJld03oM!^x08th7%0&jN=bEr-kb|#F4!var`x_wG)sPW43#2ZTJ+R7(v>^Oc)5hm#avq{MhU7~SHUEmEQV9EX@I2Qj}{AK~6A1kKQZzlE71DEd8 z=xr1IyYeZ$K{}P5b|j82HfyFXJoKldJ6&m?3*pqn+8&C!^HeN-_#sF(O5!7P{=n>R zVa`@w0DA8%aBh1R&hAnsA7zgb7q4NG+aThH0$IMo_A@!t@roH5VS?6u^G5fgols7j zEV}ppCsXdXpO!9Fsd)jT205QaXm6PU^9xw$*~W_SXf`kIXBzKzM_REZ)GBASIKa)%@ zpO6qZ^bhcqV};oGoHB0QIUXDOO~>D2edu|iy>vmZ9#gwepIPQ7%Uqc@Ku4LLq|Zlg zr%A{iD!O9=w`@TlwO>mQX4voP<>tfY_V>eB#x?9G zZcInymfv%%82%hm6^ig#^P^;YloVfQHGvP&*5bRTe_S%UVHsREg+~XK&IWTQAY^2VT&(JGan3d(KmitF*WYZ#Kj4itk)= z`WkSO{0CnJ2Wj^Z#~g6sWY+o~RjnyrPaX8T~%KMP2t93gu? z4H4_RO8l7EujH)M6>`b@HTLhyWE8?KGI`5YaA3L@Dd-|3BQ}fNU*bs8Q$`4`om6_^ zzZL9^X`k7g78A7Ux*?K0ri2#pz3iLGkJ-YYCjviJ%xXwlvYWmkM)PD8y&_^FewVC3 zJSOcXAv%r3rnZ|baC<~%p2{Se#X;nhk`mFL>5Z3aAHnM1x8Mf3@i@nM3@$ROg{yo0 zVPt3$a0_2@{=)rE+5d%TVA-I!yx}kRB0(2MZZi-()Ory3#hdF=*bT3zT!o!qO5uxt zEy%@YgY1}8xT@L$w;6dXh`I2@P3rj4LsjgPnT(y~jLA#Q6U4A1n^1PQiDs}gzco;j z$A_O1k#Z$Tl)Ftl*YpxMDOJA6Q;wIe{6@y=HWIm+hl$zfB;404#cJoxU{loXsZHvK zaW~}v7h5hf$?1M<-k5y$V(4);AVz_WzSBpalf8u6T>b?Wh33p)pw<|ESIRHYI@$ghP4<@#z30~$(FgS`~ zXF;OCkX#3f>yA-{ndxFDj}@YqFT%xd*M~#>^!Ko@@-OUtCxc%Zbi>QhPXt$Q2heX9 z!CL<)s-?w~zKj3TY2l^x-;7Ck^@YDUX-Yhq-_uW4m+11!#Dwp0nao=b8t|#MGx>f8 zOaA@uS^OHA8T>j613u1Kp8tBUnG6YrrO+xur0Nv-0aGo0+lf*9VuQz|>Z}7XdnbYY z{j6b+Yy}<^csKecA!JAGMe=TA2GP*SB5DH{h|jrgoy$D+iA91so!&J6*Iz46hHSL6SnRQo|F)~a9W208g>|D(; z(yDe$OU)lp_%(+NH(=7dM2@$vH{|cFGvQMLjrg*q)A$R~!tdZ_%9rGs^LNuH^N}r@ z{8DpmesZ@SFR|C09~tM!R}QS>?@PGyN8&7aMP?*Fy|9t^m);{srMigO)8FKkTQ@mj z*Gwk#0(o{Ng-BkGC+U4#$vnFs_?UGp9+zr|2i}~5s%bCab%Qj{_Wb~zITE<`=2*P6 zRRR}GFNP7J$*?xX4L+DDz{)}s*m2byri{A4-H}TXOV}4vTFVo~o3?GGMhxtx^UTVq zm2tXs%f?!oZI@-XTQ6YL9k(;muQoBeUb-=(N<5fgwe?KFuPKa;cN3Mh^E@anP6MwXVzYTg7sGv9pjrOqS&@yp`r#OjLMfhY5VPohko7 zW)Z(==Su#3@-lw)-Z{KezbU`5PM`M(n!qxf5tHN7hzw|K^|JtUg(A^&J}Ah?7;~hGH%5zkkm3g}x42u33`#=qACeRJ}w0 zJ#>qXxqV4+sGp>nRhQ{O-R*ST9A7GXUlez6mMj#Te6n;Y2Eh%MgJn+1lZ@6mBsc2- z`7szz7UW(awXr4SKh>+`I(wDqjyzB56AHV-jN+MdYJALqI^Wcy#-}BY26B@Rc@iQnH1#4L0vSth4P4jPnVb*pIXa>oVh zUB3tB5s#=##X0mGgG==G&3Sav%m})U;ixwkb%9Kag0zhj;LBrUZd6}3cjDlG+(!Ns z$F8vCdPFz4S#q6R*VRp2eVY_`nEm617Aln6HG6ZNZ7(d-emvqT%+^5ioiiY@$_bXc zeitV{wSuAdV$c=5HY-eqlU^^VtyiY$m+fD!1 zFoKzCH%Nc?zel@YmZ5{z)^Jm=zme{M2qBLjtRr+ttM+lNO=d6zR*-G^%(5$0 zj!u*57oA^cB)ax57)DFZ#?4;xxJJBi9vHil*_&pPug&v_$#P#pe>y}cY6sb}z6CQH!ftZ+ zSj^qDgEd}zxSxf-^jr@HzRnW5Y(+oCA(HjnX-6fxQ6-IDGp(Jz9Q}n}*`dS)*rd`| zu?po7xt`Oqa-eMVBB@_9&BY;q#)wkh|K|2)+~!WTmT~^!tA)41G>%W8AjaARzIQJG zW>Gv0J9xvlhB&S*U^Z3e9wM?EV@dm-yg}``l0nr1wx78DV3oqn^9y zbmtjd?ALg1Q|du#d36cr`>2YGVh>YRms9Dd6N_kTuXZ{oC7ZUJAV=3Qzr}@LWw;Ba z3#d7EvRroCbd(gS$|r2sJY%!r+)koRPFIH^y{8Tu4_?UGbNw4aeQ-2^uLh z=a@r3J$jvE33P0T$%dPn4Ka^G^~?6J_@sg3P@bg|4kc`TNB z&24mYqtp-eaOQ8XKv72tjP6?*Yk>-cTp2^0ShY8N~bHz!POYqS+ z5$=4v8n>wB;MGm}Sann&w)_a z`%i+SM-8k#mkP$EQXnzU1bzmLh6smkP`6eOPps0w;SO_g@TiBd+;9VQY~2d?HakH` z&^XxYp(swDs7JdS{iAj6y`+>b+E`UuZk-+G zxSBS9n?^r#zf2FPrPF&n>#2F`^ev8kvW0}Uc(^@nCp_v>1mpRh;?3*RsOI@f^wZ8E zs;#Gy({We^BzOm;Q*TAbD+lO#;luP>x2Lq4P6Zw7ZAzI1t^kStOo(W$1h=^ba9>mk z$uceQw(}zVG|d6`S-N=2Z)I$yI0?^Pb`G>=H~?Da22LM3ER{mOQ5*h8(U}HP)wN;R zJcKf4j-(=_$k}V}r4&M?L8Lc@Mh!HlLW)cgi6lfNp(3-hhMm%2NF_>ANvSldG-yEi z_V>e2&YxqiwV(UBuWPLmP+1lWRvDSH0f)~5tu^cMWKfQ`pM8&W!F_zjH4ESS6N6dl zB&?SofbEPnp+!%u1n-y*iaaU{x7XEk}Qck*Mp~(G~kV$d$9EC+32>U4>B-xM#eLG#rvk$ zGl3z;fnUc-Fn47*^WnENI%9%G)=!$mUKwhbtK5k80v-Hr@>+byj3AHC^>+FZ+tJx@ zRa~Bb2_-tJfh!NUF`urD6&ZIvv|D)jKGO)UFzF*xs~T3{M6uUy;3wx&u;b(&@u{ox zz$E3BV7B0jAN*{H*J5kDtLqn9Y}0Upk|!tA}j=okA}& z-LSHTCq9B1JaOtZBs-}AwSSKk8Mu0)4~sag)|rX7&v}dOeo2v_mI1sciD2cRHTdp1 zXKeoA3u-W&i^u2tV4tTm@CoRNZ`kb@H&3{Rrke?8SM52Wf4vT!kFy1ucNq{GxtVd_ z*})u{e}dUx@S3^y>n-Y%oq~1m;QRy1b91YpdFj_lAW-G!GPL{=vBkM5PlZKD8>yaLt#prbAG6z~+82!z?XkgEM zv~Bn$vOO_?q*o~8mQ^We#N(f0-9L-bt)Y5!MCK5#ymk<8bvTK?u5riy9n%?KB@y_# zX(A&SxdC?60Uk@vqSUf5{L@$P4^McBFXi9DwdLDza^ey!6)I#_TLRGe)I8+efRTBx z1g>A&gYG|bV)l&i6TP$0MYnGLMaeECaZgQwrS}3b`0^Jj zu}jA5%(JWFOBR4bc1gg{&lza`rHs0-@V8E^#Z`5Cv9t{<=DH)9X$EiY_IF8u>lq7~ zO^4gi^28%}?}(fD`e$dn?9d5hc%O((E}A2qMn$ya?if7uS`N*2jbLW)D?|~Fx_EAD zK5`A}6Gy2Ee!_hsyzs|kG&=Dk%Fj8B-p4zl9YzUa?fiLYr|1Q;eprVV9`8p>rj(=J zK6TLaE?5*BFc0hbw4evU3-J1UfmNEm0JCF!@dxe<)>W8|zr>1Ag5Zar>Dj{^@RA1p z*#XFZ-!ml9eHL9@ehDvs*ntgtUZX;bIoNXkbF9B31FISTLI++hvhxY@LB;vH;DzjA z;1X*Bc>C4hw@Itr$mv~-t5iMEZ><2+H%BrW)rWD+?4rvnI|a5Zw_V@oW)sQO^nCQrQ%&_=1A@F6#VkS zAbNI25wDSz#dhVH6%}FOV#7yONMp#Uy6;|t_?vK+M7bI=(^VZojEX(7IctuqedNVi zb|28-ksQ3ZJ{AA4NWj)}ZsLQVw+hbKByrwFhMDAQDysC-U=jzE@u`#;>@d{^-@3OG z7rhO{J8s<;eGwrvMJ5v~Jkh~f2iBt1{8*9yo$*X+@k3^B>mTO&YE7p6q7$RA;}P09 zwMnElArj0}Jpj(DykWo^gfeyap>gJpDD(OQ@jUlsOkaf9?up>$4AxtXOI|hN>Y7m~`mb`4NVo7Yx+|xK2SQWPzT|Rd&n9n??t+8N&Wsl3RxXb$?!{r_2Qhe> zF@vwKt;G4O3-Cpc`A8w%9<7y4MQ)C_Me`I7pdU`&SbXdR?#Xb%+X8o>h0m5SpVG}l z=dX4nU4ILtS+pIo*GoiaxpRzJojv+J>yL4=S1t@!B0;4ehw5U?x zJ)9ocir2rHi=7t;YyxV9!d#uiqBY?R(whu^CHgR1|Gh*BZTFCdx|cZTryP!1G6h?x zyE8+v_RK5OC5+{MH2|ixiSJL|i;vV?#a)3T@E6GzrsY^26CpcS{9^4l)R%h)?HzMQ zl%SH)MW~IlbjLmOc3F2Z;HtmXIycr4Lz>=!X#SggJM@t zv0{QHh^$CpY~F6dla3nWM^C*FuHJ#2<|W|GMyJq~G@h9eq6Ex#G&6GyFp6oi$8Tgt z;YE6{#S8y@WNP}_MB-Ub&}$nxT=45P3XC>E4F}~!^DX2=FYdiUeXn-lz@3Y+d)a#2 z^gt5L{_i%EbBi(_QpUjOM+c)J%0(gg9Ll<-BK{Xzfs|$}z#l)QAmxZ0W_I9FX6x&j zp!c^5P+?>7@JR{6ZQg=so*vD(#t}xkB?9$&#$$7(5F0sXi~Y}_n|?Vc#$34gLhDmUkY?wu@n zvR;#!C}WPs=$t}Jo@a{BXUpOx?_MJJr!PdacG=SQcIFPw2CXG^{(NGhWY>5QEdHFd3#BuWgo)Hap{g*2Ciip5} zY$Zq;Cq-Te@4flTCq$}#TA=&YNao+LI(ob6oG4Nnf*PxE(40ER{C?CYiaa6s-e=c} z9-cWVp1n&FPt!QdIF5gdN@CS;8_`ByHXJkK=Nw@F{xW!bJ{+X2-wl2oJOg?@C!txf zhndhH>5R|rB{=MQIj$Vtg0J2dpHzT!s_>zvd0MD9o2Ihcp# zbUSdiFzfRob!fice!IPv8yK>r3fNUEvpMx2fX*dX_WWmK_T{mE4C?h}4g{XUGvvzf z?+M1@OS7MeIz=VSx@J$jAZR(hy}5-ktXKy67q)_$uQ8&Nr#4_#s}WCM`UH;+vcS{c zF1H(WcLuUmu3+!}AEN5;LbNn3o$=EFE@*Nntjl0?L$I-(2D2G?ZLb#9i!O?vBm4-c;SvjTxX_&4cvn9k@raq z?{op=6dABw;7InfMhjT0B*)(I+XeKt*n@7{OQ22h7_ikbW2E(OA-Topu|~g($kQqX zN0ge7xKC%W+t2`BqqvnAPF+ik)pwJSTfO9Ut`6N5XGc>CSxWD!(DJXXBu4WfS#;8w z++ar%FB1#mYcq*_U!YJ`=`|KF^|WT%t~h|HujFvmg%aGn%7)C@^c?Rgh!^=OCxBS* zOmIN=ryc2Hv3T$tUN11mrRZVgYyKhDDT?6z zN!WA#EqqH>@B_b+B*vW;*r$NU&KZ*A&(f=S&(I-!Av7O-i~mr)a=8qfQ>VsW^^j%L zjb&J!khvhNYBl)h)&dd>o-2#>WSalUZd)NpoF2x$)D2 zisI~PpWj5fdQ=!qI#Nhiw~DFG<8$<#k{3NS{sZ}Z)`K+eOC(Y$iqyuyjK+UAq0F9p zq+#qT;^Mai7Z_%U_Ux@d-CtvIAU~EEjh=}O_iKT%)fu35YXOiSiWmH%##ogp#8L`- z@T-+FOnAnFPK@rXD9gE z-2i5Zhe1!#55OOjvXfQ(jMqIaAw_>jQxls7^mU>yU4Ao_Hr}YGDL%Dy_E0k|ng4^< zlnl|IHkc0GZXi_hBU`E;#cjVii#xH}fb1vJsSy)RO=m~Yb9EDGly%+`LMuGbCKY{d>4`57e6r-NA0LLAFk5d(j@V&B$NU45{SpD!w-1$Pd zs}u=5DN6=J`Q7+)R~mY_!jfq-Stjz^PmzqT8jj@-;-|+#a8BeU>~db%FPvdY$`e$` zwKJu7yTBkhwJ8A4`R;r?>7kpqd~xGymUx8ck}G#wiA=gW z{gN=7wg_`p9l^g<|GSm`EO<=kNGtFkvX%Lp7o>P|p{u9w^o@>*+D6xmwc-w~lZUaj z?oc^vE*!l?3hH&v<*aHRlD}I|(i*|bCvm`rT7HNjUlOyhoRtBVI=>n3eIx?vsc%6Q z@6CR{XUt05364&z(yzW7Lp!0oEPkK%li@v&*rq%KG>%tjo-l5MB4qri8P>KTns ztTn_)bOApv5)spAfq#frk=2eh#OaPL9UO|Iu^W%kO)2?wg-0<}=gMh!-fL=eR*@eo zIf8%ZtifNa(&RnlWcc3LH#EX0i>fP+2EvjtxAe_=7}%K$H_4Pjt#?_l$#^6D`1&Qc zY(SkjWefSU^s9gf&>2dlxfy;W%LMSDHyq6j+a& z4&z$I6?nRr=^xq( zczV9=FUhMn;r5hBK#vi7;PcN_@P5&KSlrnL|Gch)A#0Dpx|y<&jy(a27ObOF#w60f zo&{95=o?vB>rY&7WPs0>_H0k}D%QQ3fN^E7k@$?gFr)Fs3kRLB%vuw?lXt+-wH`BN zmU!;yW>lzwnWgKj0kr+c+x-c2@jGat^AYN^_&QCwCC3Ladi;oL4c;v23;nV*n;!jRMK;Dx1>ZE=mObgd*um~?9Hlu6q*t)@rtW74k9;*8~I zL3g?lYw)9nDLopBZ-!XnQTJ-mxMV#hO(T=(Zfg;3+3SMIa}E6IRS?>+yPUakK2H2Z z$Xq5ps6dHX_nAHWDw*}8a_#=f%*W3i;;=HSOm<0Ylb8cPFA!b?mZW-6?_Wvu z(YhWLD4{Uk=iomVS~|X~_QR_2fiLzH?Jn&xfG+D7@!#6;=kk zf=5%k;eQj)!?r|YxOS&KTeagbHLl8|$qJ#=S@{Klq7UeX^c=8qs0~=HNv%4pp@DC` zEJ1!z7n#G^C7>D?fZ9e`pr|+!zqlpLeU?|D{bPUF-dvI(_8xZ|-I5K!IbDAd|j_e6FY=P4UTtgWJGq(&5#|6xUM<3%(^$V?aLHc^$^O8n?@p|@dC zO%G4dqW9m(QtcBXY5hbsGQxTx8?s)oIT9yfdtx2zOlXF^Nd?fV%M5;! zyv82&-byzlU7)r5QmLBeFQWK730P(IfMEF&aK>MY39>wAw|!BrSo+R4k=ObFCfZ~l zqjpXLxi9d-gO3vMrvD`H$jo8U*@&Y^YRN_Mh*R%GDXJZ4=!XUVel8OqpVWZ=70Zyt zajS_=>k$%sxRLCbTt?!gE6DEGbI2Yxk*H`o(-m)GsqUxU)I_I;sGM`fziUTxZyn;f z9^lUPFpEi<=P{Zz@-*GGD46=*4ySF&XQ@$!nCgvwAn*b*=u^p;c-}H?F222=P5Uv2 z-RvS}^;v87lo^Lg{(WPQcE9G{*zAQ;C(gov3#Xu5+%5?k;}n9@7Lk@!WVd&q;QXl9*d7u0z5LQ2WbR{puQ~` z=w?p1*g?Ku?04`E1LEcZtNHz+t{s=emujr>>H0D}VssDo4O>cfEQltJl`n{9);mJe zB&ht6H$>QyC)z^KFu%5d`klvgLT^0v+Ro7ufvu1{RLgF>wiQ^cXX)5^muc0&bNcpd z8;$QO5;7!TdVEVgQ4GmNCvgZjey$|^d_@N)Zyx42rku-hiQ#Vc{bh5u`m<-oJZEL6 z=X3kzZQ;%tFXMxZan(bB)nGO(cKv4(eYp9D8{2i9ByI-9ETTy zmH=nK?sNz7KlPdStJmTyC#!JbJ6GbYnMSVm{3L6#ewl?(KnT>x_FQ5j7t#uaKEX_Pc1%v{3yP0sFwcwqD@TyWpIOo zqoLZ}Am}wA3HGlz41eDAg#k9B;H6cUxXV!{TEFW%7VWhAz7<~qocW1UO!<1n7P|I7C%W_YOY%105*b~3nm8C9Av+%}A@5};5I2op z+}BoxS5}1MLys6dyUqtIhgabJU#DXnYlELyDqtgWMwAk*3W{!20k`XK!GBlog4@JtPY+{8c*Dc?{^*tRA*E*U*qtTUIO4w>TQ_q_3SKi2L&sfDb z9yjAf^KR3DZhPvU`IvMH41!X#-Q?oKZNzVq9jR3?B+myUaf3C37`5?C`&=3H<`6(N zmK(6ozgXOJ+Z&(r*T*xxW*`@+AlBCU4}3R>0-MYV!Gwi*pzKa6qqXEM){`zEKZmtx z>WH~?S~?K!s&C1eF`8l@VaN4fU?N@V`i~m>s`HzKEPdt_Ut033l|4_ca%<``xPZ17 zMCti&+A+nJ534rizZ@{8b3fhXk_Fe+!cVDCgigW2@T2f-=xnI&vWqL9@rG5Qe$wKvwxQHyZ4x5djYTvd24l1KBnF@!jKjztme)a<)I6z_ycb z@`@xvMvokibR%W%qsfcTC?5^F!Ps?I)z#KcA zc5k_;aQGaWAUQ1h8rj4Y)LAgDGxY%D?a7o}u_OUueWYlMJALvkfi?_B3SNwH^w^(H z+`K{)R1TZb={*C=!Pwl6uojM31u zS?IACm2q#Dy&yq_WBFGO{`{p|t9aLrX}pa8SGvnDotl|T)A(1hzzTv6|RGxfDkzrsCQLecW$TjCw>CczW+@{QbT= zI=g!+K78UmI=O2N)>lp>YFo!pSDjeuoh7CNU$4=+{1nQT>(aQN*U50l1Uk3>8vVlw z`?GKr&6PjJy_y^aVK!sHUTVo+I-$Z0(}}cq^D8=Q(PX~z zqc{IEau)yUX&Y4!G!{BE8rNlnGgP^d6bCEs83JJ*1x% zW%&5N%KQo-#lOjaOs9Dl(2F7J^xCCPcJiMFZVtB{2JC8tQm6ai!Js~vh#tX{ze}N1 z`#LzMyL#kwzO|B;dJ^VmgR5v?&8HA-hAu!vAlgn54Cp@Qwi@%YWFLJ zcDmV7ji4K3`orx+?>R@tG+)Is{~f{G_$&C=co!U$Z;Nwo1_*M-a+E$4juz7hQLWJP zZTC9C>=oN1Yg=J%w{xynNtd1D6s@W`7he4+yj5A1**#RuSG zX?2*^@R6( zCQWlCWa>xam&MV``5|=P=zOZY>?OT=LYF6VCh;@VwD_h?rBsd00I#jOxMe5iK-NAQ zjvLH`msg*J2@&D2Dh$A4Lq$kRayi=-np~o_C%0`@7q|V`buMDIA@{yyJ2@RUiynFJ zM+aW#(U2{_^w5s^v~gM$sYo>hm!~XblHX_Gwl5pdYMUT@#!VNC z3s>T7gP*9@=pA~NtpQ&1JHd~i2Y^YME0D4{0xl0_0FMK6!G|q#$wjO}1J8NWm#^dK zsG8GsUZt3Z21@b1rowaUr#c^h{vtgU+QMcMWq9R}8?;(G2g=B}!|*-Rp{uO~l+}sn z3Vm|f(l63vj%NjF56>g6;2`^Jryu*!L6YQb9!r&a_t4;Zqv;;!F7*0ZIQv!h4tr7H zqAmn0h-AOu#Lx`pE|$r{l43PD_@I%yukkG`wCSd*1 z0I2|GT9hK}D(qv8^&htNB>&n}>sVjfWL-#^-L z;T63ox0N<4kLCi-kAh2wU19DoYZ$Os85(YW&ix57=BDNxBfGUc>Dinw+f zly;9!(fC3yt4Z-nH@fJi|29w)*GiUqaD|)p;x+d!Af1C(p0Q7#29u&>p_dePnTCiQ z=ozffp7_wjws~}+<5Lpo%sNH>gry!|8l%7yrJM9aS}qOHilY`L>Ez_uK6Za)5I1uA z32wTB6Q|{@%8t=|fJMrSakIKQj{j?b-|FU~QfW;~%@4mP17?$i5I+$zIba7=Os^(wER+u8SYtlo24q5hm6-5|$1F16%qmnXW3I4rZPm2r(U5eQWI$L3K_bgR)w3dQo}i1m4+u6C8%qY!}&QUu=TZt&%C`ZMHzUEApgEwa3V@Zv7t#Ft(nLKtmITahCoSJ*Pz$XK zR92&h=Gsc|nyWt3Y?+(1Hqe0DC0=8fn;+q>woAjm3RB^gELT{2eE(fHmNy7XBpmCtUZSAy!Ole-@E5A)>a*(yLU$(is~zjl!SPxIKReN9|@yb+9&af7hX1->2kj{Ck$o^z}aiJ$R3_|D8CY^J>*+e}@C5|bZ@ zoHna6etk2Tq|-f&t+W$xTlq<}VyGH7MNTHZ%WFvfzyD}ZODzQh3VhfFReq7iQ`&fK zIrSQ-V&_{Haea){CQ z%~(^q_&ue@31jI2^J4CJ)NFWllMD3zmBX!#K15CJRQWHm68zn+rIfqk!`Xo9m4@!!>6pabZL28OPm1Mi6?BZ0)XLtF~rxk00FUyw~+{c^!Ya6>g(p zNrwv@Ul|RjMC3sMF#w&vU5D)!w_wYm+wi(t6%1Yz2OAbngA&W@xWMm`-1k4V96x6~ zbh7e;UjwGYMu&ym-H3L&y3LlajML)pq({@Vgq2)lj4ccqy%u&FZiHy07u+haD6~~m zxDAJk*$aPb?Y3WAi0e)U;5t}C$Jdb+S~cNFx52O*q439i1J&J}%*VWa<3 zqblP^(ycnrn0)m__R!@vwmZC-3o92K%{gvRYO*7=jx&NOv-O}>gB^UjX%C!{mjw@P z$MDhSTBuoF0KQSNvdlk z!_WIVl`jh>Wzl*(by1p?Dj#p&d3cWKGK3t zCNDV`hl^Z-)=JLgkRLZ$`YCs1fhzoS+891483j|W8S!>eQgtH@sR?@A2>1G z&kLxvLI!WIST2rz;|LP9wg5-t-{9f?H^63^0ydG&B{k=^(p_)A(yzWOA3V#Bm&-mv zZLhrJwxkH%(XiN&gLvu_thNXPFzdV>)z8xNhNgt!fS+yoXRcn zui(xfd&5Z@f8zM*WgIMKxC*X_P5n8BJ-8_abh|A99#oe}e4ioywfT!(yz>pzJbtT? zpUtX_J{ZeT|9qg-)(jTEIl)-gjUx`}2DHQEDs6Z*nXlUVAFngYfOjiYV68Ie!%5|* zphbB+^s&-}1uMKu76!Tt(U+R!dc#PWX74s#;0$hBiYqE&CSrHJCwP4g|9A zrzCUlKECI^`3!P@tV_9(fywNxQfKD{qo7d`7^xtJHtnIR5jM^}K)1a(*cC4m~FGe+_OJOmB*T`lDw-g)7Ni zwxS-Gx zs{dR7MUUm-h+qw_Tj0EP+vLz!&eimIS}BdpXrKwoTZnt6^SL zsG|+lt5CSzSFv`x3&S+Wh`w-gK-Mx3`A^?a)zmBa2pkx2HZ%kb3-3<$YZS;<7SaRC z*9G>H9*?|z`6WFY`JXSw@#8K_u=($uVS-BxRB{i3*Pab?8Zt59q*)4$ou_ zvt!vYIpNfVTZ{ zqw9j)Xy(bQ^6Y*-9ah>+N2vpL@3K4Gk?5t+NhSu$+HQw3Zd&l+S0gS}x_}N% znZ!qX&f~8I1Ac?sTPi7=KnwlGQjbbs5*s4HG}RjdrA?~1G{*$55WH)pjeA)UOe~3rj-bV<$s6Y{_v!2_9tc#nbF>@eyui^-)gax)T@fHkQ+i zOXpSx$iTXnW1%N&49#9yz^wreaF)P=Tespj*LUd@n||RG4bVAH)!+Ujy{?OZWMTnU zwKJl({=FsIT0E=a`IS4=6acNq#=-W^C`bl5*t+)!ck@vg{jp*K?^863&zx_=hbhVN z_4kVCgnkiy+#X3bY%fBy)}Lie|N5ek6J6*?qO(|4@QY;{?LbRw_nS;{A<8=ISi6ymSH*Ihm@h z+^bMKZkntHCt-P)Jt}x!Hsx8>zi59fL+P;s26JwDTNd zEmy}FEZNO2FPp(m+;yLIJUy2iIL&e&4MuWfEcbIUi#~C$_A~IxdRKVhj2*1%5g41t zJGjWu0xnZw85h>{1!#VarR_OGbb7PUF)VMOKh%!V*F_5S9Op<@t^ba3X(y_Wv;rws z@d$5`XZoJlF>+B-pmSp+ND(HG_o|kGf#+Mn#M>hno32)L+IA#4@O35m9Wb3_xD8;n zkSXMM*GR!HTt?e7RQO4sMZDw&L;m>~bvnZE4wo-G7A6%x=F(;@<_6(3&L!hB_sn-Q z?D&-qrxcxmOI0G^_&ifM|F#qyHLi{;Slq>xa$mS72?HF%3v4j2S?~s01)&a>H&PPtX2*Aqm$!iX=BAYwROvt>p)@afKbZarGOXwZ0bJ@DCFCo(Muf-i>h`^MvW<76XOR zc}$#R8}oN@8>3!43Lq6taMWEHuon_>R^cP8RytL*C*wHKbaDd=EVPNrsx5S&Op>qa zb>Ll^ocOxv`?Pt%L3Xj_4Nh%2&&gNBbJybza|ZRlI9+BdYzsL7A1};?F3ceqxYYx8 z2fD$T;@NPiO(0xk6$^JiONVD0DU1@eL3Pb;I8=HEUa>n4lZAN%li7x;RAy-^efHIz%FMV|g7NP%Hnz&h*@!L}^ z%LgPa(TapzEyeB@y(oW;HeUJB3V+TYh3#IJqpq_Dm|o}KV#CUs>JXc2qRY)^#i7x^ zMMobEG7JB{Mt2WN;7+d|Al(_oc0b&~c3fP^*7a7A=RuFC@f}+}C)b%jzEYltM?R9X z7mji_v~O`&Ju|tX%y^Egy2eG8On{=@b}*SYg<3Ul#RVdpZ z(|HEM<$_tKj1tzfriHz~KOGgD2phYJ(!9!13tn}*A@5|IOk=L}vI{O`a87pCwd|FG@DkM?zO$ScVaxA9~V@Uk-lQiA;8QmD$L5Ggi zQ}g~3`b;O8+GzPvQ)@?BA$+DoE<-G@Za^ZfFmRyl66kSE0~0r=gTEG%KsKpWlrSn7 zoxJ#<`q{r~w0KrO>W`{K>(ANYPcd8Y({bkbjM+YH*YAkBysg2;7rmgfco#c+VGet6 zn&3)p4yNb#HPF~N0Y1Ad3+JCd&OOy|7FhX_^!)Q7 zst3pLH=DG1&9zE=y5NY_G4RmPDVS3VWG(9Bo zkIYJLCni{%C`7r7imv5~+tY6|?po`Bp=UYBFG&E;-L=66TYu)*b5+L5-va+}C`OO( zTZlh~oJG9A&6~HQ7CTCJ;}H$*_*Hv4F8)3hmCOHSrq<A7dh{7yAPJ~u&A=&t^z$A6|% z?YsAgm*xX@{Y*>lP-qQnw5A-tk&vU77uV6+qj7XfUJU&>BAZ^)Z>4D}+PvCVH-28( zY(5W8;5*4k-to*|`cT;GS}pWK%fA%T;(>hn%wea%5`IZs9Eyp>u@bUAeJz?@Yi`yvdALpaAW>lT~^Bk#D)~D};L7aQ`J8Bi6&v$wm@bfRc zp|2Mnq+9Gw=_Z&(6k}7^>1VvS&}P8(hwE{YoufFlm6N#`bdvkrUdwfy=;IbVt>YR! zId1AvF_Y3Vf!3FpQPY@rBHLbFP9lE;7btn2-G1dXHT`YLzw%wccf?xq{Tn6uU1OSQ z#lBPY#n5r;cOi+o7KPIC>5FJyZ#M}!7ekiq*h98V-bresHAwNNLOj%a2wQ!U!TzgI zb?W#3#0BY@q8d&W$%HJzKQv0vnQI+rXW|*=%wP;7F~bA}#g4#IhClFrfjiWnGLFot zy@Opot1}WiD9CJY0x1t4GjX%j@Z+HYGO(wBF1xP4tBf+`{d=`}U;QW4y~>}4$p^FB z((Z7xZSCN89}igicsVq;SOISq`9tTm`(TMq1YF67!nFuzbG9d`32-KxZY(65yhoAQ^;fV{R2KeqGXqa-jK^zudF(IRA=Z!FiqgJM z6RT|PV|K^1{x-`6fQ5|-0n1Nc%Hvy^pBiJFWCG7C1 z7&hV6VV35XV4I>T6vbVof4oNU>$S)7HQzP(Wn*gv&UqqUyQ7EW_IknXQZW$Or$aC* z8`gE?z}d2e&^P!ZJY{?a27lZx@MBD2^Y(w-Se*{8-s(H|{plpQP=5g&Fmi>Smwt0S zT_3;+583Cj~x1b$sc>Gp2Rf)!8rNDFnT!&q0qp7k@;^HbjQpC16h-pwk_Z7 z^gnu|NHa9FJo5L$(uD^JdX|AvzUF%3%$AH@npI| z6rG#jO*1N0`SnW;1#tM#UN!S+D~h{um8as}94J z6L&(pMS9Rdw~D(rV-;up+mmB7nR+deu3KR`#*9bY=lL+l25Oi7F=Jn5c>NO4qULuv0`24wjDO zr5{c@(B7gD{H{lxGZG!(z^yp$%;Q`p+vN!Ty#5ijEB!*F`gr>K%@KhQ;YW=_HK@Gu z9+LF60?%tu!E-cEGIRYC@woVZSl?wD(f{*Ec(%*p?6L*mdUH9@-ktzTwTl>iVh2p# z8-b1nQ}D0D0r+<7f<>+0n99)0qDLk9jOV&{BF~lEamCatc*DCg?AG6c<%PbQ$NBN- zol*-kC(8n4CXU2@$DZLUjoV1#MV=U$%F_2=1L$nyQ?yX+GWB`;h#C#Z@-`FA`FAZG zFX?2;UlcOLT!C?Ace#dY<`z)7DKT_#X(SDjeMWz+R^tC%AHj!stMI|wF};;-PYnx9 zY28s*8hY>|o#5C*FB`q0?`rSUwRr_}R$TS-~{!KUXtxOA5SE5&fIenJK4>W%=vT%y2|$<83U*?~!n z7mMcc2bq)T88h=VVhkro;laiyIPae#89iN(>^PEv(Yl*rhj0IZ*88)7p@b~{c~#i+ z+G|a=gqM>a-|mtJo8@WIeSI3b(~usV#L(=yi8TL37oC-+!*4xf#UIQy=hGEU_`5d7 ze1Mt}PxkBaUmuU;*N(R0D{U9?X+y4jmyny4N-Oc*d!JKx?oclf*h?Su{h`>l zhq{EHr6=!rQjZ6V=+bmk8mbUN$n|r0LPb0ZsH^~oRII^@?TkZURs4SOUH%JlV0IqRWv9x@*d+!^Q-AL>bZ6Kwm0vT0Km0AHduu;HnoMV_PDsd43 z*%&KST`4X*7KLvLUAP@V!d6O&u)Fbl!@G z*5o?3Xsz(Jtn~*TD&c_7(Wm?DV?h~VzkQ|%LNahKT@Hn-+m>I)}M^WI0WH>JAz@O*A7@crBfZkk*MfiEwBL6vak*OxPB zgDSy+McQPR&rFiMONO+(-i*NqPaux7L^CS{Uv7jBQLUa#4ve$Fc5NHL%+*==EOI3s zo;f5(^A<^Za)&5d7n2Lx(O74^5>N{rh3|b)Cw>~yq`e`Ll%jMpz0HsOUa;)ER(sR&G(yAqoSTf<)&447o5ya>Pfjy1AsJo^X?`YeNr|v&2mU+c94r7(U4D*ko z$fH`AH+8@UYc#OO3t3#R6v}M=dsoECRU=mgJDWB{AQy9*QTEc+Xk~?=Sd%hYKSRj}ezJey95A zCV4O351JeYbOcPm^EThG{qruS{k8aIJPjM4t>o_4z0C3FR2Z)HV~g5`D>y!lz?^lQ za$3tJsB^hSsf1J6v{1{#V#dDi6=t07K_LMo8VAmQ%^RFSu#7iK+=Z#K*o(8|kEF#ouZ{`0Ah{vhwCzlLw3Stg!aO7p^K z$wRtryNp$Qrzg#cO4_!=j%|FL#47G&kcx~3`M@~|(t}L2_u&j=ZWi2P@*z|RWAW7F z3DZ&!J_u>UCcf(;x;oA`wS%5tAd)+k9?Xf?_;IdL!&opzz}7s6I%wjF)baqz zFRVm;@m_>3{RZl_gtj6NZvRUuFerYeD#aHhAwQrQoXJLCgBv~^P6xwu?+R4r(q1bM2{@rWvgx(S>fv-SDYQ04ZUNe^xRiOG$d^e z)9iuYO*d3(s@Vv!6^yO|=tGXc>2W?B<^-tqRLIGXE6DO%d(zh(L~eza(?E8X{$8NL zqQV24yROpnb?w;o*4OYgtj4;N{;)gRLhl`Qf!c9|WY0d3w`SEMJ(q8A9}DEg5x(51 z+#aZo6=F+^1Vh3=jK=UkO?@F874(xYLhFAKMI?e6`UFgURx*-?!ur%$N`$>47=QvC49eK3`f* zvv(Qk?CozU;v&e_M>RwD!G7j*Jon5ST;>K0N8h8V{$VJVeMrVj z6wI_}gv>v&Wb7)6X=zsx5k4(qe8OuO-%>5I67lk7#bs|FFWRtp Date: Tue, 21 Nov 2023 09:15:38 -0500 Subject: [PATCH 265/294] Test for even res. Use fixtures. --- tests/saved_test_data/rln_proj_64.mrcs | Bin 0 -> 82944 bytes tests/saved_test_data/rln_proj_64.star | 18 ++++++++ .../{rln_proj.mrcs => rln_proj_65.mrcs} | Bin .../{rln_proj.star => rln_proj_65.star} | 10 ++--- tests/test_relion_interop.py | 42 +++++++++++++----- 5 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 tests/saved_test_data/rln_proj_64.mrcs create mode 100644 tests/saved_test_data/rln_proj_64.star rename tests/saved_test_data/{rln_proj.mrcs => rln_proj_65.mrcs} (100%) rename tests/saved_test_data/{rln_proj.star => rln_proj_65.star} (71%) diff --git a/tests/saved_test_data/rln_proj_64.mrcs b/tests/saved_test_data/rln_proj_64.mrcs new file mode 100644 index 0000000000000000000000000000000000000000..bdc2f9f8b2a8b5da15caccbf862244c8a1c03987 GIT binary patch literal 82944 zcmeFYc{EpF^gnD!Qp!{)k`$FGU}}(3p_C!@_^#)l=lT8h{PA1s`RBLS+JBsL);;T-bN9aYwfBCV#Udgii^o)A zOvT3Z|0#>d8vmpJGxqc4gAk@;$!NNd6x(0SD5|Ujluu)|F3dq?f=^I zzrVf4a-I6pr6M8|3m?M&S%cMKH)k(5SM^;d)%PEE*}c!hP5;nI_5W{TWME{hXK1Kr zWT7@wK2d(pryz50KY=5B8$*T#J zpXmVossqfU>@t_#=Gj!wBy1uEX6E7UIE0=8W@zGCR{)iM{+@6d0X!#Yz6Vz_x!nm{<0iUrsp4UwP0XHL=nLCdOcK_01`s+)Xk)Nty@i%=? zofXL(2$n!A&L^YZL4;mKi{pozH{nyp3AkWZh#NPqC>`6zp`>fD{kco^HDFfy=CxFG4ub732@u2N@Gm{o3x(EzcKoI8Q1pHs= z0R!o!LKXZ9-^n&2t#LNwxM&xam41hcmKO?to5Z8@7ha)@`}d+vW)F(}8b=uW;0{KA zsFbnXJId_Wh-ZLgw@};7Pf++ETJWMXmgzT_1byP7U|--S=JUlURO{G@)c(%Iasdai zvvfWd%D==B#%(y+KMoh|GsJr?#i8iqs5oO?Kgv?tgBuQQz;mZ>z)|AHcv74q9&CwZ z@>WWKol^I}OH=`rTZ0*m_FJfy3B!A|49JHh1M+0&UHo^n8ioBi2p)1T!PA5!P_O-i zQ7q0vp%Te>)xc_W8H@-QSYH7FpFaa{=n3@t?h6)({z1;&k+>j1kpw2$k`Z-P5+R(1 zKlpVEHoDXbWKFjT-HeY4o@~9ttoh&wJjKGmH2(o+`fpvK!{rWBnJi18Z2W3;_CyuXo-7p3RtP}Ks~@7& zsrQg7`GU?oRtFlluY$#u^FTq&dJq@34pah3W~D&^b}wB=JhZ0~hqMfweoqFQe-vl^ z|5&mU*EfJ+UvVIKyBT}`uEcwnUcuWFOOed;Iws`<2NbWVfQ292@$_XcaAVI@5)r?W zTu$u8qPfX<%d{z2{`Uvrko+!ivojpbm2m(Ql6L`fw--Q3{~xf-bpusRcZ(M4?8kpE z32<*`d$F+pF7s-2jNq~IBBWWXij8_D@K*N}w5jnaG7#@TUp|~e?67b4xQ`EHM&SL>7@wPKe%skJlz};1aJ);x^ zmi0F<+bm+4nC??pLjD=vsLB%MbJk>rSw8lFwqV%{6LyAW5bLRkfTNEuUcqVNFKb_6 z@Bb!~0HHr7Z&T5-ubz0XnLRFfd>w0ie~a(9=@LaR8`3rD6W)v$>DhX*nnr_WZZ6k95=+Z zp$3D==y%OZW{bEYn6uz2*xQ)}`j_N^%y-k+Lo@G#t4a-mZExQps{k?5{oafe+4ztu z`&6>)7faMP)B}aHwrp*xJX`U;6l=73lj!bE6%%PhsK?x*7~HGA-}i&E%%WvgI=c?c7|)fKqMc7fmm zQ8v=43;g$?6*!Otpy%9x&NVmV|1NCBosVy#35$0S>ur=ov>zahYb)dB1i`jFo&@au zKyG>~(9E@FRC(1RDg$QFPA6%azH%eg_wk|E!WPjAeL1>x+Z__)<4H2s6%qFXjpVd* zC{avPAsajbvE8SgVCRR2;B{dU@HJTkzN0(=v)c?i>)pm}OVV(2vI`E~autR&d zUNXhv!HnC%cE;|UA5vFT#q$3UlAD%=6c5@K$4SRC>L)HSx~^6z^vWJ=eU9Q!3kvY9 z>YKRE;|xAGpp7R67owTsp9HlH+Mw6N7R)|%3(WWY2sV$@fxn|i!0Hc+!4qvwlo@sx zGcWfs-hB($e-Sf)?9X*X+~*2O`PWaLS{l>o(@)W`lz94Zd?CFw?kWAAS3=F{RqFOT zh$h^Qr;&cC)c;{HT{-0tJ>zCXA8nmPXC61B+qE~**-9GpXMHKb8NE27R|@=H{~Kr> zmtaNv7Jza$Anb~~g!9ZF;)iSFaArp$s=6u#$_q0Xi#j>R!-@rsYfG64y1S9ba&MIH zI*hXNcocQ+r)i~us_WO-YB z=RpC(iEDxH$u}9x979mDbFpBRhXI}zKWof8$D`Ao&Ztej154>^k;lag3G;Y_EIz%C zUd@T3rum3gt2WWIMuSxMsw7`-C&J&^-bO7Y9#glt0(w%SkV?O}O`onUq6agp=$$Mr&_+{WG z%vp#tJwwUh&NfZZt}6%5pjdEen+BMd_(KpWbr97LN@J0v7}T@e0fqQHMRCyw1zLU+ zLDPcs0_$CGk=E1ocr-j3clD~^Q>P!H`KsP%kH!*QFU2s^^X7p)#ELOab_ExHooDtf z%EQZ_N)pMyQOp+x601L7iLQtaJ^$ZMntPv6f9XN`^|mN)RrQa~ahKrje5HBuKrw#( zk`h`ZnnLT>-=zC@U!-EDK2$aDG9AutqFys4`L(mA@a(HERDQUH%ICkL@jnY`?LR+y zb)f_;(;dOP&aA_m0MoN|Ko?H%mdW_?+~ne=y^`SgXAz(;P-K?-Cepxbp4h~1lPvTe&3V!Qb! z;nwaav|u$EIT4PREGh)yvLft_4Wq#J;Vc&ZR$-4Ch_d42Hvrj^05mc&82cYhz~gK$ z;u9$=u>7l`;)-3-%!(f_%&H|5n4sm|raRps8miX7O%cD)tczWY3k(3?V^4qr7cWM6 zlMhgGZUaH-@4%LSkzmTCZs7f32G|wVMYJ+Q=&p~LW}f^(cO03>FECc&X@VZVl~?4I zC)d%FdfREfwz2UQ&(KzDg5Km${_hv;j9q&%^lTZf2ae@)(wo0dq@E zGoBZw79JDK2Or+313~f(u(GERaQ5%OuM48A$gBsmGx-uS`YHebFPBq8j+LIYm1rf4w?`|L# z0>R@G1>{)sZaVS%ExJ`co5FXSsr1lQ@<9GIN;{m+MBquF`t((h<{tpwESd^7o#Ake z^#ER`qD8j7&?h?se&KIV9q_GdxoE4W2bf;z2v!WdVeav-1Xi0;iz;e@fSc`M;8(0D z7=AYcypzuYWiy|GYwy>iTR*0dV>_mjz(4L}*qQ<(3lz=6D3+-VghZpJMUE@n<0bm**s>ncP!XX9C% zm74Ji|s)sDl>;Ua?w-Z++EH5QSQ%Fk-pIU0uRdv{!`w7}YP$*l;;0cne0&;T_?@T2Kh4a|NMAiwoxcU)T z{jvZydToI&3yav2FNl>cbzt$*#Z+?LDBad8&A-3C0r_lEb1~%kfdK+2oYxDzdWIk~rpf;pPR-_?BB4^LI)juxw8NPc5zhGW9?3x9OCS z@>8+rx^!Gr5{X|*hU0D{DFWv&A%QN@MAAuw+Ra~1cO3JfS63C#N%8IUv6KwYZ=BC} z4lLk@ykz+I2Xm-egFT);6auS#I$_gUHSR?J9M0?N7ihCO2G&#9?BS#gFaw2- z;mF@)*90YskEpQct-ayXnoqD}=Xg$H^#D9!a~FOK%7d5PL*Vq?($HLeJzI7rntTsP zq+MaseDO1R{%yM`Uo@4_GJzV6w22|<>sAn@XofgzDiF)gmw5RkMIvIcoTL@ZAOXD1g^U*@I z@J$$UJYFmGj@LkTI3BHeA_Kga1c5o*^^n!N$M{!S3~Am{N<#hHiSz3qI#cl#y(TBe zA99<=H;SqAaSnBKm+b?hw^0>x_!{(Ty#>9Gx58;H>YUSn0=Fl*1v(pe!qfIqZZkZgrHbRA7MHA?S8?&g>_V48CzGC)_(>mD28^T46+u+5}To{`1 z04iE-fd&B_*%=p1T{Ft*zdsMC?Alm*^4n56YFtI6Jq?MuT^w;;Q$w^}l89NVD=|E^ zguIQSI6F`rPYYbkEN(Ugp5adgI}2lx^MlLiifff1Yu+fMI;am??bX1^mla6TK>^>g z&;bgg)@YiW4B24oN`4eqkkO^u)YmSP2A`MY&rMp$hbJ2HwWEU3S8f-Rz<5o^amzc=-L#5xARAhGrG7Vf5p6sJ#CMOrc6}rN~9HWq2Fy8}*`X zd(ElGvd_fuh$QK`@C=`~6QOa^$}}gUj5x$Ol2l~{;$cyOZC&coQ212lmR70Ba00-r ze-e@vuM%>L^MGf`FR-s#o?UdY5t6pMlJ2nh73SJO+Q*Z=l#W z3n(jjfrD$Jgl_uRaKVGAB>P?t(REg)GxloIXYT-21{suIIVH9RHd9@=b+m=G-LxNG{oVlevd42by~VjHUyH_kz#4uul!n$hT5#sV z9(GA{ILk)9XZ5$QfL={WQ1Gk?rq_LhdHzpe!kM>_@I&zK^RG}+wjADhHjMG`K^(Sh!7MCSYw_kfGG z!#wz)<}-UBaSb#d{}k%p?SL9ad*S}qlgX36;k3wnnC>|_p4S?#q5I1`sDWt}5#4YI zw|zU^(nw0bn8T&$8*I{6<}QUvaC^~x z*yrE@E7s}5%=KGgHYdeNdl51kuWMQd^#Y2BhU@_a0gkJZ#HdfOk!ct48(4aaK1&^H;j ze7iO~*IS1@oAw5%-MVV3+AE6N&jn!Z+<1H|?Ex+qpG!V0k0Eoz5!qN;OW0X6>1^Fd zs`lqCU1C2-ZKG>x<6%1*`hG6-h!Nv{*DG@WPL9Bkmv) zd-zRlCU+_41ANi`i!J30sM>cc+S}j)zvdZn;@+n?mv9GePt;{7w3nx*9=UX-cqO$N zO{0rjRq55xtH|C?oAq7x0pyRCgY6CP86=s=Tu{pbGmho~!zu)HsTk1C)x~!64NGTH-@~!AE9NWhl2GJhrqAGAw13gxuSSWp zVgyW)9e|a$+hK2y4IEJQ!+vf%fu8<3Sb0s0JMcY@GoD|@eRoRdphz#QXG`dvB0K&| z%qsrr7>22dd`k>klc8t#9Io(~K6m+&0=HwYBv+_1lk@nqh*L=s2rm!uicnCvwJq@z5fUluU|{+*1n`h(<-RV%wT%3=qvfInF}^c*RY=yN?C`U zli7L24;j~XNARL50Z8s!24>jB3eL{Iiq`Q1_{o$(9Nyf9KTBwlz2~h+{DcL>?m{+Z z<`Wz`%ZJ?i-A_suc+es3Hd;x6kE{?n>oAZXKuaq=fS>vfx&|ccB*l9p?9^Mf35l&ivg&`85BTADr=h zEoXcroNL$_!?l?PafW%PINirbxeUu$T(fK}3T_(De|4YBmrUxQgYmEN?#eXi_AVG| zUTp-&)xzn&Q=w6+V=Y$k^ud>3Ns;4z)}(u#JWb4d zL?sf{c*zz^{+ab^esG!`Z}~=nxCo@V$Uj@SDN7b}tNSb9XTF`aaa|5iR~vC3&5O9F z=m+%`r;>r{&Ib8gjdnI_nwf5z4 zCr@AIUiPlwWSSMBTtFvn$~5MKtfcv{b4#hXR~q;j)q%SUhTd6-NNjT$`iM$MV%tF~W?4#2f9KQE+dfn&-{4O&c@ohOsVW^K;|7j}tcvgySf0+i-1OH=WW!>3_3~^k(30ElK{SI~x-g1+zI_{4XW9Xuhb$xMrXkcp_Yt}MN(;Wds|(qGYGhL1 zCAvQSIo;tDOkZ7mPev-95pn(nQTNfJZ)a#xQ?8ugLpR9T>w8J6zi6KpLSj~+B{VKQ!XFk9Juprd~hyHjZo3u@}X+hwiLur*jOq{C+iSu{49;xH1;Q%^%PAR;STZ{gSYKlQI`uy^ZTD*~1OBZsE4n zujSnTGvVr`JE3;ZV>Wid3-IxBFjU;v1-l)EP*Ovkz3?%RMjt7n_w>(FBUM7aXx=6d zx~1uqGAnxLnl(){`9yjS%aAv!#(1Zj6q9g00N5z!f=~L7!G@T6@bE}CGJoTR*@tS_ z;OGiQCvPQ4*ia1&M+3m(*)zxs>n@UdXw0VW*V5+`MEK%a(!AeZNq(QTFO_lef=ZIs z+#S0p?)&cut~XJeJGah)EuNN5&)Qs~%sUp2IGSKU^uRe>jneh=yTU4+pxU2Kek!6l-T}x zO5DHBqRI8=>9nZl^zDWb+Lir>e*0KQAGLla&x4P{Yrd+Sj=-KvJYmA!Z@dZZwM{AX zYNOwFrBZvDTWnd26xXtPIk&0Ak$X(cxzCaE+=tKx7~NqHFP*%=n-&nmKT_0h&ra%;$kLm2aU{^`5>d3jjgvE~*<%(7@cPR)aPYV!n!6I{ z?IRU*Rn%2Ft=F8MD{!KrG>Wd#N~9y1`>3C08Oa&CgQq7s;X(bqn2#>RF>EA0opJ%c zsSx7(#tVr5@=4_WAi|3>uAz(HMRDUZ5u*CZpIp0CN2-N$Y5iAUI%Cyc%0GQiUpxP# zOT_Ne+|fe9;g771kw5(F_zDV49zh2+1sESblOFKANRwh^XrHVSys|=>GYxd$ibPj( z`>z{v$r>7*#{((OaBD7%LQmL71>a5kw|!=}{@Va|2Ps3b$~Ll3`VReBy>7keo6n~^k^{T1$^-xM+Ds7)tO5Ln- zsL>b}$m>?5cLMj5foM;h{KpTwz5jwYm&%Zak8?<`geJqf1{g`j)3dd%y<1>ojn z6)^kcR9rjMj@6YW5Mv_~Lh47z>Br~jc*O*ouqvI_gCzRKW*i+aKaof@za&QId~mf} z341C%n$2CM%(`4%$_Dl%vzM_ite$4XeZ6~$ecLSY*O)zIR~ikNZF->J8ngZNfJ-Y4WK*w$d#LqVO}T2*dTm@d>Ixg))%d zx-x+_O);k;9YQKP+(A1O8mRU~LI+Pgq2{Na(-&eTbkXn2bWyn_m9owvuRAO8xafCi zOOObzY>vbm!+zsk$6n);OH+`!Zzr>1^$g(MpToRfD`0rDS>XP7H8y;jHLI_a4W@Vy zoWs|Uj`DF-ZmBeN&Y4Yz^JS>%$_*5)enI{1t7&7|Qi^3VS>vB+aLef?sF9uw&lI~u zaAX{3vw170O9Qy(;Al=}=S5Ce)|``T%ZD2J9Z0;`g2t7GlM1$l)r3>^j$*Sa_1Ul_S;k>6;4bb1RLOxtr4E@`=RIdM=T@QG_pK1mMDJzWB+EAUtX77#=w7hxxIZ z*oCU`Ox*|Gbm6Z7=1a0T`{;ck`};%-+y15zEZ_DI|2LCHK-)UWX zIEqp~_c{E&^>g?(i#(bavlvPQD{}lG%N-PMMwO z#Cyjgaiu5l>bra}I%NyhmS0R)94IA0GbO0zq$1j~REt0MZ7xr$66yB$4p8CyWG?pN zWX>VN3TEHdq_3;>s98=VIoati^ z?qs_D{RPrImK!VZ8As;wf3RfiJzUxwg{z-m$Kr3KQOV+2;M;%8f&Job!14GKaHH0M zRgs8c@7X1@#Wnwd1DTNE3)5(R-8w25XGiD!JV09nC+V5{FKOunBfhHHf*1I7QiDA* z@S@RdPR(W;cS4-w8WJDDrT#w9IJ+DI-I?6p{nngp_%iO;Y<2F)&2D(@{$1+DherH{|p(Ad@UsNaF7MEKN=@UA1c0Jh=RGvZUfhWTObh=u=UIn5GC>n{Mr2u%d3Ee@Pf1IaR@Fkiz<2}_sX~@fuSo23J#Q0tQQ`mMRRc_xzZ!R<2gA+cN=cWZ; zgX7FoVMc5KuTZMPj zsHFEUT0xu2wcOt+(cF3NB-dw`13iX3sc*nrIue^nyET53hzGlf=Q3G(_tO!&F>ob~ z`jJZFR$m}@GM15NEsID|_bH+;xtR3p)#CF)udzM5W9cX7oa2rxe?Z+w`Y)RIqaI*WfEF~gq=+pYGR5bA@-4d5cCo+AM zJvEDWGBoB5#H4tmfk||3F@^fOR&dJn2&eaU2{(W8IBuFmHPrOn14H5r;MWE5@UF&3 z7<@~EJ1p(Y?OhPW`F;!HPG4HieVd&Pdk@EekrN8^&YD3b(WMF)TlD7(Q9^0?KT(<_42Cawn9(!dd4&kuw$Y{E220eqHu2x+?7l zu{e;(T3H0KN1_(973m#dO|munt+0ZPGQ7^d+B%<|S33_}OL)cXY|jF(zGZ+;uN5H4 zcRpzUw-1P=UjdO??-+9jTc$Ht0V||gV%gIRaZ}4~Jo#uX?&~rjR}8(#@Cqj~F4m6p zTl^$-*(<5b*>#jPx2B)w_|wRkMtbbeOx{r2fcK^ve43LOKYV9D9m$o0k2cr9Hd_hq z!`g4K;IuY8Y`B!D$xmlrE7ihz#?D-dXfh}ISje$`W!&$1wOo7NUCy`VBA5Nlh;uso z8lL~z3SWGk$c>!sfIt1a*tMT;(T!5Ne2J<#|1|t3{cAXnOc_}ZRi^BKKFhSA`tn>h zzqgXz)T01jm>FaB_wHXfS<)+X%5j;F=9 zgND$5braFLk2y%@(lp#N;WtVT=|n$N%oLrF=HvQXEO9um4Qu8mYsHWy`n|QIEMLzsH+-Sg4p0 zDMd3o`jxBv?+UlFZaz0W&j{XHYC_-sI!>icTBG%}c|f=A@5o-{m{t=fwl~5FW)B1|Q;kE6(5#_f)Y` z^B=*ByNdXDRq+N+>>fem4#dQpevfyg&!nm7!ct0!3(iLXn>kb8A4zNUW8? z^#caP%&UvM2r{O6`gZhM$7Z^Ip*{Wk=P;E#8cOGsmrzlAQJ$<{z#sduflpJm<6FNf z@M^vBbS2tH{0ByXD-f{x@Dl8kKEnCF?BHT|PdAHxrDIldSK%(d3V~_iA0+-fHen~ z!|IY0hzB-tRrjB9%XMVT^cGDw`|tBGr=u3i@wcnl|CTM}FFiTOYb2lI@3?H>*X0`W z$ykM-Hmbtcu_}C-|8v@8o=@y$lAveQ9IkZ1X6~}XdhUFzCil>~2KLA5Ko@Ug_QB9S z+`dtdM2vrk-?*0HSd-g0G+{{SJ-y2Gq|aqES5yYy_gsm$Ph5qsIPt}s!u$l1%Y@AI z72BCmI1xL%l*bpk4>3N9&zXb+cNjf0Ej&}T1aI9ihpc@VL2lHHqmwqxr!LJPfaEs(!&z$9*Mb_|-7){<;T#1)goXQ^<|BD{E;!AgK z_eau&w_v}wGUueZgxl93;I?cSgu1FBaE*E+drZ~}?2}8zUM13G=lu$7mTQam4clT{ z2N|Riva&d`Cz@Hh`y?9m^T781S>wHFZD{ukBV@oSps#O!Ab#36^y`x}n)*@^xE^Ae zX*Z4uzi;a@3BKu#j~rP>qVs~ufXNLKniW9KI@A)|K?SP(Xcsl!6i2V81=9H18?@w8 z8U430g>EqZOzfjHVe5%jc>U-EE{S~)_4|il)5k@eOXenS!thLP;jejcliFi?EYFd@ zyz&hH#&a+KQGF#ZqNBn)e;lCOU;LocR(Dg~#B5ryekEP=8sVoJ=b>L`8#D?S&p99N zh1>eV;WI;3SUPl#UFEV66rCJKX$gw>VCP(XdAT$&+-C#K+1t$R4ZVUNqIZ~&+e^{* zlHGXXozPP#^I19#fbYVjYWUwp_^5ujAZ#Q;hOYl{B?K+ z*>`<4Q9Sn(8<)j0B_Z3%g@+=vt5KhNzF$rMrEaBFNhfLh5?h+Ny^`E(iwCdk=E5>N z1Q(zB13P^OV9&8em^JhUeoe}PN1Fr4m&YP}PM{U1hdP#CM>GGk0q(Kj;GFfy zv91CAd^!_zFIha%m%*!Iwa}ey?o8e6QN(63I9)mf8(Lh)QiYy)=j|sr{nJXKu*9AC zjI_BJ1sB4mX$Qz=|(*u$;!j^lg3Y_;?p` zbHP2DG((>+$h78z$}IS}qm%dtU(;!T&n_A{mdj2yDWObCDV^zbhkmL*ML&nOjPZuA zfNdWyuueUeY)<}MHuBy~_R?!DR^o>^>#-{gR5k^Jl_p|fQA;8!m^d4orpw^msUCus zo8(}JP39sfx@nsu*g5&QJ;k9b=+vpMLljtPHCZe=V zZX&&iYRRmlW4QF$Ah;J|08a*OfjLf}*(f1P9)Aj^!_p%BS4~xZ|Mr=@Y~Td`aV4dP zK3GwUN(K7MW+z=A7DhKui=l6Gj!`*vCF;I7gnXG_k6rh?!9!0p36l;mkg;=@hGTNj+wb;Z`TNojKPpE&{thmH039Sw~j)zbXmRcBv z!(7jyw`U~4{FkMIKY~x_o84tJPGw`U^TQ0rIBpIIw3^Lm);~wnumN7Sp$aV-Nk)4e zH)HjWm$BC4S!5#PNE#0L5Y5@)B<^M~dHGF?q(&?zg7|MFELfVJ4wy-Mx5?41nKtCR zzz)55F^63H^o6|I7eaHS+v#sGOzS=Bsk<cY}b1UPf$Rp#i1wLnx+3DiE7#^W@{m?%;N z-SSdmM8>xYTnZPVmmvV{+1!Fw_A29n6-l^0ycF}R;&9*Hi-dStH*&9&8_4{l0Q0iQ;qwyYuk;XiP`EyCyA z_G9I}MkwlvGMM#G6sUX0G0O7UjA6VavqbKK;I%|P^RD>0KxOaaVn@Fe)0Vrtgu5p# z#IXjk_|naCJh=Wf&dROAA?*g(u~$RzB6Wq(dfjDoI`;^I4^o9Qx_1d8qk>SX)ODP) zJRP6jp^MjuPr+$#Y_VB+I1cPziH(1zphMNG@tl}rSnh%fF8Xw}a9IB_o)&3N_PCeg zbP<9pt$K-k&Kj!Yy@+17RiMoeCsJyjSCfS;;k09@x`+{u#TiDP@0ws{&RFYLSL5?*gK9oK5fA)|`T!qbWmgn`F$QE|~m z?7u7@dut5f^r+vsM#>I9tgA$KgZ<}i0ioG$v*P^j59eCoR2q}$bwIvJ^)R@!bb1Q_^|LXj^194XS{dD##avE z?D0X^q;DY_H&qIRjN1d=tuO-1}GC{n-6w=o(VemNZ=Xl3H~!;L5FHl@#eQpc;2}Q zMEXKCHhZo@w4APxT_Wd*-GY_)r}|PtG>@RoyM*MS769 z-9NPa*Gu%-O#$1?H^eP|2k`XG-gw(rZ+z^|dThH|8)v_^!jU}(aC_le>`}W}__1z_ zVBWUlj6?B|;NHcFSY_oZ{Jmx??hn3&4Tn2%S>zdfOv96TB=wHT!CFk>VRd2iaEj@$ z%2`Hyn*+0dttx1a-3vUoXoAE)c}%BQ3i=U{j8*?gk$0AV@!R~hxHJLciA}~tVTS>6 z$h?oeHc|XN?H4*L>jP2>JAmKQ?_lzocffu83}Dx}9X;P>B5;|xfVtN{3v7OR1e_Tc z1mH(oMr(@zKe!o=E$&~!jk#~|U&TIbUT^^aWc_i*!wZ!Bo(10OgndxX_KA~B4X+Vp6N-< zv)?nApH|g4^@$X5@Kzw{1y?azZXSO7P0Ehih!hZisP{-3o;r&Hn0>KC zL0+CcNX)zl$+%uIo&5G3Q(G9yR9P~NyQ(dtFMpPa);PlyBUVtf+!vj=qKxl8(ZxUa zJK*9QWk}?s1nMA_Xr6Q_(tf-So$IYyT?EcO<+zbeE2JMQ2@ zQ8CO~w4jM^rr^GSHq_LThwt6$$0m2Tpu%TB(j)mQo2B9h8fuYTLP#gXQO{-tnl^g=6K|c4wmaO#r>PqF|XW= z=ux52iC>LcR;l3Rvm0=a-zyZv*<($Sby#fqY#i~|9EFC~qQ}=McD{kpm!d+{J)D7; zX-1%kq;E{mk29SO6GJK$xXo|?OkC1rGsFhu_-vmoW!%|wPP_K1Cl8%AgjLg;MW#y z*osg*@uN5XvuZjP7=@veew&f*@*b1kP&E*jUVf7sDKO<$GUG-+F{=|Y7@P5N zg&LJtQFWC7C9Ie&v|?ohSJaHrqCKI)_U;{mIn@j!YUII8IkBCY-R#Rm8gj+Qj&!1z z5gllfM=nBfl6bS^a}?gGjz=7%@F~MlQ!i6RT>d=}nQ6oUor**-CA17E%EmI2Z`{L5 zBQ1EndoRw)aKlUXj4+|gMA&o2^6X9XaNxwpGiN6>3lHjzYh1)(lVTE^7@zKEZ zptp7dP-k|6Ws)M`?T>r-`xh-@@WY<$nmUD)yJq5em1sPP-HhiPw!pi?s)xW2o|yjfz$aMM2jUlg5rI8|R4$3=#WDJmo+M2f^c`<|^7 zrIb*qq-+gEF0^|fI&=61_b=$GD9Gi7#OF;Qthja%K5A@*2T2O#O=_`Z@vLOBJaRnDxjYqa zoa`gfvX;=lW+1$D2!f`#9FVchfd{sKVML4xb$l_AiW7#@`9EKQlbR{a?TtX+8|NW) z_Y-K^-P0&n$`)C(7bRYf2)xUGeQ7e7m$zU|>`m;K|8KOK(_uDOJK9NN+Of{n<6 z+bHT%GX#sxi@<7I0NfhcL+bsMApk5uJYXA~YYqidWnHKX&?OfoUnH}JI>5Hg|H0ts z{V=tC0F}F_N9XWnbeo|$9XmmWn#_%YyjK&s%5DpCZ`eCd%Q=PnxZ{rP5zL7;ZYUGw zoHqb9yZfYF{~u?s=8P&V=OWw6Tat0J786J1bKKh-r$xRC49J95WpuY^*d+h4Zj+A6 ze?=Qscc6Fbl}J%03*B7sm;9OF3rnWwLd?5Tc-b@$RGS*el_E+eWjl~OrOgsQi2*!} zTM20U6nIoiB`)LIxJ3?uXpT}lYB{SR-nDz6xRJYxE*@4!Do@qn9gc;x^*Qiab|s7| zjDbyZxo~5$06JzD0JvyaxKIb0hIf(PpNrx7tSjJ;2vkjdA;u7Si=x5>>{@uD3+bxChUzhs)D1M-a<$_-11B=a-l zK=*er{4;mtTx0sUdm}Zuc2{%BtI&63my!|pNAebBDQbvUIx35EY>H5Dr6t+1ZZm{5 z#6v1FfQ6Mt+_CZ!?(a1_viR&>5^5SN>3=aA`h5IA=YuAko}~_pmQEC1e42=)>@T`* z@DBwo9w1JceFJ?t)W(G+ml4^Y&7?m^1lh|6N{eQMA^kA==&3MPA|m)j!tzVD>{1ZEN73g<8AUp{nJCp=3-}} z`^cU=aumVk4r{0~aRiwSp%9=HC92=H8oA!iL45cNlp0irZoOEJgwOq=e1}kS=6DsE za(X%Z!vJB+FTlrxn?PiA6zT(?X{b~t z3y*2j-ncNDSGJZWYfq#RcXFWIc{A#}QO0>pbb$eO`@!Nq5ZBrQgkE$ZwO`T5i2ai2 z`L+?u?F5EiJ4=3zixus9u$Ys7J&4X2p$YOIwN4|m?Ai|`4(7f6u|R!z94oC zgd~M>$WNpIb}eAxAx~d+c0on_ZBVRR2IoGclUb5zl%~z(-RZA4%TFNN| z!`>0?eShKKzOQifKqwflapzP@h9H|!GU$U5f?tDEKyg(&Y~1sgY)Lw4rw{p}&NPYW z`BqcW@}9m)4e|A)afm0F$1Q?v14r2Hol4xT+a&7}mJ#E;Vu?cG?Fz$Qc}~||3u#B* zFfvk*7(yeL-aiv$G;;xhf_Ae?_v3!%F^;avDr=opd> z4h9FH77h^mFK5x##mQ*B5#wGTd&o5%F%tb2T@^XjxP#2wvv6g_da#Z8BYFS1RdP&7 z23@b&v}s`~y)eC$j*B`&)!eG-(-mPo&e^?v(L#nkB7j?BAZ%VDfc{l5^f?a& z>c=3+UjeTDkemx1!(_hpY^rTjO?@jBnT3@;>&TU1>lb~d zaz{?kf&LO0ZskH?Yyeq#UY#nMxX@|ZiZows6KT73R5GenpJoP`)9$`%IOTAhJbYEd z<@C?9YkSJb=!uSS@y-^q&FQtM@j{N>ism9RJPecHY$ineeI^x|cgfq#aiAit(Xv%O z6Mkns33~gF+%8@Tz6DQU4!sDsuN;DxYO%0vy$ar(`3xrXS$5H{Wd?!Q_iv0fP|iW{~JypK?@U!d)SHwP+ zQx(IcT{d=WVGx@VypyE{A7fX#vzT{`4->g~u=@_{nD0wV7F^Ow`&O0G*xPHU`*?4f zG442A@;-$U|NKNc1;P0M$o*eTF;8B&p!$zFXebJn1OKw69*Up+E zF^RpqrOqbwa`c$>Lom2_jQeVF7&#ceCW_&Y;p31t7?X4ds%MWxD}SUR-R^Bjw|XSf zIZ-0gubNHXb~lk*7c9UyG72>BRD&bbKwwKdSWc*h@6w*;RMk>2vFRmK=l&91zZS>6 z*{6e^cTukQt2Obn{!gTnHI=*i=P;UH@)teNPDEr=Fa!+!0luH7(77j^=}aFL`l|OV z-R(4no#5L5jlsLrTKK)E_Lo?<#A$5zUe_S2NZ6 z2xdMgpWU6&%3NB1um=Bb*0xt($SlwlqUS0Iaz*#p$og`|FAHJWy1%LAxU+O`U^tDG z9Ht5W@w8{x3OX@tE_Dfe1Vvj%q8H1bqrpGFqCwMEqlAzE&OvA*y1z^yWQQA+jJphD zqaVOVjUsrx{3Zm<%!8-Dv!T3wJzO*ygnr+w=1z<~B)U6C;I2hY;(G4Nz=nM`Agmum zv}^6rLAOd2dgmrr6n6*;Bp&cvA&6EF%b>}5657`{jd@A=sPh9av7O%<+37D*&qsGF zlRQ@vH04Z%wDqRKmLheb?Cwpb?;gNnr5tsO+jg2EA?3v z#$1vW1p5MWVW*dt(3aW9J|Dfz?vC?k;wsGi;-;{=5mVU%V^#KX(+%o!?F@C>*b6Td z<-qlZJ36r94>J9kiioNS%6-7d!wJ@~<5M=&$Fzay@hzx+vKt(VlVO7Hd^kAkICRT} z+v!{jLyzb8a{9A8$qrdtGO=TXMApuO zHPXjf25j@_AQo~_!X}r!V*mCI7Q&=+`mWeP!pg0(LT}#?;Zyx^A>_?h=GGg0g%58i}hktOScjs&%(IzX9XW(J9H*W{Jx84v9 zYu7GctcrB-9TU!~2W-q#c?l6w}^}l7cHiev&^eJ28Ug=S4ExgO8cqcO4<){Y2qG z+9YAsg%N^A_FGmQbCb~s#yt0gvqd6G)0(aM>+K+AnXm9mr)2Sy9w+gcf1mQanI5j` zHOJ=e`uJIq93~IKcmuai`t|Q)cF{>!kjpR>K3NSFoWAt1xKj^U`=fHU&|)jIK$DoP zTpMkgTSGNB?V+!CSBiqyE=9`>PjXW_BhcLX6R7`B0_VEQ4MvPg1zn?2kYw9H+!IqI z!^0R+x$~CPM_O=s*C!)~&<~=*b?KsyKM#S!%24Rh?Ie{cex!R@GF(`Z3z6IN;B)yi zqVv=k)!bT6PfD|p6)8ci{?RSwDx)u?+sqe^{8}MAd21^SKm3A?ZP>{|EiBorj~8gv z?6>0Q)dhTIiyFRsV-#+x|Htofzsu(?uj5VUYfE!pvvFI|B7EhoGd>YH8ZVn)!8dHV z1L0pN+oYu@a3zimbQU9&-0D`Ps|e8&xk?Vf=34k=)j>A8GS@COtwtq;~4%L|P@=EBHiQ$gph zqA*){%mVLbu(yjvEaHJVn`=FsX+1P&ds;;dg7sP3i#obw>`!>%^I76rp3TYizaq0w z1i}@S!6d@7fYS|9LA6J?X#*YaCc=GBpPP{mJNg5?dGBwS0i*Z ze;AtDaGTTd?UHC`biV5E>!YA93=kQYK}E7P4;z)TB6+4i_pc6VnC zOYmRCA~YP>=mDB+c1#%!-lIb2$w|5X+{}s!?Kd#>P(~+Uo)WNUM#d1O@LiV#$aWw4U?8%Ce62xl1HX$@Ts2yni&sK(t6M{B@b@( zc|iGOFYch+Zqz2{E3I=Epd^1Cv~zwE9P+NAuG}8>c-2kz7=2@dWfX+KA1Z>euZD2l zzn1B?{6e3b8u{@?L$J<`Cf;?!Y8bUWgZ-yoNxMrw^QDbG*sEeGUK947Z(D6a7v^WO z3tsx{{N+kzc|EU9*dS(w@V)c}+JLgFgq%-+^kj~{p zs=#q@45*j7!K9_8aPC|fEL0XCHO3p>#q9;3y}6|8Zvv+_PZo9P+M$K69%%XJBxE;N z18J@51?3ziw#Y1^gf6Uh4mb_toe5pVQ{?!)M3ydE<-u{bh~(m91U;bPFxK&SE7#w>JSV>WRV+ zvK{eC_l>-wLkc^%v7hDtxyC+hpTttT@~G*QPxP#0DT~gEWZ~0Ru;5>=O!0#*s}*F~ z<}P_wJ~@llEHS0u41UA*B_`={Zsse7>I_J4c>Jx6tir?SiT3)r!m zNu=V#V7{?`g}C-zFYOtzj5V&^!JL;zurk4pDNX%NpKffY>xS;7jb8p>?WzWUc6U?^ zY+WVMKCcajX39b8&39bjH3#GqR!AmXJuZ6sWfLf7Wxy!Anc#3_9Xx3m1{qy3kUD!b zja)Q=8H?_*T{n$|gcon9h01(Vv?8gzGozShT2sAfLb=q_!Ps` z_ez+FVLp?)wv&0adNa56li8fsJG3=1fXm6B#9gTg0+YxRIO9GQOylZ=!`a`h!zho%TD1?-|+d(bC4H{=!Q2AIN_Ph5h%ZivN?3pr4=u4It zSW!2aTPTdh}B1;q{1zu~+%z;K|q|E&`8`O~k$0VYu$0 z2>VSt$$#JNz*{AT@t=?0=9g5~@CoirJU&CrWS5jM^R;zsbxST==x~^&eM)712llXA z>BHCv(P;WexfxDxEd*5W2zu3+``fUN)7jj>aZ5@hqZWpev>9{BWB5$wHge>~JtsJE zW;c}8Rgw5lDnL$cq&J>Lu$!8)f^vho&~e>R5Jgq9)YEDpiN4Es-I|0Gwj-SO-*7yA zc`NUoo6VaU9O7>+BD}@^M*fyd7Qeh@H2>%BU|ve+^7jT$$C7^@xQmy=6D7aIl9Pj( z=xHQ-sQr-6yQRT5OuWb^tTMt2LKfo0wOjC*2?y|6EnmEGm>$mi@|O=eAdlz2l*QgH zp}f|jFLaH3J{ul;lMPjVz~-i%V~#(w*o`%5?D*d0EW=5IwRv5j=TsKb=;}+*Jnko1 z)2xV$+VZh(^<3&Eb{MH&%e0A7hEE{;6{~#&gCxqnlc7yu(i0MxFyqT1H z9d(7T{k%|oxa**34S$)B&=ui3-WK@noD2MC`;X$b>u%zIe%@l$>H+-gi560KiZ6eI zyT#`h8exsx#dw+94qWKG6p#O5iQ@;4!lD&pG51~%_kEqsUs?Gd8`SxcJ;_oNJX;0| z{ezz}1F0`h=kh@oSz^t4$GoCZvQ2a_e}rZn<>{?r0jhdVlX*w9k&R%Cc1PGEy`$aS z&r2$t?Yuplu9_l*8#_T%)mP%#I0jbMMAC*6Qg$$}h}Gnmu^XeE*vktK#q*p6KF3QL zcgoq}suF}RUzEkS?a%T3Ly~ygWubh}Re79gu>o&uT8!;)weagQPl?y?3gYNhnf&}f zIV=-X$4{@nA+}Lz5nmBi^QY95@CuiHUX+~1UwYXi4vt*JZ+`cU&pSB|U#*#lN95>X z&sUmQX|xtLzG;m;PjYzX$!tC_emTP)1BF`y#|bNlfv_Y=R(Nh(!(#oy*q|(ZW?s@k zCm;SwMW4>lkwI&z%AUs{yMHR|`BBT6pP9q84cp9lg+-Lde$OEab>`%4xD^cb7lDlv zf=bPIaCpmAT0LhI+tOOVT3WZU;D!*fyKlFYPqf4D*bMwNa1p+3=YWmde(~Rzm+~^L z2l9Wkf;!FS8 z;wiKD~isvcai~dQ1@ZM_UOd&s2rSs`uFI@KE+-_FVREskt;`C(HirPo*Qhl<4l^ z(coUR81C~DJJs&hq)0cKjFcG&t*)}LQ_TVDmqf$Q(n&DBQ4!pO9zpK5O8RBNepVN8 zl1={O#;WdU@>=FfScOi)#qTWfktSZqJy`lK}1yoVnmttZbtxXa=e%M%Vc-z9XZ0ueP;!hPb5R8+jl07u>eQgR-S^p!{K-Pc z1SH^@s>0Vt&8+5k8gnm*VT*Qqv!c}Dta?@%-KkOwTir#_IoS)o?34k$xLqq>a*g}EI7P`|+g~@diA&3Cn3dl|pRS2u;Yae>k!hP)K#&Gix6tDQ{a5iS ztqy$A@C06La*wZAu7X2S^s!!=CI0YkJWk&=8M_vQ;PsmVaZ8U5_CIlqckZ9ZC*GUG zH-Gcue?2@co|mpDmKmST`)pi`3zrpO_l3#0>;RAV#JBL#QIsbG&+wo6Vt7l>czV3< z4zt==%hr!~Vo}lN{INCP`2|7G_}g+}V$~@Hv1NI|U^evWMUlqqLJlxM#zYJs3q*N${?>!L5XiIpd!|J@FUliMQK|y#Q-p7^?b77_XJ@_8W z-~6uu9sK6d>uBU(eSuYN7S7IGD`;Q05}coZXSLAu<;^K+_z)}-uK@UZ0IY-ijVd1;FuBEw`C@d(h0zO zm#)Kh&ZhXH-z`4a)cExlr>v$EyMs09r@jqVEGXqyr?KcW^!~0^08>9>0zwal1 z6Xrqq&NPTvK8RB$8YohwkNXy#&P6#6sSpoNBNGlUheYe6bn&;~W+h;N=@ z#{zVX*p?sH#XjTP_#;woeCgQb_`zsne9vnPrC(hHe~V;6E+|PDd&xxzd-{*PexAcz zQ#_c$KaPEzr^x=DNTde8cvoLt0p>Sc#0M_r3 z#ZNFb#CfAJo+NcVG&S$Srk4+5C+8zrdB7oDnwx-Ed^w62*;Hd6@oQZ7;b;F!SfMHf{!W$o3+g z(z|{qn+heVxx{YlYtpZ@mdnvOFS%tThd#Qgp|-4L=+3M(u49`jIh7It)?6nZ5GSc%sX@x9tW{$`Fde?23epOgEWC!0B} zzk4fwFfA3c;&R-c(SjwH747RzYS=HS{Ei<-C*!F$HQcOIyM(>p6NC^sMd5c{79V*( z9G|3s->X$(_u2y7XC8r)fx)u z(Bntkr_MWE++bOxe8C6#HPv!1w&NtXY+~ThVgq{IBavR3eVob|l~daf%FNy}iFy3~ z!CF_#2!$DSEd6XOtGg+6qB_X2&8za5_qPjdwwZL+^=p)9ikCSoG)?2t*q-q~7 zJHdYbGb{UydR2SX9mnu%oiMuWaHQZmD@O=v@Dyh5v1Ko&YGG$V4zzng7JYX23(Ys&$DTw~vYi*In8m?k%+OEFwhmdq zMqRwc3bn@ws`sV{!=r`^0R!Dx=D{?cv`ob_nlIq31BTjrY@2HT|J_g{dq&zHvPj35 zj_jv}C`>p$^{lYQFGY|aVkul3I+(Y1i^BsBJjNE*GWJc=2ikAj^bDVGEXIppg<+?J zP1vnI2B+oc;{Lz_Jk~h^f68d$rytX2*USaBb@~;yW0}C5ZpX0sBVW?<|26=hUyi=c z9w6?h^+e7derRQ*4EmO&h)$i-LesL}b59=ikjTGhBu5+WaG}G)CELSalZ8(*z&FE` zdZtBD;~y2&X<`HY5iMeckK@^jibCdIQ_NQDL@~{W7A!Y)H*5aY#r|7vDKuud2>giY z!q`nCg^kTI%&}<{FDY4rZ{GNdUG2x)ccmeF9HeRAM&s~j8Dnbf=`9r36bV+{1;X{V z)k0kO5;kP~FwEb}!FwIru+Q9|IQhSq_|3}_?EYc{o(1;!lKNsCVv~)@xx4tf)@^Cs z7=iD*o#89Tou_&8PqM#D%9&~SW@e`Ff_^+EL#ySFipHBiM2l1!(BqjENb=+W3VvsS z){Qmew4A)TmcSJtU7bLNIb0^IVlBbwv@ZPER|cDdM^JkWODaAdOuKzb=^Uy1)Fd~6 zHD1eON-s;;;tpTdTQ-OJJkMgm8$U9yQ^SRc$|51qeTtBoXC=gLQxsayWV74VfxKpS z0`{Nq3HQAkXuq31z#coD@m#-`ROQzMp;$IbIIIvYsHS@e#-9I}sqZXqm9Z8c^)dlF zAcm8@E@6Z2BY1r=hmZbz$7g=}#J`md!e2MnVYB$xxONO+g>Hn~uHQvQkrizEof39} zI5WeDDw?+48QPN{afO4VS=JUewEB4mXO{1dB%vQTr$@!)noTY#-us5U{I-fKo0Lgp zuhf#m9RZ|9tBagHO+j*0m2SN1LDjVn(n-tInCTZ^w$H49ZQN48%s%X3E_5f`o{-B% zirQJFma3pJ*+^)}9wGco(GW&g=?PlbMoDY(OYCd8nCCt^=LCvZRdZ&igCrSSR7nZgj%za4?>6b{_6H`bXMJFyKFY zh!LkO8ieCzPT*nJ8}Q_6kUFELVM`4I^dY%`^_@*%_KM0kSh0^uz!kWtigtAfnR4*rve_G$hH@42>ixo$(`|{7(`7INKaaz-ag_&ao zzZ>Sl^?mxn^srm(VtX3>^Y<7(Tuu(VB&y-NPUrdXK1`Q1d9Wt3hj)6g6VE+ZgmGdL z-faDcU*fOMyy`BpiQG2kmS#Zh9xWFSZQ;b?nMaV#`d9$ zSH|qsu%)d0fFG-N(qwjR>)8@B$^zF*nD*TgHnH^_(>d70!ZpSSlLD3rb2Nj58GC07 z9m?85fZ1NAesHL`)N2Ept3Q$1G(2V-{0xL4yoEq#jTD@>SP8lBCJFsFl?2(Q7$!&j z#p`J$KX8Exo^;?6uWwL7^9M-lkS~7X=`+XTrKfk{dy?t6aqME=Tq}!h+tJ2q{_iFk z(*-&W=f%IZ_lifz{fCA=w1GFCQLxTYPns)_lX#B*NFJ%Qa602!xy_~~L`# z=a&sZcJJCb;~|u2OI?^plfOa!K@FNy987!0Hc=Ig9=g7HH@)0;nJ!hP)VL~uwjLL= zjc&);rurP#@II0?xTLV5AFEg-RTq>CCkh8V=Lie@J%st^CI|!CuCu*GI;?Z)1or#x zaVDbQSd_1xprk!om>QuYsJB`RI&PN26J@DWW2~>#F*Kf^hDPGm^WE?vIcaZr!Yy%R zr4@7R@)ln&`@#z%OZ?cTl^^vrj6N9A$>M`LnNoBh6PEPDQ|C~z`!7ZD;Qm@s)v5rp zM)nq2qkL78Jbfr}58e(tNDz7J*CKIP97tvw-yv@@M{$NTZ-{nI`AkNvPXv{7hhdxM zcUY`wOn3B0)5LW}^i}6rdQa^lceYyztr-55i(fK={$p?HcFUPee!m|J`n#2Fmp#e$ ztgK?b{1^7KMNul%@YU&jLK(vZ+{ESWNW=O{q&V&{tn@ zKV>Cc9Xw80w)hU)b17f^w8IdODowz0(q3p=mpA^mN5U6sdWvuklA5Z}M|? zofEegWU%k&lmsWWCbqK3irq`Czi6<1x_J5NEVR?=Id^B=Yp!YC8ZM)B6_+(ER&q*z zKQx{R0t0P(*w^O>on5wYZ}nnZ$EXC7sv8Q$Co5t9!$)8dphFq9qM?4h>=THWrC z3ba(kD+eiy%d!hm;%h%DwE@s~c?v8(X)HTgy@p+Vn#-OXD`Zt?nY1_ml{LQC77nc# zBz&$@5Y|^H2}fqUWBYeBuqPJs!br|YIKqt+&V8CJ*rwSC{s|L>o(3~vaM4NWJLM$* zsd@*VE?a@+KbPavxqGq8Sb2Oe$3}XF{1pFPIF%opdq&)Oc@~?kbb}Qwd&nFX9%0w^ z{G>BRdPw^imypTLlPIHb50@jiguME2LiFeS6z*We5kgZ{V13A8(jv|0rnYZ{+yOg5 z;m!-vXBi6R7rG%=GL*{Q*O8t-uVGtkE)+DRk)zuaD-M4Agsz8-6svrAf!52WqktW= z={A)V8hW#s)(14x?IXuaS=GI4z}Ij#Sn&|E);`0|&%4Kr?OWOM{J(6#A7x>KpOUa) zl$@{*YYH{SMuJ(@7-8;UE8%hHSmDPWJHf!)R`~BMW#3=t@}c(Fk5pm{A9-PXJ{rgt1p2y*6}dFd@blqs)si{BkAcLGddtg zMe19d1bOD6qPlO%Xtj4b+8NS^?l>rlZ)Z(HGa?N{BP-`pbyXw!R8Ng-TXjctz;ZRU z{Hn@vm(EaXXq#n7imO zds}{-jjTSv^wsT{_va#NkoOn@j^u%v&vLGOb0T`ywuGZJyJEtVvD`(=W;^F{=inh)d_7zZzeW7^_LcUhOvVwIc&txaJDXP7z=xtO0NVd(7jTg zVT;TGa{fdsrx6{2_|8L|)4K0u;94L<7wD44bPh@jzLTPhli{@78aUZ77ZNuugs>S3 zba|QseSOH0dMuksb??ej`RshMBrY6XDXT*xliN{<*Bg}i;5s^`Ka=}l{H0>Dfw^c6 z=Lb1IT2N=Zvv_*PB5|I~SMoC1jW+HbM`b63(ge`~ivIM`*!`2)&pn|mZdMdKbTOA@ zPP@hCKW<~rt~c1@M-SP;M?YC=nT#-F;$Yz@*T~X?a_GvZ(%c9)82{Kl3Kwtu!QWgu zl%M*(i`q)xjg^Xp?A7XIroC+fbK^JDZX9bbzF6=@F@ZN#hcS zBJ#R)B4oFQO0%R#AjI(iEV;TIP@xTMTQ>-bb4EkYuv+N+GLnj$(nV^AozB2)dQYRR6xYJ`l7l6-OV^lJEqS)ZpDe^PT zpYE5wgZy1|X-#P(6lrx*@h)rj=I<;vG+_aoIoO|>HXUU-o#|{s=_aOlJehq+y~#Qo zpRm^ldRXJg3U<=ZgjJ8O5UaaL_h{ti@q1IG*Mjh`)S&(-+viono{maomqhbfO?WF^ zywR4%c&kI<&@8S-M-$Z7DM8VN2PEXjeiGB;%bl>y=f?4x5D<499N$F3()IHocA++; z{8ECz#9?HBG^=gXc@I|H8BEK2)ampSh2SwWP zccN!0X)X^ZinmvTmQFk@ZMnexjNdN_EUV!525zPXN1xE>TRZ6Cxdp(luR}HF2k3>6 zmvm}=EA90j&IYwOvM&>&*!JDwY*xI8ZJO)KmQ{o?bLqWxr}jyfu`86V`=`uSS#O4U zS2*!QOM9{B=NWWj`z|K8s*qg@Nn^=-e3;wip)9;1jh^`004Fxd!v{U78nGw_MoE>e z7FCfj&*K@H7q^_7zhk@z#l0k|$txhGEE3*qcZbN!OUba`yCpZ8RY3bu5!5Mtg;W0O z)az3PoKdT>b6h_ONrKJMRW1NYp2ebpeHKW&a40&Iv|r>%o=Bo*nsGhrx`~}$J;D82 zlAl4U5O!t^xEol~|NHIVug{_|Bb+|^V-EHPonULRMcPkUOtljmY2Rierf=iM9!_kNOjkqxGpx}sYdtjn^c=L+ zCk*8~C8Mcd79-h>9h}a_G_IIqiERV2~1+%YRei8H*(Yo7s-{WyShDXVaMUY3e_g}t2nnG|kd{3O)s@q`PCjwZTV!=Nqn z8eB@q0hKd{z`{HeZf*TU;(a!9^6VV<5MSfk9e#5^Og&MDX)x*=rZ3%FSt_dP_(RNA z){vMR*GShpFYb+XqUhSjbW)q#%q2X!P9F7dr>9p2OaHqx7`wiM8}d{`{6Wf$elOi3 z-N8~UIrjPqyqE699lGfQ)V;D7`CdDTj0aS5zW+^>L}*&U$wXVaOUsdtaTrX0WE#=y z=3D9K=L_h~=fkN>)qEPU&5T}IIFQ~yp-cxYk)ww$r^CAwC1CQ~56aEVxm}Z!xzBN9 zI3J&4+oby^NTa7coU*SZM>;K#O1?P?-0a0A-j5|qMNyz8o&MGhKLl&1OWEYySdcZl zNS=C}kbGIxP1YW;hV=GhL?-tZa$5)x7((PZlM;k_xh}Yg^aa;^BNK_!zAu}Xj zR(vK4*2zKo%ta8IIgT!!)DOF?PTDyni_uo@IJ$Op5YmX0LwP|q=)s>SWJcH#66~Y_ z^W?(Gx0BDfJ%>Apa5oSPejf(w!Xu#Ap${vPYDsT=8c`ztP%labPI3|)#@~XbYvoYl zd>l5ecrWrCHYZ2TGp&O?iFp9@DK&!N8D%Kl@d(lUXF;@d?ubPBrr zAxjdJokq?bFat-u8t8l(3Av*8WLn@MLZq0()~2H*Q|g0DKl5Em0loy&-x*-x{gNy+ zeaaPw#-h73q}fXO%iLhQW5l}Glhmjw!;p(kVAmQ2*}u+#hhqus^chGsbi%-7%rKa^ z(7`U@@np1fOcZK9wGKHgor%iUKb5#RJHeI_2Z-$xMTi=14m&*`g2yvC8Xhr{{`c}0 z44XNJTXsPT{6zSm$BG@IKbbzzSGXI7tsD;*0!PF2-WA|#pAIEL1tfR3f^o1cU8<`j z^&e)DC)`{VJ>?L3zV<)VE7!;stjU+Cs@^8sTYiX)4X2Qf@s@Bfr3~suo`A%8X9@i< z87Ucm;e!8|kg{cA&?)Vg2cSW;IP(oe*FOWD!2PhYL0SiHv_t=%*&_v?b`eZEFDZYK zD_T5%7wor*gpXHFL2+6QfN4A^-R*)M-owCS&P!r-Qiyg4d(-!F$IbS{1BL8&{c91;m4#YT+@Hs&=S2fT%z3#ax60k7GC`XR~J2i zeSR;%X!JvHoU#;->JEgzzY7R>)N|8L6>u)%FwPbRah+|kU|CfPy|R?fktC_7luTa*$c0Ci}Ox{e=PJOroUsG)h$DSh2Gy;xaIRUTUO0#tu@5y8JmsFI$>2Bj+ zdp`rN8wp*yl_Je(9dvR~Bsc#1Q{wb|9vuF=3;1WTF#Sd$_>dH^IV(NKN)AD*lrwH! zo@h6s`ULl2;d8q|rrqS;kRs49?*r581L#h>1Mp(2Ik-!PNq5^y3;9Kv$Tl+pVdq6i z+Z931mi1ux&>B9ul@XLaku-^}5QDy;MMlg0iWL6i$h0rIYmAfvwH0r$FOZ6<0I@=0l z*BimFBr7ni7zi^i_K?tj>R_Rk0^|0ULWG$=oN@R^rfLDH+qs)Sdm#9|ZGtrjK$lc2iKou`4(EPg^h;Kc}b-H*%^7{Q?7S&5a+Afkgo9}aB5nkM? zw1Y^O3+F2PTtKJDm&9CEA)@BPlE%yf=-k#QWaC{bIr42gIR4Cl(hrI>G)skUbS;Lt z+mDgPf2G7|i2+zkcim1Jx(_VZhC^0jD-=IF1#5za!>^4ah`!7o?$vZ2DI9f1^@K;K z8Z$YKd@&hT(MF!EoFFmMSSzZl9S8%&mY_LF8Dzrll3$)9NUe7jH%M!m-TcH|KS(j z)i$`I#Ii6^3KqTz8zIQM9MxDp-? zxx*7++871+UZ_Jxi*Jf1m7kI%9+L-M*#h$2W+u3>|Db)|R6s+5;gs_Q`1bq>tlpa`3BY1)Ou74QP^H z#RrmvK6Hd5g=wKE_=pKA*msEQ3O*(E@C|}+h2NZ_PBB`hIU0Qf7xXe=BI#83=;5a#biq#vU3D^-luM3DxSrsGhwtOu6ea?R97xk& zE7R2lEzs|KnJD;_p;y7z(dIjEkdb6FDv|uS=*lOOmMD4jB={LR_9Gkhj<4ZNa?Qal z?Hr`3#E`@RlRTw1C8PB0NaFKu$+tB(iMZ+w*}3EuA&tuL6Klia6{=9{ zmPKy-G$cAreG(lygnT*xlHX^?6a6*&NW_B!SQW_OO zzKsd}A46vvmqQoE;r6{~->5`MNku&~bB<8fkO(DZO(I+N?6hc6D$*)cs7TWCoMq4| zZ3rbr*}X_5Nl|#`YhP-f-_x0U?)m?(3+3ee6F1n6K<$;s#c9v0*c-O7EPv}H?%Xd) zhK*|l-{M8QIrubIH8U0m>~>=MHl>Rc&Qzf_#|u&QdPN+zWgKgI;S0M4U1Jp+8yT2A z1<%`(jORZHW6o^9$#k*O`1#hmxZma*^=fm)+RlMJ-^_4$;PrJxDGG1kDjH*PDLK< z>C^(T{=dE0_1sCUwWk)-o&{Lzj5|_Z>&j#+E?`gJw`RpNXNpwc&B5cev+?&;YjK>Z zC*EH*4u6@m7KMg8qe(};pdD*#(1a8{e4o?DlbfaS?AVdm)m8}~U-S{Z`Q3_+pN~N& z;-b+0rWAB-Z4Y|&RS$Pd+hdAGVerlvCq#|FIaU(*`64;&)+mkjIC-4@@C(w8yo&l7 zikWYFrP;pLcF~w0bc+M!iqwO0y{tKauh9ndbDmbG*T}5T{wHg=%e^xZ16T-FJZ=*QeI})$7c!Br73dZ3+F3gI* z*O2wqV6^hu4EFYz6RiA-aQ1X$I8({U;Wed7Slc}UoiRBjmR>A@56_Ckfv*G}*Vl8{ zO>i<>jrYg*1|H+|cQts0X+D1RVllob@SWZ#{Xy>Cok;zy6`rlGhUfW~B1y-kNYSoU zY*-bIKAuiTtwAEBxc(l>cMnDS=U1VMqt}t#&ev$g($}cmQwtxzrHmKNH^3eFU(lLQ z2hgVRpT!Hxe;3PxpJpynG#!kkgI}7h$_z-tMC(;osNwx|Mt{k)DcnMg?geB#%pQfE=YwB09 zW9JI_j2kELPrYb7bDs_>G>Kv6{8D4@Nc%GTc5K4SpQK^8)njm2<0dTo?J8d69)?Tu zj^Sx_!8pK26Nf5mBJ0rG$Zxu>a28L%l0k{+@Wp51Q?)7R@%P85u=@$pYJH0;0-vFy z#~z}_&(-L`fdrH|l1KhWN>PdYGlXO7(Y}KFsO3Nq8s4djy6vk(?Kd>gw|mvdsoQsIF@g54d6Pw)3J(ch9gfE>GE<2#)*tE{RnBE%yDk18un@nM=AfCo{vkY54R5^CjJ!)# zv5us>z~eiM#V$>F$00{@f6$Ud-R;FLaYeXPWe8<2VDO5>OnfiqCzesQC9*pTJ9J2lo7;p$ej$9?law2tBomg{lI^dybd@tUX zj0u@Wb_c7IZ%iTXyYmp+gm@7Hrv_rQK%1(z&7if97SP>Bf|oRNJ~etdfhM`)qoVSO$^}BI4@D8EqE2cHny`ZF2YmEMBmU_Zjlax(EdF}% z1lt=H!cMGN$2vqBvLPl(X!(ax*tOsu`n)^~B{&r!)0%yl*By^787FM=FA^92-i5<2 z{6U-iZ1BNNp~$3J4i&tRM*Ct15zJ~tYn{j8t3Gk4Xvr*e>QSHg`$G$)5od;aqRyfx z=W|fiQXQOBV2ZtshS9MjvEotjBBo-~KV*1)9CBJniADp5Yvm>aK;E!m}|3!{U7}7suJgLh5t@P+>Pn!6{iJsVI zMVH*x792aDNvdWLiK?<9c^6NUV@(MpdBOrxy{iwWH#2yh)JQxvvKDQ={2raV?1T+u zor)WmE@jL!8qt%6K6K!EFKX@^hd2Cp6kqFA#nvn3alOSzyw+lxz=+F1{jsNzWdTf+UY*~;#HS57oi=27p#!*s*7C|W-21byV}PRnHH(aN&Xbjj&v^6p{@ z8I>AFLXabg&MQGxor+x3vTEi*JddqAn(z(XFg#;lCUTPKMAId-@sX|z$n>lv&irnL zUsp)sqp_u^SiusveHlQSpZZb5;8QfFNDgODQpStp$K$#m0{`Ty8!Fnb$n+h}7bU&$ zD_NnSi6(Q8P;ZYqUYju$X0fYY9%|#=q~FK(vJ<@ z-jE)fIn>j88y!{{B!6^*xq9x701r&K%Q%1JGT%sDQ)(fz~W_?>Gg_=7Dz_Rb%#y5)vl z?)HiQ9O!1MUne`bB;|?UpNwZ>s$5Z9t^)IUXEwuG%_sMyyGdQQ8jT#q3TNDIP&udz z7oYNn3HH%2dvqM^(~5wPjdsJz(j_o5a{}Z#9swW3~rDLvjQY!R2BlrEI zpEvwRzY2KVMRBP###zYUUl3R^+#+s9Y%16{<_}=q{(`crwZN(_6j)bFfct%~*`eHM zX5aP{k<-e9_>Jfy)^!6|{pw|u?Q1Hs)s$ybJVuKXT-5MqFLivbvl`WXXhE)GNvxtO ziI3|0BagQSQR@5=;#UinpGI{B^g?nGm2P-V zBu86=$F3j2-}e?Ud~h=C^D~AMoPUA4Uot^&k_pgmIL!@&N^nDIR-(m*lX2r}9Xxg2 zTv6uZEGGM~G1Iro6| zs=cw~_Aorr`UqZF*Ntp{E1`d%=Ha*rld#>!vq-#0m<=M{Vr!d4B>HPAS-moc%+T1z z_*BgX8GmJ=dfXN`<4hW?9=rl4fOwdAcndW6&ktVxaR9!#uo*Trj)#9d)4+*6HZ<^? zJ`bV+uiOFn6FK&LUe`oEdyhH)=BzF+oiD>ru`i|*8eWpYe>1>>_j2%&yc1l$Y%yd5 zgf98Hv2g0kZgB5r1}IFQ0p^}6=T!PCSbnJqHj`e6-F+`1N6X~m;-?}MlDriksW^kb z&X2?nybmtMJi7P%Eh-l0kWfZ!*KAJgQSBI=sopJUeS8VJ(3(xR2#e-B0>pWS93pxUD zev=>83fPQmF0aKg=k4*Pl*#zx`jL2Pi3X0XiNT_TVXQEyOVB4ZqOo@*36+>nA{SmI zw?jwK{ueu`^^#MxT749KtzZM>PHDh^_4A?6cVnnj<_`{=4H22G>U5%LIq=UJ4}H^v zVfdsl_+^bHoUr97$JkZTyR!{=OJjS!l(pf3vI;-wB*ACdNbz-(1#Zf|C-lW3Uz)*~ zVzrwWL6h!ixFum0tZ|wFJ;$2DOBR2@X2&~V)sPS9OTWU&%<^KVdmq7T4A0_2{{ry( z(#4E^r>odOY{R%cN@QAHSd@G10ot|O6yFav#-dBcICPpDe&e?V&oR)&uM-N8|Kn$( zXNeb3gJd2)K0O1!Hyp$RbtdFxsWoYx2#8--5;-L&WQ!v=QlBg7^h9MDH9mNjZq?n) zDfYL5QzIndx+q6*^=Uda3LVFP8mq|nj2mK^7xQ7%lWdqf<1)Npu^t{!$OY3Xg&y&% z9~4X;&6n<-z`q-(%FjPNL=&&}(dCERsY*jVl{tBt=19z*FZ z6%RmNU<_C>*9x>rXL3)+ZeSl;uSVUy3Ha-c-FWNyCunTBHS6+xF55Nqhmnw;$NZU1 z#P7nJ#amXN78NHyE%x=QL*ZNRGRw?=FuL>8*oDc(?1}Z5b#Y8)3r22WR@Cps?8-i@ z7bnJ0xd`Vw{1n`2ge;D9rDgx}Y5C(8dhk>aC7w0ZThvXIpBsaqhR>Yus&llb(uvnH z+|Q#07X1DoMeue_I228)fiK>b!C6VC;nf;RI8;|n%=uEZV;U2Iy+#8X7#tjAlI>Ic-8c%ljXS;%&8Ts%o`5ps1GVNxsia7TC*| znn!Rx|EY1m#bvCP^k&w-bR4_+=ogXP?F4LEH-b19jUa}Wk=VoHBsM+wkc=n|pgZgy z($1|z)TcaqOA3A#K? zZK0_q`gEIgG7gU(bF{9E7N(%Q3C^vkVCnrSkH&Zr(oFIc=Mz2jVoUTrp> zqxcH%X==d+fe@SHBVu2L+sx$3LT2Tg>8zt=AzL$Fn_H2g$xWZv%I36WuykDvd(^t21M6I>kx=g};-6jIRd#)|^GRtCqs{9c6)L)o` zqhBz{aX-7YzLV{!?PNU{QnpF{E87*_%8vD$%?%Wq}l z3k%bT*=Ak3P3Rczec48bcPQ{z{IvLkHKX}9;T$^=qQ}p-Gvl}HgZv`Fku6_VO$&2J zg1LH&p~M#qe~fzybz5)3SH0`tc1=a7{C7JD)E^RZ!PlsXn5Q=T&(U83Q)ut+G$P$E zI5akrAI@) zQkkk=x+ro4KjE1+e@01zR}+{`$Krm_w&57+B0rbN`XzvTa{xDVpMp;n)1ZS?C`?#o z5B*>K07)(`fUeP@8GEv5fK565&nSlmCM>1<1V3^GImVTkhq7N3_R;yyD!ifMWd5nV z4sTh()6vNsZCbLL-p_qWmBCNCqvAF-7d+xB*_pI*=nQ=hcG8|hCRAhoMbcGlNyg7k z#y`(_U^DMSI3~&!CphMdLzf4zw4jA;;77CllNfw&iYAdd^b5Dlnt>&J3{YRYy4cEm z2`-)f8mrB8B-+D1B=kWR`Bra4ADL~W`V$1td~`c~q(4m81&yDY1>0g?pBo%V+$7wEW)T| zeHi`iafJ4%pP*Yb_s|D5OX!=%ne@cTPh{19^N3RMS7A3IOO}7sA#yRV@h75&w`~~3 z&iS3ru8s1=IWvD?w_g2Q)C z!JXSrz#)l2Fy)CdZJL)z@jfYfpRF1Px3e^>Yk~(x#hu`6CX+Xzgaa|Jjkm#1o{WIfyK3 zb`toCMr7N#59qByJo{%vJ8NDYiD!HYC%2rRkj(kx=wUjQ%3nyMk#5iF{Iu_MQELkg z5?DlQ8>fSJRfe$5aT9!26#!f0ro-KregSYr515D%xl)!&BG|_y@!Sf!wkVD&`5vMB z+!s<;xvO-Rza)R|(+K{MW*40rq(#5(l7hZ=7of_Fhj5^w8iwW_hFgQfz+RtI`0vIj z&dX;7*qLPja%&Bl$IT`rdV~WhQSl-^kK>80oEE(jyPJ+qbE77&x`{Rt^11rkh@_Dn zk#f=`6CSIRPm4}sFr^4xxi$^IKBh>dcZZXi0Z~LDXe_Dz*oaRH9siyb6BGggbHm!5 zSk2BNCvNwVnp}U{RvSw{_BT@Ly)yh;x1Ut{cQN%UK24nN7l3UZ3t?qo5cIpXLI6&U zfcKPUf-I?W(xqokT`Gd{TH(ErdW`|yIq4Anm9doCWsaeKq9DrucZG^qWm13X$<#%p z3j#2NK0(Fsf^|2vwD<~l?M;Vy=bi(ZY5O<>n|EBzK7U~6vKWMJXy$VLhLD+fHwhST zLd<;LVx=xZqJR3*l?S&{&o6rPi`_P|CJkY5Rgz>_Xp(XtF;=v*!=8y3QKO{;>TIkK zjfpvd$K1b+C$>(;ZV|PNx=AVf@6>BHFL640-IPy8&gdsAO!etojg3@AJx1sOJ){qU zGNb5*52nu)J**Y>)JWsoIio`ufXU@piBXFbb%>KA`DMmn!LPp{Y{M4lNUy=WHqYSQ^$m~-J`Vru z4hLwl0$o;^OwNSe=N9U;aQC);XWjp$kR1cXWLLuvrirOu)gb_G4Xd}#pwe>h}|&bU<4qeu1LO@oUi7e97QM@zr_(yO zP_w+Fw9zkyGGbwuruULOU2TWu<<+o7D8a2$-!n1x?M#Oj%RcNbXaC0hVV54g%jVCT z#HBi(=6+QqaB}LQoTy%%n=r3{t(^Z6&674F<&hUjLd_pCDcYOL{=P<^P41%~UzziQ zUx06&W6B%a`_lVcv%n`0Q@CZ_Phh!33mo23M(CPOt}om8{^=Ej+-#v=^T8(O^3%bdx3{t4p?z)5|}F7%ZJ-$(R&Z>abM296na6@Fm2Zn zuuff?X3NylD;tOD=sSwMY@jqhz&xj6Bjf0iZJ&tWAdjmZ!_Y46PQ+VV<7Kvc@$`Pd z4bYo}Q!T3y5&SdSiCo_C(2F0sY3CFZ+Lh4<1XC%jYs-d4KV#ulZ3yRC zwSr&Gk>F>QGjKRY*gHF7aetUBFtbPoGVwe(6X^nO==YF4fps+0RfBi$AIopKtjx0? z-_jPj8uCuRGKN?bp_5hpkOVRot~vNO}SvfsV_F$QWMnA}ct9MYMHa?UES zHU>M`mQ*iR_5C~cWVgey@afqCf0e56t;6M3VF^ZE24N4~P`JAJ)8 zguCU=!G~LOppD?~4cQh4AD>WzUf+Lly3*mOe^D{8Z+C@8-aDbiEJwKDdNSDCf0C&9 zN78=^CYPqf7}p6=du44^R^VX#ypOw2hBmupzT!p}9}%Zj~VzQz>Lxo8}hI3)#} z$K7GAmLKL;blv8r{cC35Cuq@U%WqOUZFPS7l_~tSzj}Pdi|=$)K^d*p+DffbKawS< zui~e@KCJqoBkbWF(d=@CFm{LE5;n?!K`J9pF;OdvMBmDc99nY=k!^qq4va`cA+z&Q zSWPlAs^>`Mm_gFGA)2!6ORDE#$Uiw?%xAuROY7<`6Uo=@z$)bg)W1{>rH|LZGc6Ht zQb|21VP&Yx?$fkyr99}<<=~cU`{DgCKe%_m1d11)1mO>warpK``sK|a)vG$-#P%c&TgW-WEB#j?s;_{)|_xMJumvh2Q%vPxGG^*BqK_4*23zMz3d zejB7X@Dp|YyoB;o*8;odiE#P!MEH7VJ#1^FFzMMA_~K$1P@m*Quk7DO#n(21f}Dj= z?n)f=J(L3d`VYcDRe7P)JQIYjmZJ>3rjPO!|QuIpoB^X0QnG1 zH3z`|cPcQcISaPtX#mI98f?reUt+VY6|+$VTtmfCuvVnh;xUtpx-2MSQ?oXivmmYnW zUHCnMb>oXfhD$PwA6>a2y0Z8HiT^W)MlHTdn+xw!n|qJxCR?7a`16%GhhGJ|v+SV& z^@rK!xp3Fwv+&*>5$sU}Ao|H1Dt%R!hSXgH>F3wN0qJb$litt5C+!w&vBpA2^V zyGSy{RkXjnoIZ)DWKT$|!mdB_;k9-nc-=VXUfH3zsxC6Oeccc z0K)bBl>;&*;h=o_7togR2P~O+5%_!y;}VmX&}&VVv?Apm)tQ|^tAxJEV$eXQ%7&8- zU;&YPr$?~R^Il`Uk^N?_zy)pD!2Wq9!O7CmTtuHOH_Lnmcd4_NogcE3{pFTd+_Ey2 zwEEgp>b{$@@#*x%`zQ2+SqI&U&Ju!6Z*aGd88-Y%Z4_Ou$POH~zQ!7>i6xG&)6UEZ7 z?Rph>XEqgV`Zyj$dYlHcxK5De`V)AbsRFHvx4^0m31EP`0KO`A09lKXkQ`TnkE1F; zY5D{JCLAQw4cF5;`53A@DwtlkWN4>R0dYt&&FEcUn z8@j?iTxiVEH7mHM3zl>8eTv-V6_;40j}7c@d<*MddrCf|(bP-mv%O#Yj!O9SQ@Pd2 zH0Nq1Nm=livwd(Gj6CxKY%_lV+}7oS3#awL6SZ{mz;BrS()k;hevgLg&z{4E!~vMs z+y-l3oP}23<)M`6FCbiH zX#-W56kG=eR$c)ID$jsdQu!e2STE4fP>03>Gw0Bj0#N8B3*NeF5r?W#RATiGI_b>@ zT0cUU_D(!V#F@?b{s=dmmw|AeS2=FYsK7-+&&Tua2DGq6oe|Akz)F70V(;yYU_bXy zWy^&8|6$<{_MNyA`_1trUi&HWTM$IkW!}^OrjFp>Y**s_bcd*H;&oac-%F0Ow%j2< z9iX)AFc^WBfg%6H+{GRjV6$x^44_xxj91<8yQj?5^O5i1)~ji7;Hf4oZa7quSfWQQ zLhZSgHrZh6dKsvy(+j@uDFB`uGr%#UDsV6E3wTr93$g;wgH2<5xV1J9xW6$9z~B=E zl{3;o#fx^(GQJipb65&K(Ox0@uTH;s1xrtSb%H$8*%V>1v1#AO-x1~Ay;>~ll063 zM8{?lb=E1ShOb8QUkq7(>QfiqVI<^_2@E#ViGfs0c`~=)i4mBjqz*oO(*}PUF=&Q! zVMt**j2+Vm+lBq#RP-3uq{hMNpJZXg=2V<}V+Y+Mt{{2|`@w;cufb%?`=Dz>G05wE z0*qn?K}qLdP&_{yEcx*h$Ga|}w}#Wnl~1;ub72_wYpWrc#qR*7hmF9&tRUQtISP#n z=r60sv@kZ6x_3EJTJwkmpV1`;XDq|D>mLd^CV9rL@g%Du9m}pd=FS=y*s-hE-Db`N z?7|7tb;yC)2g%0S(PZbwC1lwr9#=+Q#+jAb*kE8tV8%8O3-%Ok_%6ZU95t1{`F;bR z739Z@-p}Lr!aiDm&w^G7tfjVfl*r39V7CXZpi6TY7!5nZ%Ui|zo%d2{u(2Wi z`*b4Bl|3Y!YYVC0yYr-W_6F`)Pa(LJIvXnPI|3)pSr2UvX+zW8I8bR?&zb4FgKeAt zfPtB&aQ0ShSa9AF+U!~nmwXO{@6Jqyn)mksi8Xq3%I>LDO-&iRyjTxzE^P!0Pu>C> z3zve?H96vmqG(z_r<-n#lj4^Se4~m-l4xqACe>%1NbvEwc!Z7-(mtTc-S~8b`zJob zy_+1ueM=N^XM}9y#_$I?^^i7M*)yN0N!t^hg@z;2X_6E!a_a+t-vy(gMf`_ZA^3zM)QDOjlMG?^B!9}<+G#*ZlJ_dXDZGZ_!R>I4w zb79$rKcI8`8gTKK1W*Xo0C(R^0*5W`gQIP8gxNX@Rs^O)r@C+$_SFiynqCB(D-5}i z^ICLY*=B00bAkS#6;$eOGJUs9n=an7mu!9)i~sH4g{&V0Bh8daq91>fO19eU!>bi=mC>pmknAVp^QU%Kx`ne&M&aK%- zXNx507>|u)eSJFqFfUf@`mr6ImwbR7dzHxEd`a^AUn5Swc@C2~Us2>i6MSg!F8=(a z4$FQr!3$)Z(7cii#F*bkE_>GCVU-*BQM5Cu@xMs!KN>+jp83*Mv)9vL_33oI?n#=J_GS+J-L)R>Ub+Hy zvK%b`JqFSNdH5x!31ml12a>*kigSACF8|-O+h{o*_tp_Ooz#M>i~`_r;9)4`?*i3q zDY!H-hYW=wx-|P4o!NDVHjhZ4%A3~Hi>V9gGHgQS3hT-I;l)I5K?1&?w?nk*SST7_ znT+pWeveO0!1#BgC1&@ZLrTH&OxVj&xOLtZoM0wdQaU!ir2eTBvhegqswx-(e=pn_ zl!~wa)+Dy=fn@CO_vDkY9$nNifi^bUP?-_&G+X{TovSIypDFx7Lr3hOw^Q@EBOy89 z>Yh5V@nt+1Wg#!z!?lEGnd?A}QHE)2M#CzBm0hFT4@Ms-2l-L)!1w+(@VBOcE8Td9 zblq7*4Q4+eOIkL7h*~W;xNI3LZ1#c5*?N%p`2g}0P$@i24f>3Q%A^LbE^4M$pMt67 zp$YUu>R)p2P$ID(??-0+n@;3TJ-~N^^3W2y4AB|s>*(LDqXGfa8lQ`(6Pp(qFmk=y zieqHU&_{O;x%Fa)d$;@16hAYhb|@RYs<6hW)D8c+xg6K2s*$@71IS^e9CGkhHW^$# zNHoS8(GG>XcKIs%S>N-#5Rf_2Fi*P#FG=#IQ z>;bE}AH23d0j}K*1!^67Aa``0Y!j^X~vTHa#9cy{C%pTl#2b#Lqw;Atnd8f8Z^;sK5}V&BwD*!gY6j8$^7Y6EOE0Ha=VoUNa1>q zNTe!*3jDIgCZ2oIRnAL()9%3XAC#Yg(etA`Go3{cCP|52Ya+0@E#CB3*Vg(!R% zbG5f^fvDvPS8`j6O1n1FwX=oKcE(ITHr|Gp-lWLeN#CdAxXpC@>jA=CKSn0Br{U4H z#`t3ADZEVc0&YC%i`AcQ#4Yb~@doRg*f&WYk10^Yr?N^!vRX>)+N*YKOy4@uy)t95 zDsw_KG{%DIezAx#Pw!>+Z#%=vKTTqn1pQ)md_RHaaS_ajvMR>F?iCtz3C5MnX5!8I zQ}ETPGs)o>kBR5{$+XofknR-TOApMrL9HSYZFK6Qt#`)oPa~cAq~BBdmdr`~%VSpj zpCe=V9?voSi$|usg3);1`QdmzY4j*QT9_f-w3T?>83WWI{|Ws$t(%{@maSC*h_oldy$a~pYLR!-bs z^%E`s4`g_)485Q~lX}FQr9H3j(4oJ>6facfjpxYmM_N?)`_mP9|4-Vy?LtG|YNif9 zMP$I=KWEH0$eZwO|Bd49%8hvC+hh5GB|5zKn=Yzp)8tmsJ1yp@NV0fwMW1Vq-(Ha$5y!dni zswgWG_Z~XJtb8EH8m(7k6>KMoY@}3?>ZlylH+~QL@BA+2R>U?orG;(}JI;XyU^xDtjS|E(|Q9QSWcib-y3duvf3>e|#17KUYCJvR=~xB}rcDv^@Xe zvohbBqRKDn`9}Mq3aHzoAWG|n8RyM)GI3Z$0)+4N^(E2F*Pqe2?ZXe8ygnOqbFSe+ z_fXuSJrijvP7@HKSJ}%auCj@{py=f_8$4w;gKb9%9@3x+Bx7KN9!6m%S6PECJk!IR z*w7@t+3bV1*~5~lmsQ0>?~gFo9o{lFM&sGcV`^+!^%PVPYJumyS&VlJ9&4zhC#rIu zjO@fLYK{9K?z1UGQyyjDl8fhXTAn3)TXBFr`D&ujJx(C@2Q=xEDW=pi+>Ek)V`=kW z4N9mnU3vI84cwneH&vdeOO8a-5`iVEKPQ$-f6bvuUj;{8Vj2xtJB@zR_(A-S93$f4 zN>(Y+mAhuNm`j>v#W~#h#JWvaDA~R07n-zw9O^TZ#2ZHHVu$Aa=M*!wdU zu_HVSkVB_CZui!}#ak!fDOc)I-^SC$AyQCd#qaram9$NvR!nwvo!PhKE-OH4@TyeRTSPl>+Do=Df& zOru5f9I5OWd-|+MkKUVLObg%I(zUAUv^d$G{JN#cDrQ)6huu;+PoqRm(&99iIoFk2 z^Yt~G+w@i><3AUxuQbM=wkC`juV;Mu1jd7mvVLj^ZCUOcnZ1xv|*!W^^2!&U5zy?E3sZ9!Lsu|F!3f! z*egnR*xG|O>_@p%CAVZc8UL9*%--lpZ16HuHgjSm<9+)Ph85`~R4bcUT(u@0db9Ae zd-udXcPF8x#*^7A2084pnQbiZDaC~j%5Zl=3fV8pL(F*}YlrNc%IJvG!;%4Y4P2y~ zi&hMrM>oF6VvnkFq`K%bnwoYKsaDEjG-4()+TANI0Q$`G=}$xp-xfHiKxOfom9fZd ze>bWRI*9hVD57UWBhjB2~&VMKF7Q@I0GyDVJ5KTEj*)&qb0!%53DN zxvb${gOY#A=FCBRM|@~`BR-^{O+t<0h+ETAQiku~JD+25o5~TvRrC;hM0Vnn%Z_8E z+q#T)>v49+kPJ5}TZfwnlsFafaaP(tg_+^K9T^_zK)`e&s^hBA1+{eKGn{}1JKK=n zk;kawcQX30au$!+d25J!WqjGQ69gmER~gRn z#xJ(xT_byYXAl!1Z-5qBpAgIcoP+AHhP)=nvGG;SGQ?647=CClL?gK%;G;#_7$fF?U|$DEDzHDSkCz7pX*AJCz! z_lT8viInBG3Z46CylssL&wjTZ%PGfW{eu&5Ppg@rBQs&NWX`dJjgp*_hBFsGX_#GI ztHqwS3=lghei3Cp-@z)>j9`s5-O&CC@mTYpJkcHai-!)#5U+(M^d$P6MqaTan_Q8kBI?94C4G zMf0S@NYh_Qlw6|1dYE~#z(t#t#FUwwvVc)}RV->-8YcGaevFn)RTMn)^Ks^4N!(;H zN63zwa9}GpMv8B>VX&j{el6w~h-H&T<*SQDy_pCeE zbyg9!nwEsaMhQHMt<5F&a@ivP!zYo|m;n@YCrsp;X(bwJ8-xBEV}NqJGErQVsn8`f zz$XeNFs>{{wAUVe;|);b*)dp*{-WZR26V1#IePZ=FACo-bcfdJp$7fm;;55PMfW9V ziYq|}nmXAGubfeW<{eBI7kn~9C5gJI)M9|yJp7*N_YxQeRo9t-Bn!q%w^W?*JPQYU zU%{IKpW?v45bRf>kM89MvCj>pIj<5K?){ry_Es}wLuL}xJSGUc-jBw!?y@Mhy%|Xz zbi!ap3?8@_j8oN4;7tWB_+1&ssgr`Sh5k&O71fJIaq*(%Y2(T5NDmih25RRLR|=9w;rQQ#_RCEY@)?Mt)V}@%P^V4+-5NciYL> zImQMxtW0OjNBb27&5Z&4Mm#|Yxx$_vm zh=QLkWjD*LVXb$T zvhSVC*pTz41u}dW_jrarC#<1z8LBBJT|nxKL>yj#ACRr*~%Ko+W$mjguR2+0JR0o~=e9GqwwF zCuG>I__744D<4K@pB1C3Kf#RQg(0TNp~5cZs~GaUu#Jg17Rt0q)H54*9<@_c^%wR| z76tkCfald3;v@Bi*rMYBhWh?84>MrCXh zB?bof-v9^oTEVY_WK;x$Yu=@M1AUG=5$@Ddnx82SeTn2I;*{dCuwcN zFGla9WHoJ}=g~bNV9Q!CEqyO|l$Z_Lmy7U(rhCB~HB*oi9FA_7+(X5_Vj@>nACgrV zkC%+!j5m11;{Ab}@JE|@_(j}NoO$Fn3Q4_y#ucwZi>;i{v7QpNHmnE53Prr=iR1A` z6E#7{e=SVa_gul#6D7jaE9998Yy23spHmsR{2qbf5)syYd^|Y8xq@cVEV)(0*t*#3 zj9>h?gq026;&<~oLi+w-Z7%;H^ zJu^P;Gg_=u#w>GA2X)!iOwXkeyKj{X(G#~VxZLX?zB9oW*NXfN{;TVd_s|(8YGDvF z^U_k#p_Blo+@B82%FMxh)#tWs&O;Q}KLs~0T8Hn2I^e;MdAK#h0_VMqM%DF>s4;y5 zTKgv+u`9kY|3=?4ozicariJ^Nm1|z0Jq|3MrS5_siz(qviz`u5QN9qIv=DBXB`ff6 z=wfVjl|hyBG7vi56SSoof!$uoVBw~>;BR$2n0aiK=)Cp=dK*!N<;3KPTJtPo7Pbsu zpFAHcBwWIaF3XVtx`uoZHzD?xv#{(9O)UR44^JyOf=$%Ah5vqKf;Aq00i0R{281Wk zlQ-_TvEec<4lKv2)7E37HKBN6UjUBLipKZCm9V!>x1g*@3=D_J0rf#$pqSgsT##OZ zDz`1cZ4sHcL}owkn&FKdljQIkbQT@h@5yAR%7G%iLgrUxAJe8$4F-N60NKl2nO;>p zrXgwqIvGC}SFLo$(QXs)H3fCNd_)od*d~vEj($OFAC?NDK1>41&!d62Z6@H{PJ!OI zV6dd)8E~m<1zUfFffp+e3ex58;FM4iPtvy!pB`*szT4bF|1LX{i>`;rs>n35KoqK3ykMJsuES_BFxY6X>*;Zgewpp^cH6^u&^UvgD&V z(cYWDr0z=wEqOn&=gKog!@HmGw@XOtT4$nuTaLJXnMl5f*2~f@;{^%V(}ik%k<81} z70mb15N0585SiwiVU6n|=6m}X+OkQykW4rP%(*~r##9?{hXS5$gK9>SXHx$8Mi$rF1whl5GN<*~f zE}Fjaz2InDH?#NOSWsPWEuh^TChiC~AA5ph&Nt%I-z#x{)&~5?`4w8OD+%5xF9Pb< zpMb3NNM<2TCC^kX>8cC!XwV@Ay4JRhZ2VzPpHzg?P4PK2xHy-(Wrb3k@2jc*;0d}s zrG#=NrBs{Crga1BXz-n{WWX_-7>u@)mESGsmI^bfQ+1IzUQZ=bp+$_s+?Jo&@FTjIgy^~9L2{A9YtQSwTR1Ijmqe6h80T$XLL%L z(8e%4kDWk1{%1^-*{LKXP==(Yl;8)il5usF8%{bE3eKIV#$AWLkV|wGU9KEUy(S!@ z%ZcbZ_2X&7k#d@F`x5nR&7u1yg;42NbLlsuIC`?bf(G5lqn6J~DchbyFY}>vQ~i9J zC!tCG_Aa1rlBUu?=`ylwWev%EdY@$7(4e+m@5zZ@3&_b=7WmE8<&0EKw6J?sH#+>P z3km!dV#BW=>ii9x1^Wu`FlT0@F;k2B?NoxL7`e>bjHn$B;*BK1^etNj=th~{jQD@T zpn2DkpUhO;Ha!cs>`lOa|3dJb@wZWVt22<+9|Z6Jq=ON^aAy4TuQ=IaJ*j+hj7S8W zB;PU<$+>@t#N_LK5*V|IymZ%~54vJ#`N?zil*?5bU~_{?{Z~uB=GTj|3z!BprqefL zS!!@sidLBE(6LPnjghdWYk)J&ayUXg6w2xA@ptK!)mNy@a5k+SbDru1BvUbUZ)*E% zF?}=Lfu3~PKs$skv_`XyOc$3slcbNqU%4)D^L9S4eUOi5{y0L~%qz*=uyM3}q67W(*@dPg&!9(bm(T}x zrF16jqY`xw>C7iRR9mx^PV^|DiJ!Bm|CcpXHlGmbffnJf&nMA{RwTLmQq;S?l*-8YPQSU8c{P50UYLn4Jr!T9eACUFwp%0lqip3 zmniwzeezjCerjGJvGr4FxqKqc=j&*8^+npMdyaN`)=}GoEmU(u3+12H({R^Zx@Bn~ z9k?b#H?g*C(7_4Z?72T!dFdGTzk6BikG=X_kf+V?dYt@OrzA8W9kMl5#QlEnV|8`c`W|l1@dXJF+H+5igtBi z+S>Yv8p&Rxrjz#34-bu~O|w090g?2{_5wO&7EkXs8qu`29&lNIH&-D%%_SJFd2HHYcX1X4X_q2qB~&pI^ouoPLmxRrd6@TU`%i)q->i&XaZMOu*>NnNsQ$(plD?7-X= zY)efNVPm(^@tbGTmSqu)*^dw|r2PuF?blh()o3G^koJtVXq?RsHcD`9|55J9S9Rzn zZ2=$LGk_}AzqsIAncVuIFKn5H9XPRFpH^ak>8A&8X~CXOdN$-7EkC}S z=J1p0hL;jFRN*|ynKX%bl$GI)!3%-&rb1wO4quuRqJ|Hz8$=`V8EE$BKBl070jp&Y z6ZzmXqc~y-Hc6EV^Lnh98-K?GpO?B|-X9{+t#!p}qpeu6bPB<_EOGxXPwxE{{aBX{ zS)ZFh>|*B9$fGAHcOZi*A395`7KKsA^iXnZogrr?zm@ace3t#^bdAh7afqnQ{>!F_ zoOaE*f4EHJ3*6=$6|NeDV-L&6WP%Ng+(;s4uPp|V$p7s4S_T$w6o(6o6yPj@ES#7l z2GcsLxX;a5Y=4$3_1|7kU32bH#ghB<=FuxOvho5Qb3cYkHm#+1u`XrwuagXQchUDp z#K+5=0k)c(qSWyFjK#oGW@+?1roAZ+1jnug2j?aN$5$Dk*+UM*dA?+nc3);b=tK!* zG~CgQXIq7{yF}+L@!#=|7(zj7=LwoQM7D=!5Dyj@~?OpL-XVwz`Z{-7etnsrRs#-f!a6dcQpb4IaQT>X`M+_m#l;Eq8**l;x%S~dE?sPgf!YWa8efrAY# zvOh~>e4A+ZxE8t*c2n2DW-28tqBgq|Xu_#AIbrsD$XiiVoRNJFpr ziZEEZgL^g<%IWE!<5ncg!l#>+;5~yz?%sMmd~fLmIzPLaRxNj<{f8BBz`+q9hE}l) z7Im^)H+gdfL*rn`g&_FiX$qVhngTmBR>IJh*Icq#A1ha@LG}Np(f`Em)8~ETc+(lO zJXGkQU+&jXrRz!5QbUH$`tK+a8?M2{5-yCE>}$cI;n$4f;}qfSMe9&o#xurP;U}~G zlNnQBVhkR*b~7vOteEKu^^D`FJU|QH*sXu&CGw(OL&9GK|6nwT9Jh#=PvD8nr_I!P zUJ-pYgy_V)7P{+T58Y8bNIzTU&@Zk>*_H0Exc%d2!Gyo_pn-#b6c>NS9(rC% zfVl%{FyDJGTva&>{?!@bcGgehNL4E_xO0{2Of}@20xWrNNeO-een)?S2Ktn)pb7iF zk>|3$B-UOL$E0_FhVp#iG{F)b|003gtn|<|!v&0k|9p^j_bzkM_X?BHbVqO|Ap<>d z63sT792ghTI(X3~C8lM94Vc{1gnaK8lheji=-qiqG-B{FJ??p#-p#3?xV@5QC%&a# zUcL0yk$&v2^qSk&wj5ed+5w*=I>F;o63|5LJ@=axhv|iZq+m@u`B62SY@(ia4-OjOvMrf- zR!}82>L|q4GdJU+k~DOEyBvNJEu5;k{^?b-C*@v|TG3p5U)~eCYjXwdben{IyS8#?hQc_7=N;Isi>I## zMrrb*J#=WQGbiCV3I3~G18*+h0eARB!Siu3aBrtK#PQcSC2LbEyG()i>N4QR(7&|) zEKj8abLp|B4C?m#0A&YfQs0pwa>4QwKL276{VjTiNKq($6txaN?bF3GA5O<}RURP4 z_u|5_tThuPtYq%$UaM1}t%5CTD!|*x3S{^tgK{eyL8nhONuK6HW6T5S^|97esNG8D zWG4`>0FfVb+Dm%dY?S`^x{L1Cc*z#m^lwlNj&b3rJ(w*GK1_Z1v?l2MSFv%;Y|%; z7zAy?vp3b8mz9nkA%VTq4~I&eFmCp0w%c zTw;)-#A*Bp^HcjU$i?!-VQ*P}U~>y8Tn&5vrxqWFApiwWQ~gAQ=k z)AYGGn|DNaRWltD@h((k)c6F^`@MgzIJfxa0H=S!06wle06*@|foofmVW`D&_-@K$ zF67u{Hte+|T`VS`{`*DiCB1#Larh+FR5(W^a%1RG`3W@QNHLi!r$dGo&%<>!srWk- z;%X5y^1O&&z4Wgg7DmM|{a$uJ>qi5l`n-`jKOtN|SA535+8H8#!iU6^Mw27k%&21W zX&NzfjrypbqRTbINb{_IwsFvcyP?y~wkU*%IEqr#Yw2XVby6Vte&;JlUt%Y;%{Wxm~2gP?lvytO4ZuvII|7QvZwl#A^F_R4yizY$ee-k9w zMK$${=$TJvsqu@2^v$um!mNP=7@&1ttYL3;CQ}a|AcIaU*A8Kf+GY*+@dX!|0`B>!?)#%lZ`_;BMn|PJG^B?&*#M z4g|_Tw_tC$5e37Q&Hga;j$|OTb#UO~7_`9SU^I z!tHaCah3OZd{5+KiI|^31MNT1D+ZtFqhgl&$Gqa$gE5eGOoW2cd9Wa)gOg9)$Q>5d zaNfPL;O=XOawj4t!jvbwp!|nynA*||Zy$RCg&93?X>u9#`Zy2Ho)^N+joU{o*Fj>u z?g)GD!V_?%=P>yi>A)7=UdYXZk=#g19LIJmbL(_Q*u>S%+_FGh_~YqrXqRvV8jW#; zS6nV|Z(Q?nY5V{GFHh-q`{(rejytqV_au$qK83EbY9TY@kCOE>zTgYr;&JKjaP)n+ z6`g%w%dA}afvLT&4nDU;Ged9nz`FP}0I&6ehwmzYV|614*fq-h%TL5jeLu-OnH-wm z{+W(Ven#u_E|9y=^kK^NEckI_4lH}L3T_#C!1eY{=ZeaO?B%iJxSi8eIA3KA81sB3 z+~S=9gOgjJjM8h^8{7|nSrowImz3d#$l26y_e6fC{v)coz=8zp&*R|F2yW%Xv)qEx z0pzA!=Y2%rTr+EmcfO*RCl664VhY$_&?SA8wFn;dFytJ9LgoaZB%VH*vw&-xsA z@bwkIn?8VtTV>c`^D%6nnkhTiOo2UFILcV}N>SCZqI0WRJ#?nyRT}qJs_sFeCcKvv z2K%bQ;PI#Wu=2x9&WUj$?nxhsR(%TFdZwQHn=1qL78ycG+e6U&_eHqdy9>?_Auy|7 zO=f)IiW(a^{nd_KdY~#%6LTO; znHFc^t^gfQZ7C38<*g&0zFoH!vm`mVAkjv*l!*J6`Xf+)zP!~Q=p6=T9d~+57_fH zH3{hDm?`kTFfzB(6$<(c44n?x-djtmQnJ0$Ydmx3e?x?;3boUc>midaxYKBBYG4?^G~0p zcHbN5!|&~MOhp3?IlPcMJB71*mesIj{z;^{DvlO+G*XwQebhhnI@_&FIaB=yT>j)= z+$gYyGHaH?S@T`ti1j~C%Or}fuu0>WmNoF(wc`0N=FN1+{kz;I-;*##rUc$>On?>V z%;4<>OSxropNLm(FO9m9OWQXhcF0K)4op}KEnSS^fBnDNTN}SnjlP9E__b8z1E0mm zc#H8PpZC+(W`_t|a0@IhP+^UCHGq)BZedU8NmPm*(XQ6`y5{0xB++n4aC6UcrnXXy zp^7s2*xq9NK*UFBl~==3^F4|0V{w}6=t4L4B+%C=gT|dJpkx1tdf@BE@go<+`M*Dt zD0rzvN7gISDkrfg;ou?Q2R|) zi1S*LQ+b8cMSSgMJO1(WvE1*=0nozY2AsC~0UT+rhIaOr@Ynu*_`jPssGg<;KjpFl zKO@hS7N2hBmPT;!RG0}&3SP$PyOh%(t1S4bGZ{XiK$qWX*i9$@UPq(0^bkd*9=t)@ z6et#51XtpUfZp8wU{Go+_;6+>DDMrgTeP(X4Qfrok8u&&c2F5VQ7yr5#^m8j+vS24 zuXd8f4=w07w_P-s$)E$t3Dn)Om_E&v;^VrEc*%wA!z53)=#l10CQ%r8=>iEy-toV|ZtY3A~BLF&dCx&FRl~hU1yp(4vfT zQ5%1e{AG`+Bp%CO5%uZk8;HKs=e6nX*Ii`hT{qGeC;>F@b%PBvJ^-1TBOtmZpQ)7G zDb(+IgjTFuk1fW|#EUIsvDu$GeCl2v9#B1t-TCdf{@wxd_p&X0aXXHFX)2~VnzbU| z;%)k~MS(xHe=7gZQG@rh2Q(^R7dN$C$So6}#D!}VlI>4Ekd&E5+|&#)s5Dazx=;DX zC2Wu6mInl}=@kn}@8)=lJ7suaF@w)Mpu|T?%@VOS3b{`Y4#3R!?XboC9ZY>eq22uX zutPJOZCEeG|NAwK|GVQ6oiw%pEo8)DN4YB)wVMPQL)RkvPIEMCOA0e1Knlx= z*24z+(Kz;0DL%Ti1=j@bBG)afsHb}pJ@>ncnyu)djT(J)wexp+YK9V@?JCK)53A4< z<1shy@;EqpE{(goy${FdjHg;x`dICrXIxj41guSI=c+o)xj8e;NV4e6@>=1EEPgH?E8K>Y2>ASGdVGwq{u7KwSbD?~ABKPj%Y(ZHSg zoX+#f@Mqmpxa{t1_+jB=E_m=K5QOie^^;G~L*EY535(`X!8>Uh6<3&=#6GHmEAp+e>fVQPIg^JuT6XcjNxq>D0bO;*iV-tY|R zXmg^rDk-#WS0Ozaa+mj&#fF3g8@zf-1w+gGR+Cn z5K}`|lB9CyQV}=xSppZf}^>A%U0*X zt@J+hx&8`{2oSV6>J3egUgnMkh~mAk{$Stq5bki>WJoqRz`{s#nDn!c3y5FNaRsd? z*3XC5ye*+R*?)3IY+0oud+g2~(ER2J z6W^-^u3c~j1Kwp|Y*{UM7ZwG|g5EKI(?uRA&0LcBeHz{Ev!9yehti=zN1A64OM@)$ z)1dASdiLrbdL&eb>o4x-eAGoF#$9t*G2d+ZRkDKK^3kK8J*Hs20}<>@<7}iGc$M@n ze@rx)DVy?IvX6_t87K*?@^~8!#C5PFXCzT>_oamLGp69~MSYLi6i_N@hUz|50^sScgf;4zsc}zH)_;6p6+q@0W!AyWB&{BV?7TOy4IcLyJlGNp2vr1%iILI zT(l2)T0ca0_CKVz3x#yC;TSquMG`p1+i?@ZAPl~67UoGb!C?JXs2+a<+T>6eC5V7C zRupmtlNmZsE0Df^7sriD*#R3qorHfT&4wa!2YdhV6N=5X`K7^f{Q4^o=>pOGAwRT% z_8u*u0rID4w4^t!w3Mcjo5IK(ZxtXv;~Cmt@Eo<9J!IyKbpg+^Qs6It49IJ~U~X-> z%2;>D3&q0q(esmH*iwBQ(tje2%MEVgUs35~X{ShHusx~ygz$^meB%x|AAl+k z4#INx(o}py0DT-Lyd?TBGe;^X^SJA0S`rL%M(>Qqj8u4nRG-n>7 zd!i>&>s`OurED43+?vH*kUPnCBwEvoRoPU*eJhn-bs4boRb1-WR&pppB-5#!$-?8!D_}5j@ZDGo znCk7ujbA2BSN^o27vE&F@2cN(ufJ`ECx6sIgOu0MM_$bS`N$`z6@Lj%Pxpp7nr+;m z>^ZI|*9ImeCqvbeLg;Ph46jEtk@5U8e)+6qeqdV&|5a$pk1lydgGHSQac&D8oNY*d zSwA7S#r4S2;TcSAmn?xtEXaKcGgAFj#E5f~B_&_RkRj7%yfix%OFDXD%a>+2G${{T z{=SD37qnn0X-|wCWzikwb)q?-5iTV$b*>sIKfoenM1GobhN9WZsb6kK~)i`!V{3c`H!xIYVJ zp~Qkj*b(po(qoGDKGQVqYZuDe|HF6SBbP8}KTR1L$-Lpt9i9!<93r8|#U1d&;|9)3 zqK;bHEa2ZM2JweatmaR7Oyc{MuhTIO8Pv5jl8y?zslkM?G`C8FEIa*{VQnAcX^16q zEhP^ol@+FO=e*jFU0 z+yAb+2c4H4fuq_o@P^q{E>@JxSF4{0|76#5*iDJ7Qk39_^8w$`Fr6P%Q|Fte_EUaZ z7R{{}p(@zK|F3U$`mIs^S;dcc7m0R(Z%(+N(f|3QM@>>s%OBV-B_NHFKFa z7qLgYx6!s4-gJd-0cUEJ1kJieVZYZT`}V!Y_G@*<+t1dy1S?`$I8af=-CXO%*~V<+ z)LL}7E)mYQcU&p`Z8)A^l|PYBT{o7O`g@I@R*axdC-+hFZIQIWU=2N0Hi4#@_LC3Y zXUWF*N6FvGM#NK_;s?u4;c1ZpIBsY<&eZkC5ekd(``mBHrY{)#9+YO7w!h4MFLSW4 zL;w2|A{8-8)0`A+A^io8is zDykgDgnojym;S;dr+Z-aPy~Fo>IIkibsG0N@CwNNI*KjNJtmhVBIz54dz4yA@M1l` z>EFO^dOstLzO|V~CtMsucX$hEysjm!EE^)IsERCjJ)2nV;_(>!2<#S41e0g~5GLJL z!?LPd(YrM@%wo41Mpbeet{vlxgC^GqJ?)&p^rA>`&37SKJ-Q4mxy^!+&(9g}F;dKf zh#n#NFNmj9Diq4X!S&fGezi&E+1Dg{woy!tmg`oTvLLu7JNp^y?d_kLF`) zVNp%VI#pV5I38~k7xl7J$MbdFuDqJgLjGcb1g|K*k@WS9a3_~;f;DEx;ORR0QnLo_n6{k`bctq7=^b=D|9}oj+^6L+S+vsKRFtWTB#-+r zarxX%J{y!1rRm4Wfca1C>+uThG3iD|kIcclNqa%qo^EED?=xoH-{*{de~a*L;#}Or zZNj?8rl1iQE1>o{8f-GO0@=N%nZ(#!v_$O~7I53~rEAG}+u&7vD4`ltmqFwwJj$xg zHs%Up0H^x)B)8#WI=4T>jnltbiq*AC=#@cbeqiNveqpQ?pQNY4Kd$emdyGF&G-(?D zGj9|BnP5jIk1M_JhFxiEA1iqFFObqDo!{1dDB3fBlNLw7pRYz)Zj^iaPb!02nOFe@7XS8FdR1NZ>QH)$!Gy$a>*MN6=6WM7|Wgy(c zj__;E=v~tqI*}X4pE}9$meGs(1`8)%OUjo28e`1|YB=+uf7bF;$B(abaplW5L4K{m zRNm6doZqlOzz@zc;*W3nLjx3l(;NdK6<$fE=l*8WqMZr!X6*)gaGXDll#Ha8*X^L~ zn(=htNe=yVehWnfP2|h?Dy*<(1iT*o1J2KR4jkZlP^#tyPO!twx0ooPB()6$H5mf5 zB0!+=XCCtVXf4pu{4UsjM^qfWtIvEA&B#5hUoeb&HY(i|jE9=iJZWFmH@5y1ayJX8-}UQZ(cn-r4Y8;VeI}kK?0_UHMPFZhU2l3-5V-5x+dmhL0Z}%Uj<1 zLtkyB^tJF3T_$mj4$7UOrt8+xJBMvW!PEtGLyj%|@Wq;jJzq^nBAw{df?LF=eLCT* ztWiyWBG75P0cL%01TVxon7h8(jDe{J=oRgA+B^n@%iqai2A@L4CSAy^^c}J?Gs2;t z6PT2D@j$k~7D#S7g{pRI;KF4DE3d7^lQqlmp+k;%y$g$_a)VI$irLtw3gFGfPw~>C zaPp=`oSu82M>pHG5j1^-#BEtk#T0AkvoDh&YX9U6!6y};7{oq@S=Sloo?Mh?@YZzU#ecA<_S&IdUF9?c5ff8R`#T; z)OyHEbq_LcpqI2ekEdg@zmwceVl*`RIXS!bELr!{n8*(-$9$hMuzS|R#Kd|5_ii09 zHNAn^E3E=nCT21BYpx&%wg5c{G{K)oO|Val$eDh~Nzf3U!w7_9LD7wB=IoKh%)LcR zux#BSyvA}iK4E`dC?DTq7q#yjlT$n_ymA{e`?h^V*L)O7YhxnO5pn$2%ViR`)(Ub6 zjG-~jYv{Jgg*19qHEsHxM%7)zDbEH{8;j#~b5$`NwY^36)ISojnch-^k8*r>$FI%*)khPoGrQLU2~L^;Vy>h~gwo?Wq;&N;oE-Z~pk_xT^7 z*9FdW--D^NB4iBRV0(_7I+9M*q_>cZQ%0~Eo&frX#sDjwQju5H2fufJi#8@zY#x}vnk*{b~wDU}b{M%@vAiE75lLsLlZm)BUm^c1%9Uw{XVv~gzQW$@Rn z40zpF25uhnnNl~sOV6z1m;>`%1x*24O#UKpsw0T;O<63o z+osKAM$ShyqHb4;#tdxa+lSn(M(gf)8G(OMQ@~T@Oy<_E3iSO>KDzbnwcxEF7=;}+ z#(PKX@SR$5TofyZyXH?Pr|#*K^L@o= z?!k{h-oKFbPpoJEi^gnC!XcLA`Ge(3&#=#|UBvarOAxm^m+dT5WDi<-f#m%a0#)13 z=uufWp3-m(Yr;#&sZM=Dz=}h31*G$5<$By^Mu?0 zfOdDZpm4iB^wdcQe;CxjHSs&~zK0$PSm3GI-m~e`wZV zHahU%XtgU~UUfGE=40gc{+2I7IFa{;_Bw))l*Dw|tW^2-qsmo->nU zcX`xh-iUEHA5WwGX+~ptioA~^zdWDd|^q54%pZj1b8_eX0_Z_ zv>~iR@XPs)K<#Lp=$WUYh5vMLcIGxbtym5BbWcIrT`j2K^mF8!B#RqDr(@r(qR#R@ z2Dg}4;0@>{D!OzL-L6o?_rhl3utG1)>CF(izn`N;quO;UlLi?{vDeJZ{v<{-{D;6q zqJ&AQDi`MYNW&zO@jWjL{@t zs;jY;vnhT&V-vbN+R8{wu*Xm4nvk|6GctKm4ITtKNLne0$(!!POkMDaxpVvoc(ZW^ zkbSgA=#-OkH%1;G_!j1QBD(mds`bVUrVq#~Z!lnT6& z*5fgS-njO<19R;G0n+OSf$g(3;G-r3G%u*)_5JnOb#e~&Zr_W=+7{yN8*T98=1VxP z#DK_b!}zIW8Qx+%8~>b?BiOm|J=&Md2rBy*fGb2 zbzOz!ik{Z(bZe5NN@=qkFF6aE;(LTDp!C?1gdyX=mi9yi`xcRjzO#IY#D#% z15+|)GPrk74`?d;*L~dJjMq2V;*Do!;4S@=u|e)J91~T8hYzOW;nKrceykE8aVJ4O zz5q^~8OwrGyFv1^OuJ2bS;&50y0GlOEqI^XFxEUEM!FxVkZohn;W^)&vH!pr{NvU< zoVKld(scAUIJptdo<$%8(RK?D{my|c!k3z2T%Di{JOk8Rn;Ym)4HrF1a5Z7l$KRcKn0Bze5_B{pKd zBHwaDJjLA}2cRQ3DkB^R97cHd>1@3EpE};zyca(+-&41&Mw}UJe~9iiNHeu>S2G5K z_mJ5B8r=S|1TUhxn2gOvuQI=))lT2g+H2cUaLZkEs8SAT^kkt;ao5nyi^^ED+6P}u z0Qhoo9aA#bm=P1M5}ve}A~as)XZxos6`fxbiUJ4QQR@d5FEN>fcZ`{VMR*>CnBDy3o1tWFK?sLSJBANlt;i1J zt_si7Jb@2>mcxrzbThEk8VqpvnQdJk1;t1E1wDu`W8D5@lCJ(@_D+{&zMVf~S8_)N zH6`#u9cO*q_jv{$Uag7mFEYhWdv^+JZ}E23FXE74juysFp4j7!2`U&=WfnT8FuI>3 z?W|wg;V~V@@f`PVd~V7&d_3qBb{$!Q`|il#&5soDZ2Dce{FScY%9eKY&&?b&w_Wg# z`UK|iMP1;#^f(xZ7|-gXTa3HSTzu$G7CzshOcWl;kXVlttouj}zZ2cdgW6{Vwb&Ld zh?|FFQxx$f%>v=xJwoR11Sjy65;FaBNrryUEkXjZzkYPae2191tWGQ5CHy<7NmWfmhlhMiK33#^V4y=1x8y~ph zfkyUC6P^mW$mB?t);)8S#-a=fK6g4D2QL~yhUdgU=RXd_zHLU^E6VX)wIp2I6Nq=@ zTVVY+(s=MOMvoUpqEwrBR1+18r+gE2cegZPsdEdlGMNv0q;)}rbR5{TN{V%PdmKE! zJ;hFn5&2ikBnh-MBO7{qFpe$7g`XfczAVYCxm<`+2Ar{M%6MG7&XYN)CIPH&%m6aJ zNg&iZ7MzUDVNQ!2FCqyUc0Vh?IRy?vd6kz;=}u+vaLsgJeSHG(4`V_8W*cxKSs&PF znISZ-1XWns;ZPB`Bs5hMzdQX7sj9Jf^wkplb*?w+-PciPHmQR-_TxwGpsXyqH2MqW zSn_!ISpnV^Q--!2(FDB8O=i1jZ~b3~B)*!q3q>!L#e>45xOIUyo*mLHOdfYtcuwgo zmd%U7^Oi*5>06cYn7l2(FgyxahM7c&KbCGMG^^G7mR!9HP%+MV&&!>J;l z&osQ!s|k6HD4?w5Ug5O+ZgvZ0ZZc?-B-q5t0$=T2%)yOqDD#j4PWbDEvo$wiwYnuD z2j(Q~_p%Qe22aQ9D<)yd8?xws4V{NSRc{=}k-ay`h!UkK?m72)j1m&jl$n%L$w-r; ztg?xskU~)^O1RHBsibA4{1Qb|r6Cm!?ceza?(22$>zs4H=lOm<@Avm5DNNk-07m-K zI(GZ74o2l%3ZAmF5r^mn;Xauo%#*Ba_D|?^_5sgKuy+kWPGd)rgkK(hG#%sQ?l&kd z`#CdqYd0!wFvRNz7?hD(BKkbeizSu4tnqLMYnU0xHi)lg$0UDXb<4V0zwBa^WzvgR zPPmEf1F~_%*+lCwU2WE#dBQkt@?w47d}V^al!?~aT|}j0Rx!eQ9X8)FJAp}F-3j;D(3m-(WC8Yvu9B*bESOmL%1 z0v3hD;uD*<<4r$h@yIa;?E5;2Kj&ibv3@Ch7-@@k-l$>wKV`E6cgEo#caCF&ze;%i zwg4v7C!U>KyNoT_yAF-q2*b~P&f)T?Gq~Ec6MH?LgR^ejW@eFljKvIH+^O&x{WxvP zs?DonFYS5Ccj>RQLsOnJVXttVNmCUY@7}?_Pl{n%k8*YOOY3o>bOE0EB^|wY@L<-@ zuwnl#xz6f)JF}T}vWzHI7H^Yx#daZk(8_2=BsIs`UXkHhT6&lT*UYx3;9z;x8L zEDX0@bim`k)}qDl#jx)J7yM5?A1@3zh&N7k#NH>pa8lrF^pJLnihpq^REu|I*L*|! zhwh_7)vGAdhalzmuh5@=3fS`bbS(EU6o-W^#_838n0d4xcin2lzwuG*wQ(kvKCjL^ z7o@RoThFn7pM7UkxAXn6pQh}dzwOMY{?m+pm^v=qDN9V>>l3TT4&;;F3bHOhjhIA0 z#!OZU?@q`d2YdC1vF2em-TESXFzSscPAY&b?yn#gx3ftMTTBe}!pP36S1}{Hj6zrH z;A2uUWZ{w|GQJ3sO&h~;(;+=J;${geJgC5a+>e;h8TXN6JpW%e`UhF4N@6!W4!^t; zjq69ZU>|P**1o?Qe~y#Eb<*dMP|F(k{oBcN6=vf~H)A|i=PM#zg=i_yM|d!#hb5Oz z#PR%GnEJyMKiK*k9a7rL&tI3Z7dMP+wtvK8sWEs{t_joEa)TY~wV2hiz9eY(=Fb&Q z`LpT?Qdq_05`HnBC0+tdloVy?%igb~yu5}SQv8SCDb6JJ4R?t=lSX7Ne#Mnhms^}WdhObYa|z4Pm!bi+55D&0VwxW z9{bDd5_|MUBI~U^2W_9g^L7@SA?<}JX#ec(2wRxqQ!@6v!(4!mKQhPJ>wlw}x#D=y zKN~!d&pV)v;_&_Bd+^9c8T_Or4jH!v)Vl9aLxwJrxN*B3UhzdASCmQO1p+s0mQaUR zzR)FCChL;q9U5fIy6adaK#OTv>B5>>IGZpj1!XT7QHAJr^ny(?4J*l^k_G%O zH)$um4|S5Tj&~%+vzaW@*-qM?rQ#pW+t}=~X7-;;33E!z23@*f%hVWD zS?jLLWjwlvn86kmWccwH$8275B74d4Jd-mL#Yk`a zj>4j5;=xbe_Yz3w9rQt4D}B7Lo>mWEp{kSK(WNS~+&C9$Zf^#kTbYf_J|V*)noZw=)#r4 z)&H{D-LIyzcJrgyov9t{@x*IFh4F2vqGchOGe zef3sDq3BQYW%Tm18lHIG2CJp4K?_fxXKLSWK!4HBHDlE8eNtAA3c2O88vzEkgEHi zrvaLLHlF_pQNSHrgit~-|Sx?3a{l*n(jp0fwOUn^I7bFR}NQnI+Kk% zHuGq~ z)UwDxsPDHHL{6^)!itNaP%aB>a8(9h$65$ejpwkwG6U=uJ1OB5%Y4=@VkWym_YRs= zd>&7kRf4Cz^u;j9A`y7Ok-(S`dh`(DbLM~Oi>b1lZV}Jo zZtA8T##gCNFu$vgI7--?=D;JQ5qQ0R4t!jaf%L~a#B*4I%l2QzSq|;sB0>W=uu+2( z<2~CWgEK z>RHhs^;xd)yo42-@%A`tc$8z`uiekijTRTUHEqO^Z*#HmjVlh(w?v7!ZTRnyIcX&htyku#CH6*WU_33T@t8|5;IG1v0 z45v8eF;zR}PGi(nh2J#RfR_9(fN0Est9f+ok`+2IzeEJ?uu?!nXAR4JlE*Pyl+j|I zV_={4QRKMH9q$~tha5*#ML|y|Fw2hJuT6~@6_wr+3BDKBi1b7`q76-TOu20_GiP-Ya$9)UbFCba{Mx^MsE#8`KgC@9dL!Ux_h^&5d z%=nQXOr~EJvur9xuG=HY3k8r8ZvGf_*0~Mpw2Z~_;Iq5J30A(LT-Fy7dbCI1U7+5aH4b_==!^uZf=z3Qj zviDw1#|a9DxSgkb0{OQn#3h?bf85$n(4CK>Dw;k}kLTO&haW?!4_PZpyK_Tt0V% zjyEBou4o-RKD-a6x{iaBveU@3*&VdMA(?s}@)GXr+XH$?0%)yv048B>;O2sS@UHG7 z=vH0|ON%q%x{+*{mhBJqqdb7)a5r^5ZO+}erO7Qe%b|bc9x)%XW`SncC~zt7EPw}j zmPD!n1V(x=Q&JpWjyViU7pDpz?Q|qxTJnhBvo&Pjq))hXS0O$fdloAh=wXvp2?+kY zEAn5IhwLu3A^tK#6>Bjg)wCA*M|v@C8^^K_8hqKvM=KdOjY?E@8#578-TBM$sUU^# zz@{CTO0#ZWpeDr%oLaSjJ2c|UUC(#sj)z>NyRQ_2e+|KqsGo%zmi5r@XAqPf6AxPK z|6{_O1wi@Y7od`?3Y9PQfFFnR!B?3Xz&op4DE?|GI5cVm*`?92X2X7XUwbZ`G;k0H z{t$4DcG28+;~?%{qY1aOrJW8QiKQ>+SW~S>9Yo{jG~tgx{x@WzfQfMsNc9^7TzLMW z6?ugX;u6Rz#}(v7!GBo2PKc$JjOTNnrh;6x3`Sr58N=TCEO-r$B1i2H^<8$(C|Qml zG*Y*s*UhqQa;70>`S!rnP`P(}O>G_$+{ThkI?mG)TZAn_Lrn3+S#gMM(}?*=%x9m2Ig zMu87r2WI?dD(sDJ5lY-|2amtcfWxDU;fq8Sn6EyTUjMw56UiiTvw}Ri@VzG7=PoHO z^};X(O=4VO-Us?SzLM&XE~e?1O{iU29#MPdOk72oo)a$l1*!rqiB__AHCydK|zcyeVF};lo?DQ+G-2%k{>DP+Ls^{EmX#hZYD_9WG7QH za7yrR#Tm3h>?rC4OYqKIcO2|&&d>QZ%)X5U?2pP%3>j_1%AsoH@$hkK_=m!A0MHBRnLx9!A}`4o__EhigN+VE42tSTk-5bOk!FA+jIzocRZ8 z&Kkgb3jw@eVF?vxm_nzm7VxLBGK>y83et4P0!|}GsIe7-6!`-1<+v-@UO6Oue7OQB zuF!_gk7S`kvNd=Y8^|-o`svE2GMsv$EN6L4f}8L4f^J+_MJJY((a6j+{`*JJ%a_6^ zdNh_km>*8Auf2@rquyW*i9#MV=!&JYcA$@+dCqCCG0z9r!Cili@y1aV++aKjS#3Ip z623Ky);p)NCp1S{yYp47enu$kXCuZxM>@&p(omYdTax?M=f%z2vXm$&%u0cSj3Rw^1<|u5 z-Si>%j((pZ&T%(;Xqnsx56UfL^!-0vO2^o7%e z-w#w^%l1#i_{s)4wCFh%nVEC-rsFxgcV2YwGA;OV(rNha`AsNUb(!z^oq}yR5q_vK zgQmWTpmFA2GIA+}p1SNneVV*+=uT^p)q4r-di(;2JnDensSCm%XIhEDqi@!Y@`h)OE&qSoaOAv zhz3@r?-2WYqQHO?KlGM}>GpskFI!ksZU*O^90mb0-@ycn^B`_? zEZEU6FWmL83 zM8nUx5wOvZNOeOuo$y4T>(O4sEpgw?ITpon#>@PHGXrdc1ZFmpbCj1+2;DibB)5 z53NFOt;-cM*+vG|4Do!Pw=Ap)c?X_+{13F>-47aT#lg2@<3Qu*Fd$~70)jYy(!IEu z-2OL%O*Whjq?tPa@hK+NNC+KH9)Y*vedx*ezodovu>Phtd^2$jnDl!BIc)QeRHRkY zi#Jp`tN*5Q@7~Sf;`Qvgm?C|yq-&H`3fgE9-%Y;Uo=xxC9-{|5kI<@_AGJza1d>G@m`X_h?|+*8gKY>w1NQSc?2v*a<-OdEq0M7!{hDF?9O zw-Fq>DvT_DB1Pxgt)=3}kI_mCaV|DtIrlX_pX=%^;95PWaZ79i!0iRGFz-ht>^V^Y z|6K5gL7%l?yh=HUikSfLK5MX~Z8os{vw&F6JxV?E8|V=4#O`Uljz4bZ-QLe9!bMl+ z!=^(A;gD1l?6JHCM?CW2=GSw17v5iRQA`RdEc^}px9fqrVsq+iTtg$asd3-0Oyy>I zkLOahYjDP|Rk`MulANfmgKo{gL=_jDrgF!M>0nVj)w!raHU1qT(OQ4-A--SGzsGUuDkCC#zM81DG!c>fSUUP@DqU>5 zmL|!+q~eV(+#275od3~-+>`5{`T1i!Tw2}&=ZA~iB=>%UD;fzLe=7v0pOuAf?%81M z23PQQd71EG^lZw+T%)J94bxlC&r#(!r-i+rRG|K_E!?`7=Netw4f|f@!@RRc;8S5d zwEyJ^#Ukg!)AdW?Y?qb|H#+;MvWw^wknOtn0J{Mjo z#`#A4N9&wR=tp>#D$AD8^dLu`Rr8RXskEq9Ad)`&1)6a}{t7mg(HG<`gb* zS}A?lvIe^JeusTJhBnU}b!`rLzk(N{3!trzGyEue1x7WFfGer_!d+h$QHS~ps@Qyv zK5%$I9`mkZ18)fL#Vmy;9@em7$|PuYX*z6@1Mrs9bSOhiY}S8kM4IkO;4XLqejxf z^z@5=gQ?JV!9gb%kGB{>^c_)&1)P!A}3-F)4y!Y_QOjJ%}@sU*x zSUR2W+1V@6)T$nmYX6f=nRJkLgc@?$PBGm09eX(Ax5wy{bH4Ca=@8tgtZ%cvN6+T| z?qLXC*TbmlWSFkx3T3~Hg9T#8!HgC9!eBRlY*c$n*ie-Z<|vs$Z-+QIYhw!hb7&!4 zI8Fji8@LKW@6~|sRcF8*vu0qL{u{`zRe%SMje z77Cl_`&B(u>*Zmpu+Noh?USO1eYL1qN)!$EPNG-h!sx|V6Ph7=mK>d^LEJ8cqGta| z%+caf=4iq*RH2uSfm0QJ!ou!O7`F<+DMcqCH9ZU$Ox+6= zoz38eDiLV8c^I5YeF=_#9uFtpSPQL$hv6^VY$$8bz=526(ADiGOtYvZy0-=N%3NI< z8I_C;@67-=p16WTg@eKwJ6{NE^9F>DL$<)h{3w|7C>=~ssS_ShR2IIFWdLc92ltFz z!TYKAg_C;`N?CV;_;yRs{nsUFx#neZy492HGMkBaNFHNE_g^ppkw)mmP&q0%VS@kh zJ0tO`^Qe1&C37|FHd7!gMtu2qsL{js$f;o^Dixwnl^hSy9XAX(`?AfPLBw|MXQBev zzXgKPUN1N^y$IUa6vKZu>F{Q1HazmI7{*wY!ZmiuQ2YZ6x9|?v6S zsv9ap)3>w<$L4Y1rTZ5k<OA}nzX1;m(~RwrSk*yxOSPfoM!%JP9;N!I}?8kXCEI1+ttkArL0e2?|)Z7 zhWh2fJ6-f{XR@ z>7vK-+`nZjxPwb}aS=28Ii^XCJL9>B>TR{88yqs|oY*nkqy=J}Qj89L)sYJ(o!}Xh zbR5hTEP-TdB0OM~3RlF0!tr&};JLWFK;n@zaNp}7RM+z&dJ^fxEpY}ZJ$M}{wy5$s z%)gBKf=l-PUSLMFgBNZ zb$_GU)$=%?Z)-TIVs$RT*_Ir=T@6C_lmmAQOXBxLjZS{T0Idd1__Aa^e0F3Uv}a>L zP0D<-_LT-L+TcPePl8117_1hpiRp-KXZ`sW5mUCeW{=}oNsvD4m+?9Ie zv8l|3UwOy9^<@Klpqs{5MQ zdaML%(OY5B^8{M^tCor)1!Q5`1F-bZ6xgZX2d>TZCoH^33wF2AjbAcpLDf1Ukc$SD z=bwRn3-dsir4ey?)P2ra51#*5)E4iXfan4ashKl~=5zCNuv})aNYE|jVwXB)V zb%$)C5fb@e<7OSWQFjSElbHcSKUTrb9vrMrJqr(wONKvBjfYX0Gr-tZZ=zvsMPmH! zq4}$S32fJgqHVYOn1L=y7JoRvq!nyL>Pp5;?C^Qf(T3NIdb&B=dochrPh81Ebc}2* zdcgO3UXu#fyTn~7h=wivK-cTqaKU6b*T=gG3vGAN)6Ggku{Msl?56ba^T+hjp(&JZ zKMUA!O?bLH6TBXZr@hDi(nphD)A0Mb^lw87&M3PFZd@4;6|?HVZiQe{9(A5B7?I(g zDVlPcQp+e_);hkt~3$Z_H9#}nzz^DpU9*BRX09u-cZ8k4>2TR@}pRH&!28peia zL09)$2(qq1nblP=);|%3J4wS+?Hh$tng_|xBjF@a(~?d#+xhkp>{g~7To72bRa;URo3BCRE01?q!AoHIBjI7iFVY{x-_TIl# zsC10V6vW|YgO|Y<6+bBLyBDrB91q{E@&?*3>acvFJ^ih}oF+*BWZ&0{z~D@I_&%-` z{92wPtgOnR_IJP0^8*F6VUr(7J;uKyt@nU$rf!50)AmD|l@-wYXC3@T${;zu687z` z05)TMh;6DKUEYyI)5L&eDINL5@ex&6dBD~MAyvdq55B^a2|<(YkzFS1z&8W6MLu8 zy3uaxVcm8W=$>UZyCVPz2UFnb7;vNpvgE%WO?O05V+pK6^$!WG9qD zm&Ooy;i?YYcIGjt*xUfdEs%g8Z@R$=ycc?NRy=(C*AWgH)`OUkXXN_dEdCy>WD<;} zp`(i{EPd_)-x+$sGZ_)^#g0sP=j}nb&vh>xbW(<3#(v?`40RgYEklz#y~ypmpO_yC z!>mJDIlEgv$+~O#Qv@1E>pgexXZ-fvWNJLL*(^0FMrM&58@lZd-=7>}1Mcz8w(i3y ze$ipPVplWSe!!istFNMO^W-?wW#hP;eM(&RettKd8BflPH3ehWmjl(ZA+YP-T*w%Q zL(jE-@YXy@*ki2@y0&<+`z!AVyJdL)_3~xVtuqg9+mw5J}S4~asMC-M9hiMzAQ*=Y`c*iA=s*h_pC*!cBj zzny-ZC@cwSyTosJaXX{o{uqc zM+I0gRv%wf0`zRSIGHpi3)EHjfNL{GfEf^jhabqp`nQsB_%a0>tv(73Uwt9!dsfrO zrWSNPdz=`Kj3Lvq1UOURNz=r-QV8$mdL+kfy@%+a@X2nM*_UcFkdp!R# zqjk^>{WsYVDI5!D+F4O8#8F!A@|zy>s-v>nv+0yA z{@|>gI&2*Bgd3Ch!-)B5P{%w74)|!o&F}(v`zaBKy}kl|4ywYF@7*8`PK2e(iST2A zCv0nUgKH}spu~-N(B_35^#5fIHBUikYh?~^eH#InXDmLs3qoG zw+PcFXn;m-OVB4H;Q669g=Zd~AjM1uoqvUAqSLKZZ`)t;STc|#>u2KIK~W-MhXp(M z^cE`}f0}*NwVzp;Zh*Tz)Y09OU5w%8F!tFUSN2R=6Z312Ih*V9ikVQr=Y#nUS-z_o z3F5sQ9YOtM(h)=Y)^$I%OMOE@xGcA0(I471aE+Spzd&Xl+zNsXCE=>+mT+>-TzK-u zL}X?USY1)3Ey#H?PQ#JXH9W3!EP*hg7Y@Su-B zI$h~609Q^hW$Pu`>djNwf6+ng@Nrc(1fFJ$>|2@6@+|DQ(}dKX3nqacH;7M303Dpw zM29qFxzRKeuE$QDJNf4@qxT>7SKYw?CD}4~Cp7sG#*BuAP2ZllT zKMQzer3bVR+6CR%bhzztFg#rU6Krk0#JZIyQK#Ssl+AlfQ`8SrkFLuEIoOe97OkWr zG?_lAd`qXfX>q!XH8^d}PxP46TRL%fCw&`yj~-gUGsu%L9TQ(cbt*PdG1*7tu%rrE z?;VD1zfQu-0z%Q2^A`niv+h}K+cLoX9<*hR^#%l`n~zw}$vTNXJY9rptdd1(^pBvn zu$jqMo5zZuy2l)tQ^I_@t%v8-tiF8ZO-)XnP>GwY!>63AIm)0 z#QUA9beSc*vuD;07xo7?Bsi*DhSb#C&@kWqn0iB# z^mHDkS$%tH(QgY{UZ+6GeBQ(UCkc$yOys|}FC>a{;HEXtz?z2zq{l*sQ~GAbneY5S zU8hW-Cr-8rOP{&{hu*p1Z_h;GrT=`X{hicWuuU_M1oXPyqE)~^xMWS+zp z3%Jf#C_6R+Pr5J}7kMf$?AGVZs=*w^(`{#-9@zo^LVU8urMSUN(j^}kT1Kou_f zjSRQNhL))Fjv+Y#?toQd+w zrP;@F#_WlnX2Fo6EZ(=*3tua{fS1R=#HeL5(NMv7^Qb&IGvO!ksGm!R?gi1kqHwyS zcNvWW>!~VJOkXeWqQCnUxsiTtuAxhfv(}%)Rah_P#?%LJCT6oZrKk%u7+xc7)lwvR zohdC-dq>kEXK;=Jb1pP8k6JDKPpJ2x2>9{BK%=WNz#;TLsj!cs%HPjY@w1d>Y^bNP zmojOT!35eWH10;k2R{ua+$%#kbUG`eGqnHFY^1_VuGp5nVxz=;OmU(w z6xxJec7}lNWC*01N6_pu=JZWtJng$sN>#f~)2zc8^k@4NdO@L(cy1Hm-auywPEXU0*2<2URBv3Bv z6TOk%gbXWph+-Z1v`E5+;mM znhPWL_Y5N=HePf-Gy(;QCZKk)Q)uO69n`n3UnG6^8+x?M2qo)lF%dz-%u2OH##DVP zWBB`+h*(U<0mD9cX1Oc=k|@Wg{|b>tf+0>_m55zJb;z41H^|$ss`Tva@iZpafVz3l zqONT&bg`i$ox7Od`Ry{L59=4v9Z`8y1`)c)dx&S3zNSv9glD=Oqgq@NeLc`hL#kys z;vml53aY0~yT6m{C5MGO1Ev7Sd2fVz1qX!}#76OmA|E{hHueyH`Uyq-GZ!Y=|t%~YLvoEZ``%gy{-gzC#oi0VwO!`I3?yjo;FSn6-c1D#g9G}2A`FMnybk~usv@7^9ooC!m6&-HS#aX*3cUqFt>(B7! z>BodS`wj}Ptlll05`KWa_n!pGd9|AKMre?eC%&LL)<0rvRzbs^SF zXvPcT4v@mW>qLBa2@%UIB=vHUWW(0uxIoFC=MAr5&q?gZHd59^$|#JK$@Y;3G?7lp zkESc0o}{+>(&+qiVf=oS&wRHO(|&0;svxgOjdB}FvWqtc8y~YfvnL7HllN@jiCR`! z?-6o5Xv|v(TaZcbek|=~f{$_suvK;!{&uGfkIGcys*z|Mc0U-GzKp}~hqht&X*ZGj z>M|y(sgcPBf~fKv zc2nySHp*wR@b^qRcGbyx{Pm6-o>^i+l1(v*&S)oFV^!#Z_Y3L!1@=_A+nCPOn@y#t z19hFJM(2#{B)MM`NZXMIIG%UcoLtXmeZr>**C*)+wO%wZlkZQ&&J|&ZGjl{AW~t-5 z+q1C)-*fpcQ-JZ66F5Jq9J4RZ;bkK{M`1|}HdhzN)iRBOSv7SA6LrN3s zdGoOT!HX^Iho}G8y(d(KPp9un+;`8=6b zIRaSWrwQ!Vqkvr+lg_HN?Pp&sC}!-Fl+l{+2Do>|aXk6_9o+NlHJ0U_wmk;H_*T|O zG+np`^VF{p@t%5$P2|85tWPK3vA!?@=bxUVO$*6LYYS zPQ2hj@oaXFo-N9F@CBc5=)rr8q;M+1c*e%DWM|P>d`3!|opsC#J9$1w9v7yt_s*5F zD@yX%j%S3qUg{ubItbAHEf?8bKY+p^9+liA!^6ItQE$?UsEUsi6C z9@}%~5o0vRfGHjH#tk=0@$t@lY#hnw31`;ej*p4>@vdIJA3LQ!_N`A{{mgFkVYeE7 z&3nNOzh$%EM*7*|l?KA)7iS4g!_3%qD$Ph^?>kYe#$o)u^`_u?YmhKmYl(2*U?*Dj zY7#y-dk5>iOONr}OtF4X4`MFNVm;+z@!0q}yhbR6hohe{+EFpA_izcbaj`4fcP0*- zXWzzcr?T*>)LmG4AB#IxBk(WxaBP(^nSWO0qJh>>rZwv)Gk44f1|*p>c7FTN=o<Bj38&C&?KTw9 z8;-`z7oqz`S5eciA+)*XFB%Mfg_N|Oih_05Gp~H+uxhsqQHY=c-z9(9rYl zK0c3kCf|F)MF+F&C7C%$E6zXh!uhbmV0PIyHM3iAe<`r!88@+G!fP z-1krLXk)N-Km#iRt;W{n75?lYmu}|yi&OY=UJ~{$RmMdzMU3%+H1^7re{5}418Y?y zkGdWipuR4MZ`j8(mPfy{eu5)x-4jjpZIK^(Ecb{>{1Sprx>sP^+S7R1+#{kU7egHC zbrv5CYe2_srn6tqq_CpFt;|uh2>(^TinES{;@Goy(5FZ<{E2teM#ZV&Cp{DJ$SzH6 zPfO?H*&9BPlqLbk{q5@j_ceqS;qD(X6=KKS5&KB4*6)T>R%>9v1Ul zfyOP$VQX*vWWpK~aqAcftjT)}9TP;@y=Mqt`5BBg61H3S=1fBcH`Xv$@9tukv=_5y z<*bnvw-q1!XpRMf6<`$9Z zrgd0S>o~uozlqx~%aJE#ukj-DYiNa`3On6+h>6fqXQz*9vQh2Rv6o{EezT*?TFmy0 zARsJXG{;v)bj3bIRBeXQvAQI@a$^;~#0Fs7z#^pO`+->o-!j`fbVPiV1?ic7L_SH$ K)|zL-nEwIUSo#M5 literal 0 HcmV?d00001 diff --git a/tests/saved_test_data/rln_proj_64.star b/tests/saved_test_data/rln_proj_64.star new file mode 100644 index 0000000000..b7d88e788a --- /dev/null +++ b/tests/saved_test_data/rln_proj_64.star @@ -0,0 +1,18 @@ + +# version 30001 + +data_particles + +loop_ +_rlnAngleRot #1 +_rlnAngleTilt #2 +_rlnAnglePsi #3 +_rlnOriginXAngst #4 +_rlnOriginYAngst #5 +_rlnOpticsGroup #6 +_rlnImageName #7 + 275.167758 67.327105 47.970504 0.000000 0.000000 1 000001@rln_proj_64.mrcs + 19.551395 77.303721 168.404682 0.000000 0.000000 1 000002@rln_proj_64.mrcs + 76.607773 91.268867 326.522598 0.000000 0.000000 1 000003@rln_proj_64.mrcs + 116.208186 115.859671 140.116990 0.000000 0.000000 1 000004@rln_proj_64.mrcs + 250.742147 120.229633 124.614004 0.000000 0.000000 1 000005@rln_proj_64.mrcs diff --git a/tests/saved_test_data/rln_proj.mrcs b/tests/saved_test_data/rln_proj_65.mrcs similarity index 100% rename from tests/saved_test_data/rln_proj.mrcs rename to tests/saved_test_data/rln_proj_65.mrcs diff --git a/tests/saved_test_data/rln_proj.star b/tests/saved_test_data/rln_proj_65.star similarity index 71% rename from tests/saved_test_data/rln_proj.star rename to tests/saved_test_data/rln_proj_65.star index db08a6ff26..623585b92a 100644 --- a/tests/saved_test_data/rln_proj.star +++ b/tests/saved_test_data/rln_proj_65.star @@ -12,8 +12,8 @@ _rlnOriginXAngst #4 _rlnOriginYAngst #5 _rlnOpticsGroup #6 _rlnImageName #7 - 355.858841 72.200518 240.688133 0.000000 0.000000 1 000001@rln_proj.mrcs - 63.759542 92.697219 303.796113 0.000000 0.000000 1 000002@rln_proj.mrcs - 178.751045 42.736465 137.511331 0.000000 0.000000 1 000003@rln_proj.mrcs - 248.292861 83.402259 197.749400 0.000000 0.000000 1 000004@rln_proj.mrcs - 99.680307 72.793409 270.791295 0.000000 0.000000 1 000005@rln_proj.mrcs + 355.858841 72.200518 240.688133 0.000000 0.000000 1 000001@rln_proj_65.mrcs + 63.759542 92.697219 303.796113 0.000000 0.000000 1 000002@rln_proj_65.mrcs + 178.751045 42.736465 137.511331 0.000000 0.000000 1 000003@rln_proj_65.mrcs + 248.292861 83.402259 197.749400 0.000000 0.000000 1 000004@rln_proj_65.mrcs + 99.680307 72.793409 270.791295 0.000000 0.000000 1 000005@rln_proj_65.mrcs diff --git a/tests/test_relion_interop.py b/tests/test_relion_interop.py index 9f90df0676..b744557516 100644 --- a/tests/test_relion_interop.py +++ b/tests/test_relion_interop.py @@ -1,6 +1,7 @@ import os import numpy as np +import pytest from aspire.source import RelionSource, Simulation from aspire.volume import Volume @@ -8,27 +9,46 @@ DATA_DIR = os.path.join(os.path.dirname(__file__), "saved_test_data") -def test_projections(): - # Create RelionSource from Relion generated projection images. - starfile = os.path.join(DATA_DIR, "rln_proj.star") +STARFILE = ["rln_proj_65.star", "rln_proj_64.star"] + + +@pytest.fixture(params=STARFILE, ids=lambda x: f"resolution={x}", scope="module") +def starfile(request): + filepath = request.param + return os.path.join(DATA_DIR, filepath) + + +@pytest.fixture(scope="module") +def sources(starfile): rln_src = RelionSource(starfile) - # Create Simulation source using same volume and angles. - # Note, Relion projections are shifted by 1 pixel compared to ASPIRE. - dtype = rln_src.dtype + # Generate Volume used for Relion projections. + # Note, `downsample` is a no-op for resolution 65. vol_path = os.path.join(DATA_DIR, "clean70SRibosome_vol.npy") - vol = Volume(np.load(vol_path), dtype=dtype) + vol = Volume(np.load(vol_path), dtype=rln_src.dtype).downsample(rln_src.L) + + # Create Simulation source using Volume and angles from Relion projections. + # Note, for odd resolution Relion projections are shifted by 1 pixel in x and y. + offsets = 0 + if rln_src.L % 2 == 1: + offsets = -np.ones((rln_src.n, 2), dtype=rln_src.dtype) + sim_src = Simulation( n=rln_src.n, vols=vol, - offsets=-np.ones((rln_src.n, 2), dtype=dtype), + offsets=offsets, angles=rln_src.angles, - dtype=dtype, + dtype=rln_src.dtype, ) + return rln_src, sim_src + + +def test_projections(sources): + rln_src, sim_src = sources # Compute the Fourier Ring Correlation. res, corr = rln_src.images[:].frc(sim_src.images[:], cutoff=0.143) - # Check that estimated resolution is small and correlation is close to 1. + # Check that estimated resolution is high (< 2.5 pixels) and correlation is close to 1. np.testing.assert_array_less(res, 2.5) - np.testing.assert_array_less(1 - corr[:, -2], 0.0015) + np.testing.assert_array_less(1 - corr[:, -2], 0.02) From 8569f4f37ee90ae0eaa79011c026515571c45602 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 21 Nov 2023 09:29:29 -0500 Subject: [PATCH 266/294] remove blank line --- tests/saved_test_data/rln_proj_65.star | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/saved_test_data/rln_proj_65.star b/tests/saved_test_data/rln_proj_65.star index 623585b92a..d70dc8b683 100644 --- a/tests/saved_test_data/rln_proj_65.star +++ b/tests/saved_test_data/rln_proj_65.star @@ -1,5 +1,4 @@ - # version 30001 data_particles From 095aed6558280454761509879bded6c4f2fab06b Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Mon, 27 Nov 2023 12:05:25 -0500 Subject: [PATCH 267/294] Remove unnecessary fixture. Set sim amplitudes to 1. --- tests/test_relion_interop.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_relion_interop.py b/tests/test_relion_interop.py index b744557516..d8925c59ad 100644 --- a/tests/test_relion_interop.py +++ b/tests/test_relion_interop.py @@ -12,14 +12,9 @@ STARFILE = ["rln_proj_65.star", "rln_proj_64.star"] -@pytest.fixture(params=STARFILE, ids=lambda x: f"resolution={x}", scope="module") -def starfile(request): - filepath = request.param - return os.path.join(DATA_DIR, filepath) - - -@pytest.fixture(scope="module") -def sources(starfile): +@pytest.fixture(params=STARFILE, scope="module") +def sources(request): + starfile = os.path.join(DATA_DIR, request.param) rln_src = RelionSource(starfile) # Generate Volume used for Relion projections. @@ -37,6 +32,7 @@ def sources(starfile): n=rln_src.n, vols=vol, offsets=offsets, + amplitudes=1, angles=rln_src.angles, dtype=rln_src.dtype, ) From b08c8c4fd786f0ca7e231e45a36361760faa7d84 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Tue, 28 Nov 2023 10:31:40 -0500 Subject: [PATCH 268/294] Relion projection tutorial. --- gallery/tutorials/data/rln_proj_65.mrcs | 1 + gallery/tutorials/data/rln_proj_65.star | 1 + .../tutorials/relion_projection_interop.py | 107 ++++++++++++++++++ tox.ini | 1 + 4 files changed, 110 insertions(+) create mode 120000 gallery/tutorials/data/rln_proj_65.mrcs create mode 120000 gallery/tutorials/data/rln_proj_65.star create mode 100644 gallery/tutorials/tutorials/relion_projection_interop.py diff --git a/gallery/tutorials/data/rln_proj_65.mrcs b/gallery/tutorials/data/rln_proj_65.mrcs new file mode 120000 index 0000000000..e722cc0de2 --- /dev/null +++ b/gallery/tutorials/data/rln_proj_65.mrcs @@ -0,0 +1 @@ +../../../tests/saved_test_data/rln_proj_65.mrcs \ No newline at end of file diff --git a/gallery/tutorials/data/rln_proj_65.star b/gallery/tutorials/data/rln_proj_65.star new file mode 120000 index 0000000000..5965d5d1cf --- /dev/null +++ b/gallery/tutorials/data/rln_proj_65.star @@ -0,0 +1 @@ +../../../tests/saved_test_data/rln_proj_65.star \ No newline at end of file diff --git a/gallery/tutorials/tutorials/relion_projection_interop.py b/gallery/tutorials/tutorials/relion_projection_interop.py new file mode 100644 index 0000000000..abef67156c --- /dev/null +++ b/gallery/tutorials/tutorials/relion_projection_interop.py @@ -0,0 +1,107 @@ +""" +================================== +Relion Projection Interoperability +================================== + +In this tutorial we compare projections generated by Relion +with projections generated by ASPIRE's ``Simulation`` class. +Both sets of projections are generated using a downsampled +volume map of a 70S Ribosome, absent of noise and CTF corruption. +""" + +import os + +import numpy as np + +from aspire.source import RelionSource, Simulation +from aspire.volume import Volume + +# %% +# Load Relion Projections +# ----------------------- +# We load the Relion projections as a ``RelionSource`` and view the images. + +starfile = os.path.join(os.path.dirname(os.getcwd()), "data", "rln_proj_65.star") +rln_src = RelionSource(starfile) +rln_src.images[:].show(colorbar=False) + +# %% +# .. note:: +# The projections above were generated in Relion using the following command:: +# +# relion_project --i clean70SRibosome_vol_65p.mrc --nr_uniform 3000 --angpix 5 +# +# For this tutorial we take a subset of these projections consisting of the first 5 images. + +# %% +# Generate Projections using ``Simulation`` +# ----------------------------------------- +# Using the metadata associated with the ``RelionSource`` and the same volume +# we generate an analogous set of projections with ASPIRE's ``Simulation`` class. + +# Load the volume from file as a ``Volume`` object. +filepath = os.path.join( + os.path.dirname(os.getcwd()), "data", "clean70SRibosome_vol_65p.mrc" +) +vol = Volume.load(filepath, dtype=rln_src.dtype) + +# Create a ``Simulation`` source using metadata from the RelionSource projections. +# Note, for odd resolution Relion projections are shifted from ASPIRE projections +# by 1 pixel in x and y. +sim_src = Simulation( + n=rln_src.n, + vols=vol, + offsets=-np.ones((rln_src.n, 2), dtype=rln_src.dtype), + amplitudes=rln_src.amplitudes, + angles=rln_src.angles, + dtype=rln_src.dtype, +) + +sim_src.images[:].show(colorbar=False) + +# %% +# Comparing the Projections +# ------------------------- +# We will take a few different approaches to comparing the two sets of projection images. + +# %% +# Visual Comparison +# ^^^^^^^^^^^^^^^^^ +# We'll first look at a side-by-side of the two sets of images to confirm visually that +# the projections are taken from the same viewing angles. + +rln_src.images[:].show(colorbar=False) +sim_src.images[:].show(colorbar=False) + +# %% +# Fourier Ring Correlation +# ^^^^^^^^^^^^^^^^^^^^^^^^ +# Additionally, we can compare the two sets of images using the FRC. Note that the images +# are tightly correlated up to a high resolution of 2 pixels. +rln_src.images[:].frc(sim_src.images[:], cutoff=0.143, plot=True) + +# %% +# Relative Error +# ^^^^^^^^^^^^^^ +# As Relion and ASPIRE differ in methods of generating projections, the pixel intensity of +# the images may not correspond perfectly. So we begin by first normalizing the two sets of projections. +# We then check that the relative error with respect to the frobenius norm is less than 3%. + +# Work with numpy arrays. +rln_np = rln_src.images[:].asnumpy() +sim_np = sim_src.images[:].asnumpy() + +# Normalize images. +rln_np = (rln_np - np.mean(rln_np, axis=(1, 2))[:, None, None]) / np.std( + rln_np, axis=(1, 2) +)[:, None, None] +sim_np = (sim_np - np.mean(sim_np, axis=(1, 2))[:, None, None]) / np.std( + sim_np, axis=(1, 2) +)[:, None, None] + +# Assert that error is less than 3%. +error = np.linalg.norm(rln_np - sim_np, axis=(1, 2)) / np.linalg.norm( + rln_np, axis=(1, 2) +) +assert all(error < 0.03) +print(f"Relative per-image error: {error}") diff --git a/tox.ini b/tox.ini index 723d15daf5..b81a2ae506 100644 --- a/tox.ini +++ b/tox.ini @@ -74,6 +74,7 @@ per-file-ignores = gallery/tutorials/tutorials/ctf.py: T201, E402 gallery/tutorials/tutorials/micrograph_source.py: T201, E402 gallery/tutorials/tutorials/weighted_volume_estimation.py: T201, E402 + gallery/tutorials/tutorials/relion_projection_interop.py: T201 # Ignore Sphinx gallery builds docs/build/html/_downloads/*/*.py: T201, E402, F401, E265 docs/source/auto*/*.py: T201, E402, F401, E265 From 861c657f4fa95e1f3feea0078e1c3cfa78ef2978 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Wed, 29 Nov 2023 14:08:04 -0500 Subject: [PATCH 269/294] Use global mean and std. Add relative error test. --- .../tutorials/relion_projection_interop.py | 8 ++----- tests/test_relion_interop.py | 22 ++++++++++++++++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/gallery/tutorials/tutorials/relion_projection_interop.py b/gallery/tutorials/tutorials/relion_projection_interop.py index abef67156c..96122282db 100644 --- a/gallery/tutorials/tutorials/relion_projection_interop.py +++ b/gallery/tutorials/tutorials/relion_projection_interop.py @@ -92,12 +92,8 @@ sim_np = sim_src.images[:].asnumpy() # Normalize images. -rln_np = (rln_np - np.mean(rln_np, axis=(1, 2))[:, None, None]) / np.std( - rln_np, axis=(1, 2) -)[:, None, None] -sim_np = (sim_np - np.mean(sim_np, axis=(1, 2))[:, None, None]) / np.std( - sim_np, axis=(1, 2) -)[:, None, None] +rln_np = (rln_np - np.mean(rln_np)) / np.std(rln_np) +sim_np = (sim_np - np.mean(sim_np)) / np.std(sim_np) # Assert that error is less than 3%. error = np.linalg.norm(rln_np - sim_np, axis=(1, 2)) / np.linalg.norm( diff --git a/tests/test_relion_interop.py b/tests/test_relion_interop.py index d8925c59ad..a1a2796675 100644 --- a/tests/test_relion_interop.py +++ b/tests/test_relion_interop.py @@ -39,7 +39,27 @@ def sources(request): return rln_src, sim_src -def test_projections(sources): +def test_projections_relative_error(sources): + """Check the relative error between Relion and ASPIRE projection images.""" + rln_src, sim_src = sources + + # Work with numpy arrays. + rln_np = rln_src.images[:].asnumpy() + sim_np = sim_src.images[:].asnumpy() + + # Normalize images. + rln_np = (rln_np - np.mean(rln_np)) / np.std(rln_np) + sim_np = (sim_np - np.mean(sim_np)) / np.std(sim_np) + + # Check that relative error is less than 3%. + error = np.linalg.norm(rln_np - sim_np, axis=(1, 2)) / np.linalg.norm( + rln_np, axis=(1, 2) + ) + np.testing.assert_array_less(error, 0.03) + + +def test_projections_frc(sources): + """Compute the FRC between Relion and ASPIRE projection images.""" rln_src, sim_src = sources # Compute the Fourier Ring Correlation. From 2ed0094e50e31d43a51fb7e5846942626de45a01 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 4 Dec 2023 11:20:13 -0500 Subject: [PATCH 270/294] Make cutoff+resolution plotting optional for frc/fsc --- src/aspire/image/image.py | 5 ++- src/aspire/utils/resolution_estimation.py | 37 ++++++++++++++++------- src/aspire/volume/volume.py | 4 ++- tests/test_fourier_correlation.py | 8 +++-- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/aspire/image/image.py b/src/aspire/image/image.py index c85fc8188f..11ee7f412a 100644 --- a/src/aspire/image/image.py +++ b/src/aspire/image/image.py @@ -571,7 +571,7 @@ def show(self, columns=5, figsize=(20, 10), colorbar=True): plt.show() - def frc(self, other, cutoff, pixel_size=None, method="fft", plot=False): + def frc(self, other, cutoff=None, pixel_size=None, method="fft", plot=False): r""" Compute the Fourier ring correlation between two images. @@ -586,6 +586,9 @@ def frc(self, other, cutoff, pixel_size=None, method="fft", plot=False): :param other: `Image` instance to compare. :param cutoff: Cutoff value, traditionally `.143`. + Default `None` implies `cutoff=1` and excludes + plotting cutoff line. + :param pixel_size: Pixel size in angstrom. Default `None` implies unit in pixels, equivalent to pixel_size=1. :param method: Selects either 'fft' (on cartesian grid), diff --git a/src/aspire/utils/resolution_estimation.py b/src/aspire/utils/resolution_estimation.py index 5a54c269b8..b7a06f68df 100644 --- a/src/aspire/utils/resolution_estimation.py +++ b/src/aspire/utils/resolution_estimation.py @@ -250,7 +250,11 @@ def analyze_correlations(self, cutoff): Convert from the Fourier correlations to frequencies and resolution. :param cutoff: Cutoff value, traditionally `.143`. + Note `cutoff=None` evaluates as `cutoff=1`. """ + # Handle optional cutoff plotting. + if cutoff is None: + cutoff = 1 cutoff = float(cutoff) if not (0 <= cutoff <= 1): @@ -289,17 +293,25 @@ def _freq(self, k): # Similar to wavenumbers. Larger is higher frequency. return k / (self.L * self.pixel_size) - def plot(self, cutoff, save_to_file=False, labels=None): + def plot(self, cutoff=None, save_to_file=False, labels=None): """ Generates a Fourier correlation plot. :param cutoff: Cutoff value, traditionally `.143`. + Default `None` implies `cutoff=1` and excludes + plotting cutoff line. :param save_to_file: Optionally, save plot to file. Defaults False, enabled by providing a string filename. User is responsible for providing reasonable filename. See `https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.savefig.html`. """ - cutoff = float(cutoff) + + # Handle optional cutoff plotting. + _plot_cutoff = True + if cutoff is None: + cutoff = 1 + _plot_cutoff = False + if not (0 <= cutoff <= 1): raise ValueError("Supplied correlation `cutoff` not in [0,1], {cutoff}") @@ -344,17 +356,20 @@ def plot(self, cutoff, save_to_file=False, labels=None): _label = labels[i] plt.plot(freqs_units, line, label=_label) - # Display cutoff - plt.axhline(y=cutoff, color="r", linestyle="--", label=f"cutoff={cutoff}") estimated_resolution = self.analyze_correlations(cutoff)[0] - # Display resolution - plt.axvline( - x=estimated_resolution, - color="b", - linestyle=":", - label=f"Resolution={estimated_resolution:.3f}", - ) + # Display cutoff + if _plot_cutoff: + plt.axhline(y=cutoff, color="r", linestyle="--", label=f"cutoff={cutoff}") + + # Display resolution + plt.axvline( + x=estimated_resolution, + color="b", + linestyle=":", + label=f"Resolution={estimated_resolution:.3f}", + ) + # x-axis decreasing plt.gca().invert_xaxis() plt.legend(title=f"Method: {self.method}") diff --git a/src/aspire/volume/volume.py b/src/aspire/volume/volume.py index 5993a1dd4e..cc29710045 100644 --- a/src/aspire/volume/volume.py +++ b/src/aspire/volume/volume.py @@ -549,7 +549,7 @@ def load(cls, filename, permissive=True, dtype=None, symmetry_group=None): return cls(loaded_data, symmetry_group=symmetry_group, dtype=dtype) - def fsc(self, other, cutoff, pixel_size=None, method="fft", plot=False): + def fsc(self, other, cutoff=None, pixel_size=None, method="fft", plot=False): r""" Compute the Fourier shell correlation between two volumes. @@ -564,6 +564,8 @@ def fsc(self, other, cutoff, pixel_size=None, method="fft", plot=False): :param other: `Volume` instance to compare. :param cutoff: Cutoff value, traditionally `.143`. + Default `None` implies `cutoff=1` and excludes + plotting cutoff line. :param pixel_size: Pixel size in angstrom. Default `None` implies unit in pixels, equivalent to pixel_size=1. :param method: Selects either 'fft' (on cartesian grid), diff --git a/tests/test_fourier_correlation.py b/tests/test_fourier_correlation.py index 476e2f5548..282089ce12 100644 --- a/tests/test_fourier_correlation.py +++ b/tests/test_fourier_correlation.py @@ -145,9 +145,10 @@ def test_frc_img_plot(image_fixture): _ = img_a.frc(img_n, pixel_size=1, cutoff=0.143, plot=True) # Plot to file + # Also tests `cutoff=None` with tempfile.TemporaryDirectory() as tmp_input_dir: file_path = os.path.join(tmp_input_dir, "img_frc_curve.png") - img_a.frc(img_n, pixel_size=1, cutoff=0.143, plot=file_path) + img_a.frc(img_n, pixel_size=1, cutoff=None, plot=file_path) assert os.path.exists(file_path) @@ -204,9 +205,10 @@ def test_fsc_vol_plot(volume_fixture): _ = vol_a.fsc(vol_b, pixel_size=1, cutoff=0.5, plot=True) # Plot to file + # Also tests `cutoff=None` with tempfile.TemporaryDirectory() as tmp_input_dir: - file_path = os.path.join(tmp_input_dir, "img_fsc_curve.png") - vol_a.fsc(vol_b, pixel_size=1, cutoff=0.143, plot=file_path) + file_path = os.path.join(tmp_input_dir, "vol_fsc_curve.png") + vol_a.fsc(vol_b, pixel_size=1, cutoff=None, plot=file_path) assert os.path.exists(file_path) From f722817ae652a68dc6a2cdf075ef5d61a1156939 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 4 Dec 2023 11:26:54 -0500 Subject: [PATCH 271/294] Resolve new Flake8 concern, unprinted tuple --- gallery/tutorials/tutorials/image_class.py | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gallery/tutorials/tutorials/image_class.py b/gallery/tutorials/tutorials/image_class.py index 36d689ddc0..ca4df61961 100644 --- a/gallery/tutorials/tutorials/image_class.py +++ b/gallery/tutorials/tutorials/image_class.py @@ -14,7 +14,7 @@ file_path = os.path.join(os.path.dirname(os.getcwd()), "data", "monuments.npy") img_data = np.load(file_path) -img_data.shape, img_data.dtype +print(img_data.shape, img_data.dtype) # %% # Create an Image Instance diff --git a/tox.ini b/tox.ini index b81a2ae506..3970e6e317 100644 --- a/tox.ini +++ b/tox.ini @@ -72,6 +72,7 @@ per-file-ignores = gallery/tutorials/pipeline_demo.py: T201 gallery/tutorials/turorials/data_downloader.py: E402 gallery/tutorials/tutorials/ctf.py: T201, E402 + gallery/tutorials/tutorials/image_class.py: T201 gallery/tutorials/tutorials/micrograph_source.py: T201, E402 gallery/tutorials/tutorials/weighted_volume_estimation.py: T201, E402 gallery/tutorials/tutorials/relion_projection_interop.py: T201 From 6f9c58fa2ca41cf9cc8c7ba3780d7925ed3b42f5 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 4 Dec 2023 15:38:24 -0500 Subject: [PATCH 272/294] Cleanup warnings --- src/aspire/utils/resolution_estimation.py | 23 ++++++++++++++--------- tests/test_utils.py | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/aspire/utils/resolution_estimation.py b/src/aspire/utils/resolution_estimation.py index b7a06f68df..8e2fee99c9 100644 --- a/src/aspire/utils/resolution_estimation.py +++ b/src/aspire/utils/resolution_estimation.py @@ -2,6 +2,7 @@ This module contains code for estimating resolution achieved by reconstructions. """ import logging +import warnings import matplotlib.pyplot as plt import numpy as np @@ -38,13 +39,13 @@ class FourierCorrelation: def __init__(self, a, b, pixel_size=None, method="fft"): """ - :param a: Input array a, shape(..., *dim). - :param b: Input array b, shape(..., *dim). - :param pixel_size: Pixel size in angstrom. - Default `None` implies "pixel" units. - :param method: Selects either 'fft' (on Cartesian grid), - or 'nufft' (on polar grid). Defaults to 'fft'. - 7""" + :param a: Input array a, shape(..., *dim). + :param b: Input array b, shape(..., *dim). + :param pixel_size: Pixel size in angstrom. + Default `None` implies "pixel" units. + :param method: Selects either 'fft' (on Cartesian grid), + or 'nufft' (on polar grid). Defaults to 'fft'. + """ # Sanity checks if not hasattr(self, "dim"): @@ -275,8 +276,12 @@ def analyze_correlations(self, cutoff): # Convert indices to frequency (as 1/angstrom) frequencies = self._freq(c_ind) - # Convert to resolution in angstrom, smaller is higher frequency. - self._resolutions = 1 / frequencies + with warnings.catch_warnings(): + # When using high cutoff (eg. 1) it is possible `frequencies` + # contains 0; capture and ignore that division warning. + warnings.filterwarnings("ignore", r".*divide by zero.*") + # Convert to resolution in angstrom, smaller is higher frequency. + self._resolutions = 1 / frequencies return self._resolutions diff --git a/tests/test_utils.py b/tests/test_utils.py index ecd87c0122..ffad5bc9f6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -395,7 +395,7 @@ def matplotlib_no_gui(): # Save and restore current warnings list. with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Matplotlib is currently using agg") + warnings.filterwarnings("ignore", r"Matplotlib is currently using agg.*") yield From fd34087a010e1fb4abafa0fd78fdcefbb1f24693 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 27 Nov 2023 14:41:48 -0500 Subject: [PATCH 273/294] Add simple rotation test across bases --- tests/test_rotation.py | 330 ++++++++++++++++++++++++++--------------- 1 file changed, 210 insertions(+), 120 deletions(-) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index caffb17f0c..10e32e7487 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -1,131 +1,191 @@ import logging -from unittest import TestCase import numpy as np +import PIL.Image as PILImage import pytest +import scipy as sp from scipy.spatial.transform import Rotation as sp_rot -from aspire.utils import Rotation, utest_tolerance +from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D, PSWFBasis2D +from aspire.image import Image +from aspire.utils import Rotation, grid_2d, utest_tolerance logger = logging.getLogger(__name__) +# Parameters -class UtilsTestCase(TestCase): - def setUp(self): - self.dtype = np.float32 - self.num_rots = 32 - self.rot_obj = Rotation.generate_random_rotations( - self.num_rots, seed=0, dtype=self.dtype - ) - self.angles = self.rot_obj.angles - self.matrices = self.rot_obj.matrices - - def testRotMatrices(self): - rot_ref = sp_rot.from_matrix(self.matrices) - matrices = rot_ref.as_matrix().astype(self.dtype) - self.assertTrue( - np.allclose(self.matrices, matrices, atol=utest_tolerance(self.dtype)) +NUM_ROTS = 32 +SEED = 0 + +DTYPES = [ + np.float64, + np.float32, +] + +BASES = [ + FFBBasis2D, + FBBasis2D, + FLEBasis2D, + PSWFBasis2D, + FPSWFBasis2D, +] + +IMG_SIZES = [ + 32, + pytest.param(31, marks=pytest.mark.expensive), +] + +# Fixtures + + +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}", scope="module") +def dtype(request): + return request.param + + +@pytest.fixture(scope="module") +def rot_obj(dtype): + return Rotation.generate_random_rotations(NUM_ROTS, seed=SEED, dtype=dtype) + + +@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}", scope="module") +def img_size(request): + return request.param + + +@pytest.fixture(params=BASES, ids=lambda x: f"basis={x}", scope="module") +def basis(request, img_size, dtype): + cls = request.param + # Setup a Basis + basis = cls(img_size, dtype=dtype) + return basis + + +# Rotation Class Tests + + +def test_matrices(rot_obj): + rot_ref = sp_rot.from_matrix(rot_obj.matrices) + matrices = rot_ref.as_matrix() + np.testing.assert_allclose( + rot_obj.matrices, matrices, atol=utest_tolerance(rot_obj.dtype) + ) + + +def test_as_angles(rot_obj): + rot_ref = sp_rot.from_euler("ZYZ", rot_obj.angles, degrees=False) + angles = rot_ref.as_euler("ZYZ", degrees=False) + np.testing.assert_allclose(rot_obj.angles, angles) + + +def test_from_matrix(rot_obj): + rot_ref = sp_rot.from_matrix(rot_obj.matrices) + angles = rot_ref.as_euler("ZYZ", degrees=False) + rot = Rotation.from_matrix(rot_obj.matrices) + np.testing.assert_allclose(rot.angles, angles) + + +def test_from_euler(rot_obj): + rot_ref = sp_rot.from_euler("ZYZ", rot_obj.angles, degrees=False) + matrices = rot_ref.as_matrix() + rot = Rotation.from_euler(rot_obj.angles, dtype=rot_obj.dtype) + np.testing.assert_allclose(rot._matrices, matrices) + + +def test_invert(rot_obj): + rot_mat = rot_obj.matrices + rot_mat_t = rot_obj.invert() + np.testing.assert_allclose(rot_mat_t, np.transpose(rot_mat, (0, 2, 1))) + + +def test_multiplication(rot_obj): + result = (rot_obj * rot_obj.invert()).matrices + for i in range(len(rot_obj)): + np.testing.assert_allclose( + np.eye(3), result[i], atol=utest_tolerance(rot_obj.dtype) ) - def testRotAngles(self): - rot_ref = sp_rot.from_euler("ZYZ", self.angles, degrees=False) - angles = rot_ref.as_euler("ZYZ", degrees=False).astype(self.dtype) - self.assertTrue(np.allclose(self.angles, angles)) - - def testFromMatrix(self): - rot_ref = sp_rot.from_matrix(self.matrices) - angles = rot_ref.as_euler("ZYZ", degrees=False).astype(self.dtype) - rot = Rotation.from_matrix(self.matrices, dtype=self.dtype) - self.assertTrue(np.allclose(rot.angles, angles)) - - def testFromEuler(self): - rot_ref = sp_rot.from_euler("ZYZ", self.angles, degrees=False) - matrices = rot_ref.as_matrix().astype(self.dtype) - rot = Rotation.from_euler(self.angles, dtype=self.dtype) - self.assertTrue(np.allclose(rot._matrices, matrices)) - - def testInvert(self): - rot_mat = self.rot_obj.matrices - rot_mat_t = self.rot_obj.invert() - self.assertTrue(np.allclose(rot_mat_t, np.transpose(rot_mat, (0, 2, 1)))) - - def testMultiplication(self): - result = (self.rot_obj * self.rot_obj.invert()).matrices - for i in range(len(self.rot_obj)): - self.assertTrue( - np.allclose(np.eye(3), result[i], atol=utest_tolerance(self.dtype)) - ) - - def testRegisterRots(self): - q_mat = Rotation.generate_random_rotations(1, dtype=self.dtype)[0] - for flag in [0, 1]: - regrots_ref = self.rot_obj.apply_registration(q_mat, flag) - q_mat_est, flag_est = self.rot_obj.find_registration(regrots_ref) - self.assertTrue( - np.allclose(flag_est, flag) - and np.allclose(q_mat_est, q_mat, atol=utest_tolerance(self.dtype)) - ) - - def testRegister(self): - # These will yield two more distinct sets of random rotations wrt self.rot_obj - set1 = Rotation.generate_random_rotations(self.num_rots, dtype=self.dtype) - set2 = Rotation.generate_random_rotations( - self.num_rots, dtype=self.dtype, seed=7 + +def test_register_rots(rot_obj): + q_mat = Rotation.generate_random_rotations(1, dtype=rot_obj.dtype)[0] + for flag in [0, 1]: + regrots_ref = rot_obj.apply_registration(q_mat, flag) + q_mat_est, flag_est = rot_obj.find_registration(regrots_ref) + np.testing.assert_allclose(flag_est, flag) + np.testing.assert_allclose( + q_mat_est, q_mat, atol=utest_tolerance(rot_obj.dtype) ) - # Align both sets of random rotations to rot_obj - aligned_rots1 = self.rot_obj.register(set1) - aligned_rots2 = self.rot_obj.register(set2) - self.assertTrue(aligned_rots1.mse(aligned_rots2) < utest_tolerance(self.dtype)) - self.assertTrue(aligned_rots2.mse(aligned_rots1) < utest_tolerance(self.dtype)) - - def testMSE(self): - q_ang = [np.random.random(3)] - q_mat = sp_rot.from_euler("ZYZ", q_ang, degrees=False).as_matrix()[0] - for flag in [0, 1]: - regrots_ref = self.rot_obj.apply_registration(q_mat, flag) - mse = self.rot_obj.mse(regrots_ref) - self.assertTrue(mse < utest_tolerance(self.dtype)) - - def testCommonLines(self): - ell_ij, ell_ji = self.rot_obj.common_lines(8, 11, 360) - self.assertTrue(ell_ij == 235 and ell_ji == 284) - - def testString(self): - logger.debug(str(self.rot_obj)) - - def testRepr(self): - logger.debug(repr(self.rot_obj)) - - def testLen(self): - self.assertTrue(len(self.rot_obj) == self.num_rots) - - def testSetterGetter(self): - # Excute set - tmp = np.arange(9).reshape((3, 3)) - self.rot_obj[13] = tmp - # Execute get - self.assertTrue(np.all(self.rot_obj[13] == tmp)) - - def testDtype(self): - self.assertTrue(self.dtype == self.rot_obj.dtype) - - def testFromRotvec(self): - # Build random rotation vectors. - axis = np.array([1, 0, 0], dtype=self.dtype) - angles = np.random.uniform(0, 2 * np.pi, 10) - rot_vecs = np.array([angle * axis for angle in angles], dtype=self.dtype) - - # Build rotations using from_rotvec and about_axis (as reference). - rotations = Rotation.from_rotvec(rot_vecs, dtype=self.dtype) - ref_rots = Rotation.about_axis("x", angles, dtype=self.dtype) - - self.assertTrue(isinstance(rotations, Rotation)) - self.assertTrue(rotations.matrices.dtype == self.dtype) - self.assertTrue(np.allclose(rotations.matrices, ref_rots.matrices)) - - -def test_angle_dist(): - dtype = np.float32 + + +def test_register(rot_obj): + # These will yield two more distinct sets of random rotations wrt rot_obj + set1 = Rotation.generate_random_rotations(NUM_ROTS, dtype=rot_obj.dtype) + set2 = Rotation.generate_random_rotations(NUM_ROTS, dtype=rot_obj.dtype, seed=7) + # Align both sets of random rotations to rot_obj + aligned_rots1 = rot_obj.register(set1) + aligned_rots2 = rot_obj.register(set2) + tol = utest_tolerance(rot_obj.dtype) + np.testing.assert_array_less(aligned_rots1.mse(aligned_rots2), tol) + np.testing.assert_array_less(aligned_rots2.mse(aligned_rots1), tol) + + +def test_mse(rot_obj): + q_ang = [np.random.random(3)] + q_mat = sp_rot.from_euler("ZYZ", q_ang, degrees=False).as_matrix()[0] + for flag in [0, 1]: + regrots_ref = rot_obj.apply_registration(q_mat, flag) + mse = rot_obj.mse(regrots_ref) + np.testing.assert_array_less(mse, utest_tolerance(rot_obj.dtype)) + + +def test_common_lines(rot_obj): + ell_ij, ell_ji = rot_obj.common_lines(8, 11, 360) + np.testing.assert_equal([ell_ij, ell_ji], [235, 284]) + + +def test_string(rot_obj): + logger.debug(str(rot_obj)) + + +def test_repr(rot_obj): + logger.debug(repr(rot_obj)) + + +def test_len(rot_obj): + assert len(rot_obj) == NUM_ROTS + + +def test_setter_getter(rot_obj): + # Excute set + tmp = np.arange(9).reshape((3, 3)) + rot_obj[13] = tmp + # Execute get + np.testing.assert_equal(rot_obj[13], tmp) + + +def test_dtype(dtype, rot_obj): + assert dtype == rot_obj.dtype + + +def test_from_rotvec(rot_obj): + # Build random rotation vectors. + axis = np.array([1, 0, 0], dtype=rot_obj.dtype) + angles = np.random.uniform(0, 2 * np.pi, 10) + rot_vecs = np.array([angle * axis for angle in angles], dtype=rot_obj.dtype) + + # Build rotations using from_rotvec and about_axis (as reference). + rotations = Rotation.from_rotvec(rot_vecs, dtype=rot_obj.dtype) + ref_rots = Rotation.about_axis("x", angles, dtype=rot_obj.dtype) + + assert isinstance(rotations, Rotation) + assert rotations.matrices.dtype == rot_obj.dtype + np.testing.assert_allclose(rotations.matrices, ref_rots.matrices) + + +# Angular Distance Tests + + +def test_angle_dist(dtype): angles = np.array([i * np.pi / 360 for i in range(360)], dtype=dtype) rots = Rotation.about_axis("x", angles, dtype=dtype) @@ -140,12 +200,42 @@ def test_angle_dist(): _ = Rotation.angle_dist(rots[:3], rots[:5]) -def test_mean_angular_distance(): - rots_z = Rotation.about_axis( - "z", [0, np.pi / 4, np.pi / 2], dtype=np.float32 - ).matrices - rots_id = Rotation.about_axis("z", [0, 0, 0], dtype=np.float32).matrices +def test_mean_angular_distance(dtype): + rots_z = Rotation.about_axis("z", [0, np.pi / 4, np.pi / 2], dtype=dtype).matrices + rots_id = Rotation.about_axis("z", [0, 0, 0], dtype=dtype).matrices mean_ang_dist = Rotation.mean_angular_distance(rots_z, rots_id) assert np.allclose(mean_ang_dist, np.pi / 4) + + +# Basis Rotations + + +def test_basis_rotation_2d(basis): + """ + Test steerable basis rotation performs similar operation to PIL real space image rotation. + + Checks both orientation and rough values. + """ + # Set a rotation amount + rot_radians = np.pi / 6 + + # Create an empty image + L = basis.nres + img = np.zeros((L, L), dtype=basis.dtype) + # Set one pixel (between I and IV quadrants) + img[L // 2, 3 * L // 4] = 1 + # Roundtrip using the basis. Smooths out the discontinuous pixel. + img = basis.expand(Image(img)).evaluate() + + # Rotate with ASPIRE Steerable Basis, returning to real space. + rot_img = basis.expand(img).rotate(rot_radians).evaluate() + + # Rotate image with PIL, returning to Numpy array. + pil_rot_img = np.asarray( + PILImage.fromarray(img.asnumpy()[0]).rotate(rot_radians * 180 / np.pi) + ) + + # Rough compare arrays. + np.testing.assert_allclose(rot_img.asnumpy()[0], pil_rot_img, atol=0.25) From 0e13ffaab81b8e139c71d5b01d7522f94fc44aa0 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 27 Nov 2023 14:57:46 -0500 Subject: [PATCH 274/294] Convert basis rotation test to use Gaussian blob --- tests/test_rotation.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 10e32e7487..4f4756667e 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -3,12 +3,11 @@ import numpy as np import PIL.Image as PILImage import pytest -import scipy as sp from scipy.spatial.transform import Rotation as sp_rot from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D, PSWFBasis2D from aspire.image import Image -from aspire.utils import Rotation, grid_2d, utest_tolerance +from aspire.utils import Rotation, gaussian_2d, utest_tolerance logger = logging.getLogger(__name__) @@ -221,20 +220,18 @@ def test_basis_rotation_2d(basis): # Set a rotation amount rot_radians = np.pi / 6 - # Create an empty image + # Create an Image containing a smooth blob. L = basis.nres - img = np.zeros((L, L), dtype=basis.dtype) - # Set one pixel (between I and IV quadrants) - img[L // 2, 3 * L // 4] = 1 - # Roundtrip using the basis. Smooths out the discontinuous pixel. - img = basis.expand(Image(img)).evaluate() + img = Image(gaussian_2d(L, mu=(L // 4, 0), dtype=basis.dtype)) # Rotate with ASPIRE Steerable Basis, returning to real space. rot_img = basis.expand(img).rotate(rot_radians).evaluate() # Rotate image with PIL, returning to Numpy array. pil_rot_img = np.asarray( - PILImage.fromarray(img.asnumpy()[0]).rotate(rot_radians * 180 / np.pi) + PILImage.fromarray(img.asnumpy()[0]).rotate( + rot_radians * 180 / np.pi, resample=PILImage.BILINEAR + ) ) # Rough compare arrays. From 47e5aa370ea817af6fb2a880773722a265a01c2b Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 28 Nov 2023 09:15:01 -0500 Subject: [PATCH 275/294] Update rot test params to run doubles in long running reduces regular ut time --- tests/test_rotation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 4f4756667e..46fdbdf403 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -11,14 +11,15 @@ logger = logging.getLogger(__name__) + # Parameters NUM_ROTS = 32 SEED = 0 DTYPES = [ - np.float64, np.float32, + pytest.param(np.float64, marks=pytest.mark.expensive), ] BASES = [ From e296011d22d241eea4c96f90a95515de23d7e1c3 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 28 Nov 2023 09:22:14 -0500 Subject: [PATCH 276/294] Remove old FFB and FLE specific rotation tests --- tests/test_FFBbasis2D.py | 90 ---------------------------------------- tests/test_FLEbasis2D.py | 72 -------------------------------- 2 files changed, 162 deletions(-) diff --git a/tests/test_FFBbasis2D.py b/tests/test_FFBbasis2D.py index 04b7a9b4c7..73bb8667e8 100644 --- a/tests/test_FFBbasis2D.py +++ b/tests/test_FFBbasis2D.py @@ -81,96 +81,6 @@ def testElements(self, basis): for ell, k, sgn in zip(ells, ks, sgns): self._testElement(basis, ell, k, sgn) - def testRotate(self, basis): - # Now low res (8x8) had problems; - # better with odd (7x7), but still not good. - # We'll use a higher res test image. - # fh = np.load(os.path.join(DATA_DIR, 'ffbbasis2d_xcoef_in_8_8.npy'))[:7,:7] - # Use a real data volume to generate a clean test image. - v = Volume( - np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype( - basis.dtype - ) - ) - src = Simulation(L=v.resolution, n=1, vols=v, dtype=v.dtype) - # Extract, this is the original image to transform. - x1 = src.images[0] - - # Rotate 90 degrees CCW in cartesian coordinates. - x2 = Image(np.rot90(x1.asnumpy(), axes=(1, 2))) - - # Express in an FB basis - basis = FFBBasis2D(x1.resolution, dtype=x1.dtype) - v1 = basis.evaluate_t(x1) - v2 = basis.evaluate_t(x2) - - # Reflect in the FB basis space - v4 = basis.rotate(v1, 0, refl=[True]) - - # Rotate in the FB basis space - v3 = basis.rotate(v1, 2 * np.pi) - v1 = basis.rotate(v1, np.pi / 2) - - # Evaluate back into cartesian - y1 = basis.evaluate(v1).asnumpy() - y2 = basis.evaluate(v2).asnumpy() - y3 = basis.evaluate(v3).asnumpy() - y4 = basis.evaluate(v4).asnumpy() - - # Rotate 90 - assert np.allclose(y1[0], y2[0], atol=1e-5) - - # 2*pi Identity - assert np.allclose(x1[0], y3[0], atol=1e-5) - - # Refl (flipped using flipud) - assert np.allclose(np.flipud(x1.asnumpy()[0]), y4[0], atol=1e-5) - - def testRotateComplex(self, basis): - # Now low res (8x8) had problems; - # better with odd (7x7), but still not good. - # We'll use a higher res test image. - # fh = np.load(os.path.join(DATA_DIR, 'ffbbasis2d_xcoef_in_8_8.npy'))[:7,:7] - # Use a real data volume to generate a clean test image. - v = Volume( - np.load(os.path.join(DATA_DIR, "clean70SRibosome_vol.npy")).astype( - basis.dtype - ) - ) - src = Simulation(L=v.resolution, n=1, vols=v, dtype=v.dtype) - # Extract, this is the original image to transform. - x1 = src.images[0] - - # Rotate 90 degrees CCW in cartesian coordinates. - x2 = Image(np.rot90(x1.asnumpy(), axes=(1, 2))) - - # Express in an FB basis - basis = FFBBasis2D(x1.resolution, dtype=x1.dtype) - v1 = basis.evaluate_t(x1) - v2 = basis.evaluate_t(x2) - - # Reflect in the FB basis space - v4 = basis.to_real(basis.complex_rotate(basis.to_complex(v1), 0, refl=[True])) - - # Complex Rotate in the FB basis space - v3 = basis.to_real(basis.complex_rotate(basis.to_complex(v1), 2 * np.pi)) - v1 = basis.to_real(basis.complex_rotate(basis.to_complex(v1), np.pi / 2)) - - # Evaluate back into cartesian - y1 = basis.evaluate(v1).asnumpy() - y2 = basis.evaluate(v2).asnumpy() - y3 = basis.evaluate(v3).asnumpy() - y4 = basis.evaluate(v4).asnumpy() - - # Rotate 90 - assert np.allclose(y1[0], y2[0], atol=1e-5) - - # 2*pi Identity - assert np.allclose(x1[0].asnumpy(), y3[0], atol=1e-5) - - # Refl (flipped using flipud) - assert np.allclose(np.flipud(x1.asnumpy()[0]), y4[0], atol=1e-5) - def testShift(self, basis): """ Compare shifting using Image with shifting provided by the Basis. diff --git a/tests/test_FLEbasis2D.py b/tests/test_FLEbasis2D.py index 9fb5504202..f25885684e 100644 --- a/tests/test_FLEbasis2D.py +++ b/tests/test_FLEbasis2D.py @@ -228,78 +228,6 @@ def testLowPass(): _ = basis.lowpass(coefs[0, :], L) -def testRotate(): - # test ability to accurately rotate images via - # FLE coefficients - - L = 128 - basis = FLEBasis2D(L, match_fb=False) - - # sample image - ims = create_images(L, 1) - # rotate 90 degrees in cartesian coordinates - ims_90 = Image(np.rot90(ims.asnumpy(), axes=(1, 2))) - - # get FLE coefficients - coefs = basis.evaluate_t(ims) - coefs_cart_rot = basis.evaluate_t(ims_90) - - # rotate original image in FLE space using Steerable rotate method - coefs_fle_rot = basis.rotate(coefs, np.pi / 2) - - # back to cartesian - ims_cart_rot = basis.evaluate(coefs_cart_rot) - ims_fle_rot = basis.evaluate(coefs_fle_rot) - - # test rot90 close - assert np.allclose(ims_cart_rot[0], ims_fle_rot[0], atol=1e-4) - - # 2Pi identity in FLE space (rotate by 2Pi) - coefs_fle_2pi = basis.rotate(coefs, 2 * np.pi) - ims_fle_2pi = basis.evaluate(coefs_fle_2pi) - - # test 2Pi identity - assert np.allclose(ims[0], ims_fle_2pi[0], atol=utest_tolerance(basis.dtype)) - - # Reflect in FLE space (rotate by Pi) - coefs_fle_pi = basis.rotate(coefs, np.pi) - ims_fle_pi = basis.evaluate(coefs_fle_pi) - - # test reflection - assert np.allclose(np.fliplr(np.flipud(ims.asnumpy()[0])), ims_fle_pi[0], atol=1e-4) - - # make sure you can pass in a 1-D array if you want - _ = basis.lowpass(Coef(basis, np.zeros((basis.count,))), np.pi) - - -def testRotate45(): - # test ability to accurately rotate images via - # FLE coefficients - dtype = np.float64 - - L = 128 - fb_basis = FFBBasis2D(L, dtype=dtype) - basis = FLEBasis2D(L, match_fb=True, dtype=dtype) - - # sample image - ims = create_images(L, 1) - - # get FLE coefficients - fb_coefs = fb_basis.evaluate_t(ims) - coefs = basis.evaluate_t(ims) - - # rotate original image in FLE space using Steerable rotate method - fb_coefs_rot = fb_basis.rotate(fb_coefs, np.pi / 4) - coefs_rot = basis.rotate(coefs, np.pi / 4) - - # back to cartesian - fb_ims_rot = fb_basis.evaluate(fb_coefs_rot) - ims_rot = basis.evaluate(coefs_rot) - - # test close - assert np.allclose(ims_rot[0], fb_ims_rot[0], atol=1e-4) - - def testRadialConvolution(): # test ability to accurately convolve with a radial # (e.g. CTF) function via FLE coefficients From dc395e19fd84c9d25f752935ff21b94c19cd6aa0 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 28 Nov 2023 09:47:09 -0500 Subject: [PATCH 277/294] move new steerable rotate tests out of test_rotation --- tests/test_FFBbasis2D.py | 1 - tests/test_FLEbasis2D.py | 3 +- tests/test_rotation.py | 62 +----------------------- tests/test_steerable_bases_2d.py | 82 ++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 63 deletions(-) create mode 100644 tests/test_steerable_bases_2d.py diff --git a/tests/test_FFBbasis2D.py b/tests/test_FFBbasis2D.py index 73bb8667e8..8acf7201d1 100644 --- a/tests/test_FFBbasis2D.py +++ b/tests/test_FFBbasis2D.py @@ -6,7 +6,6 @@ from scipy.special import jv from aspire.basis import Coef, FFBBasis2D -from aspire.image import Image from aspire.source import Simulation from aspire.utils.misc import grid_2d from aspire.volume import Volume diff --git a/tests/test_FLEbasis2D.py b/tests/test_FLEbasis2D.py index f25885684e..7d6b3f5c47 100644 --- a/tests/test_FLEbasis2D.py +++ b/tests/test_FLEbasis2D.py @@ -4,12 +4,11 @@ import numpy as np import pytest -from aspire.basis import Coef, FBBasis2D, FFBBasis2D, FLEBasis2D +from aspire.basis import Coef, FBBasis2D, FLEBasis2D from aspire.image import Image from aspire.nufft import backend_available from aspire.numeric import fft from aspire.source import Simulation -from aspire.utils import utest_tolerance from aspire.volume import Volume from ._basis_util import UniversalBasisMixin diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 46fdbdf403..aaa1262695 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -1,13 +1,10 @@ import logging import numpy as np -import PIL.Image as PILImage import pytest from scipy.spatial.transform import Rotation as sp_rot -from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D, PSWFBasis2D -from aspire.image import Image -from aspire.utils import Rotation, gaussian_2d, utest_tolerance +from aspire.utils import Rotation, utest_tolerance logger = logging.getLogger(__name__) @@ -19,21 +16,9 @@ DTYPES = [ np.float32, - pytest.param(np.float64, marks=pytest.mark.expensive), + np.float64, ] -BASES = [ - FFBBasis2D, - FBBasis2D, - FLEBasis2D, - PSWFBasis2D, - FPSWFBasis2D, -] - -IMG_SIZES = [ - 32, - pytest.param(31, marks=pytest.mark.expensive), -] # Fixtures @@ -48,19 +33,6 @@ def rot_obj(dtype): return Rotation.generate_random_rotations(NUM_ROTS, seed=SEED, dtype=dtype) -@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}", scope="module") -def img_size(request): - return request.param - - -@pytest.fixture(params=BASES, ids=lambda x: f"basis={x}", scope="module") -def basis(request, img_size, dtype): - cls = request.param - # Setup a Basis - basis = cls(img_size, dtype=dtype) - return basis - - # Rotation Class Tests @@ -207,33 +179,3 @@ def test_mean_angular_distance(dtype): mean_ang_dist = Rotation.mean_angular_distance(rots_z, rots_id) assert np.allclose(mean_ang_dist, np.pi / 4) - - -# Basis Rotations - - -def test_basis_rotation_2d(basis): - """ - Test steerable basis rotation performs similar operation to PIL real space image rotation. - - Checks both orientation and rough values. - """ - # Set a rotation amount - rot_radians = np.pi / 6 - - # Create an Image containing a smooth blob. - L = basis.nres - img = Image(gaussian_2d(L, mu=(L // 4, 0), dtype=basis.dtype)) - - # Rotate with ASPIRE Steerable Basis, returning to real space. - rot_img = basis.expand(img).rotate(rot_radians).evaluate() - - # Rotate image with PIL, returning to Numpy array. - pil_rot_img = np.asarray( - PILImage.fromarray(img.asnumpy()[0]).rotate( - rot_radians * 180 / np.pi, resample=PILImage.BILINEAR - ) - ) - - # Rough compare arrays. - np.testing.assert_allclose(rot_img.asnumpy()[0], pil_rot_img, atol=0.25) diff --git a/tests/test_steerable_bases_2d.py b/tests/test_steerable_bases_2d.py new file mode 100644 index 0000000000..863ad9c829 --- /dev/null +++ b/tests/test_steerable_bases_2d.py @@ -0,0 +1,82 @@ +import logging + +import numpy as np +import PIL.Image as PILImage +import pytest + +from aspire.basis import FBBasis2D, FFBBasis2D, FLEBasis2D, FPSWFBasis2D, PSWFBasis2D +from aspire.image import Image +from aspire.utils import gaussian_2d + +logger = logging.getLogger(__name__) + + +# Parameters + +DTYPES = [ + np.float32, + pytest.param(np.float64, marks=pytest.mark.expensive), +] + +BASES = [ + FFBBasis2D, + FBBasis2D, + FLEBasis2D, + PSWFBasis2D, + FPSWFBasis2D, +] + +IMG_SIZES = [ + 32, + pytest.param(31, marks=pytest.mark.expensive), +] + +# Fixtures + + +@pytest.fixture(params=DTYPES, ids=lambda x: f"dtype={x}", scope="module") +def dtype(request): + return request.param + + +@pytest.fixture(params=IMG_SIZES, ids=lambda x: f"img_size={x}", scope="module") +def img_size(request): + return request.param + + +@pytest.fixture(params=BASES, ids=lambda x: f"basis={x}", scope="module") +def basis(request, img_size, dtype): + cls = request.param + # Setup a Basis + basis = cls(img_size, dtype=dtype) + return basis + + +# Basis Rotations + + +def test_basis_rotation_2d(basis): + """ + Test steerable basis rotation performs similar operation to PIL real space image rotation. + + Checks both orientation and rough values. + """ + # Set a rotation amount + rot_radians = np.pi / 6 + + # Create an Image containing a smooth blob. + L = basis.nres + img = Image(gaussian_2d(L, mu=(L // 4, 0), dtype=basis.dtype)) + + # Rotate with ASPIRE Steerable Basis, returning to real space. + rot_img = basis.expand(img).rotate(rot_radians).evaluate() + + # Rotate image with PIL, returning to Numpy array. + pil_rot_img = np.asarray( + PILImage.fromarray(img.asnumpy()[0]).rotate( + rot_radians * 180 / np.pi, resample=PILImage.BILINEAR + ) + ) + + # Rough compare arrays. + np.testing.assert_allclose(rot_img.asnumpy()[0], pil_rot_img, atol=0.25) From 96f967e14fbcd17d9cd14c099aa318b68b9c11b3 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 28 Nov 2023 11:50:10 -0500 Subject: [PATCH 278/294] Add a shared steerable reflection test. --- tests/test_steerable_bases_2d.py | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_steerable_bases_2d.py b/tests/test_steerable_bases_2d.py index 863ad9c829..9a9d181b0d 100644 --- a/tests/test_steerable_bases_2d.py +++ b/tests/test_steerable_bases_2d.py @@ -80,3 +80,38 @@ def test_basis_rotation_2d(basis): # Rough compare arrays. np.testing.assert_allclose(rot_img.asnumpy()[0], pil_rot_img, atol=0.25) + + +def test_basis_reflection_2d(basis): + """ + Test steerable basis reflection performs similar operation to Numpy flips. + + Checks both orientation and rough values. + """ + + # Create an Image containing a smooth blob. + L = basis.nres + img = Image(gaussian_2d(L, mu=(L // 4, L // 5), dtype=basis.dtype)) + + # Reflect with ASPIRE Steerable Basis, returning to real space. + refl_img = basis.expand(img).rotate(0, refl=True).evaluate() + + # Reflect image with Numpy. + # Note for odd images we can simply use Numpy, + # but evens have the expected offset issue + # when compared to a row/col based flip. + flip = np.flipud + if isinstance(basis, PSWFBasis2D): + # TODO, reconcile PSWF reflection axis + flip = np.fliplr + + refl_img_np = flip(img.asnumpy()[0]) + + # Rough compare arrays. + atol = 0.01 + if L % 2 == 0: + # Even images test is crude, + # but is enough ensure flipping without complicating test. + atol = 0.5 + + np.testing.assert_allclose(refl_img.asnumpy()[0], refl_img_np, atol=atol) From e7a8f87e98d962dbbe03569b35f5a8305f5d497c Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 28 Nov 2023 13:14:03 -0500 Subject: [PATCH 279/294] Touch up tests --- tests/test_rotation.py | 4 +++- tests/test_steerable_bases_2d.py | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index aaa1262695..9e0dba4ec6 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -92,7 +92,9 @@ def test_register_rots(rot_obj): def test_register(rot_obj): # These will yield two more distinct sets of random rotations wrt rot_obj set1 = Rotation.generate_random_rotations(NUM_ROTS, dtype=rot_obj.dtype) - set2 = Rotation.generate_random_rotations(NUM_ROTS, dtype=rot_obj.dtype, seed=7) + set2 = Rotation.generate_random_rotations( + NUM_ROTS, dtype=rot_obj.dtype, seed=SEED + 7 + ) # Align both sets of random rotations to rot_obj aligned_rots1 = rot_obj.register(set1) aligned_rots2 = rot_obj.register(set2) diff --git a/tests/test_steerable_bases_2d.py b/tests/test_steerable_bases_2d.py index 9a9d181b0d..88a90ac15d 100644 --- a/tests/test_steerable_bases_2d.py +++ b/tests/test_steerable_bases_2d.py @@ -27,8 +27,8 @@ ] IMG_SIZES = [ - 32, - pytest.param(31, marks=pytest.mark.expensive), + 31, + pytest.param(32, marks=pytest.mark.expensive), ] # Fixtures @@ -68,7 +68,7 @@ def test_basis_rotation_2d(basis): L = basis.nres img = Image(gaussian_2d(L, mu=(L // 4, 0), dtype=basis.dtype)) - # Rotate with ASPIRE Steerable Basis, returning to real space. + # Rotate with an ASPIRE steerable basis, returning to real space. rot_img = basis.expand(img).rotate(rot_radians).evaluate() # Rotate image with PIL, returning to Numpy array. @@ -93,13 +93,13 @@ def test_basis_reflection_2d(basis): L = basis.nres img = Image(gaussian_2d(L, mu=(L // 4, L // 5), dtype=basis.dtype)) - # Reflect with ASPIRE Steerable Basis, returning to real space. + # Reflect with an ASPIRE steerable basis, returning to real space. refl_img = basis.expand(img).rotate(0, refl=True).evaluate() # Reflect image with Numpy. - # Note for odd images we can simply use Numpy, + # Note for odd images we can accurately use Numpy, # but evens have the expected offset issue - # when compared to a row/col based flip. + # when compared to a purely row/col based flip. flip = np.flipud if isinstance(basis, PSWFBasis2D): # TODO, reconcile PSWF reflection axis From 3402710850334bd421bae2fade1de1994500d995 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Wed, 29 Nov 2023 14:20:09 -0500 Subject: [PATCH 280/294] normalize the images in steerable tests --- tests/test_steerable_bases_2d.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_steerable_bases_2d.py b/tests/test_steerable_bases_2d.py index 88a90ac15d..d2cf3cbb4b 100644 --- a/tests/test_steerable_bases_2d.py +++ b/tests/test_steerable_bases_2d.py @@ -66,7 +66,8 @@ def test_basis_rotation_2d(basis): # Create an Image containing a smooth blob. L = basis.nres - img = Image(gaussian_2d(L, mu=(L // 4, 0), dtype=basis.dtype)) + img = gaussian_2d(L, mu=(L // 4, 0), dtype=basis.dtype) + img = Image(img / np.linalg.norm(img)) # Normalize # Rotate with an ASPIRE steerable basis, returning to real space. rot_img = basis.expand(img).rotate(rot_radians).evaluate() @@ -91,7 +92,8 @@ def test_basis_reflection_2d(basis): # Create an Image containing a smooth blob. L = basis.nres - img = Image(gaussian_2d(L, mu=(L // 4, L // 5), dtype=basis.dtype)) + img = gaussian_2d(L, mu=(L // 4, L // 5), dtype=basis.dtype) + img = Image(img / np.linalg.norm(img)) # Normalize # Reflect with an ASPIRE steerable basis, returning to real space. refl_img = basis.expand(img).rotate(0, refl=True).evaluate() From 9876e59b3dc26d375d5740295216e04b3d985bc2 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 30 Nov 2023 08:41:23 -0500 Subject: [PATCH 281/294] Tighten up steerable rotate tol after normalization --- tests/test_steerable_bases_2d.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_steerable_bases_2d.py b/tests/test_steerable_bases_2d.py index d2cf3cbb4b..9777c3d2ed 100644 --- a/tests/test_steerable_bases_2d.py +++ b/tests/test_steerable_bases_2d.py @@ -75,12 +75,12 @@ def test_basis_rotation_2d(basis): # Rotate image with PIL, returning to Numpy array. pil_rot_img = np.asarray( PILImage.fromarray(img.asnumpy()[0]).rotate( - rot_radians * 180 / np.pi, resample=PILImage.BILINEAR + rot_radians * 180 / np.pi, resample=PILImage.BICUBIC ) ) # Rough compare arrays. - np.testing.assert_allclose(rot_img.asnumpy()[0], pil_rot_img, atol=0.25) + np.testing.assert_allclose(rot_img.asnumpy()[0], pil_rot_img, atol=0.15) def test_basis_reflection_2d(basis): From bc3481b827257ab02570e64035108b295385734a Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 30 Nov 2023 11:14:46 -0500 Subject: [PATCH 282/294] refactor eval_coords to handle >2 vols --- src/aspire/source/simulation.py | 57 ++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index f27597b21d..fb1c94078b 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -487,36 +487,49 @@ def eval_coords(self, mean_vol, eig_vols, coords_est): # 0-indexed states vector states = self.states - 1 - coords_true = coords_true[states] + + coords_true = coords_true.T[states] res_norms = res_norms[states] res_inners = res_inners[:, states] - mean_eigs_inners = (mean_vol.to_vec() @ eig_vols.to_vec().T).item() + if coords_est.ndim == 1: + coords_est = coords_est[:, None] + coords_true = coords_true[:, None] + + mean_eigs_inners = mean_vol.to_vec() @ eig_vols.to_vec().T coords_err = coords_true - coords_est - err = np.hypot(res_norms, coords_err) + K = coords_true.shape[-1] + err = np.zeros((K, len(coords_true))) + rel_err = np.zeros((K, len(coords_true))) + corr = np.zeros((K, len(coords_true))) - mean_vol_norm2 = anorm(mean_vol) ** 2 - norm_true = np.sqrt( - coords_true**2 - + mean_vol_norm2 - + 2 * res_inners - + 2 * mean_eigs_inners * coords_true - ) - norm_true = np.hypot(res_norms, norm_true) + for k in range(K): + err[k] = np.hypot(res_norms, coords_err[:, k]) - rel_err = err / norm_true - inner = ( - mean_vol_norm2 - + mean_eigs_inners * (coords_true + coords_est) - + coords_true * coords_est - + res_inners - ) - norm_est = np.sqrt( - coords_est**2 + mean_vol_norm2 + 2 * mean_eigs_inners * coords_est - ) + mean_vol_norm2 = anorm(mean_vol) ** 2 + norm_true = np.sqrt( + coords_true[:, k] ** 2 + + mean_vol_norm2 + + 2 * res_inners + + 2 * mean_eigs_inners[:, k] * coords_true[:, k] + ) + norm_true = np.hypot(res_norms, norm_true) + + rel_err[k] = err[k] / norm_true + inner = ( + mean_vol_norm2 + + mean_eigs_inners[:, k] * (coords_true[:, k] + coords_est[:, k]) + + coords_true[:, k] * coords_est[:, k] + + res_inners + ) + norm_est = np.sqrt( + coords_est[:, k] ** 2 + + mean_vol_norm2 + + 2 * mean_eigs_inners[:, k] * coords_est[:, k] + ) - corr = inner / (norm_true * norm_est) + corr[k] = inner / (norm_true * norm_est) return {"err": err, "rel_err": rel_err, "corr": corr} From b4e5e2006d37dabcb543284b829ed4aef606a27c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 30 Nov 2023 11:17:53 -0500 Subject: [PATCH 283/294] add description for return values. --- src/aspire/source/simulation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/aspire/source/simulation.py b/src/aspire/source/simulation.py index fb1c94078b..e2ef10da12 100644 --- a/src/aspire/source/simulation.py +++ b/src/aspire/source/simulation.py @@ -477,9 +477,10 @@ def eval_coords(self, mean_vol, eig_vols, coords_est): :param mean_vol: A mean volume in the form of a Volume instance. :param eig_vols: A set of eigenvolumes in an Volume instance. - :param coords_est: The estimated coordinates in the affine space defined centered at `mean_vol` and spanned - by `eig_vols`. - :return: + :param coords_est: The estimated coordinates in the affine space defined centered + at `mean_vol` and spanned by `eig_vols`. + :return: Dictionary containing error, relative error, and correlation for each set + of estimated coordinates. """ assert isinstance(mean_vol, Volume) assert isinstance(eig_vols, Volume) From 51577d69448b00d4d381dc607214e90bdc9d1d1c Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 30 Nov 2023 11:43:03 -0500 Subject: [PATCH 284/294] Use 3 volumes in the gallery. --- gallery/tutorials/tutorials/cov3d_simulation.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gallery/tutorials/tutorials/cov3d_simulation.py b/gallery/tutorials/tutorials/cov3d_simulation.py index c30653d5d9..da75f998c8 100644 --- a/gallery/tutorials/tutorials/cov3d_simulation.py +++ b/gallery/tutorials/tutorials/cov3d_simulation.py @@ -33,10 +33,11 @@ num_eigs = 16 # number of eigen-vectors to keep dtype = np.float32 -# Generate a ``Volume`` object for use in the simulation. Here we use a ``LegacyVolume`` which -# by default generates 2 unique random volumes. +# Generate a ``Volume`` object for use in the simulation. Here we use a ``LegacyVolume`` and +# set C = 3 to generate 3 unique random volumes. vols = LegacyVolume( L=img_size, + C=3, dtype=dtype, ).generate() @@ -49,7 +50,7 @@ dtype=dtype, ) -# The Simulation object was created using 2 volumes. +# The Simulation object was created using 3 volumes. num_vols = sim.C # Specify the normal FB basis method for expending the 2D images @@ -159,6 +160,6 @@ logger.info(f'Coordinates (mean correlation) = {np.mean(coords_perf["corr"])}') # Basic Check -assert covar_perf["rel_err"] <= 0.60 -assert np.mean(coords_perf["corr"]) >= 0.98 +assert covar_perf["rel_err"] <= 0.80 +assert np.mean(coords_perf["corr"]) >= 0.97 assert clustering_accuracy >= 0.99 From 5f2748ad272510822d6ac0e35a8d3d63f1e9b40d Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 30 Nov 2023 13:18:02 -0500 Subject: [PATCH 285/294] Fix test --- tests/test_simulation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simulation.py b/tests/test_simulation.py index c3144ae598..659ce95603 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -515,7 +515,7 @@ def testSimulationEvalCoords(self): self.assertTrue( np.allclose( - result["err"][:10], + result["err"][0, :10], [ 1.58382394, 1.58382394, From cf2ef25d39298cde8f87d8ed091bef9f8b63b452 Mon Sep 17 00:00:00 2001 From: Josh Carmichael Date: Thu, 7 Dec 2023 13:38:18 -0500 Subject: [PATCH 286/294] =?UTF-8?q?Bump=20version:=200.12.0=20=E2=86=92=20?= =?UTF-8?q?0.12.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 4 ++-- docs/source/conf.py | 2 +- docs/source/index.rst | 2 +- pyproject.toml | 2 +- src/aspire/__init__.py | 2 +- src/aspire/config_default.yaml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e8869dac22..3d6c2a6025 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.12.0 +current_version = 0.12.1 commit = True tag = True diff --git a/README.md b/README.md index ffdeb55387..278ad11df5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5657281.svg)](https://doi.org/10.5281/zenodo.5657281) [![Downloads](https://static.pepy.tech/badge/aspire/month)](https://pepy.tech/project/aspire) -# ASPIRE - Algorithms for Single Particle Reconstruction - v0.12.0 +# ASPIRE - Algorithms for Single Particle Reconstruction - v0.12.1 The ASPIRE-Python project supersedes [Matlab ASPIRE](https://github.com/PrincetonUniversity/aspire). @@ -20,7 +20,7 @@ For more information about the project, algorithms, and related publications ple Please cite using the following DOI. This DOI represents all versions, and will always resolve to the latest one. ``` -ComputationalCryoEM/ASPIRE-Python: v0.12.0 https://doi.org/10.5281/zenodo.5657281 +ComputationalCryoEM/ASPIRE-Python: v0.12.1 https://doi.org/10.5281/zenodo.5657281 ``` diff --git a/docs/source/conf.py b/docs/source/conf.py index cd83c6a7ff..f9c69025c4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -86,7 +86,7 @@ # built documents. # # The full version, including alpha/beta/rc tags. -release = version = "0.12.0" +release = version = "0.12.1" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/index.rst b/docs/source/index.rst index 15a7335521..3802166549 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,4 +1,4 @@ -Aspire v0.12.0 +Aspire v0.12.1 ============== Algorithms for Single Particle Reconstruction diff --git a/pyproject.toml b/pyproject.toml index a83b4251bc..d45af190a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "aspire" -version = "0.12.0" +version = "0.12.1" description = "Algorithms for Single Particle Reconstruction" readme = "README.md" # Optional requires-python = ">=3.8" diff --git a/src/aspire/__init__.py b/src/aspire/__init__.py index 654112e47d..fbb3ac2d2a 100644 --- a/src/aspire/__init__.py +++ b/src/aspire/__init__.py @@ -12,7 +12,7 @@ from aspire.exceptions import handle_exception # version in maj.min.bld format -__version__ = "0.12.0" +__version__ = "0.12.1" # Setup `confuse` config diff --git a/src/aspire/config_default.yaml b/src/aspire/config_default.yaml index 1768d18f08..cb82637a9b 100644 --- a/src/aspire/config_default.yaml +++ b/src/aspire/config_default.yaml @@ -1,4 +1,4 @@ -version: 0.12.0 +version: 0.12.1 common: # numeric module to use - one of numpy/cupy numeric: numpy From 4eafdec1bf0277f0d720a31b4d2b1f031e885d98 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 8 Dec 2023 08:56:10 -0500 Subject: [PATCH 287/294] Enforce blk diag dtype on assign/insert. --- src/aspire/operators/blk_diag_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aspire/operators/blk_diag_matrix.py b/src/aspire/operators/blk_diag_matrix.py index 7612d38c38..3e493f9525 100644 --- a/src/aspire/operators/blk_diag_matrix.py +++ b/src/aspire/operators/blk_diag_matrix.py @@ -91,7 +91,7 @@ def append(self, blk): :param blk: Block to append (ndarray). """ - self.data.append(blk) + self.data.append(blk.astype(self.dtype, copy=False)) self.nblocks += 1 self.reset_cache() @@ -130,7 +130,7 @@ def __setitem__(self, key, value): Convenience wrapper, setter on self.data. """ - self.data[key] = value + self.data[key] = value.astype(self.dtype, copy=False) self.reset_cache() def __len__(self): From 35264683999a39e5c3045495c6be6076fb1efc37 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Tue, 12 Dec 2023 08:02:44 -0500 Subject: [PATCH 288/294] Workaround Ray dependency conflict Published versions of ray require pydantic<2, but they have not pinned versions --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d45af190a5..3ddeb7ac1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,12 +40,13 @@ dependencies = [ "numpy>=1.21.5", "packaging", "pooch>=1.7.0", + "pillow", "psutil", + "pydantic<2", # Workaround for Ray<2.9 "pyfftw", "pymanopt", "pyshtools", "PyWavelets", - "pillow", "ray", "scipy >= 1.10.0", "scikit-learn", From 347c77954741a7623db1da7b0fe58ead84669c71 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 14 Dec 2023 09:35:07 -0500 Subject: [PATCH 289/294] Fix incorrect basis in Coef wrapper --- src/aspire/basis/fspca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/basis/fspca.py b/src/aspire/basis/fspca.py index d7cc4c96a6..16500918af 100644 --- a/src/aspire/basis/fspca.py +++ b/src/aspire/basis/fspca.py @@ -377,7 +377,7 @@ def evaluate(self, c): # corrected_c[:, self.angular_indices!=0] *= 2 # return corrected_c @ eigvecs.T - return Coef(c.basis, c @ eigvecs.T) + return Coef(self.basis, c @ eigvecs.T) # TODO: Python>=3.8 @cached_property def _get_compressed_indices(self): From e222b1b0d2e87f6c53150e5be80f92a7ef0a5ef4 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 11 Dec 2023 08:53:57 -0500 Subject: [PATCH 290/294] Remove importlib pre-import --- src/aspire/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/aspire/__init__.py b/src/aspire/__init__.py index fbb3ac2d2a..b0d3b5f234 100644 --- a/src/aspire/__init__.py +++ b/src/aspire/__init__.py @@ -1,4 +1,3 @@ -import importlib import logging.config import os import pkgutil @@ -73,4 +72,3 @@ __all__ = [] for _, modname, _ in pkgutil.iter_modules(aspire.__path__): __all__.append(modname) # Add module to __all_ - importlib.import_module(f"aspire.{modname}") # Import the module From 89f1ce1077ea666bd8a73f011a37a9558634b1f9 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 11 Dec 2023 08:55:16 -0500 Subject: [PATCH 291/294] Resolve circular import --- src/aspire/utils/bot_align.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aspire/utils/bot_align.py b/src/aspire/utils/bot_align.py index 68f6f9047c..fc8fbe4fdc 100644 --- a/src/aspire/utils/bot_align.py +++ b/src/aspire/utils/bot_align.py @@ -11,7 +11,6 @@ from numpy.linalg import norm from scipy.optimize import minimize -from aspire.operators import wemd_embed from aspire.utils.rotation import Rotation # Store parameters specific to each loss_type. @@ -64,6 +63,9 @@ def align_BO( Default `None` infers dtype from `vol_ref`. :return: Rotation matrix R_init (without refinement) or (R_init, R_est) (with refinement). """ + # Avoid utils/operators/utils circular import + from aspire.operators import wemd_embed + # Infer dtype dtype = np.dtype(dtype or vol_ref.dtype) From 1053eeb90cc3f07dfb80a257df87f5a18bac3672 Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Mon, 11 Dec 2023 12:02:04 -0500 Subject: [PATCH 292/294] Import aspire.downloader before use --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index c211b5234c..44d799e4ae 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -155,7 +155,7 @@ jobs: - name: Cache Data run: | ASPIREDIR=${{ env.WORK_DIR }} python -c \ - "import aspire; print(aspire.config['common']['cache_dir']); aspire.downloader.emdb_2660()" + "import aspire; print(aspire.config['common']['cache_dir']); import aspire.downloader; aspire.downloader.emdb_2660()" - name: Cleanup run: rm -rf ${{ env.WORK_DIR }} From 8951060bd17b9e8536c0d01edaeba5a994f47d7b Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Thu, 14 Dec 2023 10:55:25 -0500 Subject: [PATCH 293/294] Dynamically load sub module via getattr overload --- src/aspire/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/aspire/__init__.py b/src/aspire/__init__.py index b0d3b5f234..69790bdbb3 100644 --- a/src/aspire/__init__.py +++ b/src/aspire/__init__.py @@ -1,3 +1,4 @@ +import importlib import logging.config import os import pkgutil @@ -68,7 +69,18 @@ sys.excepthook = handle_exception +# Collect set of all module names in package +_modules = set(item[1] for item in pkgutil.iter_modules(aspire.__path__)) +# Automatically add modules __all__ = [] -for _, modname, _ in pkgutil.iter_modules(aspire.__path__): +for modname in _modules: __all__.append(modname) # Add module to __all_ + + +# Dynamically load and return attributes +def __getattr__(attr): + if attr in _modules: + return importlib.import_module(f"aspire.{attr}") + else: + raise AttributeError(f"module {__name__} has no attribute {attr}.") From 8f94a24cc1e6f7d8655566ed6c03eefe19bb407a Mon Sep 17 00:00:00 2001 From: Garrett Wright Date: Fri, 15 Dec 2023 07:26:02 -0500 Subject: [PATCH 294/294] Backtik string err str change --- src/aspire/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aspire/__init__.py b/src/aspire/__init__.py index 69790bdbb3..8e412fbab8 100644 --- a/src/aspire/__init__.py +++ b/src/aspire/__init__.py @@ -83,4 +83,4 @@ def __getattr__(attr): if attr in _modules: return importlib.import_module(f"aspire.{attr}") else: - raise AttributeError(f"module {__name__} has no attribute {attr}.") + raise AttributeError(f"module `{__name__}` has no attribute `{attr}`.")