Resolve #412 (#428)

* Add pytest option --level and corresponding stratification of tests in test_radon_svmbir.py

* Test level option determines alpha range

* Test level option determines dtype range

* Tests internally depend on level parameter

* Fix style violation

* Set test level on osx

* Add note on level semantics and set levels in CI tests

* Add note on level semantics and set levels in CI tests

* Move Diagonal tests to separate file

* Add pytest version requirement

* Update sbmodule

* Use new durations files with pytest-split

* Remove old test durations file

* Debugging attempt

* Debugging attempt

* Move pytest_addoption call to root conftest.py

* Restore --level options

* Remove secondary conftest.py

* Clean up

* Remove unused function

* Update docs

* Update developer list

.github/workflows/pytest_macos.yml

@@ -70,7 +70,10 @@ jobs:
         run: pip install -e .
       # Run unit tests
       - name: Run main unit tests
-        run: pytest -x --splits 5 --group ${{ matrix.group }}
+        run: |
+          DURATIONS_FILE=$(mktemp)
+          bzcat data/pytest/durations_macos.bz2 > $DURATIONS_FILE
+          pytest -x --durations-path $DURATIONS_FILE --splits 5 --group ${{ matrix.group }} --level 1
       # Run doc tests
       - name: Run doc tests
         if: matrix.group == 1

.github/workflows/pytest_ubuntu.yml

@@ -73,7 +73,10 @@ jobs:
         run: pip install -e .
       # Run unit tests
       - name: Run main unit tests
-        run: pytest -x --cov --splits 5 --group ${{ matrix.group }}
+        run: |
+          DURATIONS_FILE=$(mktemp)
+          bzcat data/pytest/durations_ubuntu.bz2 > $DURATIONS_FILE
+          pytest -x --cov --durations-path $DURATIONS_FILE --splits 5 --group ${{ matrix.group }} --level 2
       # Upload coverage data
       - name: Upload coverage
         uses: actions/upload-artifact@v3

conftest.py

@@ -3,7 +3,8 @@
 because `np` is used in doc strings for jax functions
 (e.g. `linear_transpose`) that get pulled into `scico/__init__.py`.
 
-Also allows `snp` to be used without explicitly importing
+Also allow `snp` to be used without explicitly importing, and add
+`level` parameter.
 """
 
 import numpy as np
@@ -18,3 +19,16 @@ def add_modules(doctest_namespace):
     """Add common modules for use in docstring examples."""
     doctest_namespace["np"] = np
     doctest_namespace["snp"] = snp
+
+
+def pytest_addoption(parser, pluginmanager):
+    """Add --level pytest option.
+
+    Level definitions:
+      1  Critical tests only
+      2  Skip tests that do have a significant impact on coverage
+      3  All tests
+    """
+    parser.addoption(
+        "--level", action="store", default=3, type=int, help="Set test level to be run"
+    )

dev_requirements.txt

@@ -1,5 +1,5 @@
 pylint
-pytest
+pytest>=7.3.0
 pytest-runner
 packaging
 pre-commit

docs/source/contributing.rst

@@ -290,14 +290,20 @@ version of ``scico`` by
    pytest --pyargs scico
 
 When any significant changes are made to the test suite, the ``pytest-split`` test
-time database file ``.test_durations`` in the repository root directory should be
-updated
+time database files in ``data/pytest`` should be updated using
 
 ::
 
-   pytest --store-durations
+   pytest --store-durations --durations-path data/pytest/durations_ubuntu --level 2
 
-and the changes should be committed into the repository.
+(for Ubuntu CI), and
+
+::
+
+   pytest --store-durations --durations-path data/pytest/durations_macos --level 1
+
+(for MacOS CI). These updated files should be bzipped and committed into the
+``scico-data`` repository, replacing the current versions.
 
 
 Test Coverage

docs/source/team.rst

@@ -4,13 +4,18 @@ Developers
 Core Developers
 ---------------
 
+- `Cristina Garcia Cardona <https://github.com/crstngc>`_
+- `Michael McCann <https://github.com/Michael-T-McCann>`_
+- `Brendt Wohlberg <https://github.com/bwohlberg>`_
+
+
+Emeritus Developers
+-------------------
+
 - `Thilo Balke <https://github.com/tbalke>`_
 - `Fernando Davis <https://github.com/FernandoDavis>`_
-- `Cristina Garcia Cardona <https://github.com/crstngc>`_
 - `Soumendu Majee <https://github.com/smajee>`_
-- `Michael McCann <https://github.com/Michael-T-McCann>`_
 - `Luke Pfister <https://github.com/lukepfister>`_
-- `Brendt Wohlberg <https://github.com/bwohlberg>`_
 
 
 Contributors

scico/flax/examples/data_generation.py

@@ -120,7 +120,7 @@ def generate_foam2_images(seed: float, size: int, ndata: int) -> Array:
         foam = Foam2(size_range=[0.075, 0.0025], gap=1e-3, porosity=1)
         saux[i, ..., 0] = discrete_phantom(foam, size=size)
 
-    # Normalize
+    # normalize
     saux = saux / np.max(saux, axis=(1, 2), keepdims=True)
 
     return saux

scico/linop/radon_svmbir.py

@@ -475,10 +475,3 @@ def __init__(
             raise ValueError(
                 "Parameter is_masked must be False for the TomographicProjector in SVMBIRSquaredL2Loss."
             )
-
-
-def _unsqueeze(x: snp.Array, input_shape: Shape) -> snp.Array:
-    """If x is 2D, make it 3D according to the SVMBIR convention."""
-    if len(input_shape) == 2:
-        x = x[snp.newaxis, :, :]
-    return x

scico/test/functional/test_core.py

@@ -17,9 +17,23 @@
 from scico.random import randn
 
 NO_BLOCK_ARRAY = [functional.L21Norm, functional.L1MinusL2Norm, functional.NuclearNorm]
-NO_COMPLEX = [
-    functional.NonNegativeIndicator,
-]
+NO_COMPLEX = [functional.NonNegativeIndicator]
+
+
+def pytest_generate_tests(metafunc):
+    level = int(metafunc.config.getoption("--level"))
+    alpha_range = [1e-2, 1e-1, 1e0, 1e1]
+    dtype_range = [np.float32, np.complex64, np.float64, np.complex128]
+    if level == 2:
+        alpha_range = [1e-2, 1e1]
+        dtype_range = [np.float32, np.complex64, np.float64]
+    elif level < 2:
+        alpha_range = [1e-2]
+        dtype_range = [np.float32, np.complex64]
+    if "alpha" in metafunc.fixturenames:
+        metafunc.parametrize("alpha", alpha_range)
+    if "test_dtype" in metafunc.fixturenames:
+        metafunc.parametrize("test_dtype", dtype_range)
 
 
 class ProxTestObj:
@@ -31,9 +45,9 @@ def __init__(self, dtype):
         self.vz = snp.zeros((3, 4), dtype=dtype)
 
 
-@pytest.fixture(params=[np.float32, np.complex64, np.float64, np.complex128])
-def test_prox_obj(request):
-    return ProxTestObj(request.param)
+@pytest.fixture
+def test_prox_obj(test_dtype):
+    return ProxTestObj(test_dtype)
 
 
 class SeparableTestObject:
@@ -51,9 +65,9 @@ def __init__(self, dtype):
         self.vb = snp.blockarray([self.v1, self.v2])
 
 
-@pytest.fixture(params=[np.float32, np.complex64, np.float64, np.complex128])
-def test_separable_obj(request):
-    return SeparableTestObject(request.param)
+@pytest.fixture
+def test_separable_obj(test_dtype):
+    return SeparableTestObject(test_dtype)
 
 
 def test_separable_eval(test_separable_obj):
@@ -102,7 +116,6 @@ def __init__(self, delta=1.0):
 
 class TestNormProx:
 
-    alphalist = [1e-2, 1e-1, 1e0, 1e1]
     normlist = [
         functional.L0Norm,
         functional.L1Norm,
@@ -119,15 +132,13 @@ class TestNormProx:
     normlist_blockarray_ready = set(normlist.copy()) - set(NO_BLOCK_ARRAY)
 
     @pytest.mark.parametrize("norm", normlist)
-    @pytest.mark.parametrize("alpha", alphalist)
     def test_prox(self, norm, alpha, test_prox_obj):
         nrmobj = norm()
         nrm = nrmobj.__call__
         prx = nrmobj.prox
         pf = prox_test(test_prox_obj.v, nrm, prx, alpha)
 
     @pytest.mark.parametrize("norm", normlist)
-    @pytest.mark.parametrize("alpha", alphalist)
     def test_conj_prox(self, norm, alpha, test_prox_obj):
         nrmobj = norm()
         v = test_prox_obj.v
@@ -137,7 +148,6 @@ def test_conj_prox(self, norm, alpha, test_prox_obj):
         np.testing.assert_allclose(lhs, rhs, rtol=1e-6, atol=0.0)
 
     @pytest.mark.parametrize("norm", normlist_blockarray_ready)
-    @pytest.mark.parametrize("alpha", alphalist)
     def test_prox_blockarray(self, norm, alpha, test_prox_obj):
         nrmobj = norm()
         nrm = nrmobj.__call__
@@ -168,7 +178,6 @@ def test_scaled_attrs(self, norm, test_prox_obj):
         assert scaled.scale == test_prox_obj.scalar
 
     @pytest.mark.parametrize("norm", normlist)
-    @pytest.mark.parametrize("alpha", alphalist)
     def test_scaled_eval(self, norm, alpha, test_prox_obj):
 
         unscaled = norm()
@@ -179,7 +188,6 @@ def test_scaled_eval(self, norm, alpha, test_prox_obj):
         np.testing.assert_allclose(a, b)
 
     @pytest.mark.parametrize("norm", normlist)
-    @pytest.mark.parametrize("alpha", alphalist)
     def test_scaled_prox(self, norm, alpha, test_prox_obj):
         # Test prox
         unscaled = norm()
@@ -237,7 +245,6 @@ class TestProj:
 
     cnstrlist = [functional.NonNegativeIndicator, functional.L2BallIndicator]
     sdistlist = [functional.SetDistance, functional.SquaredSetDistance]
-    alphalist = [1e-2, 1e-1, 1e0, 1e1]
 
     @pytest.mark.parametrize("cnstr", cnstrlist)
     def test_prox(self, cnstr, test_proj_obj):
@@ -264,7 +271,6 @@ def test_prox_scale_invariance(self, cnstr, test_proj_obj):
 
     @pytest.mark.parametrize("sdist", sdistlist)
     @pytest.mark.parametrize("cnstr", cnstrlist)
-    @pytest.mark.parametrize("alpha", alphalist)
     def test_setdistance(self, sdist, cnstr, alpha, test_proj_obj):
         if cnstr in NO_COMPLEX and snp.util.is_complex_dtype(test_proj_obj.v.dtype):
             return