Merge pull request #12 from mdekstrand/tweak/mem-tests

Gate large-memory tests based on available memory

.github/workflows/test.yml

@@ -19,7 +19,6 @@ jobs:
         - windows
         - ubuntu
         python:
-        - 3.6
         - 3.7
         - 3.8
         - 3.9

conftest.py

@@ -1,6 +1,6 @@
 import logging
 
-from hypothesis import settings
+from hypothesis import settings, HealthCheck
 from pytest import fixture, skip
 from csr import CSR
 from csr.kernels import use_kernel, get_kernel
@@ -38,10 +38,10 @@ def kernel(request):
 
 
 # set up profiles
-settings.register_profile('default', deadline=1500)
+settings.register_profile('default', deadline=2500)
 settings.register_profile('large', settings.get_profile('default'),
                           max_examples=5000, deadline=None)
-settings.register_profile('fast', max_examples=10)
+settings.register_profile('fast', max_examples=20)
 settings.register_profile('nojit', settings.get_profile('fast'),
-                          deadline=None)
+                          deadline=None, suppress_health_check=HealthCheck.all())
 settings.load_profile('default')

csr/test_utils.py

@@ -5,6 +5,7 @@
 import numpy as np
 import scipy.sparse as sps
 
+import psutil
 from hypothesis import settings, HealthCheck
 import hypothesis.strategies as st
 import hypothesis.extra.numpy as nph
@@ -20,12 +21,12 @@ def fractions(**kwargs):
 def csrs(draw, nrows=None, ncols=None, nnz=None, values=None):
     "Draw CSR matrices by generating COO data."
     if ncols is None:
-        ncols = draw(st.integers(1, 100))
+        ncols = draw(st.integers(1, 80))
     elif not isinstance(ncols, int):
         ncols = draw(ncols)
 
     if nrows is None:
-        nrows = draw(st.integers(1, 100))
+        nrows = draw(st.integers(1, 80))
     elif not isinstance(nrows, int):
         nrows = draw(nrows)
 
@@ -84,3 +85,9 @@ def csr_slow(divider=2):
     dft = settings.default
     return settings(dft, deadline=None, suppress_health_check=HealthCheck.all(),
                     max_examples=dft.max_examples // divider)
+
+
+def has_memory(req_gb=32):
+    req_bytes = req_gb * 1024 * 1024 * 1024
+    vm = psutil.virtual_memory()
+    return vm.total >= req_bytes

pyproject.toml

@@ -9,10 +9,10 @@ author-email = "michaelekstrand@boisestate.edu"
 home-page = "https://csr.lenskit.org"
 classifiers = ["License :: OSI Approved :: MIT License"]
 description-file = "README.md"
-requires-python = ">= 3.6.1"
+requires-python = ">= 3.7"
 requires = [
     "numba >=0.51,<0.54",
-    "numpy >=1.16",
+    "numpy >=1.17",
     "scipy ==1.*"
 ]
 
@@ -22,7 +22,8 @@ test = [
     "pytest-doctestplus >=0.9",
     "pytest-benchmark >=3",
     "pytest-cov >=2.12",
-    "hypothesis >=6"
+    "hypothesis >=6",
+    "psutil >=5"
 ]
 dev = [
     "flake8",

tests/test_initialize.py

@@ -1,4 +1,5 @@
 from csr import CSR, constructors
+from csr.test_utils import has_memory
 import numpy as np
 
 import pytest
@@ -52,6 +53,7 @@ def test_create_from_sizes(data, nrows, ncols):
     assert len(csr.colinds) == np.sum(sizes)
 
 
+@pytest.mark.skipif(not has_memory(12), reason='insufficient memory')
 def test_large_init():
     # 10M * 500 = 5B >= INT_MAX
     nrows = 10000000
@@ -74,6 +76,7 @@ def test_large_init():
     assert csr.rowptrs.dtype == np.dtype('i8')
 
 
+@pytest.mark.skipif(not has_memory(12), reason='insufficient memory')
 def test_large_empty():
     # 10M * 250 = 2.5B >= INT_MAX
     nrows = 10000000

tests/test_mkl.py

@@ -8,7 +8,7 @@
 import numpy as np
 
 from csr import CSR
-from csr.test_utils import csrs
+from csr.test_utils import csrs, has_memory
 
 from pytest import skip, mark
 from hypothesis import given
@@ -20,7 +20,7 @@
 try:
     from csr.kernels import mkl
 except ImportError:
-    pytestmark = skip("MKL is not available")
+    pytestmark = mark.skip("MKL is not available")
 
 _log = logging.getLogger(__name__)
 
@@ -90,6 +90,7 @@ def test_multiply_transpose_lim():
         tmm.test_multiply_transpose(mkl)
 
 
+@mark.skipif(not has_memory(32), reason='insufficient memory')
 def test_large_mult_vec():
     # 10M * 500 = 2.5B >= INT_MAX
     nrows = 10000000