Development side improvements (#226)

* add python3.10 as a fallback to basepython in tox

* move tests/test_*/*.py > tests/*.py

* fix test_partial_config (fails if cloned directory is not named 'snap')

* estimate an offset for pickled size based on data directory path

bluepysnap/circuit.py

@@ -17,6 +17,7 @@
 
 """Access to circuit data."""
 import logging
+from pathlib import Path
 
 from cached_property import cached_property
 
@@ -41,7 +42,7 @@ def __init__(self, config):
         Returns:
             Circuit: A Circuit object.
         """
-        self._circuit_config_path = config
+        self._circuit_config_path = str(Path(config).absolute())
         self._config = CircuitConfig.from_config(config)
 
         if self.partial_config:

bluepysnap/simulation.py

@@ -58,7 +58,7 @@ def __init__(self, config):
         Returns:
             Simulation: A Simulation object.
         """
-        self._simulation_config_path = Path(config).absolute()
+        self._simulation_config_path = str(Path(config).absolute())
         self._config = SimulationConfig.from_config(config)
 
     @property

tests/test_circuit.py

@@ -9,7 +9,7 @@
 from bluepysnap.exceptions import BluepySnapError
 from bluepysnap.nodes import NodePopulation, Nodes
 
-from utils import TEST_DATA_DIR, copy_test_data, edit_config
+from utils import PICKLED_SIZE_ADJUSTMENT, TEST_DATA_DIR, copy_test_data, edit_config
 
 
 def test_all():
@@ -92,7 +92,7 @@ def test_pickle(tmp_path):
     with open(pickle_path, "rb") as fd:
         circuit = pickle.load(fd)
 
-    assert pickle_path.stat().st_size < 200
+    assert pickle_path.stat().st_size < 60 + PICKLED_SIZE_ADJUSTMENT
     assert list(circuit.edges) == ["default", "default2"]
 
 

tests/test_edges/test_edge_population.py → tests/test_edge_population.py

@@ -17,7 +17,7 @@
 from bluepysnap.sonata_constants import DEFAULT_EDGE_TYPE, Edge
 from bluepysnap.utils import IDS_DTYPE
 
-from utils import TEST_DATA_DIR, copy_test_data, edit_config
+from utils import PICKLED_SIZE_ADJUSTMENT, TEST_DATA_DIR, copy_test_data, edit_config
 
 
 def index_as_ids_dtypes(values):
@@ -655,7 +655,7 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             edge_population = pickle.load(fd)
 
-        assert pickle_path.stat().st_size < 260
+        assert pickle_path.stat().st_size < 130 + PICKLED_SIZE_ADJUSTMENT
         assert edge_population.name == "default"
 
 

tests/test_edges/test_edges.py → tests/test_edges.py

@@ -15,7 +15,7 @@
 from bluepysnap.exceptions import BluepySnapError
 from bluepysnap.utils import IDS_DTYPE
 
-from utils import TEST_DATA_DIR
+from utils import PICKLED_SIZE_ADJUSTMENT, TEST_DATA_DIR
 
 
 class TestEdges:
@@ -784,5 +784,5 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             test_obj = pickle.load(fd)
 
-        assert pickle_path.stat().st_size < 180
+        assert pickle_path.stat().st_size < 100 + PICKLED_SIZE_ADJUSTMENT
         assert test_obj.size == 8

tests/test_nodes/test_node_population.py → tests/test_node_population.py

@@ -21,7 +21,12 @@
 from bluepysnap.sonata_constants import DEFAULT_NODE_TYPE, Node
 from bluepysnap.utils import IDS_DTYPE
 
-from utils import TEST_DATA_DIR, assert_array_equal_strict, create_node_population
+from utils import (
+    PICKLED_SIZE_ADJUSTMENT,
+    TEST_DATA_DIR,
+    assert_array_equal_strict,
+    create_node_population,
+)
 
 
 class TestNodePopulation:
@@ -639,7 +644,7 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             test_obj = pickle.load(fd)
 
-        assert pickle_path.stat().st_size < 210
+        assert pickle_path.stat().st_size < 130 + PICKLED_SIZE_ADJUSTMENT
         assert test_obj.size == 3
 
     def test_filter_properties(self):

tests/test_nodes/test_nodes.py → tests/test_nodes.py

@@ -15,7 +15,7 @@
 from bluepysnap.node_sets import NodeSets
 from bluepysnap.utils import IDS_DTYPE
 
-from utils import TEST_DATA_DIR
+from utils import PICKLED_SIZE_ADJUSTMENT, TEST_DATA_DIR
 
 
 class TestNodes:
@@ -448,5 +448,5 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             test_obj = pickle.load(fd)
 
-        assert pickle_path.stat().st_size < 180
+        assert pickle_path.stat().st_size < 100 + PICKLED_SIZE_ADJUSTMENT
         assert test_obj.size == 7

tests/test_config/test_partial_config.py → tests/test_partial_config.py

@@ -59,7 +59,7 @@ def test_partial_circuit_config_minimal():
     assert circuit.nodes["default"].count() == 3
     assert circuit.nodes["default"].morph is not None
     assert circuit.nodes["default"].models is not None
-    assert circuit.nodes["default"].h5_filepath.endswith("/snap/tests/data/nodes.h5")
+    assert circuit.nodes["default"].h5_filepath == str(TEST_DATA_DIR / "nodes.h5")
     assert circuit.nodes["default"]._properties.spatial_segment_index_dir == ""
 
     assert circuit.nodes.population_names == ["default"]
@@ -87,7 +87,7 @@ def test_partial_circuit_config_minimal():
     assert list(circuit.edges["default"].afferent_edges(0)) == [0]
     assert list(circuit.edges["default"].efferent_edges(0)) == [1, 2]
     assert circuit.edges["default"].iter_connections() is not None
-    assert circuit.edges["default"].h5_filepath.endswith("/snap/tests/data/edges.h5")
+    assert circuit.edges["default"].h5_filepath == str(TEST_DATA_DIR / "edges.h5")
     assert circuit.nodes["default"]._properties.spatial_segment_index_dir == ""
 
     assert circuit.edges.population_names == ["default"]

tests/test_simulation.py

@@ -17,7 +17,7 @@
 from bluepysnap.node_sets import NodeSets
 from bluepysnap.spike_report import PopulationSpikeReport, SpikeReport
 
-from utils import TEST_DATA_DIR, copy_test_data, edit_config
+from utils import PICKLED_SIZE_ADJUSTMENT, TEST_DATA_DIR, copy_test_data, edit_config
 
 
 def test_all():
@@ -115,5 +115,5 @@ def test_pickle(tmp_path):
     with open(pickle_path, "rb") as fd:
         simulation = pickle.load(fd)
 
-    assert pickle_path.stat().st_size < 200
+    assert pickle_path.stat().st_size < 70 + PICKLED_SIZE_ADJUSTMENT
     assert simulation.dt == 0.01

tests/utils.py

@@ -1,6 +1,7 @@
 """Module providing utility functions for the tests"""
 
 import json
+import pickle
 import shutil
 import tempfile
 from contextlib import contextmanager
@@ -17,6 +18,11 @@
 TEST_DIR = Path(__file__).resolve().parent
 TEST_DATA_DIR = TEST_DIR / "data"
 
+# Pickle size tests often fail when run locally. At the moment, the only thing affecting the
+# pickled size is the path length. This is to estimate a safe offset for the size limit
+# based on the pickled size of the path of the test data directory.
+PICKLED_SIZE_ADJUSTMENT = len(pickle.dumps(str(TEST_DATA_DIR.absolute())))
+
 
 @contextmanager
 def setup_tempdir(cleanup=True):

tox.ini

@@ -9,7 +9,7 @@ envlist =
 ignore_basepython_conflict = true
 
 [testenv]
-basepython=python3.11
+basepython=python3.11,python3.10
 deps =
     pytest
     pytest-cov