From 343ab1d2b00b60ef6388f875cb585238f61437bf Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 14 Jun 2021 03:11:03 +0100 Subject: [PATCH 01/33] ENH: Validate experiment parameters before loading --- fissa/core.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/fissa/core.py b/fissa/core.py index 1ba2ab08..d83d3706 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -880,7 +880,7 @@ def clear_separated(self, verbosity=None): if verbosity >= 1 and keys_cleared: print("Cleared {}".format(", ".join(repr(k) for k in keys_cleared))) - def load(self, path=None): + def load(self, path=None, force=False): r""" Load data from cache file in npz format. @@ -894,8 +894,12 @@ def load(self, path=None): Default behaviour is to use the :attr:`folder` parameter which was provided when the object was initialised is used (``experiment.folder``). + force : bool, optional + Whether to load the cache even if its experiment parameters differ + from the properties of this experiment. Default is ``False``. """ dynamic_properties = ["nCell", "nTrials"] + validate_fields = ["alpha", "expansion", "method", "nRegions"] if path is None: if self.folder is None: raise ValueError( @@ -912,6 +916,19 @@ def load(self, path=None): if self.verbosity >= 1: print("Reloading data from cache {}".format(path)) cache = np.load(path, allow_pickle=True) + if not force: + for field in validate_fields: + if ( + field in cache.files + and getattr(self, field, None) is not None + and cache[field] != getattr(self, field) + ): + raise ValueError( + "Cache value {} ({}) does not match the current" + " experiment value {}.".format( + field, cache[field], getattr(self, field) + ) + ) for field in cache.files: if field in dynamic_properties: continue From 9a19eca112cf0b8524597c315d709673100fda81 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 17 Jun 2021 01:57:52 +0100 Subject: [PATCH 02/33] MNT: Include expansion and nRegions in separated npz --- fissa/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fissa/core.py b/fissa/core.py index d83d3706..3640798b 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1327,11 +1327,13 @@ def save_separated(self, destination=None): "alpha", "deltaf_raw", "deltaf_result", + "expansion", "info", "max_iter", "max_tries", "method", "mixmat", + "nRegions", "sep", "tol", "result", From b1baa445b91b2103cb18b3942c721e5bfe17dfdd Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 17 Jun 2021 01:58:02 +0100 Subject: [PATCH 03/33] ENH: More validate when loading --- fissa/core.py | 124 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 22 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 3640798b..5e4dbd4e 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -880,7 +880,7 @@ def clear_separated(self, verbosity=None): if verbosity >= 1 and keys_cleared: print("Cleared {}".format(", ".join(repr(k) for k in keys_cleared))) - def load(self, path=None, force=False): + def load(self, path=None, force=False, skip_clear=False): r""" Load data from cache file in npz format. @@ -897,9 +897,38 @@ def load(self, path=None, force=False): force : bool, optional Whether to load the cache even if its experiment parameters differ from the properties of this experiment. Default is ``False``. + skip_clear : bool, optional + Whether to skip clearing values before loading. Default is ``False``. """ dynamic_properties = ["nCell", "nTrials"] - validate_fields = ["alpha", "expansion", "method", "nRegions"] + ValGroup = collections.namedtuple( + "ValGroup", + ["category", "validators", "fields", "clearif", "clearfn"], + ) + validation_groups = [ + ValGroup( + "prepared", + ["expansion", "nRegions"], + ["deltaf_raw", "means", "nCell", "raw", "roi_polys"], + ["raw"], + self.clear, + ), + ValGroup( + "separated", + [ + "alpha", + "nRegions", + "expansion", + "max_iter", + "max_tries", + "method", + "tol", + ], + ["deltaf_result", "info", "mixmat", "sep", "result"], + ["result"], + self.clear_separated, + ), + ] if path is None: if self.folder is None: raise ValueError( @@ -916,29 +945,80 @@ def load(self, path=None, force=False): if self.verbosity >= 1: print("Reloading data from cache {}".format(path)) cache = np.load(path, allow_pickle=True) - if not force: - for field in validate_fields: - if ( - field in cache.files - and getattr(self, field, None) is not None - and cache[field] != getattr(self, field) - ): - raise ValueError( - "Cache value {} ({}) does not match the current" - " experiment value {}.".format( - field, cache[field], getattr(self, field) + if force: + for field in cache.files: + if field in dynamic_properties: + continue + setattr(self, field, cache[field]) + return + set_fields = set() + for category, validators, fields, clearif, clearfn in validation_groups: + valid = True + validation_errors = [] + for validator in validators: + if validator not in cache or cache[validator] is None: + # If the validator is not set in the cache, we can't + # verify that the cached data is compatible. + valid = False + break + if getattr(self, validator, None) is None: + # If the validator is not yet set locally, it is fine to + # overwrite it. + continue + if getattr(self, validator) != cache[validator]: + # If the validator is set and doesn't match the value in + # the cache, we will raise an error. + validation_errors.append( + " {}: Experiment (ours) {}, Cache (theirs) {}".format( + validator, getattr(self, validator), cache[validator] ) ) - for field in cache.files: - if field in dynamic_properties: + if len(validation_errors) > 0: + raise ValueError( + "Experiment parameter value(s) in {} do not match the" + " current experiment values:\n{}".format( + path, "\n".join(validation_errors) + ) + ) + if not valid: continue - value = cache[field] - if np.array_equal(value, None): - value = None - elif value.ndim == 0: - # Handle loading scalars - value = value.item() - setattr(self, field, value) + # Wipe the values currently held before setting new values + if not skip_clear and clearif in cache.files: + clearfn() + # All the validators were valid, so we are okay to load the fields + for validator in validators: + # Load all the validators, overwriting our local values if None + if getattr(self, validator, None) is None: + print( + "Adopting value {}={} from {}".format( + validator, cache[validator], path + ) + ) + setattr(self, validator, cache[validator]) + set_fields.add(validator) + for field in fields: + if field not in cache or field in dynamic_properties: + continue + value = cache[field] + if np.array_equal(value, None): + value = None + elif value.ndim == 0: + # Handle loading scalars + value = value.item() + setattr(self, field, value) + set_fields.add(field) + print("Loaded {} data from {}".format(category, path)) + + # Check there weren't any left over fields in the cache which + # were left unloaded + unset_fields = [] + for field in cache.files: + if field not in set_fields: + unset_fields.append(field) + if len(unset_fields) > 0: + print( + "Warning: field(s) {} in {} were not loaded.".format(unset_fields, path) + ) def separation_prep(self, redo=False): r""" From 9cae594c59fbf007162b7680deb05e38810f8e8c Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 16:27:38 +0100 Subject: [PATCH 04/33] TST: Need to use force during load None test now --- fissa/tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 2980e1d9..e171d428 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -983,7 +983,7 @@ def test_load_none(self): fname = os.path.join(self.output_dir, "dummy.npz") np.savez_compressed(fname, **{field: None for field in fields}) # Load the file and check the data appears as None, not np.array(None) - exp.load(fname) + exp.load(fname, force=True) for field in fields: self.assertIs(getattr(exp, field), None) From 47f185dcbfa906fdff79be6c017cc4bfb5eb8d4f Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 16:27:53 +0100 Subject: [PATCH 05/33] TST: Add scalar loading test --- fissa/tests/test_core.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index e171d428..c51dccd6 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -987,6 +987,21 @@ def test_load_none(self): for field in fields: self.assertIs(getattr(exp, field), None) + def test_load_scalar(self): + """Behaviour when loading a cache containing scalar values.""" + fields = ["raw", "result", "deltaf_result"] + exp = core.Experiment(self.images_dir, self.roi_zip_path, self.output_dir) + # Set the fields to be something other than `None` + for field in fields: + setattr(exp, field, 42) + # Make a save file which contains values set to `None` + fname = os.path.join(self.output_dir, "dummy.npz") + np.savez_compressed(fname, **{field: 1337 for field in fields}) + # Load the file and check the data appears as None, not np.array(None) + exp.load(fname, force=True) + for field in fields: + self.assertEqual(getattr(exp, field), 1337) + @unittest.expectedFailure def test_badprepcache_init1(self): """ From 3d61337e540f6e4f98dfb27a7717dc8151762b3d Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 16:34:02 +0100 Subject: [PATCH 06/33] TST: Test with non-force loading of None and scalar --- fissa/tests/test_core.py | 48 +++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index c51dccd6..a88f5f5c 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -972,36 +972,72 @@ def test_load_empty_sep(self): exp.separate() self.compare_output(exp) - def test_load_none(self): - """Behaviour when loading a cache containing None.""" + def test_load_none_force(self): + """Behaviour when forcibly loading a cache containing None.""" fields = ["raw", "result", "deltaf_result"] - exp = core.Experiment(self.images_dir, self.roi_zip_path, self.output_dir) + exp = core.Experiment(self.images_dir, self.roi_zip_path) # Set the fields to be something other than `None` for field in fields: setattr(exp, field, 42) # Make a save file which contains values set to `None` fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) np.savez_compressed(fname, **{field: None for field in fields}) # Load the file and check the data appears as None, not np.array(None) exp.load(fname, force=True) for field in fields: self.assertIs(getattr(exp, field), None) - def test_load_scalar(self): - """Behaviour when loading a cache containing scalar values.""" + def test_load_none(self): + """Behaviour when loading a cache containing None.""" + kwargs = {"expansion": 0.213, "nRegions": 2} + fields = {"raw": None} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Set the fields to be something other than `None` + for field in fields: + setattr(exp, field, 42) + # Make a save file which contains values set to `None` + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data appears as None, not np.array(None) + exp.load(fname) + for field in fields: + self.assertIs(getattr(exp, field), None) + + def test_load_scalar_force(self): + """Behaviour when forcibly loading a cache containing scalar values.""" fields = ["raw", "result", "deltaf_result"] - exp = core.Experiment(self.images_dir, self.roi_zip_path, self.output_dir) + exp = core.Experiment(self.images_dir, self.roi_zip_path) # Set the fields to be something other than `None` for field in fields: setattr(exp, field, 42) # Make a save file which contains values set to `None` fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) np.savez_compressed(fname, **{field: 1337 for field in fields}) # Load the file and check the data appears as None, not np.array(None) exp.load(fname, force=True) for field in fields: self.assertEqual(getattr(exp, field), 1337) + def test_load_scalar(self): + """Behaviour when loading a cache containing None.""" + kwargs = {"expansion": 0.213, "nRegions": 2} + fields = {"raw": 1337} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Set the fields to be something other than `None` + for field in fields: + setattr(exp, field, 42) + # Make a save file which contains values set to `None` + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data appears as None, not np.array(None) + exp.load(fname) + for field in fields: + self.assertEqual(getattr(exp, field), 1337) + @unittest.expectedFailure def test_badprepcache_init1(self): """ From dd3b9e9931af7d14e10c549481d5e40f3d07a98f Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 17:24:17 +0100 Subject: [PATCH 07/33] TST: Loosen checks for text in stdout not necessarily at the start --- fissa/tests/test_core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index a88f5f5c..0a06afe6 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -851,17 +851,17 @@ def test_load_cache_piecemeal(self): capture_pre = self.capsys.readouterr() # Clear stdout exp = core.Experiment(image_path, roi_path, self.output_dir) capture_post = self.recapsys(capture_pre) # Capture and then re-output - self.assert_starts_with(capture_post.out, "Reloading data") + self.assertTrue("oading data" in capture_post.out) # Ensure previous cache is loaded again when we run separation_prep capture_pre = self.capsys.readouterr() # Clear stdout exp.separation_prep() capture_post = self.recapsys(capture_pre) - self.assert_starts_with(capture_post.out, "Reloading data") + self.assertTrue("oading data" in capture_post.out) # Ensure previous cache is loaded again when we run separate capture_pre = self.capsys.readouterr() # Clear stdout exp.separate() capture_post = self.recapsys(capture_pre) - self.assert_starts_with(capture_post.out, "Reloading data") + self.assertTrue("oading data" in capture_post.out) # Check the contents loaded from cache self.compare_output(exp) @@ -878,12 +878,12 @@ def test_load_cached_prep(self): capture_pre = self.capsys.readouterr() # Clear stdout exp = core.Experiment(image_path, roi_path, self.output_dir, verbosity=3) capture_post = self.recapsys(capture_pre) # Capture and then re-output - self.assertTrue("Reloading data" in capture_post.out) + self.assertTrue("oading data" in capture_post.out) # Ensure previous cache is loaded again when we run separation_prep capture_pre = self.capsys.readouterr() # Clear stdout exp.separation_prep() capture_post = self.recapsys(capture_pre) # Capture and then re-output - self.assertTrue("Reloading data" in capture_post.out) + self.assertTrue("oading data" in capture_post.out) # Since we did not run and cache separate, this needs to run now capture_pre = self.capsys.readouterr() # Clear stdout exp.separate() From 7e3c030b72b70798217f171e88b9f59557a09c10 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 17:26:21 +0100 Subject: [PATCH 08/33] TST: More loading tests --- fissa/tests/test_core.py | 101 ++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 18 deletions(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 0a06afe6..4b0bc2c7 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -972,6 +972,23 @@ def test_load_empty_sep(self): exp.separate() self.compare_output(exp) + def test_load_npz(self): + """Test whether npz file is loaded by load method.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.array([[1, 2], [3, 4]])} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Set the fields to be something other than `None` + for key in fields: + setattr(exp, key, 42.24) + # Make a save file which contains values set to `None` + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data appears correctly + exp.load(fname) + for key, value in fields.items(): + self.assertEqual(getattr(exp, key), value) + def test_load_none_force(self): """Behaviour when forcibly loading a cache containing None.""" fields = ["raw", "result", "deltaf_result"] @@ -990,53 +1007,101 @@ def test_load_none_force(self): def test_load_none(self): """Behaviour when loading a cache containing None.""" - kwargs = {"expansion": 0.213, "nRegions": 2} + kwargs = {"expansion": 0.213, "nRegions": 3} fields = {"raw": None} exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) # Set the fields to be something other than `None` - for field in fields: - setattr(exp, field, 42) + for key in fields: + setattr(exp, key, 42) # Make a save file which contains values set to `None` fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) np.savez_compressed(fname, **kwargs, **fields) # Load the file and check the data appears as None, not np.array(None) exp.load(fname) - for field in fields: - self.assertIs(getattr(exp, field), None) + for key, value in fields.items(): + self.assertIs(getattr(exp, key), value) def test_load_scalar_force(self): """Behaviour when forcibly loading a cache containing scalar values.""" fields = ["raw", "result", "deltaf_result"] exp = core.Experiment(self.images_dir, self.roi_zip_path) - # Set the fields to be something other than `None` - for field in fields: - setattr(exp, field, 42) - # Make a save file which contains values set to `None` + # Make a save file which contains values set to a scalar fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) np.savez_compressed(fname, **{field: 1337 for field in fields}) - # Load the file and check the data appears as None, not np.array(None) + # Load the file and check the data appears correctly exp.load(fname, force=True) for field in fields: self.assertEqual(getattr(exp, field), 1337) def test_load_scalar(self): """Behaviour when loading a cache containing None.""" - kwargs = {"expansion": 0.213, "nRegions": 2} + kwargs = {"expansion": 0.213, "nRegions": 3} fields = {"raw": 1337} exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) - # Set the fields to be something other than `None` - for field in fields: - setattr(exp, field, 42) - # Make a save file which contains values set to `None` + # Make a save file which contains values set to a scalar` fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) np.savez_compressed(fname, **kwargs, **fields) - # Load the file and check the data appears as None, not np.array(None) + # Load the file and check the data appears correctly exp.load(fname) - for field in fields: - self.assertEqual(getattr(exp, field), 1337) + for key, value in fields.items(): + self.assertEqual(getattr(exp, key), value) + + def test_load_wrong_nRegions(self): + """Test load doesn't load analysis from wrong nRegions param.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.array([[1, 2], [3, 4]])} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"], + nRegions=kwargs["expansion"] + 1, + ) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data is not loaded + with self.assertRaises(ValueError): + exp.load(fname) + + def test_load_wrong_in_init(self): + """Test load doesn't load analysis from wrong nRegions param during init.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.array([[1, 2], [3, 4]])} + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "preparation.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data is not loaded + with self.assertRaises(ValueError): + core.Experiment( + self.images_dir, + self.roi_zip_path, + self.output_dir, + expansion=kwargs["expansion"], + nRegions=kwargs["nRegions"] + 1, + ) + + def test_load_wrong_expansion(self): + """Test load doesn't load analysis from wrong expansion param.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.array([[1, 2], [3, 4]])} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"] + 1, + nRegions=kwargs["nRegions"], + ) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data is not loaded + with self.assertRaises(ValueError): + exp.load(fname) @unittest.expectedFailure def test_badprepcache_init1(self): From 8bc3ba3cef53243b055eba599e3b0f512bf9aca1 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 17:31:10 +0100 Subject: [PATCH 09/33] BUG: Loading None/scalar when validator or forced --- fissa/core.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 5e4dbd4e..245f1c23 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -949,7 +949,13 @@ def load(self, path=None, force=False, skip_clear=False): for field in cache.files: if field in dynamic_properties: continue - setattr(self, field, cache[field]) + value = cache[field] + if np.array_equal(value, None): + value = None + elif value.ndim == 0: + # Handle loading scalars + value = value.item() + setattr(self, field, value) return set_fields = set() for category, validators, fields, clearif, clearfn in validation_groups: @@ -988,13 +994,15 @@ def load(self, path=None, force=False, skip_clear=False): # All the validators were valid, so we are okay to load the fields for validator in validators: # Load all the validators, overwriting our local values if None + value = cache[validator] + if np.array_equal(value, None): + value = None + elif value.ndim == 0: + # Handle loading scalars + value = value.item() if getattr(self, validator, None) is None: - print( - "Adopting value {}={} from {}".format( - validator, cache[validator], path - ) - ) - setattr(self, validator, cache[validator]) + print("Adopting value {}={} from {}".format(validator, value, path)) + setattr(self, validator, value) set_fields.add(validator) for field in fields: if field not in cache or field in dynamic_properties: From dacae62084e12ec5332ffc0511ecb552a8bb7efc Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 17:32:39 +0100 Subject: [PATCH 10/33] MNT: Only print loading debug messages with verbosity --- fissa/core.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 245f1c23..489f1759 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1001,7 +1001,12 @@ def load(self, path=None, force=False, skip_clear=False): # Handle loading scalars value = value.item() if getattr(self, validator, None) is None: - print("Adopting value {}={} from {}".format(validator, value, path)) + if self.verbosity >= 2: + print( + "Adopting value {}={} from {}".format( + validator, value, path + ) + ) setattr(self, validator, value) set_fields.add(validator) for field in fields: @@ -1015,7 +1020,8 @@ def load(self, path=None, force=False, skip_clear=False): value = value.item() setattr(self, field, value) set_fields.add(field) - print("Loaded {} data from {}".format(category, path)) + if self.verbosity >= 2: + print("Loaded {} data from {}".format(category, path)) # Check there weren't any left over fields in the cache which # were left unloaded @@ -1023,7 +1029,7 @@ def load(self, path=None, force=False, skip_clear=False): for field in cache.files: if field not in set_fields: unset_fields.append(field) - if len(unset_fields) > 0: + if len(unset_fields) > 0 and self.verbosity >= 1: print( "Warning: field(s) {} in {} were not loaded.".format(unset_fields, path) ) From 38c452bfe068b9c4e517e5f6d092106bee593678 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 17:33:31 +0100 Subject: [PATCH 11/33] MNT: Remove re- from loading message --- fissa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fissa/core.py b/fissa/core.py index 489f1759..835496ce 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -943,7 +943,7 @@ def load(self, path=None, force=False, skip_clear=False): self.load(fullfname) return if self.verbosity >= 1: - print("Reloading data from cache {}".format(path)) + print("Loading data from cache {}".format(path)) cache = np.load(path, allow_pickle=True) if force: for field in cache.files: From 0d48ca5ec30aafb2e8b12e3ba372e75671c069fd Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Mon, 12 Jul 2021 17:49:16 +0100 Subject: [PATCH 12/33] TST: Test loading with bad separation params --- fissa/tests/test_core.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 4b0bc2c7..64419936 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -1103,6 +1103,45 @@ def test_load_wrong_expansion(self): with self.assertRaises(ValueError): exp.load(fname) + def test_load_wrong_alpha_only_sep_results(self): + """Test load doesn't load analysis from wrong alpha param.""" + kwargs = {"expansion": 0.213, "nRegions": 3, "alpha": 5.23} + fields = {"result": np.array([[3, 4]])} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"], + nRegions=kwargs["nRegions"], + alpha=kwargs["alpha"] + 1, + ) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data is not loaded + with self.assertRaises(ValueError): + exp.load(fname) + + def test_load_wrong_alpha_mixed_prep_sep(self): + """Test load doesn't load analysis from wrong alpha param.""" + kwargs = {"expansion": 0.213, "nRegions": 3, "alpha": 5.23} + fields = {"raw": np.array([[1, 2]]), "result": np.array([[3, 4]])} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"], + nRegions=kwargs["nRegions"], + alpha=kwargs["alpha"] + 1, + ) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **kwargs, **fields) + # Load the file and check the data appears correctly + exp.load(fname) + self.assert_equal(exp.raw, fields["raw"]) + self.assertIs(exp.result, None) + @unittest.expectedFailure def test_badprepcache_init1(self): """ From 3c984f0b3ae46e766c705c219fc97a4faaa48292 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 13 Jul 2021 01:23:04 +0100 Subject: [PATCH 13/33] JNB: Use a different folder for new analysis --- examples/Basic usage - Functional.ipynb | 5 +++-- examples/Basic usage.ipynb | 17 +++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/examples/Basic usage - Functional.ipynb b/examples/Basic usage - Functional.ipynb index 43b211fc..8d29ee63 100644 --- a/examples/Basic usage - Functional.ipynb +++ b/examples/Basic usage - Functional.ipynb @@ -582,8 +582,9 @@ "alpha = 0.02\n", "\n", "# If you change the experiment parameters, you need to change the cache directory too.\n", - "# Otherwise you will reload the results from the previous run instead of computing\n", - "# the new results.\n", + "# Otherwise FISSA will try to reload the results from the previous run instead of\n", + "# computing the new results. FISSA will throw an error if you try to load data which\n", + "# was generated with different analysis parameters to its parameters.\n", "output_folder2 = output_folder + \"_alt\"\n", "\n", "# Run FISSA with these parameters\n", diff --git a/examples/Basic usage.ipynb b/examples/Basic usage.ipynb index 2bff2569..818341dc 100644 --- a/examples/Basic usage.ipynb +++ b/examples/Basic usage.ipynb @@ -817,25 +817,26 @@ "# The degree of signal sparsity can be controlled with the alpha parameter.\n", "alpha = 0.02\n", "\n", + "# If you change the experiment parameters, you need to change the cache directory too.\n", + "# Otherwise FISSA will try to reload the results from the previous run instead of\n", + "# computing the new results. FISSA will throw an error if you try to load data which\n", + "# was generated with different analysis parameters to its parameters.\n", + "output_folder2 = output_folder + \"_alt\"\n", + "\n", "# Set up a FISSA experiment with these parameters\n", "experiment = fissa.Experiment(\n", " images_location,\n", " rois_location,\n", - " output_folder,\n", + " output_folder2,\n", " nRegions=nRegions,\n", " expansion=expansion,\n", " alpha=alpha,\n", " ncores_preparation=ncores_preparation,\n", " ncores_separation=ncores_separation,\n", ")\n", + "\n", "# Extract the data with these new parameters.\n", - "# Note that we are using the same output folder as before. Since FISSA has cached\n", - "# a result alrady to this directory, its default behaviour is to restore the\n", - "# previously generated results.\n", - "# To make sure FISSA runs a fresh decontamination process with the new parameters,\n", - "# we need to make sure to specify to redo the preparation and separation.\n", - "# FISSA will then ignore the cached output and overwrite it with new results.\n", - "experiment.separate(redo_prep=True)" + "experiment.separate()" ] }, { From f8448876601ef5a8f206b58f918922cfa7108b12 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 13 Jul 2021 01:38:07 +0100 Subject: [PATCH 14/33] BUG: Can't do multiple dictionary expansions into fn on python2 --- fissa/tests/test_core.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 64419936..b40646c7 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -17,6 +17,14 @@ from .base_test import BaseTestCase +def merge_dicts(x, *args): + """Merge multiple dictionaries together.""" + z = x.copy() + for arg in args: + z.update(arg) + return z + + class ExperimentTestMixin: """Base tests for Experiment class.""" @@ -983,7 +991,7 @@ def test_load_npz(self): # Make a save file which contains values set to `None` fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data appears correctly exp.load(fname) for key, value in fields.items(): @@ -1016,7 +1024,7 @@ def test_load_none(self): # Make a save file which contains values set to `None` fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data appears as None, not np.array(None) exp.load(fname) for key, value in fields.items(): @@ -1043,7 +1051,7 @@ def test_load_scalar(self): # Make a save file which contains values set to a scalar` fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data appears correctly exp.load(fname) for key, value in fields.items(): @@ -1062,7 +1070,7 @@ def test_load_wrong_nRegions(self): # Make a save file which contains values set badly fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data is not loaded with self.assertRaises(ValueError): exp.load(fname) @@ -1074,7 +1082,7 @@ def test_load_wrong_in_init(self): # Make a save file which contains values set badly fname = os.path.join(self.output_dir, "preparation.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data is not loaded with self.assertRaises(ValueError): core.Experiment( @@ -1098,7 +1106,7 @@ def test_load_wrong_expansion(self): # Make a save file which contains values set badly fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data is not loaded with self.assertRaises(ValueError): exp.load(fname) @@ -1117,7 +1125,7 @@ def test_load_wrong_alpha_only_sep_results(self): # Make a save file which contains values set badly fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data is not loaded with self.assertRaises(ValueError): exp.load(fname) @@ -1136,7 +1144,7 @@ def test_load_wrong_alpha_mixed_prep_sep(self): # Make a save file which contains values set badly fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) - np.savez_compressed(fname, **kwargs, **fields) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data appears correctly exp.load(fname) self.assert_equal(exp.raw, fields["raw"]) From 8c42b7ef71c235629fbefb8f1176e371009b71e7 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 13 Jul 2021 01:42:57 +0100 Subject: [PATCH 15/33] TST: Change expected behaviour with mixed init --- fissa/tests/test_core.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index b40646c7..920d34db 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -1145,10 +1145,9 @@ def test_load_wrong_alpha_mixed_prep_sep(self): fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) np.savez_compressed(fname, **merge_dicts(kwargs, fields)) - # Load the file and check the data appears correctly - exp.load(fname) - self.assert_equal(exp.raw, fields["raw"]) - self.assertIs(exp.result, None) + # Load the file and check the data is not loaded + with self.assertRaises(ValueError): + exp.load(fname) @unittest.expectedFailure def test_badprepcache_init1(self): From b1260622c74ddb3f75482451cb834f470c20e619 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 13 Jul 2021 23:37:16 +0100 Subject: [PATCH 16/33] MNT: Pad adoption message when loading --- fissa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fissa/core.py b/fissa/core.py index 835496ce..54896111 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1003,7 +1003,7 @@ def load(self, path=None, force=False, skip_clear=False): if getattr(self, validator, None) is None: if self.verbosity >= 2: print( - "Adopting value {}={} from {}".format( + " Adopting value {}={} from {}".format( validator, value, path ) ) From 4c2d0c2a36b98514efe91d44ef080677770c190e Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 13 Jul 2021 23:13:22 +0100 Subject: [PATCH 17/33] API:MNT: Re-order Experiment arguments --- fissa/core.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 54896111..7eda5d24 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -464,6 +464,11 @@ class Experiment: ROI area. Default is ``1``. The total neuropil area will be ``nRegions * expansion * area(ROI)``. + method : "nmf" or "ica", default="nmf" + Which blind source-separation method to use. Either ``"nmf"`` + for non-negative matrix factorization, or ``"ica"`` for + independent component analysis. Default is ``"nmf"`` (recommended). + alpha : float, default=0.1 Sparsity regularizaton weight for NMF algorithm. Set to zero to remove regularization. Default is ``0.1``. @@ -511,11 +516,6 @@ class Experiment: the preparation routine, and so `ncores_separation` be often be set higher than `ncores_preparation`. - method : "nmf" or "ica", default="nmf" - Which blind source-separation method to use. Either ``"nmf"`` - for non-negative matrix factorization, or ``"ica"`` for - independent component analysis. Default is ``"nmf"`` (recommended). - lowmemory_mode : bool, optional If ``True``, FISSA will load TIFF files into memory frame-by-frame instead of holding the entire TIFF in memory at once. This @@ -681,13 +681,13 @@ def __init__( folder=None, nRegions=4, expansion=1, + method="nmf", alpha=0.1, max_iter=20000, tol=1e-4, max_tries=1, ncores_preparation=-1, ncores_separation=-1, - method="nmf", lowmemory_mode=False, datahandler=None, verbosity=1, @@ -728,13 +728,13 @@ def __init__( self.folder = folder self.nRegions = nRegions self.expansion = expansion + self.method = method self.alpha = alpha self.max_iter = max_iter self.tol = tol self.max_tries = max_tries self.ncores_preparation = ncores_preparation self.ncores_separation = ncores_separation - self.method = method self.verbosity = verbosity # check if any data already exists From bf33a6684a685e9c1ce6c697acc65e35e1d1ea3d Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 13 Jul 2021 23:28:56 +0100 Subject: [PATCH 18/33] ENH: Soft-default parameters can be taken from load instead --- fissa/core.py | 65 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 7eda5d24..7c5368ee 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -674,18 +674,28 @@ class Experiment: until then, it is set to ``None``. """ + _defaults = { + "nRegions": 4, + "expansion": 1, + "method": "nmf", + "alpha": 0.1, + "max_iter": 20000, + "tol": 1e-4, + "max_tries": 1, + } + def __init__( self, images, rois, folder=None, - nRegions=4, - expansion=1, - method="nmf", - alpha=0.1, - max_iter=20000, - tol=1e-4, - max_tries=1, + nRegions=None, + expansion=None, + method=None, + alpha=None, + max_iter=None, + tol=None, + max_tries=None, ncores_preparation=-1, ncores_separation=-1, lowmemory_mode=False, @@ -880,6 +890,41 @@ def clear_separated(self, verbosity=None): if verbosity >= 1 and keys_cleared: print("Cleared {}".format(", ".join(repr(k) for k in keys_cleared))) + def _adopt_default_parameters(self, only_preparation=False, force=False): + r""" + Adopt default values for unset analysis parameters. + + .. versionadded:: 1.0.0 + + Parameters + ---------- + only_preparation : bool, optional + Whether to restrict the parameters to only those used for data + extraction during the preparation step. Default is ``False``. + force : bool, optional + If `True`, all parameters will be overridden with default values + even if they had already been set. Default is ``False``. + """ + defaults = self._defaults + if only_preparation: + # Prune down to only the preparation parameters + preparation_fields = ["expansion", "nRegions"] + defaults = {k: v for k, v in defaults.items() if k in preparation_fields} + # Check through each parameter and set unset values from defaults + keys_adopted = [] + for key, value in defaults.items(): + if getattr(self, key, None) is not None and not force: + continue + setattr(self, key, value) + keys_adopted.append(key) + + if self.verbosity >= 5 and keys_adopted: + print( + "Adopted default values for {}".format( + ", ".join(repr(k) for k in keys_adopted) + ) + ) + def load(self, path=None, force=False, skip_clear=False): r""" Load data from cache file in npz format. @@ -1094,6 +1139,9 @@ def separation_prep(self, redo=False): # Wipe outputs self.clear() + # Adopt default values + self._adopt_default_parameters(only_preparation=True) + # Extract signals n_trial = len(self.images) if self.verbosity >= 2: @@ -1296,6 +1344,9 @@ def separate(self, redo_prep=False, redo_sep=False): # Wipe outputs self.clear_separated() + # Adopt default values + self._adopt_default_parameters() + # Check size of the input arrays n_roi = len(self.raw) n_trial = len(self.raw[0]) From 53086facaab84ba6ad6b5502c9d945888d72ae25 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 13 Jul 2021 23:58:44 +0100 Subject: [PATCH 19/33] MNT: Use repr around values being adopted --- fissa/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 7c5368ee..ad0046cb 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1021,7 +1021,9 @@ def load(self, path=None, force=False, skip_clear=False): # the cache, we will raise an error. validation_errors.append( " {}: Experiment (ours) {}, Cache (theirs) {}".format( - validator, getattr(self, validator), cache[validator] + validator, + getattr(self, validator), + cache[validator], ) ) if len(validation_errors) > 0: @@ -1049,7 +1051,7 @@ def load(self, path=None, force=False, skip_clear=False): if self.verbosity >= 2: print( " Adopting value {}={} from {}".format( - validator, value, path + validator, repr(value), path ) ) setattr(self, validator, value) From 1cb80dfe5cc975b20a3d7c991b713a6261f60693 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 00:00:00 +0100 Subject: [PATCH 20/33] BUG: Don't warn about unloaded dynamic properties --- fissa/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fissa/core.py b/fissa/core.py index ad0046cb..ddd49efa 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1074,6 +1074,8 @@ def load(self, path=None, force=False, skip_clear=False): # were left unloaded unset_fields = [] for field in cache.files: + if field in dynamic_properties: + continue if field not in set_fields: unset_fields.append(field) if len(unset_fields) > 0 and self.verbosity >= 1: From b899a90bf8957ca0bc28f9c72810b5d8356fbfc3 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 00:00:29 +0100 Subject: [PATCH 21/33] MNT: Only load validators and show message if data was loaded from cache --- fissa/core.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index ddd49efa..517edcb0 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1039,8 +1039,25 @@ def load(self, path=None, force=False, skip_clear=False): if not skip_clear and clearif in cache.files: clearfn() # All the validators were valid, so we are okay to load the fields + any_field_loaded = False + for field in fields: + if field not in cache or field in dynamic_properties: + continue + value = cache[field] + if np.array_equal(value, None): + value = None + elif value.ndim == 0: + # Handle loading scalars + value = value.item() + setattr(self, field, value) + set_fields.add(field) + any_field_loaded = True + # If we didn't load any output data, no need to set the validators + # or print that we loaded something. + if not any_field_loaded: + continue + # Load all the validators, overwriting our local values if None for validator in validators: - # Load all the validators, overwriting our local values if None value = cache[validator] if np.array_equal(value, None): value = None @@ -1056,17 +1073,6 @@ def load(self, path=None, force=False, skip_clear=False): ) setattr(self, validator, value) set_fields.add(validator) - for field in fields: - if field not in cache or field in dynamic_properties: - continue - value = cache[field] - if np.array_equal(value, None): - value = None - elif value.ndim == 0: - # Handle loading scalars - value = value.item() - setattr(self, field, value) - set_fields.add(field) if self.verbosity >= 2: print("Loaded {} data from {}".format(category, path)) From 360866eb4d81ddf71accbcfc1b712e63a7c6553b Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 00:20:46 +0100 Subject: [PATCH 22/33] MNT: Don't try to load nCell --- fissa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fissa/core.py b/fissa/core.py index 517edcb0..6ee77ab4 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -954,7 +954,7 @@ def load(self, path=None, force=False, skip_clear=False): ValGroup( "prepared", ["expansion", "nRegions"], - ["deltaf_raw", "means", "nCell", "raw", "roi_polys"], + ["deltaf_raw", "means", "raw", "roi_polys"], ["raw"], self.clear, ), From 2662d72e9c56dc5a4f6c6bad0e06f7fb15fe7cb3 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 00:23:51 +0100 Subject: [PATCH 23/33] ENH: Check image and roi size is appropriate before loading --- fissa/core.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fissa/core.py b/fissa/core.py index 6ee77ab4..ba0a295e 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1035,6 +1035,23 @@ def load(self, path=None, force=False, skip_clear=False): ) if not valid: continue + # Check the image and roi size is appropriate + for k in ["raw", "result"]: + if k not in cache.files: + continue + if cache[k].shape[1] != self.nTrials: + raise ValueError( + "Data mismatch between {} and our images." + " Cached {} has {} trials, but our Experiment has {}" + " trials.".format(path, k, cache[k].shape[1], self.nTrials) + ) + if self.nCell is not None and cache[k].shape[0] != self.nCell: + raise ValueError( + "Data mismatch between {} and our roisets." + " Cached {} has {} ROIs, but our Experiment has {}" + " ROIs.".format(path, k, cache[k].shape[1], self.nCell) + ) + # Wipe the values currently held before setting new values if not skip_clear and clearif in cache.files: clearfn() From 3d4ab25bc2efd8b026515994902e5ba144cf494f Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 00:30:25 +0100 Subject: [PATCH 24/33] RF: Move common code into _unpack_scalar --- fissa/core.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index ba0a295e..ec2c3eb1 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -990,17 +990,20 @@ def load(self, path=None, force=False, skip_clear=False): if self.verbosity >= 1: print("Loading data from cache {}".format(path)) cache = np.load(path, allow_pickle=True) + + def _unpack_scalar(x): + if np.array_equal(x, None): + return None + if x.ndim == 0: + # Handle loading scalars + return x.item() + return x + if force: for field in cache.files: if field in dynamic_properties: continue - value = cache[field] - if np.array_equal(value, None): - value = None - elif value.ndim == 0: - # Handle loading scalars - value = value.item() - setattr(self, field, value) + setattr(self, field, _unpack_scalar(cache[field])) return set_fields = set() for category, validators, fields, clearif, clearfn in validation_groups: @@ -1060,13 +1063,7 @@ def load(self, path=None, force=False, skip_clear=False): for field in fields: if field not in cache or field in dynamic_properties: continue - value = cache[field] - if np.array_equal(value, None): - value = None - elif value.ndim == 0: - # Handle loading scalars - value = value.item() - setattr(self, field, value) + setattr(self, field, _unpack_scalar(cache[field])) set_fields.add(field) any_field_loaded = True # If we didn't load any output data, no need to set the validators @@ -1075,12 +1072,7 @@ def load(self, path=None, force=False, skip_clear=False): continue # Load all the validators, overwriting our local values if None for validator in validators: - value = cache[validator] - if np.array_equal(value, None): - value = None - elif value.ndim == 0: - # Handle loading scalars - value = value.item() + value = _unpack_scalar(cache[validator]) if getattr(self, validator, None) is None: if self.verbosity >= 2: print( From 657f7b899429368e08969afef9977fff160e2d19 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 00:39:17 +0100 Subject: [PATCH 25/33] MNT: More flexible validator checks --- fissa/core.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index ec2c3eb1..3d35833a 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1010,23 +1010,27 @@ def _unpack_scalar(x): valid = True validation_errors = [] for validator in validators: - if validator not in cache or cache[validator] is None: + if validator not in cache: # If the validator is not set in the cache, we can't # verify that the cached data is compatible. valid = False break + value = _unpack_scalar(cache[validator]) + if value is None: + valid = False + break if getattr(self, validator, None) is None: # If the validator is not yet set locally, it is fine to # overwrite it. continue - if getattr(self, validator) != cache[validator]: + if not np.array_equal(getattr(self, validator), value): # If the validator is set and doesn't match the value in # the cache, we will raise an error. validation_errors.append( " {}: Experiment (ours) {}, Cache (theirs) {}".format( validator, getattr(self, validator), - cache[validator], + value, ) ) if len(validation_errors) > 0: From b839953648f76855ce4b9641cfba48d0f1a2599f Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 01:09:29 +0100 Subject: [PATCH 26/33] BUG: Handle raw=None in cache --- fissa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fissa/core.py b/fissa/core.py index 3d35833a..d45206e5 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1044,7 +1044,7 @@ def _unpack_scalar(x): continue # Check the image and roi size is appropriate for k in ["raw", "result"]: - if k not in cache.files: + if k not in cache.files or np.array_equal(cache[k], None): continue if cache[k].shape[1] != self.nTrials: raise ValueError( From b86760cf4ddd6d8483067a405884eae75d0163d0 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 01:13:33 +0100 Subject: [PATCH 27/33] TST: Fix up and expand tests for new load features --- fissa/tests/test_core.py | 171 ++++++++++++++++++++++++++++++++------- 1 file changed, 144 insertions(+), 27 deletions(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 920d34db..750a326b 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -980,10 +980,10 @@ def test_load_empty_sep(self): exp.separate() self.compare_output(exp) - def test_load_npz(self): + def test_load_force(self): """Test whether npz file is loaded by load method.""" kwargs = {"expansion": 0.213, "nRegions": 3} - fields = {"raw": np.array([[1, 2], [3, 4]])} + fields = {"foobar": np.array([[1, 2], [3, 4]])} exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) # Set the fields to be something other than `None` for key in fields: @@ -993,13 +993,13 @@ def test_load_npz(self): os.makedirs(self.output_dir) np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data appears correctly - exp.load(fname) + exp.load(fname, force=True) for key, value in fields.items(): - self.assertEqual(getattr(exp, key), value) + self.assert_equal(getattr(exp, key), value) def test_load_none_force(self): """Behaviour when forcibly loading a cache containing None.""" - fields = ["raw", "result", "deltaf_result"] + fields = ["foobar", "result", "deltaf_result"] exp = core.Experiment(self.images_dir, self.roi_zip_path) # Set the fields to be something other than `None` for field in fields: @@ -1043,29 +1043,15 @@ def test_load_scalar_force(self): for field in fields: self.assertEqual(getattr(exp, field), 1337) - def test_load_scalar(self): - """Behaviour when loading a cache containing None.""" - kwargs = {"expansion": 0.213, "nRegions": 3} - fields = {"raw": 1337} - exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) - # Make a save file which contains values set to a scalar` - fname = os.path.join(self.output_dir, "dummy.npz") - os.makedirs(self.output_dir) - np.savez_compressed(fname, **merge_dicts(kwargs, fields)) - # Load the file and check the data appears correctly - exp.load(fname) - for key, value in fields.items(): - self.assertEqual(getattr(exp, key), value) - def test_load_wrong_nRegions(self): """Test load doesn't load analysis from wrong nRegions param.""" kwargs = {"expansion": 0.213, "nRegions": 3} - fields = {"raw": np.array([[1, 2], [3, 4]])} + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} exp = core.Experiment( self.images_dir, self.roi_zip_path, expansion=kwargs["expansion"], - nRegions=kwargs["expansion"] + 1, + nRegions=kwargs["nRegions"] + 1, ) # Make a save file which contains values set badly fname = os.path.join(self.output_dir, "dummy.npz") @@ -1075,12 +1061,29 @@ def test_load_wrong_nRegions(self): with self.assertRaises(ValueError): exp.load(fname) + def test_load_missing_param(self): + """Test load doesn't load (without error) without param.""" + kwargs = {"expansion": 0.213} # nRegions is unset + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"], + ) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file and check the data is not loaded + exp.load(fname) + self.assertIs(exp.raw, None) + def test_load_wrong_in_init(self): """Test load doesn't load analysis from wrong nRegions param during init.""" kwargs = {"expansion": 0.213, "nRegions": 3} - fields = {"raw": np.array([[1, 2], [3, 4]])} + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} # Make a save file which contains values set badly - fname = os.path.join(self.output_dir, "preparation.npz") + fname = os.path.join(self.output_dir, "prepared.npz") os.makedirs(self.output_dir) np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data is not loaded @@ -1093,10 +1096,113 @@ def test_load_wrong_in_init(self): nRegions=kwargs["nRegions"] + 1, ) + def test_load_unset_in_init(self): + """Test load analysis from wrong nRegions param during init.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "prepared.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file and check the data loads + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + self.output_dir, + ) + self.assert_equal(exp.raw, fields["raw"]) + + def test_load_wrong_trial_count(self): + """Test load doesn't load analysis when the shape is wrong.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names) + 1))} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"], + nRegions=kwargs["nRegions"], + ) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file and check the data is not loaded + with self.assertRaises(ValueError): + exp.load(fname) + + def test_load_sequential(self): + """Test load works correctly when we load twice.""" + kwargs = { + "expansion": 0.213, + "nRegions": 3, + "alpha": 5.23, + "max_iter": 12038, + "max_tries": 2, + "method": "nmf", + "tol": 1.123e-3, + } + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"], + nRegions=kwargs["nRegions"], + ) + # Make a save file which contains values + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + exp.load(fname) + self.assert_equal(exp.raw, fields["raw"]) + # Make a cache with result + fields2 = {"result": np.zeros((len(self.roi_paths), len(self.image_names)))} + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy2.npz") + np.savez_compressed(fname, **merge_dicts(kwargs, fields2)) + # Load the file + exp.load(fname) + self.assert_equal(exp.result, fields2["result"]) + self.assert_equal(exp.raw, fields["raw"]) + + def test_load_wrong_roi_count(self): + """Test load works correctly when we load twice.""" + kwargs = { + "expansion": 0.213, + "nRegions": 3, + "alpha": 5.23, + "max_iter": 12038, + "max_tries": 2, + "method": "nmf", + "tol": 1.123e-3, + } + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=kwargs["expansion"], + nRegions=kwargs["nRegions"], + ) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + exp.load(fname) + self.assert_equal(exp.raw, fields["raw"]) + # Make a cache with result + fields2 = {"result": np.zeros((len(self.roi_paths) + 1, len(self.image_names)))} + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy2.npz") + np.savez_compressed(fname, **merge_dicts(kwargs, fields2)) + # Load the file + with self.assertRaises(ValueError): + exp.load(fname) + def test_load_wrong_expansion(self): """Test load doesn't load analysis from wrong expansion param.""" kwargs = {"expansion": 0.213, "nRegions": 3} - fields = {"raw": np.array([[1, 2], [3, 4]])} + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} exp = core.Experiment( self.images_dir, self.roi_zip_path, @@ -1114,7 +1220,7 @@ def test_load_wrong_expansion(self): def test_load_wrong_alpha_only_sep_results(self): """Test load doesn't load analysis from wrong alpha param.""" kwargs = {"expansion": 0.213, "nRegions": 3, "alpha": 5.23} - fields = {"result": np.array([[3, 4]])} + fields = {"result": np.ones((len(self.roi_paths), len(self.image_names)))} exp = core.Experiment( self.images_dir, self.roi_zip_path, @@ -1132,8 +1238,19 @@ def test_load_wrong_alpha_only_sep_results(self): def test_load_wrong_alpha_mixed_prep_sep(self): """Test load doesn't load analysis from wrong alpha param.""" - kwargs = {"expansion": 0.213, "nRegions": 3, "alpha": 5.23} - fields = {"raw": np.array([[1, 2]]), "result": np.array([[3, 4]])} + kwargs = { + "expansion": 0.213, + "nRegions": 3, + "alpha": 5.23, + "max_iter": 12038, + "max_tries": 2, + "method": "nmf", + "tol": 1.123e-3, + } + fields = { + "raw": np.ones((len(self.roi_paths), len(self.image_names))), + "result": np.zeros((len(self.roi_paths), len(self.image_names))), + } exp = core.Experiment( self.images_dir, self.roi_zip_path, From f60e570330751a518172860fee152726b17aed49 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 14 Jul 2021 13:41:31 +0100 Subject: [PATCH 28/33] MNT: Re-order method parameter in str and repr output --- fissa/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index d45206e5..c559e8e3 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -790,13 +790,13 @@ def __str__(self): "folder", "nRegions", "expansion", + "method", "alpha", "max_iter", "tol", "max_tries", "ncores_preparation", "ncores_separation", - "method", "datahandler", "verbosity", ] @@ -818,13 +818,13 @@ def __repr__(self): "folder", "nRegions", "expansion", + "method", "alpha", "max_iter", "tol", "max_tries", "ncores_preparation", "ncores_separation", - "method", "datahandler", "verbosity", ] From b57f3da359f34e823b3e01dca2b5a5665e2778e9 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 15 Jul 2021 00:04:19 +0100 Subject: [PATCH 29/33] MNT: Allow npz to omit validators if also unset object attribute --- fissa/core.py | 16 ++++++++++------ fissa/tests/test_core.py | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index c559e8e3..8983a837 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1010,19 +1010,21 @@ def _unpack_scalar(x): valid = True validation_errors = [] for validator in validators: + if getattr(self, validator, None) is None: + # If the validator is not yet set locally, it is fine to + # overwrite it. + continue if validator not in cache: - # If the validator is not set in the cache, we can't - # verify that the cached data is compatible. + # If the validator is not set in the cache and is set + # locally, we can't verify that the cached data is + # compatible. We don't raise an error for this because the + # contents are probably not this category. valid = False break value = _unpack_scalar(cache[validator]) if value is None: valid = False break - if getattr(self, validator, None) is None: - # If the validator is not yet set locally, it is fine to - # overwrite it. - continue if not np.array_equal(getattr(self, validator), value): # If the validator is set and doesn't match the value in # the cache, we will raise an error. @@ -1076,6 +1078,8 @@ def _unpack_scalar(x): continue # Load all the validators, overwriting our local values if None for validator in validators: + if validator not in cache.files: + continue value = _unpack_scalar(cache[validator]) if getattr(self, validator, None) is None: if self.verbosity >= 2: diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 750a326b..444e7447 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -1062,7 +1062,7 @@ def test_load_wrong_nRegions(self): exp.load(fname) def test_load_missing_param(self): - """Test load doesn't load (without error) without param.""" + """Test load when missing a param from cache.""" kwargs = {"expansion": 0.213} # nRegions is unset fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} exp = core.Experiment( @@ -1076,7 +1076,7 @@ def test_load_missing_param(self): np.savez_compressed(fname, **merge_dicts(kwargs, fields)) # Load the file and check the data is not loaded exp.load(fname) - self.assertIs(exp.raw, None) + self.assert_equal(exp.raw, fields["raw"]) def test_load_wrong_in_init(self): """Test load doesn't load analysis from wrong nRegions param during init.""" From 623d0ebdbb55e8c138c829496d196889d93686a5 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 15 Jul 2021 01:59:47 +0100 Subject: [PATCH 30/33] TST: Add more tests for validated loading --- fissa/tests/test_core.py | 116 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 3 deletions(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 444e7447..3d51f83c 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -980,6 +980,53 @@ def test_load_empty_sep(self): exp.separate() self.compare_output(exp) + def test_load_dynamic(self): + """Test behaviour when loading a cache containing a dynamic property.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"nTrials": 182} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Make a save file + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + exp.load(fname) + # Check the data still appears correctly + self.assert_equal(exp.nTrials, len(self.image_names)) + + def test_load_dynamic_force(self): + """Test behaviour when loading a cache containing a dynamic property.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"nTrials": 182} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Make a save file + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + exp.load(fname, force=True) + # Make sure there is no error + + def test_load_wrong_data_field(self): + """Test whether npz file is loaded by load method.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"foobar": np.array([[1, 2], [3, 4]])} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Set the fields to be something other than `None` + for key in fields: + setattr(exp, key, 42.24) + # Make a save file + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + capture_pre = self.capsys.readouterr() # Clear stdout + exp.load(fname) + capture_post = self.recapsys(capture_pre) # Capture and then re-output + # Check the warning message appears + self.assertTrue("foobar" in capture_post.out) + self.assertTrue(fname + " were not loaded" in capture_post.out) + def test_load_force(self): """Test whether npz file is loaded by load method.""" kwargs = {"expansion": 0.213, "nRegions": 3} @@ -1062,22 +1109,85 @@ def test_load_wrong_nRegions(self): exp.load(fname) def test_load_missing_param(self): - """Test load when missing a param from cache.""" + """Behaviour when a param is missing from npz and unset in experiment.""" kwargs = {"expansion": 0.213} # nRegions is unset fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file and check the data is loaded + exp.load(fname) + self.assert_equal(exp.raw, fields["raw"]) + + def test_load_missing_param_set(self): + """Behaviour when a param is missing from npz but set in experiment.""" + # Initial value for raw + initial_value = np.zeros((len(self.roi_paths), len(self.image_names))) + kwargs = {"nRegions": 3} # expansion is not defined + fields = {"raw": initial_value + 1} exp = core.Experiment( self.images_dir, self.roi_zip_path, - expansion=kwargs["expansion"], + expansion=123, # expansion is defined + nRegions=kwargs["nRegions"], ) + exp.raw = initial_value # Make a save file which contains values set badly fname = os.path.join(self.output_dir, "dummy.npz") os.makedirs(self.output_dir) np.savez_compressed(fname, **merge_dicts(kwargs, fields)) - # Load the file and check the data is not loaded + # Load the file + capture_pre = self.capsys.readouterr() # Clear stdout + exp.load(fname) + capture_post = self.recapsys(capture_pre) # Capture and then re-output + # Check the warning message appears + self.assertTrue("raw" in capture_post.out) + self.assertTrue(fname + " were not loaded" in capture_post.out) + # Check the value of raw has not changed + self.assert_equal(exp.raw, initial_value) + + def test_load_None_param(self): + """Behaviour when a param is None in npz and unset in experiment.""" + kwargs = {"expansion": 0.213, "nRegions": None} # nRegions is None + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file and check the data is loaded exp.load(fname) self.assert_equal(exp.raw, fields["raw"]) + def test_load_None_param_set(self): + """Behaviour when a param is None in npz and set in experiment.""" + # Initial value for raw + initial_value = np.zeros((len(self.roi_paths), len(self.image_names))) + kwargs = {"nRegions": 3, "expansion": None} # expansion is None + fields = {"raw": initial_value + 1} + exp = core.Experiment( + self.images_dir, + self.roi_zip_path, + expansion=123, # expansion is defined + nRegions=kwargs["nRegions"], + ) + exp.raw = initial_value + # Make a save file which contains values set badly + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + capture_pre = self.capsys.readouterr() # Clear stdout + exp.load(fname) + capture_post = self.recapsys(capture_pre) # Capture and then re-output + # Check the warning message appears + self.assertTrue("raw" in capture_post.out) + self.assertTrue(fname + " were not loaded" in capture_post.out) + # Check the value of raw has not changed + self.assert_equal(exp.raw, initial_value) + def test_load_wrong_in_init(self): """Test load doesn't load analysis from wrong nRegions param during init.""" kwargs = {"expansion": 0.213, "nRegions": 3} From 320d50094c9a9db5efa31b29a43a7d24e9f5d051 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 15 Jul 2021 02:35:20 +0100 Subject: [PATCH 31/33] TST: Test loading raw clears other prep --- fissa/tests/test_core.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 3d51f83c..698234a1 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -980,6 +980,37 @@ def test_load_empty_sep(self): exp.separate() self.compare_output(exp) + def test_load_clear(self): + """Test loading raw clears other existing values.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + exp.means = 31802 # Assign means to be a random value + # Make a save file + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + exp.load(fname) + # Check means was wiped + self.assertIs(exp.means, None) + + def test_load_clear_skip(self): + """Test loading raw doesn't clears existing values with skip_clear.""" + kwargs = {"expansion": 0.213, "nRegions": 3} + fields = {"raw": np.ones((len(self.roi_paths), len(self.image_names)))} + exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) + value = 31802 + exp.means = value # Assign means to be a random value + # Make a save file + fname = os.path.join(self.output_dir, "dummy.npz") + os.makedirs(self.output_dir) + np.savez_compressed(fname, **merge_dicts(kwargs, fields)) + # Load the file + exp.load(fname, skip_clear=True) + # Check means was not wiped + self.assert_equal(exp.means, value) + def test_load_dynamic(self): """Test behaviour when loading a cache containing a dynamic property.""" kwargs = {"expansion": 0.213, "nRegions": 3} From a72729f11b51a4c025b1981bc92b3c6b722c07f0 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 15 Jul 2021 02:35:45 +0100 Subject: [PATCH 32/33] BUG: Fix clearif when loading --- fissa/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fissa/core.py b/fissa/core.py index 8983a837..528aed3d 100644 --- a/fissa/core.py +++ b/fissa/core.py @@ -1062,8 +1062,11 @@ def _unpack_scalar(x): ) # Wipe the values currently held before setting new values - if not skip_clear and clearif in cache.files: - clearfn() + if not skip_clear: + for field in clearif: + if field in cache.files: + clearfn() + break # All the validators were valid, so we are okay to load the fields any_field_loaded = False for field in fields: From 4643ff4b3cc5b3633c680826bfcd0e44022dfa30 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 15 Jul 2021 02:46:28 +0100 Subject: [PATCH 33/33] TST: Fix dynamic property load test --- fissa/tests/test_core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fissa/tests/test_core.py b/fissa/tests/test_core.py index 698234a1..533c64f6 100644 --- a/fissa/tests/test_core.py +++ b/fissa/tests/test_core.py @@ -1014,7 +1014,10 @@ def test_load_clear_skip(self): def test_load_dynamic(self): """Test behaviour when loading a cache containing a dynamic property.""" kwargs = {"expansion": 0.213, "nRegions": 3} - fields = {"nTrials": 182} + fields = { + "raw": np.ones((len(self.roi_paths), len(self.image_names))), + "nTrials": 182, + } exp = core.Experiment(self.images_dir, self.roi_zip_path, **kwargs) # Make a save file fname = os.path.join(self.output_dir, "dummy.npz")