From bc596349ff1506f524ed24f9c3d0acad04c38083 Mon Sep 17 00:00:00 2001 From: rythorpe Date: Fri, 21 Aug 2020 10:37:02 -0400 Subject: [PATCH 1/6] update feed-specific param dictionaries in Network attribute of NeuronNetwork --- hnn_core/network_builder.py | 8 ++++++-- hnn_core/parallel_backends.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hnn_core/network_builder.py b/hnn_core/network_builder.py index 333a88aa4..50c8d9ebf 100644 --- a/hnn_core/network_builder.py +++ b/hnn_core/network_builder.py @@ -11,6 +11,7 @@ from .cell import _ArtificialCell from .pyramidal import L2Pyr, L5Pyr from .basket import L2Basket, L5Basket +from .params import create_pext # a few globals _PC = None @@ -121,8 +122,6 @@ def simulation_time(): dpl.scale(neuron_net.net.params['dipole_scalefctr']) dpl.smooth(neuron_net.net.params['dipole_smooth_win'] / h.dt) - neuron_net.net.trial_idx += 1 - return dpl @@ -353,6 +352,11 @@ def _create_cells_and_feeds(self): External inputs are not targets. """ params = self.net.params + # Re-create external feed param dictionaries + # Note that the only thing being updated here are the param['prng_*'] + # values + self.net.p_common, self.net.p_unique = create_pext(params, + params['tstop']) # loop through gids on this node for gid in self.net._gid_list: diff --git a/hnn_core/parallel_backends.py b/hnn_core/parallel_backends.py index 2ce714cc7..8df2a66c0 100644 --- a/hnn_core/parallel_backends.py +++ b/hnn_core/parallel_backends.py @@ -90,6 +90,7 @@ def _clone_and_simulate(self, net, trial_idx): net.params['prng_*'] = trial_idx neuron_net = NetworkBuilder(net) + neuron_net.net.trial_idx = trial_idx dpl = _simulate_single_trial(neuron_net) spikedata = neuron_net.get_data_from_neuron() From dfa6445630422eb7101d2fcdbb5e49b184b2da2e Mon Sep 17 00:00:00 2001 From: rythorpe Date: Fri, 11 Sep 2020 17:09:26 -0400 Subject: [PATCH 2/6] fix trial message for MPI_BACKEND --- hnn_core/mpi_child.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hnn_core/mpi_child.py b/hnn_core/mpi_child.py index 6672301f4..f666cc2dd 100644 --- a/hnn_core/mpi_child.py +++ b/hnn_core/mpi_child.py @@ -66,6 +66,7 @@ def run_mpi_simulation(): for trial in range(params['N_trials']): dpl = _simulate_single_trial(neuron_net) if rank == 0: + net.trial_idx += 1 spikedata = neuron_net.get_data_from_neuron() sim_data.append((dpl, spikedata)) From 7118e29f55beb8c7269d511165172fae26472fc7 Mon Sep 17 00:00:00 2001 From: rythorpe Date: Fri, 11 Sep 2020 18:44:33 -0400 Subject: [PATCH 3/6] iterate rng seed param for each trial in mpi_child --- hnn_core/mpi_child.py | 10 +++++++--- hnn_core/parallel_backends.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hnn_core/mpi_child.py b/hnn_core/mpi_child.py index f666cc2dd..e21634102 100644 --- a/hnn_core/mpi_child.py +++ b/hnn_core/mpi_child.py @@ -60,13 +60,17 @@ def run_mpi_simulation(): params = comm.bcast(params, root=0) net = Network(params) - neuron_net = NetworkBuilder(net) sim_data = [] - for trial in range(params['N_trials']): + for trial_idx in range(params['N_trials']): + # update rng seed state with new trial + #if rank == 0: + net.trial_idx = trial_idx + for param_key in net.params['prng_*'].keys(): + net.params[param_key] += trial_idx + neuron_net = NetworkBuilder(net) dpl = _simulate_single_trial(neuron_net) if rank == 0: - net.trial_idx += 1 spikedata = neuron_net.get_data_from_neuron() sim_data.append((dpl, spikedata)) diff --git a/hnn_core/parallel_backends.py b/hnn_core/parallel_backends.py index 8df2a66c0..bdba7f144 100644 --- a/hnn_core/parallel_backends.py +++ b/hnn_core/parallel_backends.py @@ -86,8 +86,8 @@ def _clone_and_simulate(self, net, trial_idx): from hnn_core.network_builder import NetworkBuilder from hnn_core.network_builder import _simulate_single_trial - if trial_idx != 0: - net.params['prng_*'] = trial_idx + for param_key in net.params['prng_*'].keys(): + net.params[param_key] += trial_idx neuron_net = NetworkBuilder(net) neuron_net.net.trial_idx = trial_idx From b340fe0150c7b333d40a56f3e87ea0c0ea85004f Mon Sep 17 00:00:00 2001 From: rythorpe Date: Mon, 14 Sep 2020 18:19:34 -0400 Subject: [PATCH 4/6] MPI and Joblib backends now produce the same results --- hnn_core/mpi_child.py | 14 ++++++++------ hnn_core/network_builder.py | 4 ++-- hnn_core/parallel_backends.py | 5 +++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/hnn_core/mpi_child.py b/hnn_core/mpi_child.py index e21634102..4b8fdcd8e 100644 --- a/hnn_core/mpi_child.py +++ b/hnn_core/mpi_child.py @@ -60,16 +60,18 @@ def run_mpi_simulation(): params = comm.bcast(params, root=0) net = Network(params) + # XXX store the initial prng_seedcore params to be referenced in each trial + prng_seedcore_initial = net.params['prng_*'].copy() sim_data = [] for trial_idx in range(params['N_trials']): - # update rng seed state with new trial - #if rank == 0: - net.trial_idx = trial_idx - for param_key in net.params['prng_*'].keys(): - net.params[param_key] += trial_idx + # XXX this should be built into NetworkBuilder + # update prng_seedcore params to provide jitter between trials + for param_key in prng_seedcore_initial.keys(): + net.params[param_key] = (prng_seedcore_initial[param_key] + + trial_idx) neuron_net = NetworkBuilder(net) - dpl = _simulate_single_trial(neuron_net) + dpl = _simulate_single_trial(neuron_net, trial_idx) if rank == 0: spikedata = neuron_net.get_data_from_neuron() sim_data.append((dpl, spikedata)) diff --git a/hnn_core/network_builder.py b/hnn_core/network_builder.py index 50c8d9ebf..97d4e5377 100644 --- a/hnn_core/network_builder.py +++ b/hnn_core/network_builder.py @@ -24,7 +24,7 @@ _LAST_NETWORK = None -def _simulate_single_trial(neuron_net): +def _simulate_single_trial(neuron_net, trial_idx): """Simulate one trial.""" from .dipole import Dipole @@ -41,7 +41,7 @@ def _simulate_single_trial(neuron_net): _PC.barrier() # sync for output to screen if rank == 0: print("running trial %d on %d cores" % - (neuron_net.net.trial_idx + 1, nhosts)) + (trial_idx + 1, nhosts)) # create or reinitialize scalars in NEURON (hoc) context h("dp_total_L2 = 0.") diff --git a/hnn_core/parallel_backends.py b/hnn_core/parallel_backends.py index bdba7f144..2a64149b4 100644 --- a/hnn_core/parallel_backends.py +++ b/hnn_core/parallel_backends.py @@ -86,12 +86,13 @@ def _clone_and_simulate(self, net, trial_idx): from hnn_core.network_builder import NetworkBuilder from hnn_core.network_builder import _simulate_single_trial + # XXX this should be built into NetworkBuilder + # update prng_seedcore params to provide jitter between trials for param_key in net.params['prng_*'].keys(): net.params[param_key] += trial_idx neuron_net = NetworkBuilder(net) - neuron_net.net.trial_idx = trial_idx - dpl = _simulate_single_trial(neuron_net) + dpl = _simulate_single_trial(neuron_net, trial_idx) spikedata = neuron_net.get_data_from_neuron() From b8f5d1d5935cc756c478b5ef2bae92e1cec17508 Mon Sep 17 00:00:00 2001 From: rythorpe Date: Tue, 15 Sep 2020 15:16:52 -0400 Subject: [PATCH 5/6] add comparison tests across parallel backends --- hnn_core/tests/test_compare_hnn.py | 48 ++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/hnn_core/tests/test_compare_hnn.py b/hnn_core/tests/test_compare_hnn.py index ca9839c8c..a47152813 100644 --- a/hnn_core/tests/test_compare_hnn.py +++ b/hnn_core/tests/test_compare_hnn.py @@ -1,7 +1,7 @@ import os.path as op from numpy import loadtxt -from numpy.testing import assert_array_equal +from numpy.testing import assert_array_equal, assert_allclose, assert_raises from mne.utils import _fetch_file import hnn_core @@ -25,18 +25,30 @@ def run_hnn_core(backend=None, n_jobs=1): # default params params_fname = op.join(hnn_core_root, 'param', 'default.json') params = read_params(params_fname) - - # run the simulation + params_reduced = params.copy() + params_reduced.update({'N_pyr_x': 3, + 'N_pyr_y': 3, + 'tstop': 25, + 't_evprox_1': 5, + 't_evdist_1': 10, + 't_evprox_2': 20, + 'N_trials': 2}) + + # run the simulation on full model (1 trial) and a reduced model (2 trials) net = Network(params) + net_reduced = Network(params_reduced) if backend == 'mpi': with MPIBackend(n_procs=2, mpi_cmd='mpiexec'): dpl = simulate_dipole(net)[0] + dpls_reduced = simulate_dipole(net_reduced) elif backend == 'joblib': with JoblibBackend(n_jobs=n_jobs): dpl = simulate_dipole(net)[0] + dpls_reduced = simulate_dipole(net_reduced) else: dpl = simulate_dipole(net)[0] + dpls_reduced = simulate_dipole(net_reduced) # write the dipole to a file and compare fname = './dpl2.txt' @@ -63,18 +75,30 @@ def run_hnn_core(backend=None, n_jobs=1): 'L5_basket': 85, 'evdist1': 234, 'evprox2': 269} + return dpls_reduced + +def test_compare_across_backends(): + """Test that trials are generated consistently across parallel backends.""" -def test_hnn_core(): - """Test that running hnn-core succeeds (implicit n_jobs=1).""" - run_hnn_core(None) + # test consistency between default backend simulation and master + dpls_reduced_default = run_hnn_core(None) + # test consistency between mpi backend simulation (n_procs=2) and master + dpls_reduced_mpi = run_hnn_core(backend='mpi') -def test_mpi(): - """Test that running hnn-core with MPI context manager when n_jobs=1.""" - run_hnn_core(backend='mpi') + # test consistency between joblib backend simulation (n_jobs=2) with master + dpls_reduced_joblib = run_hnn_core(backend='joblib', n_jobs=2) + # test consistency across all parallel backends for multiple trials + assert_raises(AssertionError, assert_array_equal, + dpls_reduced_default[0].data['agg'], + dpls_reduced_default[1].data['agg']) -def test_joblib(): - """Test that running hnn-core with Joblib context manager when n_jobs=2.""" - run_hnn_core(backend='joblib', n_jobs=2) + for trial_idx in range(len(dpls_reduced_default)): + # account for rounding error incured during MPI parallelization + assert_allclose(dpls_reduced_default[trial_idx].data['agg'], + dpls_reduced_mpi[trial_idx].data['agg'], rtol=0, + atol=1e-14) + assert_array_equal(dpls_reduced_default[trial_idx].data['agg'], + dpls_reduced_joblib[trial_idx].data['agg']) From 98a405047965ea6e2c0be381ade3113e0d53a5c8 Mon Sep 17 00:00:00 2001 From: rythorpe Date: Tue, 15 Sep 2020 15:29:34 -0400 Subject: [PATCH 6/6] update whats_new.rst --- doc/whats_new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats_new.rst b/doc/whats_new.rst index f99a55016..154ac0bd8 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -42,6 +42,8 @@ Bug - Connections now cannot be removed by setting the weights to 0., by `Mainak Jas`_ and `Ryan Thorpe`_ in `#162 `_ +- MPI and Joblib backends now apply jitter across multiple trials identically, by `Ryan Thorpe`_ in `#171 `_ + API ~~~