[MRG] Enable MPI parallelism with ParallelContext

.circleci/config.yml

@@ -15,40 +15,45 @@ jobs:
                   wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh;
                   chmod +x ~/miniconda.sh;
                   ~/miniconda.sh -b -p ~/miniconda;
-                  echo "export PATH=~/miniconda/bin:$PATH" >> $BASH_ENV;
+
+          - run:
+              name: Install openmpi
+              command: |
+                  sudo apt-get install libopenmpi-dev openmpi-bin
 
           - run:
               name: Setup Python environment
               command: |
+                  export PATH=~/miniconda/bin:$PATH
                   conda update --yes --quiet conda
                   conda create -n testenv --yes pip python=3.6
                   source activate testenv
                   conda install --yes scipy numpy matplotlib
-                  pip install mne
+                  pip install mne psutil mpi4py joblib
 
           - run:
               name: Setup doc building stuff
               command: |
-                  source activate testenv
+                  source ~/miniconda/bin/activate testenv
                   pip install sphinx numpydoc sphinx-gallery sphinx_bootstrap_theme pillow
 
           - run:
               name: Setup Neuron
               command: |
-                  source activate testenv
+                  source ~/miniconda/bin/activate testenv
                   pip install NEURON
 
           - run:
               name: Setup hnn-core
               command: |
-                  source activate testenv
+                  source ~/miniconda/bin/activate testenv
                   make
                   python setup.py develop
 
           - run:
               name: Build the documentation
               command: |
-                  source activate testenv
+                  source ~/miniconda/bin/activate testenv
                   cd doc/ && make html
 
           - persist_to_workspace:

.travis.yml

@@ -1,45 +1,56 @@
-language: python
+language: c
 
 env:
-  - PYTHON_VERSION=3.6
+  - PYTHON_VERSION=3.7
 
-addons:
-  apt:
-    sources:
-    - ubuntu-toolchain-r-test
-    packages:
-    - libopenmpi-dev
+matrix:
+  include:
+      # Linux
+      - os: linux
+        name: "Linux full"
+        addons:
+          apt:
+            sources:
+              - ubuntu-toolchain-r-test
+            packages:
+              - libopenmpi-dev
+              - openmpi-bin
+      # OSX
+      - os: osx
+        name: "MacOS full"
 
 
 before_install:
-  - wget -q http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
+  - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+        wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+    else
+        wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
+    fi;
   - chmod +x miniconda.sh
-  - ./miniconda.sh -b -p /home/travis/miniconda
-  - export PATH=/home/travis/miniconda/bin:$PATH
+  - ./miniconda.sh -b -p ${HOME}/miniconda
+  - export PATH=${HOME}/miniconda/bin:$PATH
   - conda update --yes --quiet conda
 
 install:
     - install_prefix=~
     - conda create -n testenv --yes pip python=${PYTHON_VERSION}
     - source activate testenv
     - conda install --yes scipy numpy matplotlib
+    - |
+      if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+        conda install --yes mpi4py openmpi
+      else
+        pip install mpi4py
+      fi
     - pip install flake8 pytest pytest-cov
-    - pip install mpi4py mne
+    - pip install mne psutil joblib
     - pip install coverage coveralls
-    - |
-      git clone https://github.com/neuronsimulator/nrn
-      cd nrn
-      ./build.sh
-      ./configure --with-nrnpython=python3 --without-iv --prefix=${install_prefix}
-      make -j4
-      make install -j4
-      cd src/nrnpython/
-      python3 setup.py install
+    - pip install NEURON
     - |
       cd $TRAVIS_BUILD_DIR
-      export PATH=$PATH:${install_prefix}/x86_64/bin
       make
       python setup.py develop
+
 script:
     - flake8 --count hnn_core
     - py.test --cov=hnn_core hnn_core/tests/

CONTRIBUTING.rst

@@ -35,7 +35,7 @@ Building the documentation
 The documentation can be built using sphinx. For that, please additionally
 install the following::
 
-    $ pip install matplotlib sphinx numpydoc sphinx-gallery sphinx_bootstrap_theme pillow
+    $ pip install matplotlib sphinx numpydoc sphinx-gallery sphinx_bootstrap_theme pillow mpi4py joblib psutil
 
 You can build the documentation locally using the command::
 

README.rst

@@ -28,7 +28,11 @@ Dependencies
 * scipy
 * numpy
 * matplotlib
-* joblib (optional for parallel processing)
+* joblib (optional for running trials simultaneously)
+* mpi4py (optional for running each trial in parallel across cores). Also depends on:
+
+  * openmpi or other mpi platform installed on system
+  * psutil
 
 Installation
 ============
@@ -37,9 +41,7 @@ We recommend the `Anaconda Python distribution <https://www.continuum.io/downloa
 
 	$ conda install numpy matplotlib scipy
 
-For joblib, you can do::
-
-	$ pip install joblib
+For using more than one CPU core, see :ref:`Parallel backends` below.
 
 Additionally, you would need Neuron which is available here: `https://neuron.yale.edu/neuron/ <https://neuron.yale.edu/neuron/>`_
 
@@ -64,6 +66,77 @@ To check if everything worked fine, you can do::
 
 and it should not give any error messages.
 
+.. _Parallel backends:
+
+Parallel backends
+=================
+
+Two options are available for making use of multiple CPU cores. The first runs multiple trials in parallel with joblib. Alternatively, you can run each trial across multiple cores to reduce the runtime.
+
+Joblib
+------
+
+This is the default backend and will execute multiple trials at the same time, with each trial running on a separate core in "embarrassingly parallel" execution. Note that with only 1 trial, there will be no parallelism.
+
+**Dependencies**::
+
+	$ pip install joblib
+
+**Usage**::
+
+	dpls = simulate_dipole(net, n_trials=2)
+
+MPI
+------
+
+This backend will use MPI (Message Passing Interface) on the system to split neurons across CPU cores (processors) and reduce the simulation time as more cores are used.
+
+**Linux Dependencies**::
+
+	$ sudo apt-get install libopenmpi-dev openmpi-bin
+	$ pip install mpi4py psutil
+
+**MacOS Dependencies**::
+
+	$ conda install yes openmpi mpi4py
+	$ pip install psutil
+
+**MacOS Environment**::
+
+	$ export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib
+
+Alternatively, run the commands below will avoid needing to run the export command every time a new shell is opened::
+
+	cd ${CONDA_PREFIX}
+	mkdir -p etc/conda/activate.d etc/conda/deactivate.d
+	echo "export OLD_LD_LIBRARY_PATH=\$LD_LIBRARY_PATH" >> etc/conda/activate.d/env_vars.sh
+	echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\${CONDA_PREFIX}/lib" >> etc/conda/activate.d/env_vars.sh
+	echo "export LD_LIBRARY_PATH=\$OLD_LD_LIBRARY_PATH" >> etc/conda/deactivate.d/env_vars.sh
+	echo "unset OLD_LD_LIBRARY_PATH" >> etc/conda/deactivate.d/env_vars.sh
+
+**Test MPI**::
+
+	$ mpiexec -np 2 nrniv -mpi -python -c 'from neuron import h; from mpi4py import MPI; \
+	                                       print("Hello from proc %d" % MPI.COMM_WORLD.Get_rank()); \
+                                               h.quit()'
+	numprocs=2
+	NEURON -- VERSION 7.7.2 7.7 (2b7985ba) 2019-06-20
+	Duke, Yale, and the BlueBrain Project -- Copyright 1984-2018
+	See http://neuron.yale.edu/neuron/credits
+
+	Hello from proc 0
+	Hello from proc 1
+
+Verifies that MPI, NEURON, and Python are all working together.
+
+**Usage**::
+
+	from hnn_core import MPIBackend
+
+	# set n_procs to the number of processors MPI can use (up to number of cores on system)
+	with MPIBackend(n_procs=2):
+		dpls = simulate_dipole(net, n_trials=1)
+
 Bug reports
 ===========
 

doc/api.rst

@@ -30,6 +30,15 @@ Simulation (:py:mod:`hnn_core`):
    Params
    read_params
 
+.. currentmodule:: hnn_core.parallel_backends
+
+.. autosummary::
+   :toctree: generated/
+
+   MPIBackend
+   JoblibBackend
+
+
 Inputs and Outputs (:py:mod:`hnn_core`):
 
 .. currentmodule:: hnn_core

doc/whats_new.rst

@@ -17,11 +17,13 @@ Changelog
 
 - Add ability to simulate multiple trials in parallel using joblibs, by `Mainak Jas`_ in `#44 <https://github.com/jonescompneurolab/hnn-core/pull/44>`_
 
+- Rhythmic inputs can now be turned off by setting their conductance weights to 0 instead of setting their start times to exceed the simulation stop time, by `Ryan Thorpe`_ in `#105 <https://github.com/jonescompneurolab/hnn-core/pull/105>`_
+
 - Reader for parameter files, by `Blake Caldwell`_ in `#80 <https://github.com/jonescompneurolab/hnn-core/pull/80>`_
 
-- Add plotting of voltage at soma to inspect firing pattern of cells, by `Mainak Jas`_ in `#86 <https://github.com/jasmainak/hnn-core/pull/86>`_
+- Add plotting of voltage at soma to inspect firing pattern of cells, by `Mainak Jas`_ in `#86 <https://github.com/jonescompneurolab/hnn-core/pull/86>`_
 
-- Rhythmic inputs can now be turned off by setting their conductance weights to 0 instead of setting their start times to exceed the simulation stop time, by `Ryan Thorpe`_ in `#105 <https://github.com/jonescompneurolab/hnn-core/pull/105>`_
+- Add ability to simulate a single trial in parallel across cores using MPI, by `Blake Caldwell`_ in `#79 <https://github.com/jonescompneurolab/hnn-core/pull/79>`_
 
 Bug
 ~~~
@@ -35,10 +37,14 @@ Bug
 API
 ~~~
 
-- Make a context manager for Network class, by `Mainak Jas`_ and `Blake Caldwell`_ in `#86 <https://github.com/jasmainak/hnn-core/pull/86>`_
+- Make a context manager for Network class, by `Mainak Jas`_ and `Blake Caldwell`_ in `#86 <https://github.com/jonescompneurolab/hnn-core/pull/86>`_
 
 - Create Spikes class, add write methods and read functions for Spikes and Dipole classes, by `Ryan Thorpe`_ in `#96 <https://github.com/jonescompneurolab/hnn-core/pull/96>`_
 
+- Remove `n_jobs` parameter for instantiating Network class, by `Blake Caldwell`_ in `#79 <https://github.com/jonescompneurolab/hnn-core/pull/79>`_
+
+- Make a context manager for parallel backends (JoblibBackend, MPIBackend), by `Blake Caldwell`_ in `#79 <https://github.com/jonescompneurolab/hnn-core/pull/79>`_
+
 .. _Mainak Jas: http://jasmainak.github.io/
 .. _Blake Caldwell: https://github.com/blakecaldwell
 .. _Ryan Thorpe: https://github.com/rythorpe

examples/plot_firing_pattern.py

@@ -16,6 +16,7 @@
 
 import hnn_core
 from hnn_core import read_params, Network
+from hnn_core.neuron import NeuronNetwork
 
 hnn_core_root = op.join(op.dirname(hnn_core.__file__), '..')
 
@@ -28,11 +29,12 @@
 # Now let's build the network
 import matplotlib.pyplot as plt
 
-with Network(params) as net:
-    net.build()
+net = Network(params)
+with NeuronNetwork(net) as neuron_network:
+    neuron_network.cells[0].plot_voltage()
 
     # The cells are stored in the network object as a list
-    cells = net.cells
+    cells = neuron_network.cells
     print(cells[:5])
 
     # We have different kinds of cells with different cell IDs (gids)
@@ -41,15 +43,13 @@
         print(cells[gid].name)
 
     # We can plot the firing pattern of individual cells
-    net.cells[0].plot_voltage()
+    neuron_network.cells[0].plot_voltage()
     plt.title('%s (gid=%d)' % (cells[0].name, gid))
 
 ###############################################################################
-# Let's do this for the rest of the cell types with a new Network object
-with Network(params) as net:
-    net.build()
-
+# Let's do this for the rest of the cell types with a new NeuronNetwork object
+with NeuronNetwork(net) as neuron_network:
     fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8, 4))
     for gid, ax in zip([35, 170], axes):
-        net.cells[gid].plot_voltage(ax)
+        neuron_network.cells[gid].plot_voltage(ax)
         ax.set_title('%s (gid=%d)' % (cells[gid].name, gid))

examples/plot_simulate_evoked.py

@@ -9,6 +9,7 @@
 
 # Authors: Mainak Jas <mainak.jas@telecom-paristech.fr>
 #          Sam Neymotin <samnemo@gmail.com>
+#          Blake Caldwell <blake_caldwell@brown.edu>
 
 import os.path as op
 import tempfile
@@ -63,6 +64,16 @@
 
 params.update({'sync_evinput': True})
 net_sync = Network(params)
-dpls_sync = simulate_dipole(net_sync, n_jobs=1, n_trials=1)
+
+###############################################################################
+# To simulate the dipole, we will use the MPI backend. This will
+# start the simulation across the number of processors (cores)
+# specified by n_procs using MPI. The 'mpiexec' launcher is for
+# openmpi, which must be installed on the system
+from hnn_core import MPIBackend
+
+with MPIBackend(n_procs=2, mpi_cmd='mpiexec'):
+    dpls_sync = simulate_dipole(net_sync, n_trials=1)
+
 dpls_sync[0].plot()
 net_sync.plot_input()

examples/plot_simulate_somato.py

@@ -77,7 +77,7 @@
 params = read_params(params_fname)
 
 net = Network(params)
-dpl = simulate_dipole(net)
+dpl = simulate_dipole(net, n_trials=1)
 
 import matplotlib.pyplot as plt
 fig, axes = plt.subplots(2, 1, sharex=True, figsize=(6, 6))

hnn_core/__init__.py

@@ -1,10 +1,7 @@
-from .utils import load_custom_mechanisms
-
-load_custom_mechanisms()
-
 from .dipole import simulate_dipole, read_dipole
 from .feed import ExtFeed
 from .params import Params, read_params
 from .network import Network, Spikes, read_spikes
 from .pyramidal import L2Pyr, L5Pyr
 from .basket import L2Basket, L5Basket
+from .parallel_backends import MPIBackend, JoblibBackend

hnn_core/cell.py

@@ -361,9 +361,9 @@ def parconnect_from_src(self, gid_presyn, nc_dict, postsyn):
         nc : instance of h.NetCon
             A network connection object.
         """
-        from .parallel import pc
+        from .neuron import PC
 
-        nc = pc.gid_connect(gid_presyn, postsyn)
+        nc = PC.gid_connect(gid_presyn, postsyn)
         # calculate distance between cell positions with pardistance()
         d = self._pardistance(nc_dict['pos_src'])
         # set props here