Merge branch 'add-technical-docs' into add-sphinx-mermaid

.github/styles/nest-styles/Units.yml

@@ -2,7 +2,6 @@
 extends: existence
 message: "Add a space between a numeral and a unit of measure."
 level: error
-noword: true
 tokens:
   - '\d+(?:MB|Gbit|GB|kbps|KB|kb)'
 

.github/workflows/ebrains-push.yml

@@ -10,14 +10,14 @@ jobs:
     if: ${{ github.repository_owner == 'nest' }}
     steps:
       - name: sycnmaster
-        uses: wei/git-sync@v3
+        uses: wei/git-sync@55c6b63b4f21607da0e9877ca9b4d11a29fc6d83
         with:
           source_repo: "nest/nest-simulator"
           source_branch: "master"
           destination_repo: "https://ghpusher:${{ secrets.EBRAINS_GITLAB_ACCESS_TOKEN }}@gitlab.ebrains.eu/nest/nest-simulator.git"
           destination_branch: "master"
       - name: synctags
-        uses: wei/git-sync@v3
+        uses: wei/git-sync@55c6b63b4f21607da0e9877ca9b4d11a29fc6d83
         with:
           source_repo: "nest/nest-simulator"
           source_branch: "refs/tags/*"

.github/workflows/nestbuildmatrix.yml

@@ -80,10 +80,11 @@ jobs:
 
       - name: "Install dependencies"
         run: |
+          sudo apt update
           sudo apt install python3-docutils
-          wget --progress=dot:mega 'https://github.com/errata-ai/vale/releases/download/v2.15.2/vale_2.15.2_Linux_64-bit.tar.gz'
-          echo '7528175c995818bcb88b07574dd2e17c1aad13f5676880f43927bb8f673095aa  vale_2.15.2_Linux_64-bit.tar.gz' | sha256sum -c
-          tar -xzf 'vale_2.15.2_Linux_64-bit.tar.gz'
+          wget --progress=dot:mega 'https://github.com/errata-ai/vale/releases/download/v2.26.0/vale_2.26.0_Linux_64-bit.tar.gz'
+          echo '956577b214ce3db8fb11483f99a183cf65673e3bd47423c6d4ebe37f085cadc7 vale_2.26.0_Linux_64-bit.tar.gz' | sha256sum -c
+          tar -xzf 'vale_2.26.0_Linux_64-bit.tar.gz'
 
       - name: "Run vale..."
         run: |
@@ -335,22 +336,28 @@ jobs:
       - name: "Set up Python 3.x"
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          # Using 3.8 because Read the docs does not work with higher versions.
+          # See also: https://github.com/nest/nest-simulator/pull/2744
+          python-version: 3.8
 
       - name: "Install dependencies"
         run: |
+          sudo apt update
           pip install -r doc/requirements.txt
+          # The pandoc executable, which is required, cannot be installed via pip see: https://stackoverflow.com/a/71585691
+          sudo apt install pandoc
 
       - name: "Test-build documentation"
         run: |
           mkdir -pv build
           cd build
-          cmake -DCMAKE_INSTALL_PREFIX=../install ..
-          make html |& tee sphinx-output.log
+          cmake -Dwith-userdoc=ON -DCMAKE_INSTALL_PREFIX=../install ..
+          make docs |& tee sphinx-output.log
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
+          name: "sphinx-rtd output"
           path: |
             build/sphinx-output.log
             build/userdoc/html/
@@ -415,7 +422,7 @@ jobs:
           - "boost, optimize, warning"
           - "openmp, python, gsl, ltdl, boost, optimize, warning"
           - "mpi, python, gsl, ltdl, boost, optimize, warning"
-          - "openmp, mpi, python, gsl, ltdl, boost, sionlib, libneurosim, optimize, warning"
+          - "openmp, mpi, python, gsl, ltdl, boost, hdf5, sionlib, libneurosim, optimize, warning"
 
     steps:
       - name: "Checkout repository content"
@@ -450,6 +457,7 @@ jobs:
           sudo apt-get install libltdl-dev libreadline6-dev libncurses5-dev libgsl0-dev python3-all-dev jq libpcre3 libpcre3-dev
           sudo apt-get install tcl8.6 tcl8.6-dev tk8.6-dev
           sudo apt-get install libboost-filesystem-dev libboost-regex-dev libboost-wave-dev libboost-python-dev libboost-program-options-dev libboost-test-dev
+          sudo apt-get install pkg-config
           sudo ldconfig
 
       - name: "Install GSL system dependencies"
@@ -462,6 +470,11 @@ jobs:
         run: |
           sudo apt-get install openmpi-bin libopenmpi-dev
 
+      - name: "Install HDF5 system dependencies"
+        if: "${{ contains(matrix.use, 'hdf5') }}"
+        run: |
+          sudo apt-get install libhdf5-dev
+
       - name: "Restore ccache"
         # Using CCache to re-use compiled objects from prior runs that have the same
         # source (hashed), compiler (mtime+size) and compile flags.
@@ -507,6 +520,11 @@ jobs:
         run: |
           python -m pip install --force-reinstall --upgrade mpi4py
 
+      - name: "Install h5py"
+        if: "${{ contains(matrix.use, 'hdf5') }}"
+        run: |
+          python -m pip install --force-reinstall --upgrade --no-binary=h5py h5py
+
       - name: "Install MUSIC"
         if: "${{ contains(matrix.use, 'music') }}"
         run: |
@@ -552,6 +570,7 @@ jobs:
               -Dwith-gsl=${{ contains(matrix.use, 'gsl') && 'ON' || 'OFF' }} \
               -Dwith-ltdl=${{ contains(matrix.use, 'ltdl') && 'ON' || 'OFF' }} \
               -Dwith-readline=${{ contains(matrix.use, 'readline') && 'ON' || 'OFF' }} \
+              -Dwith-hdf5=${{ contains(matrix.use, 'hdf5') && 'ON' || 'OFF' }} \
               -Dwith-sionlib=${{ contains(matrix.use, 'sionlib') && '$HOME/.cache/sionlib.install' || 'OFF' }} \
               -Dwith-libneurosim=${{ contains(matrix.use, 'libneurosim') && '$HOME/.cache/libneurosim.install' || 'OFF' }} \
               ..
@@ -624,7 +643,7 @@ jobs:
 
       - name: "Install MacOS system dependencies"
         run: |
-          brew install coreutils gsl open-mpi libomp automake autoconf libtool boost
+          brew install coreutils gsl open-mpi libomp automake autoconf libtool boost hdf5
 
       - name: "Restore pip cache"
         env:
@@ -643,7 +662,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools
           python -c "import setuptools; print('package location:', setuptools.__file__)"
-          python -m pip install --force-reinstall --upgrade scipy 'junitparser>=2' numpy pytest pytest-timeout pytest-xdist mpi4py cython matplotlib terminaltables pandoc pandas
+          python -m pip install --force-reinstall --upgrade scipy 'junitparser>=2' numpy pytest pytest-timeout pytest-xdist mpi4py h5py cython matplotlib terminaltables pandoc pandas
           test \! -e "=2"   # assert junitparser is correctly quoted and '>' is not interpreted as shell redirect
           python -c "import pytest; print('package location:', pytest.__file__)"
           pip list
@@ -671,6 +690,7 @@ jobs:
               -Dwith-gsl=${{ contains(matrix.use, 'gsl') && 'ON' || 'OFF' }} \
               -Dwith-ltdl=${{ contains(matrix.use, 'ltdl') && 'ON' || 'OFF' }} \
               -Dwith-readline=${{ contains(matrix.use, 'readline') && 'ON' || 'OFF' }} \
+              -Dwith-hdf5=${{ contains(matrix.use, 'hdf5') && 'ON' || 'OFF' }} \
               -Dwith-sionlib=${{ contains(matrix.use, 'sionlib') && '$HOME/.cache/sionlib.install' || 'OFF' }} \
               -Dwith-libneurosim=${{ contains(matrix.use, 'libneurosim') && '$HOME/.cache/libneurosim.install' || 'OFF' }} \
               ..

.readthedocs.yml

@@ -1,13 +1,14 @@
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+
 sphinx:
   builder: html
   configuration: doc/htmldoc/conf.py
 
 python:
-   version: "3.8"
    install:
    - requirements: doc/requirements.txt
-
-build:
-  image: latest

CMakeLists.txt

@@ -59,6 +59,7 @@ set( with-libneurosim OFF CACHE STRING "Build with libneurosim [default=OFF]. Op
 set( with-music OFF CACHE STRING "Build with MUSIC [default=OFF]. Optionally give the directory where MUSIC is installed." )
 set( with-sionlib OFF CACHE STRING "Build with SIONlib [default=OFF]. Optionally give the directory where sionlib is installed." )
 set( with-boost ON CACHE STRING "Build with Boost [default=ON]. To set a specific Boost installation, give the install path." )
+set( with-hdf5 OFF CACHE STRING "Find a HDF5 library. To set a specific HDF5 installation, set install path. [default=ON]" )
 set( with-readline ON CACHE STRING "Build with GNU Readline library [default=ON]. To set a specific library, give the install path." )
 set( with-ltdl ON CACHE STRING "Build with ltdl library [default=ON]. To set a specific ltdl, give the  install path. NEST uses ltdl for dynamic loading of external user modules." )
 set( with-gsl ON CACHE STRING "Build with the GSL library [default=ON]. To set a specific library, give the install path." )
@@ -148,6 +149,7 @@ nest_process_with_music()
 nest_process_with_sionlib()
 nest_process_with_mpi4py()
 nest_process_with_boost()
+nest_process_with_hdf5()
 nest_process_target_bits_split()
 nest_process_userdoc()
 nest_process_devdoc()

README.md

@@ -13,7 +13,7 @@
 [![Fedora package](https://img.shields.io/fedora/v/nest?logo=fedora)](https://src.fedoraproject.org/rpms/nest)
 [![Conda version](https://img.shields.io/conda/vn/conda-forge/nest-simulator.svg?logo=conda-forge&logoColor=white)](https://anaconda.org/conda-forge/nest-simulator)
 [![Homebrew version](https://img.shields.io/homebrew/v/nest.svg?logo=apple)](https://formulae.brew.sh/formula/nest)
-[![Docker Image Version](https://img.shields.io/docker/v/nestsim/nest?color=blue&label=docker&logo=docker&logoColor=white&sort=semver)](https://hub.docker.com/r/nestsim/nest)
+[![Docker Image Version](https://img.shields.io/docker/v/nest/nest-simulator?color=blue&label=docker&logo=docker&logoColor=white&sort=semver)](https://hub.docker.com/r/nest/nest-simulator)
 [![Virtual applicance](https://img.shields.io/badge/VM-v3.4-blue?logo=CodeSandbox)](https://nest-simulator.readthedocs.io/en/latest/installation/livemedia.html#live-media)
 
 [![YouTube Video Views](https://img.shields.io/youtube/views/K7KXmIv6ROY?style=social)](https://www.youtube.com/results?search_query=nest-simulator+neurons)

build_support/version_info.sh

@@ -22,6 +22,12 @@
 # This script extracts version control information to be used in
 # cmake/NestVersionInfo.cmake
 
+# If we can't run git at all, set everything to "unknown"
+if ! command -v git > /dev/null 2>&1; then
+  echo unknown\;unknown\;unknown
+  exit 0
+fi
+
 HASH=$(git rev-parse HEAD)
 
 # Might fail if not on a branch, or no remote tracking branch is set

cmake/ConfigureSummary.cmake

@@ -190,6 +190,15 @@ function( NEST_PRINT_CONFIG_SUMMARY )
     message( "Use SIONlib         : No")
   endif ()
 
+  message( "" )
+  if ( HAVE_HDF5 )
+    message( "Use HDF5            : Yes (HDF5 ${HDF5_VERSION})" )
+    message( "    Includes        : ${HDF5_INCLUDE_DIR}" )
+    message( "    Libraries       : ${HDF5_LIBRARIES}" )
+  else()
+    message( "Use HDF5            : No" )
+  endif()
+
   if ( with-libraries )
     message( "" )
     message( "Additional libraries:" )

cmake/ProcessOptions.cmake

@@ -596,6 +596,32 @@ function( NEST_PROCESS_WITH_BOOST )
   endif ()
 endfunction()
 
+function( NEST_PROCESS_WITH_HDF5 )
+
+  set( HAVE_HDF5 OFF PARENT_SCOPE )
+  if ( with-hdf5 )
+    if ( NOT ${with-hdf5} STREQUAL "ON" )
+      # a path is set
+      set( HDF5_ROOT "${with-hdf5}" )
+    endif ()
+
+    find_package( HDF5 REQUIRED COMPONENTS C CXX )
+    if ( HDF5_FOUND )
+      # export found variables to parent scope
+      set( HAVE_HDF5 ON PARENT_SCOPE )
+      set( HDF5_FOUND "${HDF5_FOUND}" PARENT_SCOPE )
+      set( HDF5_LIBRARIES "${HDF5_LIBRARIES}" PARENT_SCOPE )
+      set( HDF5_INCLUDE_DIR "${HDF5_INCLUDE_DIRS}" PARENT_SCOPE )
+      set( HDF5_VERSION "${HDF5_VERSION}" PARENT_SCOPE )
+      set( HDF5_HL_LIBRARIES "${HDF5_HL_LIBRARIES}" PARENT_SCOPE )
+      set( HDF5_DEFINITIONS "${HDF5_DEFINITIONS}" PARENT_SCOPE )
+
+      include_directories( ${HDF5_INCLUDE_DIRS} )
+
+    endif ()
+  endif ()
+endfunction()
+
 function( NEST_PROCESS_TARGET_BITS_SPLIT )
   if ( target-bits-split )
     # set to value according to defines in config.h

doc/README.md

@@ -5,6 +5,9 @@ as well as the infrastructure and tools for extracting documentation
 dynamically from the source code and building a unified HTML site
 using Sphinx.
 
+To read the NEST documentation, please visit
+https://nest-simulator.readthedocs.org.
+
 For more information about the different workflows for documentation,
 please see
-https://nest-simulator.readthedocs.io/en/latest/documentation_workflow/index.html
+https://nest-simulator.readthedocs.io/en/latest/developer_space/workflows/documentation_workflow/

doc/htmldoc/connect_nest/nest_server.rst

@@ -106,7 +106,7 @@ container:
 
 .. code-block:: text
 
-  docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` -p 52425:52425 nestsim/nest:latest nest-server start
+  docker run -it --rm -e LOCAL_USER_ID=`id -u $USER` -p 52425:52425 nest/nest-simulator:dev nest-server start
 
 The generic invocation command line for the ``nest-server`` command
 looks as follows:

doc/htmldoc/examples/index.rst

@@ -136,6 +136,10 @@ PyNEST examples
 
            * :doc:`../auto_examples/CampbellSiegert`
 
+    .. grid-item-card:: SONATA network
+           :img-top: ../static/img/300_pointneurons.png
+
+           * :doc:`../auto_examples/sonata_example/sonata_network`
 
 
 .. grid:: 1 1 2 3

doc/htmldoc/hpc/benchmarking.rst

@@ -0,0 +1,48 @@
+.. _benchmark:
+
+Benchmarking NEST
+=================
+
+
+beNNch
+~~~~~~
+
+Computational efficiency is essential to simulate complex neuronal networks and study long-term effects such as learning.
+The scaling performance of neuronal network simulators on high-performance computing systems can be assessed with benchmark simulations.
+However, maintaining comparability of benchmark results across different systems, software environments, network models, and researchers from potentially different labs poses a challenge.  
+
+The software framework `beNNch <https://github.com/INM-6/beNNch>`_ tackles this challenge by implementing a unified, modular workflow for configuring, executing, and analyzing such benchmarks.  
+beNNch builds around the `JUBE Benchmarking Environment <https://www.fz-juelich.de/ias/jsc/EN/Expertise/Support/Software/JUBE/_node.html>`_, installs simulation software, provides an interface to benchmark models, automates data and metadata annotation, and accounts for storage and presentation of results.
+
+For more details on the conceptual ideas behind beNNch, refer to Albers et al. (2022) [1]_.
+
+.. figure:: ../static/img/multi-area-model_5faa0e9c.png
+
+    Example ``beNNch`` output (Figure 5C of [1]_) 
+
+    Strong-scaling performance of the `multi-area model <https://github.com/INM-6/multi-area-model>`_ simulated with NEST on JURECA-DC.
+    The left graph shows the absolute wall-clock time measured with Python-level timers for both network construction and state propagation.
+    Error bars indicate variability across three simulation repeats with different random seeds.
+    The top right graph displays the real-time factor defined as wall-clock time normalized by the model time.
+    Built-in timers resolve four different phases of the state propagation: update, collocation, communication, and delivery.
+    Pink error bars show the same variability of state propagation as the left graph.
+    The lower right graph shows the relative contribution of these phases to the state-propagation time.
+
+
+.. seealso:: 
+
+    For further details, see the accompanying `beNNch GitHub Page <https://inm-6.github.io/beNNch>`_.
+    And for a detailed step-by-step walk though see `Walk through guide <https://inm-6.github.io/beNNch/walk-through.html>`_.
+
+    Example PyNEST script: :doc:`../auto_examples/hpc_benchmark`
+
+
+References
+----------
+
+
+.. [1] Albers J., et al (2022). A Modular Workflow for Performance Benchmarking of Neuronal Network Simulations.
+       Frontiers in Neuroinformatics(16):837549. https://doi.org/10.3389/fninf.2022.837549
+
+
+

doc/htmldoc/hpc/index.rst

@@ -1,9 +1,7 @@
 :orphan:
 
-.. _hpc_index:
-
-All about high performance computing
-====================================
+High performance computing
+==========================
 
 .. toctree::
   :maxdepth: 1