Part 1 of the big GPU merge

CMakeLists.txt

@@ -31,13 +31,17 @@ if(WITH_TRACE)
     add_definitions("-DWITH_TRACE")
 endif()
 
+# list of libraries to be linked against targets
+set(EXTERNAL_LIBRARIES "")
+
 #threading model selection
 set(THREADING_MODEL "serial" CACHE STRING "set the threading model, one of serial/tbb/omp")
 if(THREADING_MODEL MATCHES "tbb")
     # TBB support
     find_package(TBB REQUIRED)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TBB_DEFINITIONS}")
     add_definitions(-DWITH_TBB)
+    set(EXTERNAL_LIBRARIES ${EXTERNAL_LIBRARIES} ${TBB_LIBRARIES})
 
 elseif(THREADING_MODEL MATCHES "omp")
     # OpenMP support
@@ -47,17 +51,18 @@ elseif(THREADING_MODEL MATCHES "omp")
 
 elseif(THREADING_MODEL MATCHES "serial")
     #setup previously done
-   
+
 else()
     message( FATAL_ERROR "-- Threading model '${THREADING_MODEL}' not supported, use one of serial/tbb/omp")
+
 endif()
 
 # libunwind for pretty printing stack traces
-set(WITH_UNWIND OFF CACHE BOOL "use libunwind for debug messages" )
-if(WITH_UNWIND)
-    find_package(Unwind REQUIRED)
+find_package(Unwind)
+if(UNWIND_FOUND)
     add_definitions(-DWITH_UNWIND)
     include_directories(${UNWIND_INCLUDE_DIR})
+    set(EXTERNAL_LIBRARIES ${EXTERNAL_LIBRARIES} ${UNWIND_LIBRARIES})
 endif()
 
 # CUDA support
@@ -80,6 +85,7 @@ if(WITH_CUDA)
 
     add_definitions(-DWITH_GPU)
     include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
+    set(EXTERNAL_LIBRARIES ${EXTERNAL_LIBRARIES} ${CUDA_LIBRARIES})
 endif()
 
 # MPI support
@@ -93,15 +99,12 @@ if(WITH_MPI)
     set_property(DIRECTORY APPEND_STRING PROPERTY COMPILE_OPTIONS "${MPI_C_COMPILE_FLAGS}")
 endif()
 
-
 # Internal profiler support
 set(WITH_PROFILING OFF CACHE BOOL "use built-in profiling of miniapp" )
 if(WITH_PROFILING)
     add_definitions(-DWITH_PROFILING)
 endif()
 
-
-
 # Cray systems
 set(SYSTEM_CRAY OFF CACHE BOOL "add flags for compilation on Cray systems")
 if(SYSTEM_CRAY)

cmake/FindUnwind.cmake

@@ -8,41 +8,41 @@
 # respectively can be used to help CMake finding the library if it
 # is not installed in any of the usual locations.
 
-if (NOT UNWIND_FOUND)
-    set(UNWIND_SEARCH_DIR ${UNWIND_ROOT_DIR} $ENV{UNWIND_ROOT})
-
-    find_path(UNWIND_INCLUDE_DIR libunwind.h
-        HINTS ${UNWIND_SEARCH_DIR}
-        PATH_SUFFIXES include
-    )
-
-    # libunwind requires that we link agains both libunwind.so/a and a
-    # a target-specific library libunwind-target.so/a.
-    # This code sets the "target" string above in libunwind_arch.
-    if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
-        set(libunwind_arch "arm")
-    elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
-        set(libunwind_arch "x86_64")
-    elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
-        set(libunwind_arch "x86")
-    endif()
-
-    find_library(unwind_library_generic unwind
-        HINTS ${UNWIND_SEARCH_DIR}
-        PATH_SUFFIXES lib64 lib
-    )
-
-    find_library(unwind_library_target unwind-${libunwind_arch}
-        HINTS ${UNWIND_SEARCH_DIR}
-        PATH_SUFFIXES lib64 lib
-    )
-
-    set(UNWIND_LIBRARIES ${unwind_library_generic} ${unwind_library_target})
-
-    mark_as_advanced(UNWIND_LIBRARIES UNWIND_INCLUDE_DIR)
-
-    unset(unwind_search_dir)
-    unset(unwind_library_generic)
-    unset(unwind_library_target)
-    unset(libunwind_arch)
-endif ()
+set(UNWIND_FOUND ON)
+
+set(UNWIND_SEARCH_DIR ${UNWIND_ROOT_DIR} $ENV{UNWIND_ROOT})
+
+find_path(UNWIND_INCLUDE_DIR libunwind.h
+    HINTS ${UNWIND_SEARCH_DIR}
+    PATH_SUFFIXES include
+)
+
+# libunwind requires that we link agains both libunwind.so/a and a
+# a target-specific library libunwind-target.so/a.
+# This code sets the "target" string above in libunwind_arch.
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
+    set(libunwind_arch "arm")
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
+    set(libunwind_arch "x86_64")
+elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
+    set(libunwind_arch "x86")
+endif()
+
+find_library(unwind_library_generic unwind
+    HINTS ${UNWIND_SEARCH_DIR}
+    PATH_SUFFIXES lib64 lib
+)
+
+find_library(unwind_library_target unwind-${libunwind_arch}
+    HINTS ${UNWIND_SEARCH_DIR}
+    PATH_SUFFIXES lib64 lib
+)
+
+set(UNWIND_LIBRARIES ${unwind_library_generic} ${unwind_library_target})
+
+mark_as_advanced(UNWIND_LIBRARIES UNWIND_INCLUDE_DIR)
+
+unset(unwind_search_dir)
+unset(unwind_library_generic)
+unset(unwind_library_target)
+unset(libunwind_arch)

miniapp/CMakeLists.txt

@@ -18,21 +18,16 @@ else()
     add_executable(miniapp.exe ${MINIAPP_SOURCES} ${HEADERS})
 endif()
 
-target_link_libraries(miniapp.exe LINK_PUBLIC nestmc)
+set(aaa nestmc)
 
-if(WITH_TBB)
-    target_link_libraries(miniapp.exe LINK_PUBLIC ${TBB_LIBRARIES})
-endif()
+target_link_libraries(miniapp.exe LINK_PUBLIC nestmc)
+target_link_libraries(miniapp.exe LINK_PUBLIC ${EXTERNAL_LIBRARIES})
 
 if(WITH_MPI)
     target_link_libraries(miniapp.exe LINK_PUBLIC ${MPI_C_LIBRARIES})
     set_property(TARGET miniapp.exe APPEND_STRING PROPERTY LINK_FLAGS "${MPI_C_LINK_FLAGS}")
 endif()
 
-if(WITH_UNWIND)
-    target_link_libraries(miniapp.exe LINK_PUBLIC ${UNWIND_LIBRARIES})
-endif()
-
 set_target_properties(miniapp.exe
    PROPERTIES
    RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/miniapp"

scripts/print_backtrace

@@ -5,12 +5,43 @@ import argparse
 import os
 import subprocess
 
+class color:
+       purple = '\033[95m'
+       white  = '\033[37m'
+       cyan = '\033[96m'
+       darkcyan = '\033[36m'
+       blue = '\033[94m'
+       green = '\033[92m'
+       yellow = '\033[93m'
+       red = '\033[91m'
+       bold = '\033[1m'
+       underline = '\033[4m'
+       end = '\033[0m'
+
+class nocolor:
+       purple = ''
+       white = ''
+       cyan = ''
+       darkcyan = ''
+       blue = ''
+       green = ''
+       yellow = ''
+       red = ''
+       bold = ''
+       underline = ''
+       end = ''
+
+
 def parse_clargs():
     P = argparse.ArgumentParser(description='pretty print stack traces')
     P.add_argument('input', metavar='FILE',
                    help='name of file with stack trace')
-    P.add_argument('-b', '--brief', action='store_false',
+    P.add_argument('-b', '--brief', action='store_true',
                    help='print only the file locations')
+    P.add_argument('-e', '--executable', metavar='FILE',
+                   help='name of the executable or object file to look up symbols')
+    P.add_argument('-c', '--color', action='store_true',
+                   help='use color output in terminal')
 
     return P.parse_args()
 
@@ -23,27 +54,36 @@ def parse_backtrace(source):
             tokens = line.split()
             trace.append({'location':tokens[0], 'function':tokens[1]})
     else:
-        print "error: unable to open file ", source
+        print "error: unable to back trace file ", source
 
     return trace
 
-def get_function_name(location):
-    result = os.popen('addr2line ' + location + ' -e miniapp.exe').read()
+def get_function_name(location, executable):
+    result = os.popen('addr2line ' + location + ' -e ' + executable).read()
     descriptor = result.split()[0].split(':')
     return {'filename': descriptor[0], 'line': descriptor[1]}
 
 def unmangle(mangled):
     unmangled = os.popen('c++filt ' + mangled).read().strip()
-    # remove the nest::mc:: namespace from all types
     return unmangled.replace('nest::mc::', '')
 
+#
+# main
+#
 args = parse_clargs()
-trace = parse_backtrace(args.input)
 
-for frame in trace:
-    location = get_function_name(frame['location'])
-    name = unmangle(frame['function'])
-    if args.brief:
-        print location['filename'] + ':' + location['line'], name
-    else:
-        print location['filename'] + ':' + location['line']
+# check that a valid executable was provided
+executable = args.executable
+if not os.path.isfile(executable):
+    print "error:", executable, "is not a valid executable"
+else:
+    for frame in parse_backtrace(args.input):
+        location = get_function_name(frame['location'], executable)
+        name = unmangle(frame['function'])
+        c = color if args.color else nocolor
+        fname = c.yellow + location['filename'] + c.end
+        line = c.cyan + location['line'] + c.end
+        if args.brief:
+            print  fname + ':' + line
+        else:
+            print  fname + ':' + line, '\n ', name

src/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(nestmc ${BASE_SOURCES} ${HEADERS})
 add_dependencies(nestmc build_all_mods)
 if(WITH_CUDA)
     cuda_add_library(gpu ${CUDA_SOURCES})
+    set(NESTMC_LIBRARIES ${NESTMC_LIBRARIES} gpu)
     add_dependencies(gpu build_all_gpu_mods)
 endif()
 

src/fvm_multicell.hpp

@@ -15,6 +15,7 @@
 #include <ion.hpp>
 #include <math.hpp>
 #include <matrix.hpp>
+#include <memory/memory.hpp>
 #include <profiling/profiler.hpp>
 #include <segment.hpp>
 #include <stimulus.hpp>
@@ -24,8 +25,6 @@
 #include <util/rangeutil.hpp>
 #include <util/span.hpp>
 
-#include <memory/memory.hpp>
-
 namespace nest {
 namespace mc {
 namespace fvm {
@@ -63,7 +62,6 @@ class fvm_multicell {
 
     using matrix_assembler = typename backend::matrix_assembler;
 
-    /// API for cell_group (see above):
     using detector_handle = size_type;
     using target_handle = std::pair<size_type, size_type>;
     using probe_handle = std::pair<const array fvm_multicell::*, size_type>;
@@ -97,6 +95,7 @@ class fvm_multicell {
         return (this->*h.first)[h.second];
     }
 
+    /// integrate all cell state forward in time
     void advance(double dt);
 
     /// Following types and methods are public only for testing:
@@ -401,6 +400,15 @@ void fvm_multicell<Backend>::initialize(
     std::vector<value_type> tmp_cv_areas(ncomp);
     std::vector<value_type> tmp_cv_capacitance(ncomp);
 
+    // Iterate over the input cells and build the indexes etc that descrbe the
+    // fused cell group. On completion:
+    //  - group_paranet_index contains the full parent index for the fused cells.
+    //  - mech_map and syn_mech_map provide a map from mechanism names to an
+    //    iterable container of compartment ranges, which are used later to
+    //    generate the node index for each mechanism kind.
+    //  - the tmp_* vectors contain compartment-specific information for each
+    //    compartment in the fused cell group (areas, capacitance, etc).
+    //  - each probe, stimulus and detector is attached to its compartment.
     for (auto i: make_span(0, ncell)) {
         const auto& c = cells[i];
         auto comp_ival = cell_comp_part[i];
@@ -488,6 +496,11 @@ void fvm_multicell<Backend>::initialize(
         }
     }
 
+    // confirm user-supplied containers for detectors and probes were
+    // appropriately sized.
+    EXPECTS(detectors_size==detectors_count);
+    EXPECTS(probes_size==probes_count);
+
     // normalize capacitance across cell
     for (auto i: util::make_span(0, ncomp)) {
         tmp_cv_capacitance[i] /= tmp_cv_areas[i];
@@ -505,9 +518,9 @@ void fvm_multicell<Backend>::initialize(
         matrix_.d(), matrix_.u(), matrix_.rhs(), matrix_.p(),
         cv_areas_, face_alpha_, voltage_, current_, cv_capacitance_);
 
-    // create density mechanisms
+    // For each density mechanism build the full node index, i.e the list of
+    // compartments with that mechanism, then build the mechanism instance.
     std::vector<size_type> mech_comp_indices(ncomp);
-
     std::map<std::string, std::vector<size_type>> mech_index_map;
     for (auto& mech: mech_map) {
         mech_comp_indices.clear();
@@ -523,7 +536,7 @@ void fvm_multicell<Backend>::initialize(
         mech_index_map[mech.first] = mech_comp_indices;
     }
 
-    // create point (synapse) mechanisms
+    // Create point (synapse) mechanisms
     for (const auto& syni: syn_mech_indices) {
         const auto& mech_name = syni.first;
         size_type mech_index = mechanisms_.size();
@@ -553,17 +566,16 @@ void fvm_multicell<Backend>::initialize(
         target_hi = std::copy_n(std::begin(handles), n_indices, target_hi);
         targets_count += n_indices;
 
-        //auto mech = mechanism_catalogue::make(
-        auto mech = backend::make_mechanism(
-            mech_name, voltage_, current_, comp_indices);
+        auto mech = backend::make_mechanism(mech_name, voltage_, current_, comp_indices);
         mech->set_areas(cv_areas_);
         mechanisms_.push_back(std::move(mech));
+
+        // save the compartment indexes for this synapse type
+        mech_index_map[mech_name] = comp_indices;
     }
 
-    // confirm write-parameters were appropriately sized
-    EXPECTS(detectors_size==detectors_count);
+    // confirm user-supplied containers for targets are appropriately sized
     EXPECTS(targets_size==targets_count);
-    EXPECTS(probes_size==probes_count);
 
     // build the ion species
     for (auto ion : mechanisms::ion_kinds()) {
@@ -641,19 +653,16 @@ void fvm_multicell<Backend>::advance(double dt) {
         PL();
     }
 
+    // TODO KERNEL: the stimulus might have to become a "proper" mechanism
+    // so that the update kernel is fully implemented on GPU.
+
     // add current contributions from stimuli
     for (auto& stim : stimuli_) {
         auto ie = stim.second.amplitude(t_); // [nA]
         auto loc = stim.first;
 
-        // TODO KERNEL
-        // is a kernel actually needed?
-        // for now I only make the update if the injected current in nonzero to
-        // avoid a redundant host->device copy on the gpu
-        //
         // note: current_ in [mA/cm^2], ie in [nA], cv_areas_ in [µm^2].
         // unit scale factor: [nA/µm^2]/[mA/cm^2] = 100
-        //current_[loc] -= 100*ie/cv_areas_[loc];
         if (ie!=0.) {
             current_[loc] = current_[loc] - 100*ie/cv_areas_[loc];
         }