NEST Help Generator

.gitignore

@@ -16,7 +16,6 @@ build/
 /juqueen/
 /extras/nest-config
 /extras/create_release.sh
-/extras/userdoc/cmds/
 /libnestutil/config.h
 /librandom/randomtest
 /libtool

CMakeLists.txt

@@ -173,6 +173,7 @@ nest_check_have_stl_vector_capacity_doubling()
 nest_check_have_xlc_ice_on_using()
 nest_check_have_std_nan()
 nest_check_have_std_isnan()
+nest_generate_help()
 
 ################################################################################
 ##################         Enable Testing Targets             ##################

cmake/GenerateHelp.cmake

@@ -48,29 +48,11 @@ function(NEST_GENERATE_HELP)
   endforeach ()
 
   if ( NOT CMAKE_CROSSCOMPILING )
-    # install help (depends on sli and lib/sli/* installed)
     install( CODE
-        "execute_process(
-          COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_INSTALL_FULL_DOCDIR}/help/sli
-          COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_INSTALL_FULL_DOCDIR}/help/cc
-          COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_INSTALL_FULL_DOCDIR}/help/helpindex.html
-          COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_INSTALL_FULL_DOCDIR}/help/helpindex.hlp
-
-          COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_DOCDIR}/help
-          COMMAND ${CMAKE_COMMAND}
-              -DDOC_DIR='${CMAKE_INSTALL_FULL_DOCDIR}'
-              -DDATA_DIR='${CMAKE_INSTALL_FULL_DATADIR}'
-              -DHELPDIRS='${HELPDIRS}'
-              -DINSTALL_DIR='${CMAKE_INSTALL_PREFIX}'
-              -P ${PROJECT_SOURCE_DIR}/cmake/generate_help.cmake
-            WORKING_DIRECTORY \"${PROJECT_BINARY_DIR}\"
-          )"
-        )
-        install( CODE
-            "execute_process(
-                COMMAND ${PYTHON} parse_help.py \"${CMAKE_INSTALL_FULL_DOCDIR}/help2\"
-                WORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/extras/userdoc/generator\"
-              )"
-            )
+      "execute_process(
+         COMMAND python -B parse_help.py \"${PROJECT_SOURCE_DIR}\" \"${PROJECT_BINARY_DIR}\" \"${CMAKE_INSTALL_FULL_DOCDIR}/help\"
+         WORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/extras/help_generator\"
+         )"
+      )
   endif ()
 endfunction()

examples/MyModule/CMakeLists.txt

@@ -268,18 +268,14 @@ install( FILES ${MODULE_HEADER} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} )
 install( DIRECTORY sli DESTINATION ${CMAKE_INSTALL_DATADIR} )
 
 # Install help.
-set( HELPDIRS "${PROJECT_SOURCE_DIR}:${PROJECT_SOURCE_DIR}/sli" )
 install( CODE
-    "execute_process(COMMAND ${CMAKE_COMMAND}
-          -DDOC_DIR='${CMAKE_INSTALL_FULL_DOCDIR}'
-          -DDATA_DIR='${CMAKE_INSTALL_FULL_DATADIR}'
-          -DHELPDIRS='${HELPDIRS}'
-          -DINSTALL_DIR='${CMAKE_INSTALL_PREFIX}'
-          -P ${CMAKE_INSTALL_FULL_DOCDIR}/generate_help.cmake
-        WORKING_DIRECTORY \"${PROJECT_BINARY_DIR}\"
-      )"
+    "execute_process(
+        COMMAND python -B parse_help.py ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} \"${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_DOCDIR}/help\"
+        WORKING_DIRECTORY \"${CMAKE_INSTALL_PREFIX}/${NEST_DATADIR}/help_generator\"
+        )"
     )
 
+
 message( "" )
 message( "-------------------------------------------------------" )
 message( "${MODULE_NAME} Configuration Summary" )

extras/CMakeLists.txt

@@ -28,6 +28,10 @@ install( DIRECTORY bibliography logos
     DESTINATION ${CMAKE_INSTALL_DOCDIR}
     )
 
+install( DIRECTORY help_generator
+    DESTINATION ${CMAKE_INSTALL_DATADIR}
+    )
+
 install( FILES EditorSupport/emacs/postscript-sli.el EditorSupport/emacs/psvn.el ${CMAKE_CURRENT_BINARY_DIR}/EditorSupport/emacs/sli.el
     DESTINATION ${CMAKE_INSTALL_DATADIR}/extras/EditorSupport/emacs
     )

extras/EditorSupport/vim/syntax/sli.vim

@@ -16,7 +16,7 @@ setlocal cindent
 syntax keyword sliTodo TODO XXX FIXME NOTE
 hi def link sliTodo        Todo
 
-syn keyword sliKeyword abort abs abs_d abs_i acos add addpath addtotrie allocations and append AppendTo Apply apropos arange area2 area array arrayload ArrayQ ArrayShape arraystore asin assert available backtrace_off backtrace_on begin bind break breakup callback call capacity case cd ceil cerr ChangeSubnet cin clear cleardict cleardictstack clic clock cloc clonedict close closeinput CompareFiles CompileMath Connect connruledict continue copy CopyFile CopyModel cos count countdictstack counttomark cout Create CreateRDV CreateRNG cst ctermid currentdict currentname CurrentSubnet cv1d convert 2-dimensional coordinates to 1-dim index cv2d cva cva_d cva_t cvd cvdict cvd_s cv_dv cvgidcollection cvi cvi_s cv_iv cvlit cvn cvo cvs cvs_f cvt_a cvx cycles CyclicValue DataConnect_a DataConnect DataConnect_i_D_s debug debugoff debugon debug.sli def DeleteFile dexp dict DictQ dictstack Dimensions Directory dirname div Dot1D Dot DoubleQ drand Drop dup2 dup edit elementstates empty end endl endusing environment eof eq_dv eq_iv eq erase Erfc Erf errordict eval EvaluateLiteralInfixes E exch exec ExecFunction ExecMath execstack executive exit exithook exp Export FileNames file FindRoot FiniteQ First FixedPoint Flatten floor flush FoldList Fold forall forallindexed for fork FractionalPart frexp Function gabor_ Gammainc gauss2d_ GaussDiskConv geq GetConnections getc GetDefaults get_d getenv GetGlobalChildren GetGlobalLeaves GetGlobalNodesInfo GetGlobalNodes GetGlobalRNG getinterval getline GetLocalChildren GetLocalLeaves GetLocalNodes GetMax GetMin GetNetwork GetOption GetOptions getPGRP getPID getPPID get gets GetStatus_dict GetStatus GetStatus_v GetSynapseStatus GetVpRNG GNUaddhistory GNUreadline good grep gt handleerror HasDifferentMemberQ helpdesk helpindex help iclear ieof ifail ifstream ignore igood in_avail index info info_ds initialize_module Inline insertelement insert inspect Install IntegerPart IntegerQ inv irand iround isatty is_mpi is_threaded is_threaded_with_openmp join joinpath JoinTo kernel keys kill known LambertW0 LambertWm1 LambertW Last LayoutArray LayoutNetwork ldexp length_a length length_d length_lp length_p length_s leq license LiteralQ ln load LocateFileNames log lookup loop ls lt MakeDirectory makehelp Map MapAt MapIndexed MapThread mark MathematicaToSliIndex mathexecutive MatrixQ Max max Mean MemberQ MemoryInfo memory_thisjob_bg memory_thisjob_darwin memory_thisjob MergeDictionary MergeLists message Min min mkfifo mod modeldict modf Most MoveDirectory MoveFile ms2hms mul namespace neg_d neg_i neg neq Nest nest_indirect NestList nest_serial Node noop not npop NumberQ NumProcesses oclear oeof ofsopen ofstream ogood oldgetline ones operandstack Options OptionsDictionary or osstream ostrstream OuterProduct over page pageoutput parsestdin Partition Part path pclockspersec pclocks pcvs pgetrusage pick pipe Pi Plus pop pow ppage pprint prepend print_error PrintNetwork print ProcessorName proxynode pstack ptimes put_d putinterval put pwd quit raiseagain raiseerror RandomArray Random RandomSubset Range Rank rdevdict ReadDouble ReadInt readline ReadList ReadModes readPGM Read ReadWord realtime references regcomp regexdict regexec regex_find regex_find_r regex_find_rf regex_find_s regex_find_sf regex_replace RemoveDirectory removeguard repeatany repeat ReplaceOccurrences ReplacePart replace reserve ResetKernel ResetNetwork ResetOptions reset RestoreDictionary restoreestack RestoreOptions restoreostack restore Rest resume ResumeSimulation reverse rngdict rolld roll rollu rot round run SaveDictionary SaveOptions save ScanThread searchfile searchif searchifstream search seed Select SetAcceptableLatency SetDefaults Set SetDirectory SetFakeNumProcesses setguard setNONBLOCK SetNumRecProcesses SetOptions setpath setprecision SetStatus_dict SetStatus SetStatus_v SetSynapseStatus setverbosity SFWdumpparameters ShowDefaults ShowOptions ShowStatus shpawn shrink signaldict Sign Simulate sin size sleep_i sleep SLIFunctionWrapper SliToMathematicaIndex Sort spawn spikes Split spoon sqr sqrt stack StandardDeviation start statusdict stopped stop StringQ str subnet SubsetQ sub switchdefault switch symbol symbol_s synapsedict SyncProcesses sysexec system systemtime Table Take taskset taskset_thisjob TensorRank tic TimeCommunicationAlltoall TimeCommunicationAlltoall TimeCommunicationAlltoallv TimeCommunicationAlltoallv TimeCommunicationOffgrid TimeCommunication TimeCommunication TimeCommunicationv time Times tmpnam toc token_is token token_s ToLowercase ToMathematicaExpression topinfo_d Total ToUppercase Transpose trie trieinfo trim trunc typebind typeinfo type typestack undef unit_conversion UnitStep usertime using validate values Variance variant verbosity volume_transmitter waitPID wait welcome which who whos writePGM xifstream xor zeros
+syn keyword sliKeyword abort abs abs_d abs_i acos add addpath addtotrie allocations and append AppendTo Apply apropos arange area2 area array arrayload ArrayQ ArrayShape arraystore asin assert available backtrace_off backtrace_on begin bind break breakup callback call capacity case cd ceil cerr ChangeSubnet cin clear cleardict cleardictstack clic clock cloc clonedict close closeinput CompareFiles CompileMath Connect connruledict continue copy CopyFile CopyModel cos count countdictstack counttomark cout Create CreateRDV CreateRNG cst ctermid currentdict currentname CurrentSubnet cv1d convert 2-dimensional coordinates to 1-dim index cv2d cva cva_d cva_t cvd cvdict cvd_s cv_dv cvgidcollection cvi cvi_s cv_iv cvlit cvn cvo cvs cvs_f cvt_a cvx cycles CyclicValue DataConnect_a DataConnect DataConnect_i_D_s debug debugoff debugon debug.sli def DeleteFile dexp dict DictQ dictstack Dimensions Directory dirname div Dot1D Dot DoubleQ drand Drop dup2 dup edit elementstates empty end endl endusing environment eof eq_dv eq_iv eq erase Erfc Erf errordict eval EvaluateLiteralInfixes E exch exec ExecFunction ExecMath execstack executive exit exithook exp Export FileNames file FindRoot FiniteQ First FixedPoint Flatten floor flush FoldList Fold forall forallindexed for fork FractionalPart frexp Function gabor_ Gammainc gauss2d_ GaussDiskConv geq GetConnections getc GetDefaults get_d getenv GetGlobalChildren GetGlobalLeaves GetGlobalNodesInfo GetGlobalNodes GetGlobalRNG getinterval getline GetLocalChildren GetLocalLeaves GetLocalNodes GetMax GetMin GetNetwork GetOption GetOptions getPGRP getPID getPPID get gets GetStatus_dict GetStatus GetStatus_v GetSynapseStatus GetVpRNG GNUaddhistory GNUreadline good grep gt handleerror HasDifferentMemberQ helpdesk helpindex help iclear ieof ifail ifstream ignore igood in_avail index info info_ds initialize_module Inline insertelement insert inspect Install IntegerPart IntegerQ inv irand iround isatty is_mpi is_threaded is_threaded_with_openmp join joinpath JoinTo kernel keys kill known LambertW0 LambertWm1 LambertW Last LayoutArray LayoutNetwork ldexp length_a length length_d length_lp length_p length_s leq license LiteralQ ln load LocateFileNames log lookup loop ls lt MakeDirectory Map MapAt MapIndexed MapThread mark MathematicaToSliIndex mathexecutive MatrixQ Max max Mean MemberQ MemoryInfo memory_thisjob_bg memory_thisjob_darwin memory_thisjob MergeDictionary MergeLists message Min min mkfifo mod modeldict modf Most MoveDirectory MoveFile ms2hms mul namespace neg_d neg_i neg neq Nest nest_indirect NestList nest_serial Node noop not npop NumberQ NumProcesses oclear oeof ofsopen ofstream ogood oldgetline ones operandstack Options OptionsDictionary or osstream ostrstream OuterProduct over page pageoutput parsestdin Partition Part path pclockspersec pclocks pcvs pgetrusage pick pipe Pi Plus pop pow ppage pprint prepend print_error PrintNetwork print ProcessorName proxynode pstack ptimes put_d putinterval put pwd quit raiseagain raiseerror RandomArray Random RandomSubset Range Rank rdevdict ReadDouble ReadInt readline ReadList ReadModes readPGM Read ReadWord realtime references regcomp regexdict regexec regex_find regex_find_r regex_find_rf regex_find_s regex_find_sf regex_replace RemoveDirectory removeguard repeatany repeat ReplaceOccurrences ReplacePart replace reserve ResetKernel ResetNetwork ResetOptions reset RestoreDictionary restoreestack RestoreOptions restoreostack restore Rest resume ResumeSimulation reverse rngdict rolld roll rollu rot round run SaveDictionary SaveOptions save ScanThread searchfile searchif searchifstream search seed Select SetAcceptableLatency SetDefaults Set SetDirectory SetFakeNumProcesses setguard setNONBLOCK SetNumRecProcesses SetOptions setpath setprecision SetStatus_dict SetStatus SetStatus_v SetSynapseStatus setverbosity SFWdumpparameters ShowDefaults ShowOptions ShowStatus shpawn shrink signaldict Sign Simulate sin size sleep_i sleep SLIFunctionWrapper SliToMathematicaIndex Sort spawn spikes Split spoon sqr sqrt stack StandardDeviation start statusdict stopped stop StringQ str subnet SubsetQ sub switchdefault switch symbol symbol_s synapsedict SyncProcesses sysexec system systemtime Table Take taskset taskset_thisjob TensorRank tic TimeCommunicationAlltoall TimeCommunicationAlltoall TimeCommunicationAlltoallv TimeCommunicationAlltoallv TimeCommunicationOffgrid TimeCommunication TimeCommunication TimeCommunicationv time Times tmpnam toc token_is token token_s ToLowercase ToMathematicaExpression topinfo_d Total ToUppercase Transpose trie trieinfo trim trunc typebind typeinfo type typestack undef unit_conversion UnitStep usertime using validate values Variance variant verbosity volume_transmitter waitPID wait welcome which who whos writePGM xifstream xor zeros
 hi def link sliKeyword     Keyword
 
 syn keyword sliConstant M_INFO M_ERROR M_DEBUG M_WARNING M_STATUS M_FATAL

extras/help_generator/README.md

@@ -0,0 +1,4 @@
+# README for the NEST help generator
+
+The parser goes through all .sli and .cc files to find documentation
+and converts it into .html and .hlp files.

extras/userdoc/generator/modules/helpers.py → extras/help_generator/helpers.py

@@ -20,25 +20,29 @@
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
+import os
+import errno
 
 
 def cut_it(separator, text):
     """
-    Cut it.
+    Cut it
+    ------
 
-    cut text by separator
+    Cut text by separator.
     """
     if separator:
         return re.split(separator, text)
     else:
-        return text
+        return [separator, text]
 
 
 def check_ifdef(item, filetext, docstring):
         """
-        Check the ifdef context.
+        Check the 'ifdef' context
+        -------------------------
 
-        If there is an ifdef requirement write it to the data.
+        If there is an 'ifdef' requirement write it to the data.
         """
         ifdefstring = r'(\#ifdef((.*?)\n(.*?)\n*))\#endif'
         require_reg = re.compile('HAVE\_((.*?)*)\n')
@@ -51,3 +55,26 @@ def check_ifdef(item, filetext, docstring):
                     if item == initem:
                         features = require_reg.search(str_ifdef)
                         return features.group()
+
+
+def makedirs(path):
+    """
+    Forgiving version of os.makedirs, emulating the behavior of the
+    shell command 'mkdir -p'. The function tries to create the
+    directory at the given path including all subdirectories and
+    returns silently if the directory already exists.
+    """
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno != errno.EEXIST or os.path.isdir(path):
+            raise
+
+
+def create_helpdirs(path):
+    """
+    Create the directories for the help files.
+    """
+
+    makedirs(os.path.join(path, 'sli'))
+    makedirs(os.path.join(path, 'cc'))