help_dat.dat automatic handling (#998)

* help_dat.dat automatic handling
* support python2 pickling
* windows pathlib fix

Co-authored-by: ramcdougal <robert.mcdougal@yale.edu>

.github/workflows/neuron-ci.yml

@@ -74,7 +74,7 @@ jobs:
 
       - name: Install Python2 dependencies
         if: ${{ matrix.config.use_python2 == 'ON' || matrix.config.python_dynamic == 'ON' }}
-        run: python2 -m pip install --upgrade pip wheel setuptools scikit-build pytest matplotlib bokeh ipython cython pytest pytest-cov mpi4py
+        run: python2 -m pip install --upgrade pip wheel setuptools scikit-build pytest matplotlib bokeh ipython cython pytest pytest-cov mpi4py pathlib
 
       - name: Set up Python3
         uses: actions/setup-python@v2

.gitignore

@@ -37,3 +37,4 @@ src/nrnoc/syn.c
 src/nrnoc/vclmp.c
 src/oc/parse.c
 src/oc/parse.h
+share/lib/python/neuron/help_data.dat

CMakeLists.txt

@@ -450,6 +450,23 @@ add_custom_target(
   VERBATIM)
 add_dependencies(nrniv_lib hh_update)
 
+# =============================================================================
+# Generate help_data.dat
+# =============================================================================
+if(NRN_ENABLE_PYTHON)
+  add_custom_target(
+      help_data_dat
+      COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/docs/parse_rst.py
+              ${PROJECT_SOURCE_DIR}/docs/python
+              ${PROJECT_SOURCE_DIR}/share/lib/python/neuron/help_data.dat
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different
+              ${PROJECT_SOURCE_DIR}/share/lib/python/neuron/help_data.dat
+              ${PROJECT_BINARY_DIR}/lib/python/neuron/help_data.dat
+      COMMENT "Generating help_data.dat"
+      VERBATIM)
+  add_dependencies(nrniv_lib help_data_dat)
+endif()
+
 # =============================================================================
 # Add tests if enabled
 # =============================================================================

Makefile.am

@@ -40,6 +40,13 @@ else
 	PARANEURON='no'
 endif
 
+# Generate help_data.dat since it's no longer committed in the repo.
+.PHONY: help_data_dat
+help_data_dat:
+	$(PYTHON_BLD) $(top_srcdir)/docs/parse_rst.py $(top_srcdir)/docs/python $(top_srcdir)/share/lib/python/neuron/help_data.dat
+
+-include help_data_dat
+
 if BUILD_CYGWIN
 #install from the build directories to the mswin destination
 #uses the classical positions of files

ci/win_install_deps.cmd

@@ -29,6 +29,9 @@ C:\Python37\python.exe -m pip install numpy==1.14.6 || goto :error
 C:\Python38\python.exe -m pip install numpy==1.17.5 || goto :error
 C:\Python27\python.exe -m pip install numpy || goto :error
 
+:: install pathlib
+C:\Python27\python.exe -m pip install pathlib || goto :error
+
 :: install nsis
 nsis-3.05-setup.exe /S || goto :error
 pwsh -command Expand-Archive EnVar_pugin.zip -DestinationPath "${env:ProgramFiles(x86)}\NSIS" || goto :error

docs/parse_rst.py

@@ -1,76 +1,96 @@
 """
-Convert rst to a compressed dictionary suitable for NEURON + Python
-help system.
+Convert rst to a compressed dictionary suitable for NEURON + Python help system.
 Run via:
-python parse_rst.py `find . -name \*.rst -print`
+>>> python3 parse_rst.py ./python/ help_data.dat
 """
 
+import os
 import sys
+from pathlib import Path
 
-help_dictionary = {}
 
-filenames = sys.argv[1:]
 
-def handle_identifier(lines, i, identifier):
-    identifier = '.. %s::' % identifier
-    line = lines[i]
-    start = line.find(identifier)
-    #print line, identifier, start
-    #print identifier
-    if start >= 0:
-        name = line[start + len(identifier):].strip()
+class ParseRst(object):
 
-        print '%s -- %s' % (name, identifier)
+    help_dictionary = {}
+
+    def __init__(self, rst_path, out_file):
+        self._rst_path = rst_path
+        self._out_file = out_file
+        self._filenames = Path(self._rst_path).glob('**/*.rst')
+
+    @classmethod
+    def handle_identifier(cls, lines, i, identifier):
+        identifier = '.. %s::' % identifier
+        line = lines[i]
+        start = line.find(identifier)
+        #print line, identifier, start
+        #print identifier
+        if start >= 0:
+            name = line[start + len(identifier):].strip()
+
+            #print('%s -- %s' % (name, identifier))
 
-        indent_line = lines[i + 1]
-        while not indent_line.strip():
-            i += 1
             indent_line = lines[i + 1]
-        start = 0
-        while start < len(indent_line):
-            if indent_line[start] != ' ': break
-            start += 1
-
-        # TODO: store the body text
-        body = []
-        while i < len(lines) - 1:
-            i += 1
-            if lines[i].strip():
-                if lines[i][:start] == indent_line[:start]:
-                    next_line = lines[i][start:]
-                    if next_line[-1] == '\n':
-                        next_line = next_line[: -1]
-                    body.append(next_line)
+            while not indent_line.strip():
+                i += 1
+                indent_line = lines[i + 1]
+            start = 0
+            while start < len(indent_line):
+                if indent_line[start] != ' ': break
+                start += 1
+
+            # TODO: store the body text
+            body = []
+            while i < len(lines) - 1:
+                i += 1
+                if lines[i].strip():
+                    if lines[i][:start] == indent_line[:start]:
+                        next_line = lines[i][start:]
+                        if next_line[-1] == '\n':
+                            next_line = next_line[: -1]
+                        body.append(next_line)
+                    else:
+                        break
                 else:
-                    break
-            else:
-                if not body or body[-1] != '\n':
-                    body.append('\n')
-        help_dictionary[name] = '\n'.join(body)
-
-
-for filename in filenames:
-    with open(filename) as f:
-        lines = []
-        for line in f:
-            if line[-1] == '\n':
-                line = line[:-1]
-            lines.append(line)
-
-    i = 0
-    while i < len(lines):
-        for kind in ['method', 'data', 'class', 'function']:
-            handle_identifier(lines, i, kind)
-        i += 1
-
-import cPickle
-import zlib
-compressed = zlib.compress(cPickle.dumps(help_dictionary))
-try:
-    import os
-    os.mkdir('_build')
-except:
-    # directory already exists, so nothing to do
-    pass
-with open('_build/help_data.dat', 'wb') as f:
-    f.write(compressed)
+                    if not body or body[-1] != '\n':
+                        body.append('\n')
+            cls.help_dictionary[name] = '\n'.join(body)
+
+    def parse(self):
+        for filename in self._filenames:
+            with open(str(filename)) as f:
+                lines = []
+                for line in f:
+                    if line[-1] == '\n':
+                        line = line[:-1]
+                    lines.append(line)
+            i = 0
+            while i < len(lines):
+                for kind in ['method', 'data', 'class', 'function']:
+                    ParseRst.handle_identifier(lines, i, kind)
+                i += 1
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 3:
+        rst_path = sys.argv[1]
+        out_file = sys.argv[2]
+    else:
+        print('usage: python3 parse_rst.py <rst_folder_path> <output_file>')
+        exit(1)
+
+    try:
+        ParseRst(rst_path, out_file).parse()
+        with open(out_file, 'wb') as f:
+            import pickle
+            import zlib
+
+            # TODO - protocol parameter shall be dropped along with Python2 support
+            compressed = zlib.compress(pickle.dumps(ParseRst.help_dictionary, protocol=2))
+            f.write(compressed)
+    except Exception:
+        import traceback
+
+        print(traceback.format_exc())
+        exit(1)

src/nrnpython/CMakeLists.txt

@@ -152,7 +152,7 @@ endif()
 # Install package files that were created in build (e.g. .py.in)
 install(DIRECTORY ${PROJECT_BINARY_DIR}/share/lib/python
         DESTINATION ${NRN_LIBDIR}
-        FILES_MATCHING PATTERN *.py PATTERN *.so PATTERN *.dylib)
+        FILES_MATCHING PATTERN *.py PATTERN *.so PATTERN *.dylib PATTERN *.dat)
 
 
 # =============================================================================
@@ -225,9 +225,6 @@ if(NRN_ENABLE_MODULE_INSTALL)
             ${PROJECT_BINARY_DIR}/share/nrn/lib
     COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/share/demo
             ${PROJECT_BINARY_DIR}/share/nrn/demo)
-
-  file(COPY ${PROJECT_SOURCE_DIR}/share/lib/python/neuron/help_data.dat
-        DESTINATION ${PROJECT_BINARY_DIR}/lib/python/neuron/)
 
   # =============================================================================
   # Build python module
@@ -238,6 +235,9 @@ if(NRN_ENABLE_MODULE_INSTALL)
       TARGET hoc_module POST_BUILD
       COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/inithoc.cpp
               ${CMAKE_CURRENT_BINARY_DIR}/inithoc.cpp
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different
+              ${PROJECT_SOURCE_DIR}/share/lib/python/neuron/help_data.dat
+              ${CMAKE_CURRENT_BINARY_DIR}/lib/python/neuron/help_data.dat
       COMMAND ${pyexe} setup.py --quiet build --build-lib=${NRN_PYTHON_BUILD_LIB}
       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
       COMMENT "Building python module with: ${pyexe}")