Transition to pyproject.toml (#1147)

* replace `setup.py` with `pyproject.toml`
* move all Python package requirements to single `requirements.txt`
* remove checking Python package requirements in `CMakeLists.txt`
* move Python bindings to own dir (python/nmodl), fixes #462
* add separate script for generating docs
* add `packaging/change_name.py` script to as workaround to enable both
  NMODL and NMODL-nightly wheels in CI
* update documentation and CI to reflect above changes

---------

Co-authored-by: Luc Grosheintz <luc.grosheintz@gmail.com>

.github/workflows/coverage.yml

@@ -6,7 +6,7 @@ concurrency:
 
 on:
   push:
-    branches: 
+    branches:
       - master
       - live-debug*
       - release/**
@@ -41,7 +41,7 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 2    
+          fetch-depth: 2
       - name: Install Python3 dependencies
         working-directory: ${{runner.workspace}}/nmodl
         run: |

.github/workflows/nmodl-ci.yml

@@ -191,7 +191,7 @@ jobs:
           path: ${{runner.workspace}}/nmodl/build/Testing/*/Test.xml
 
       # This step will set up an SSH connection on tmate.io for live debugging.
-      # To enable it, you have to: 
+      # To enable it, you have to:
       #   * add 'live-debug-tests' to your PR title
       #   * push something to your PR branch (note that just re-running disregards the title update)
       - name: live debug session on failure (manual steps required, check `nmodl-ci.yml`)

.github/workflows/nmodl-doc.yml

@@ -6,7 +6,7 @@ concurrency:
 
 on:
   push:
-    branches: 
+    branches:
       - master
       - release/**
   pull_request:
@@ -37,7 +37,7 @@ jobs:
         uses: jwlawson/actions-setup-cmake@v2
         with:
           cmake-version: ${{ env.DESIRED_CMAKE_VERSION }}
-          
+
       - name: Install apt packages
         run: |
           sudo apt-get update
@@ -55,49 +55,28 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install Python3 dependencies
+      - name: Documentation
+        id: documentation
         working-directory: ${{runner.workspace}}/nmodl
         run: |
-          pip3 install -U pip setuptools
-          pip3 install --user -r requirements.txt
+          echo "------- Build Documentation -------";
+          bash docs/generate_docs.sh public $(command -v python${PYTHON_VERSION})
+          touch public/docs/.nojekyll
+          echo "<meta http-equiv=\"refresh\" content=\"0; url=./html/index.html\" />" > public/docs/index.html
+          echo "status=done" >> $GITHUB_OUTPUT
+        env:
+          CCACHE_DIR: ${{runner.workspace}}/ccache
 
       # This step will set up an SSH connection on tmate.io for live debugging.
       # To trigger it, simply add 'live-debug-docs' to your last pushed commit message.
       - name: live debug session on failure
         if: failure() && contains(github.event.head_commit.message, 'live-debug-docs')
         uses: mxschmitt/action-tmate@v3
 
-      - name: Restore compiler cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ${{runner.workspace}}/ccache
-          key: docs-${{github.ref}}-${{github.sha}}
-          restore-keys: |
-            docs-${{github.ref}}-
-            docs-
-
-      - name: Documentation 
-        id: documentation
-        working-directory: ${{runner.workspace}}/nmodl
-        run: |
-          echo "------- Build Documentation -------";
-          ccache -z
-          ccache -s
-          python3 setup.py build_ext --inplace docs -j 2 -G Ninja \
-            -- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache;
-          ccache -s
-          cd _skbuild/linux-x86_64-3.8/setuptools/sphinx;
-          rm -rf doctest doctrees && touch .nojekyll;
-          echo "<meta http-equiv=\"refresh\" content=\"0; url=./html/index.html\" />" > index.html;
-          echo "status=done" >> $GITHUB_OUTPUT
-        env:
-          CCACHE_DIR: ${{runner.workspace}}/ccache
-
       - name: Deploy 🚀
         uses: JamesIves/github-pages-deploy-action@v4
         if: steps.documentation.outputs.status == 'done' && startsWith(github.ref, 'refs/heads/master')
         with:
           branch: gh-pages # The branch the action should deploy to.
-          folder: ${{runner.workspace}}/nmodl/_skbuild/linux-x86_64-3.8/setuptools/sphinx # The folder the action should deploy.
+          folder: ${{runner.workspace}}/nmodl/public/docs
           clean: false # Automatically remove deleted files from the deploy branch

CMakeLists.txt

@@ -26,6 +26,12 @@ set(NMODL_EXTRA_CXX_FLAGS
     ""
     CACHE STRING "Add extra compile flags for NMODL sources")
 separate_arguments(NMODL_EXTRA_CXX_FLAGS)
+option(LINK_AGAINST_PYTHON "Should the Python library be linked or not" ON)
+option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF)
+if(NMODL_BUILD_WHEEL)
+  set(LINK_AGAINST_PYTHON OFF)
+  set(NMODL_ENABLE_TESTS OFF)
+endif()
 
 # =============================================================================
 # Settings to enable project as submodule
@@ -166,12 +172,6 @@ endif()
 message(STATUS "CHECKING FOR PYTHON")
 find_package(PythonInterp 3.8 REQUIRED)
 cpp_cc_strip_python_shims(EXECUTABLE "${PYTHON_EXECUTABLE}" OUTPUT PYTHON_EXECUTABLE)
-include(cmake/hpc-coding-conventions/cpp/cmake/bbp-find-python-module.cmake)
-cpp_cc_find_python_module(jinja2 2.9.3 REQUIRED)
-cpp_cc_find_python_module(pytest 3.3.0 REQUIRED)
-cpp_cc_find_python_module(sympy 1.3 REQUIRED)
-cpp_cc_find_python_module(textwrap 0.9 REQUIRED)
-cpp_cc_find_python_module(yaml 3.12 REQUIRED)
 
 # =============================================================================
 # Compiler specific flags for external submodules

CONTRIBUTING.rst

@@ -210,9 +210,8 @@ the Python API:
 1. setup a sandbox environment with either *virtualenv*, *pyenv*, or
    *pipenv*. For instance with *virtualenv*:
    ``python -m venv .venv && source .venv/bin/activate``
-2. build the Python package with the command: ``python setup.py build``
-3. install *pytest* Python package: ``pip install pytest``
-4. execute the unit-tests: ``pytest``
+2. build the Python wheel with the command: ``python -m pip wheel . --no-deps``
+3. execute the unit-tests for the wheel: ``bash packaging/test_wheel.bash $(command -v python) WHEEL``, where ``WHEEL`` is the path to the wheel generated in the previous step.
 
 Memory Leaks and clang-tidy
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

INSTALL.rst

@@ -45,8 +45,8 @@ of all dependencies we recommend using `homebrew <https://brew.sh/>`__:
 
    brew install flex bison cmake python3
 
-The necessary Python packages can then easily be added using the pip3
-command.
+All of the Python dependencies (build, run, and development) can be installed
+using:
 
 .. code:: sh
 
@@ -74,7 +74,8 @@ installed along with the system toolchain:
 
    apt-get install flex bison gcc python3 python3-pip
 
-The Python dependencies are installed using:
+All of the Python dependencies (build, run, and development) can be installed
+using:
 
 .. code:: sh
 
@@ -124,8 +125,8 @@ to cmake as:
             -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison \
             -DCMAKE_INSTALL_PREFIX=$HOME/nmodl
 
-Using Python setuptools
-~~~~~~~~~~~~~~~~~~~~~~~
+Using the Python build system
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If you are mainly interested in the NMODL Framework parsing and analysis
 tools and wish to use them from Python, we recommend building and
@@ -139,6 +140,19 @@ This should build the NMODL framework and install it into your pip user
 ``site-packages`` folder such that it becomes available as a Python
 module.
 
+Building a wheel
+~~~~~~~~~~~~~~~~
+
+You can also build a wheel you can test and install in another environment using:
+
+.. code:: sh
+
+   pip3 wheel . --no-deps [-C OPTION1=VALUE1 -C OPTION2=VALUE2...] [--wheel-dir DIRECTORY]
+
+where the various ``OPTION`` values describe the build options (for a list of
+all available options, please consult the `reference <https://scikit-build-core.readthedocs.io/en/latest/configuration.html>`_).
+Notably, due to a bug in CMake, on MacOS one should pass ``-C build-dir=DIRECTORY`` to the above.
+
 When building without linking against libpython
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -210,6 +224,16 @@ example in your Python 3 interpeter as follows:
        SUFFIX hh
    }
 
+You can also run all of the Python tests for a given wheel using:
+
+.. code:: sh
+
+   bash packaging/test_wheel.bash PYTHON_EXECUTABLE WHEEL
+
+where ``PYTHON_EXECUTABLE`` should be replaced by the path to the Python
+executable, and ``WHEEL`` should be replaced by the path to the wheel you wish
+to test.
+
 NMODL is now setup correctly!
 
 Generating Documentation
@@ -219,9 +243,14 @@ In order to build the documentation you must have additionally
 ``pandoc`` installed. Use your system’s package manager to do this
 (e.g. ``sudo apt-get install pandoc``).
 
-You can build the entire documentation simply by using sphinx from
-``setup.py``:
+You can build the entire documentation simply by using the ``generate_docs.sh``
+script:
 
 .. code:: sh
 
-   python3 setup.py build_ext --inplace docs
+   bash docs/generate_docs.sh DIRECTORY [PYTHON_EXECUTABLE]
+
+where ``DIRECTORY`` is where you want to put the output files. The HTML
+documentation will then be available in ``DIRECTORY/docs``, and the temporary
+build will be stored in ``DIRECTORY/build``. You can also specify the path to
+the Python executable if it is not picked up automatically.

azure-pipelines.yml

@@ -88,7 +88,7 @@ stages:
         sudo apt-get install -y python3.8 python3.8-dev python3.8-venv ninja-build
         sudo apt-get remove -y python3-importlib-metadata
         python3.8 -m pip install --upgrade pip setuptools
-        python3.8 -m pip install --user -r $(Build.Repository.LocalPath)/requirements.txt
+        python3.8 -m pip install --user -r requirements.txt
         # we manually get version 3.15.0 to make sure that changes in the cmake
         # files do not require unsupported versions of cmake in our package.
         wget --quiet --output-document=- "https://github.com/Kitware/CMake/releases/download/$CMAKE_VER/$CMAKE_PKG.tar.gz" | tar xzpf -
@@ -149,7 +149,7 @@ stages:
     - script: |
         brew install flex bison cmake python@3
         python3 -m pip install --upgrade pip setuptools
-        python3 -m pip install --user -r $(Build.Repository.LocalPath)/requirements.txt
+        python3 -m pip install --user -r requirements.txt
       displayName: 'Install Dependencies'
     - script: |
         export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH;
@@ -222,10 +222,16 @@ stages:
         else
           export TAG=""
         fi
+        # the following 2 lines are a workaround for PEP 621 not allowing a
+        # dynamic `name` in `pyproject.toml`
+        python3 -m pip install --user tomli tomli-w
+        python3 packaging/change_name.py pyproject.toml "NMODL${TAG}"
+        export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)"
         docker run --rm \
           -w /root/nmodl \
           -v $PWD:/root/nmodl \
           -e NMODL_NIGHTLY_TAG=$TAG \
+          -e SETUPTOOLS_SCM_PRETEND_VERSION=$SETUPTOOLS_SCM_PRETEND_VERSION \
           'bluebrain/nmodl:wheel' \
           packaging/build_wheels.bash linux $(python.version)
       condition: succeeded()
@@ -295,7 +301,11 @@ stages:
         else
           export NMODL_NIGHTLY_TAG=""
         fi
-        packaging/build_wheels.bash osx $(python.version)
+        # the following 2 lines are a workaround for PEP 621 not allowing a
+        # dynamic `name` in `pyproject.toml`
+        python3 -m pip install tomli tomli-w
+        python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}"
+        SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" packaging/build_wheels.bash osx $(python.version)
       condition: succeeded()
       displayName: 'Build macos Wheel'
     - task: PublishBuildArtifacts@1

docs/conf.py

@@ -12,29 +12,14 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-
-
-
-import os
-import subprocess
-import sys
 import textwrap
 
-# The project needs to be built before documentation in the usual build folder
-sys.path.insert(0, os.path.abspath('..'))
-
 import nmodl  # isort:skip
 
-os.environ['PYTHONPATH'] = ':'.join(sys.path)
-
-# Run doxygen
-subprocess.call('doxygen Doxyfile', shell=True)
-
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -134,11 +119,11 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-#html_static_path = ["_static"]
+# html_static_path = ["_static"]
 
-# A list of paths that contain extra files not directly related to the 
-# documentation, such as robots.txt or .htaccess. Relative paths are taken 
-# as relative to the configuration directory. They are copied to the output 
+# A list of paths that contain extra files not directly related to the
+# documentation, such as robots.txt or .htaccess. Relative paths are taken
+# as relative to the configuration directory. They are copied to the output
 # directory. They will overwrite any existing file of the same name.
 html_extra_path = ["sphinx_doxygen"]
 

docs/generate_docs.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# script for generating documentation for NMODL
+
+set -eu
+
+if [ $# -lt 1 ]
+then
+    echo "Usage: $(basename "$0") output_dir [python_exe]"
+    exit 1
+fi
+
+# the dir where we put the temporary build and the docs
+output_dir="$1"
+# path to the Python executable
+python_exe="${2:-"$(command -v python3)"}"
+
+if ! [ -d "${output_dir}" ]
+then
+    mkdir -p "${output_dir}"
+fi
+
+build_dir="build"
+docs_dir="docs"
+
+echo "== Building documentation files in: ${output_dir}/${docs_dir} =="
+echo "== Temporary project build directory is: ${output_dir}/${build_dir} =="
+
+venv_name="${output_dir}/env"
+${python_exe} -m venv "${venv_name}"
+. "${venv_name}/bin/activate"
+python_exe="$(command -v python)"
+${python_exe} -m pip install -U pip
+${python_exe} -m pip install ".[docs]" -C build-dir="${output_dir}/${build_dir}"
+
+# the abs dir where this script is located (so we can call it from wherever)
+script_dir="$(cd "$(dirname "$0")"; pwd -P)"
+
+cd "${script_dir}"
+doxygen Doxyfile
+cd -
+sphinx-build docs/ "${output_dir}/${docs_dir}"
+
+deactivate