1110 Migrate Python packages to use pyproject.toml for builds

.github/actions/build-py/action.yml

@@ -19,9 +19,11 @@ runs:
       shell: bash
       run: |
         cd pycode/memilio-${{ inputs.package }}/
-        /opt/python/cp38-cp38/bin/python -m pip install scikit-build
+        /opt/python/cp38-cp38/bin/python -m pip install --upgrade pip setuptools wheel
+        /opt/python/cp38-cp38/bin/python -m pip install scikit-build scikit-build-core
         /opt/python/cp38-cp38/bin/python -m build --no-isolation --wheel
-        /opt/python/cp311-cp311/bin/python -m pip install scikit-build
+        /opt/python/cp311-cp311/bin/python -m pip install --upgrade pip setuptools wheel
+        /opt/python/cp311-cp311/bin/python -m pip install scikit-build scikit-build-core
         /opt/python/cp311-cp311/bin/python -m build --no-isolation --wheel
         # Exclude memilio-generation, because its a pure python package, cmake is only used in the build process to retrieve data from cpp
         if [[ -f "CMakeLists.txt" ]] && [ "${{ inputs.package }}" != "generation" ]; then 

.github/actions/test-pylint/action.yml

@@ -13,6 +13,12 @@ runs:
         sudo apt-get -qq update
         sudo apt-get -qq -y install python3-pip gnupg
         python -m pip install --upgrade pip
+    # Pylint runs against the prebuilt wheels in pycode/wheelhouse/*cp311*.
+    # So, use current latest supported Python version.
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
     - name: Download Python Wheels
       uses: actions/download-artifact@v4
       with:
@@ -26,10 +32,8 @@ runs:
     - name: Run pylint
       shell: bash
       run: |
-        cd pycode/memilio-${{ inputs.package }}
-        mkdir -p build_pylint
-        python setup.py pylint
-        pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
+        python pycode/run_pylint.py --package-dir memilio-${{ inputs.package }}
+        pylint-json2html -f jsonextended -o pycode/memilio-${{ inputs.package }}/build_pylint/pylint.html < pycode/memilio-${{ inputs.package }}/build_pylint/pylint_extended.json
     - name: Upload Pylint Report
       uses: actions/upload-artifact@v4
       with:

.readthedocs.yaml

@@ -11,7 +11,7 @@ build:
   jobs:
     install:
       - pip install --exists-action=w --no-cache-dir -r docs/requirements-build.txt
-      - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt --no-build-isolation
+      - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt
 
 sphinx:
   configuration: docs/source/conf.py

cpp/thirdparty/CMakeLists.txt

@@ -37,6 +37,7 @@ if(MEMILIO_USE_BUNDLED_SPDLOG)
 
     if(NOT spdlog_POPULATED)
         FetchContent_Populate(spdlog)
+
         add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR} EXCLUDE_FROM_ALL)
     endif()
 
@@ -198,6 +199,7 @@ else()
 endif()
 
 find_package(sbml ${MEMILIO_SBML_VERSION} CONFIG)
+
 if(sbml_FOUND)
     set(MEMILIO_HAS_SBML ON)
     message(STATUS "Found libSBML: ${libsbml_INCLUDE_DIRS}")

docs/source/python/m-epidata.rst

@@ -85,7 +85,7 @@ When you start creating a new script:
 
 When you add a new script
 
-- Add an executable to the setup.py in "pycode/memilio-epidata".
+- Add a console script entry to the ``pycode/memilio-epidata/pyproject.toml`` file.
 - Add it to the cli_dict in getDataIntoPandasDataFrame.py.
     - Add a meaningful key for the new script.
     - for the dict value add a list in the form [comment to print when script is started, list of used parser arguments (optional)].

docs/source/python/m-generation.rst

@@ -30,7 +30,7 @@ For a successful build, the development libraries for Python need to be installe
 .. warning::
     Generation currently requires specifically version ``18.1.1`` of `libclang.so`, since the function ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_ generates the abstract syntax tree using `clang-18`. Different versions may lead to unsupported abstractions.
 
-    If you want to try a different version, set your `libclang` version under ``install_requires`` in the `setup.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/setup.py>`_ and change the clang command in ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_.
+    If you want to try a different version, set your `libclang` version under ``dependencies`` in the `pyproject.toml <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/pyproject.toml>`_ and change the clang command in ``create_ast`` in `ast.py <https://github.com/SciCompMod/memilio/blob/main/pycode/memilio-generation/memilio/generation/ast.py>`_.
 
 Usage
 -----

docs/source/python/python_packages.rst

@@ -111,21 +111,21 @@ Please see the individual package documentation for more details on the function
 Installation
 ------------
 
-Each package provides a `setup.py` script that installs the package and its dependencies. 
+Each package provides a ``pyproject.toml`` that installs the package and its dependencies with pip.
 The dependencies of the individual packages are denoted in their documentation.
-The installation can be run with the following command (from the directory containing the `setup.py`)
+The installation can be run with the following command (from the directory containing the ``pyproject.toml`` file)
 
 .. code-block:: console 
 
-    pip install .
+    python -m pip install .
 
 This copies the package and the required dependencies to your site-packages.
 
 For development of code use this command instead
 
 .. code-block:: console 
 
-    pip install -e .[dev]
+    python -m pip install -e .[dev]
 
 This command allows you to work on the code without having to reinstall the package after a change. It also installs all additional dependencies required for development and maintenance.
 
@@ -157,7 +157,7 @@ To get the coverage report do in the package folder
 
 Coverage report for actual master:
 
-`Coverage Report <https://scicompmod.github.io/memilio/coverage/python/>`_
+`Coverage Report <https://scicompmod.github.io/memilio/coverage/python/>`__
 
 Inspection via pylint
 ---------------------
@@ -171,9 +171,12 @@ Run pylint with the commands in the package folder
 
 .. code-block:: console
 
-    python setup.py pylint
+    python ../run_pylint.py
     pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
 
+From the repository root you can also target a package explicitly, for example
+``python pycode/run_pylint.py --package-dir memilio-plot``.
+
 Pylint report for actual master:
 
-`Pylint Report <https://dlr-sc.github.io/memilio/pylint/>`_
+`Pylint Report <https://dlr-sc.github.io/memilio/pylint/>`__

pycode/memilio-epidata/README.rst

@@ -23,9 +23,9 @@ A more detailed description of the sources can be found in the `epidata subfolde
 Installation
 ------------
 
-Use the provided ``setup.py`` script to install the package and its dependencies.
+This project uses ``pyproject.toml`` to install the package and its dependencies.
 
-To install the package, use (from the directory that contains ``setup.py``)
+To install the package, run (from the directory that contains ``pyproject.toml``)
 
 .. code:: sh
 
@@ -84,12 +84,19 @@ The following packages are used by the tests:
 
 See Installation on how to install all these dependencies automatically.
 
-To run the tests make 
+To run the tests make sure the package is installed in editable mode (``pip install -e .``) and run
 
 .. code:: sh
 
     python -m unittest
 
+Alternatively, if the package is installed normally (``pip install .``), you have to run the tests from outside the source directory to avoid import errors:
+
+.. code:: sh
+
+    cd /path/to/another/directory
+    python -m unittest discover -s /path/to/memilio/pycode/memilio-epidata/memilio/epidata_test
+
 To get the coverage report do
 
 .. code:: sh
@@ -116,7 +123,7 @@ Run pylint with the commands
 
 .. code:: sh
 
-    python setup.py pylint
+    python ../run_pylint.py
     pylint-json2html -f jsonextended -o build_pylint/pylint.html < build_pylint/pylint_extended.json
 
 Pylint report for actual master:
@@ -163,7 +170,7 @@ When you start creating a new script:
 
 When you add a new script
 
-- add a executable to the setup.py in "pycode/memilio-epidata"
+- add a script entry to ``pyproject.toml`` in "pycode/memilio-epidata"
 - add it to the cli_dict in getDataIntoPandasDataFrame.py
     - add a meaningfull key for the new script
     - as the value add a list in the form [comment to print when script is started, list of used parser arguments (optional)]

pycode/memilio-epidata/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "memilio-epidata"
+version = "1.0.0"
+description = "Part of MEmilio project, reads epidemiological data from different official and unofficial sources."
+readme = "README.rst"
+requires-python = ">=3.8"
+license = { text = "Apache-2.0" }
+authors = [{ name = "MEmilio Team" }]
+maintainers = [
+  { email = "martin.kuehn@dlr.de" }
+]
+dependencies = [
+  "pandas>=2.0.0",
+  "pyarrow",
+  "matplotlib",
+  "tables",
+  "numpy>=1.22,<1.25",
+  "openpyxl",
+  "xlrd",
+  "xlsxwriter",
+  "requests",
+  "pyxlsb",
+  "wget",
+  "twill==3.1",
+  "PyQt6-sip<13.9",
+  "PyQt6",
+  "python-calamine",
+  "python-magic-bin; sys_platform == 'win32'",
+  "python-magic; sys_platform != 'win32'"
+]
+
+[project.optional-dependencies]
+dev = [
+  "pyfakefs>=4.6,<5.3.3",
+  "coverage>=7.0.1",
+  "pylint>=2.13.0,<2.16",
+  "pylint_json2html==0.4.0"
+]
+
+[project.scripts]
+getcasedata = "memilio.epidata.getCaseData:main"
+getpopuldata = "memilio.epidata.getPopulationData:main"
+getjhdata = "memilio.epidata.getJHData:main"
+getdividata = "memilio.epidata.getDIVIData:main"
+getsimdata = "memilio.epidata.getSimulationData:main"
+cleandata = "memilio.epidata.cleanData:main"
+getcommutermobility = "memilio.epidata.getCommuterMobility:main"
+getvaccinationdata = "memilio.epidata.getVaccinationData:main"
+gethospitalizationdata = "memilio.epidata.getHospitalizationData:main"
+
+[project.urls]
+Homepage = "https://github.com/SciCompMod/memilio"
+Team = "https://memilio.readthedocs.io/en/latest/team.html"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["memilio*"]