Improve infra and add step for type checking. (#605)

Author: tobiasraabe

.github/workflows/main.yml

@@ -15,20 +15,20 @@ on:
 
 jobs:
 
-  build-package:
-    name: Build & verify package
+  run-type-checking:
+
+    name: Run tests for type-checking
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          fetch-depth: 0
-
-      - uses: hynek/build-and-inspect-python-package@v2
-        id: baipp
-
-    outputs:
-      python-versions: ${{ steps.baipp.outputs.supported_python_classifiers_json_array }}
+          python-version-file: .python-version
+          allow-prereleases: true
+          cache: pip
+      - run: pip install tox-uv
+      - run: tox -e typing
 
   run-tests:
 

.pre-commit-config.yaml

@@ -24,10 +24,6 @@ repos:
     -   id: python-check-mock-methods
     -   id: python-no-log-warn
     -   id: text-unicode-replacement-char
-# -   repo: https://github.com/tox-dev/pyproject-fmt
-#     rev: 1.2.0
-#     hooks:
-#     -   id: pyproject-fmt
 -   repo: https://github.com/aio-libs/sort-all
     rev: v1.2.0
     hooks:
@@ -37,26 +33,10 @@ repos:
     hooks:
       - id: ruff-format
       - id: ruff
-        args: [--unsafe-fixes]
 -   repo: https://github.com/dosisod/refurb
     rev: v2.0.0
     hooks:
     -   id: refurb
-        args: [--ignore, FURB126]
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
-    hooks:
-    -   id: mypy
-        additional_dependencies: [
-            attrs>=21.3.0,
-            click,
-            optree,
-            pluggy>=1.3.0,
-            rich,
-            sqlalchemy>2,
-            types-setuptools,
-        ]
-        pass_filenames: false
 -   repo: https://github.com/executablebooks/mdformat
     rev: 0.7.17
     hooks:
@@ -77,11 +57,6 @@ repos:
             mdformat-pyproject,
         ]
         files: (docs/.)
--   repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.8.5
-    hooks:
-    -   id: nbqa-mypy
-        args: [--ignore-missing-imports]
 -   repo: https://github.com/kynan/nbstripout
     rev: 0.7.1
     hooks:

docs/source/changes.md

@@ -46,6 +46,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 - {pull}`600` refactors test using subprocesses.
 - {pull}`603` fixes an example in the documentation about capturing warnings.
 - {pull}`604` fixes some examples with `PythonNode`s in the documentation.
+- {pull}`605` improves checks and CI.
 
 ## 0.4.7 - 2024-03-19
 

docs_src/tutorials/using_a_data_catalog_5_py310_return.py

@@ -8,6 +8,5 @@
 def task_transform_csv(
     path: Annotated[Path, data_catalog["csv"]],
 ) -> Annotated[pd.DataFrame, data_catalog["transformed_csv"]]:
-    df = pd.read_csv(path)
+    return pd.read_csv(path)
     # ... transform the data
-    return df

pyproject.toml

@@ -1,23 +1,23 @@
 [project]
 name = "pytask"
-description = "In its highest aspirations, pytask tries to be pytest as a build system."
+description = "pytask is a workflow management system that facilitates reproducible data analyses."
 requires-python = ">=3.8"
 classifiers = [
-  "Development Status :: 4 - Beta",
-  "Environment :: Console",
-  "Intended Audience :: Science/Research",
-  "License :: OSI Approved :: MIT License",
-  "Operating System :: MacOS :: MacOS X",
-  "Operating System :: Microsoft :: Windows",
-  "Operating System :: POSIX",
-  "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.8",
-  "Programming Language :: Python :: 3.9",
-  "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
-  "Programming Language :: Python :: 3.12",
-  "Topic :: Scientific/Engineering",
-  "Topic :: Software Development :: Build Tools",
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Build Tools",
 ]
 dynamic = ["version"]
 dependencies = [
@@ -48,18 +48,19 @@ email = "raabe@posteo.de"
 
 [project.optional-dependencies]
 docs = [
-  "furo",
-  "ipython",
-  "matplotlib",
-  "myst-parser",
-  "nbsphinx",
-  "sphinx",
-  "sphinx-click",
-  "sphinx-copybutton",
-  "sphinx-design>=0.3",
-  "sphinx-toolbox",
-  "sphinxext-opengraph",
+    "furo",
+    "ipython",
+    "matplotlib",
+    "myst-parser",
+    "nbsphinx",
+    "sphinx",
+    "sphinx-click",
+    "sphinx-copybutton",
+    "sphinx-design>=0.3",
+    "sphinx-toolbox",
+    "sphinxext-opengraph",
 ]
+plugin-list = ["httpx", "tabulate[widechars]", "tqdm"]
 test = [
     "deepdiff",
     "nbmake",
@@ -68,15 +69,10 @@ test = [
     "pytest-cov",
     "pytest-xdist",
     "syrupy",
-    # For HTTPPath tests.
-    "aiohttp",
+    "aiohttp",      # For HTTPPath tests.
     "coiled",
 ]
-plugin-list = [
-    "httpx>=0.27.0",
-    "tabulate[widechars]>=0.9.0",
-    "tqdm>=4.66.2",
-]
+typing = ["mypy>=1.9.0", "nbqa[mypy]>=1.8.5"]
 
 [project.urls]
 Changelog = "https://pytask-dev.readthedocs.io/en/stable/changes.html"
@@ -94,7 +90,7 @@ build-backend = "hatchling.build"
 
 [tool.rye]
 managed = true
-dev-dependencies = []
+dev-dependencies = ["tox-uv>=1.7.0"]
 
 [tool.hatch.build.hooks.vcs]
 version-file = "src/_pytask/_version.py"
@@ -124,15 +120,15 @@ select = ["ALL"]
 ignore = [
     "ANN101",
     "ANN102",
-    "ANN401",  # flake8-annotate typing.Any
-    "COM812",  # Comply with ruff-format.
-    "ISC001",  # Comply with ruff-format.
+    "ANN401", # flake8-annotate typing.Any
+    "COM812", # Comply with ruff-format.
+    "ISC001", # Comply with ruff-format.
     "FBT",
     "PD901",  # Avoid generic df for dataframes.
-    "S101",  # raise errors for asserts.
-    "S603",  # Call check with subprocess.run.
-    "S607",  # Call subprocess.run with partial executable path.
-    "SLF001",  # access private members.
+    "S101",   # raise errors for asserts.
+    "S603",   # Call check with subprocess.run.
+    "S607",   # Call subprocess.run with partial executable path.
+    "SLF001", # access private members.
 ]
 
 [tool.ruff.lint.per-file-ignores]
@@ -148,10 +144,13 @@ ignore = [
 "docs/source/conf.py" = ["D401", "INP001"]
 "docs_src/*" = ["ARG001", "D", "INP001", "S301"]
 "docs_src/*/*.py" = ["FA100", "FA102", "PLR2004", "TCH"]
-"docs/source/how_to_guides/functional_interface*" = ["B018", "D", "INP", "ARG005"]
+"docs/source/how_to_guides/functional_interface*" = [
+    "B018",
+    "D",
+    "INP",
+    "ARG005",
+]
 "docs_src/how_to_guides/using_task_returns_*_task.py" = ["ARG005", "E731"]
-"docs_src/how_to_guides/writing_custom_nodes_*.py" = ["S301"]
-"docs_src/tutorials/using_a_data_catalog_*.py" = ["RET504"]
 
 [tool.ruff.lint.isort]
 force-single-line = true
@@ -191,6 +190,10 @@ ignore_errors = true
 module = ["click_default_group", "networkx"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = ["_pytask.coiled_utils"]
+disable_error_code = ["import-not-found"]
+
 [[tool.mypy.overrides]]
 module = ["_pytask.hookspecs"]
 disable_error_code = ["empty-body"]

tox.ini

@@ -6,15 +6,21 @@ envlist = docs, test
 passenv = CI
 package = editable
 
+[testenv:docs]
+extras = docs, test
+commands =
+    - sphinx-build -n -T -b html -d {envtmpdir}/doctrees docs/source docs/build/html
+    - sphinx-build -n -T -b doctest -d {envtmpdir}/doctrees docs/source docs/build/html
+
+[testenv:typing]
+extras = typing
+commands =
+    mypy
+    nbqa mypy --ignore-missing-imports .
+
 [testenv:test]
 extras = test
 deps =
     pygraphviz;platform_system != "Windows"
 commands =
     pytest --nbmake {posargs}
-
-[testenv:docs]
-extras = docs, test
-commands =
-    - sphinx-build -n -T -b html -d {envtmpdir}/doctrees docs/source docs/build/html
-    - sphinx-build -n -T -b doctest -d {envtmpdir}/doctrees docs/source docs/build/html