move to poetry for dependency management; consolidate more settings into pyproject.toml

.devcontainer.json

@@ -19,5 +19,6 @@
     "EditorConfig.EditorConfig",
     "paulvarache.vscode-taskfile",
     "stardog-union.vscode-stardog-languages"
-  ]
+  ],
+  "postCreateCommand": "poetry install --all-extras"
 }

.github/dependabot.yml

@@ -27,6 +27,6 @@ updates:
     schedule:
       interval: weekly
   - package-ecosystem: pip
-    directory: docker/unstable/
+    directory: devtools/
     schedule:
       interval: weekly

.github/workflows/docker-images.yaml

@@ -44,6 +44,10 @@ jobs:
         uses: arduino/setup-task@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Python Poetry Action
+        uses: abatilo/actions-poetry@v2.2.0
+        with:
+          poetry-version: 1.3.2
       - name: Build images
         shell: bash
         run: |
@@ -66,6 +70,10 @@ jobs:
         uses: arduino/setup-task@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Python Poetry Action
+        uses: abatilo/actions-poetry@v2.2.0
+        with:
+          poetry-version: 1.3.2
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:

.github/workflows/validate.yaml

@@ -8,8 +8,10 @@ on:
 
 env:
   FORCE_COLOR: 1
-  XDG_CACHE_HOME: ${{ github.workspace }}/cache
-  PIP_CACHE_DIR: ${{ github.workspace }}/pip-cache
+  XDG_CACHE_HOME: ${{ github.workspace }}/.var/cache
+  POETRY_CACHE_DIR: ${{ github.workspace }}/.var/cache/pypoetry
+  PIP_CACHE_DIR: ${{ github.workspace }}/.var/cache/pip
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -25,11 +27,19 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         os: [ubuntu-latest, macos-latest, windows-latest]
+        # This is used for injecting additional tests for a specific python
+        # version and OS.
+        suffix: [""]
         include:
           - python-version: "3.7"
             os: ubuntu-latest
             extensive-tests: true
             TOXENV_SUFFIX: "-docs"
+          - python-version: "3.7"
+            os: ubuntu-latest
+            extensive-tests: true
+            suffix: "-min"
+            TOXENV_SUFFIX: "-min"
           - python-version: "3.8"
             os: ubuntu-latest
             TOX_EXTRA_COMMAND: "- isort --check-only --diff ."
@@ -42,32 +52,32 @@ jobs:
             os: ubuntu-latest
             TOX_EXTRA_COMMAND: "flake8 --exit-zero rdflib"
             TOXENV_SUFFIX: "-docs"
+          - python-version: "3.11"
+            os: ubuntu-latest
+            TOXENV_SUFFIX: "-docs"
     steps:
       - uses: actions/checkout@v3
+      - name: Cache XDG_CACHE_HOME
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.XDG_CACHE_HOME }}
+          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
+          restore-keys: |
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Python Poetry Action
+        uses: abatilo/actions-poetry@v2.2.0
+        with:
+          poetry-version: 1.3.2
       - uses: actions/setup-java@v3
         if: ${{ matrix.extensive-tests }}
         with:
           distribution: "temurin"
           java-version: "17"
-      - name: Cache pip
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-${{
-            hashFiles('**/setup.py', '**/*requirements*.txt') }}
-          restore-keys: |
-            ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-
-      - name: Cache xdg
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.XDG_CACHE_HOME }}
-          key: ${{ matrix.os }}-xdg-v1-${{ hashFiles('**/with-fuseki.sh') }}
-          restore-keys: |
-            ${{ matrix.os }}-xdg-v1-
       - name: Install Task
         uses: arduino/setup-task@v1
         with:
@@ -78,24 +88,27 @@ jobs:
           task \
             TOX_EXTRA_COMMAND="${{ matrix.TOX_EXTRA_COMMAND }}" \
             OS=${{ matrix.os }} \
+            MATRIX_SUFFIX=${{ matrix.suffix }} \
             EXTENSIVE=${{ matrix.extensive-tests || 'false' }} \
             TOX_PYTHON_VERSION=${{ matrix.python-version }} \
             TOXENV_SUFFIX=${{ matrix.TOXENV_SUFFIX }} \
-            TOX_JUNIT_XML_PREFIX=${{ matrix.python-version }}-${{ matrix.os }}- \
+            TOX_JUNIT_XML_PREFIX=${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}- \
             gha:validate
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/upload-artifact@v3
         if: ${{ (success() || failure()) }}
         with:
-          name: ${{ matrix.python-version }}-${{ matrix.os }}-mypy-junit-xml
-          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}-mypy-junit.xml
+          name: ${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-mypy-junit-xml
+          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-mypy-junit.xml
       - uses: actions/upload-artifact@v3
         if: ${{ (success() || failure()) }}
         with:
-          name: ${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit-xml
-          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit.xml
+          name: ${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-pytest-junit-xml
+          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-pytest-junit.xml
   extra-tasks:
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -105,18 +118,22 @@ jobs:
             python-version: 3.8
     steps:
       - uses: actions/checkout@v3
+      - name: Cache XDG_CACHE_HOME
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.XDG_CACHE_HOME }}
+          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
+          restore-keys: |
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Set up Python ${{env.DEFAULT_PYTHON}}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache pip
-        uses: actions/cache@v3
+      - name: Python Poetry Action
+        uses: abatilo/actions-poetry@v2.2.0
         with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: tox-${{ matrix.task }}-pip-v1-${{
-            hashFiles('**/setup.py', '**/requirements*.txt') }}
-          restore-keys: |
-            tox-${{ matrix.task }}-pip-v1-
+          poetry-version: 1.3.2
       - name: Install Task
         uses: arduino/setup-task@v1
         with:

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ ci:
   autoupdate_schedule: weekly
   autofix_prs: false
 
+# https://pre-commit.com/#adding-pre-commit-plugins-to-your-project
 repos:
   - repo: https://github.com/pycqa/isort
     rev: 5.11.4
@@ -15,11 +16,18 @@ repos:
       require_serial: true
       args: ["."]
   - repo: https://github.com/psf/black
-    # WARNING: version should be the same as in `pyproject.toml` and `requirements.dev.txt`.
+    # WARNING: version should be the same as in `pyproject.toml`
     # Using git ref spec because of https://github.com/psf/black/issues/2493
     rev: 'refs/tags/22.12.0:refs/tags/22.12.0'
     hooks:
     - id: black
       pass_filenames: false
       require_serial: true
       args: ["."]
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.3.2
+    hooks:
+    - id: poetry-check
+    - id: poetry-lock
+      # sadly `--no-update` does not work on pre-commit.ci
+      args: ["--check"]

.readthedocs.yaml

@@ -15,13 +15,20 @@ build:
     # Using 3.9 as earlier versions have trouble generating documentation for
     # `@typing.overload`` with type aliases.
     python: "3.9"
+  jobs:
+    post_create_environment:
+    # Using requirements-poetry.in as requirements-poetry.txt has conflicts with
+    # the readthedocs environment.
+    - pip install -r devtools/requirements-poetry.in
+    post_install:
+    - poetry config virtualenvs.create false
+    - poetry install --only=main --only=docs --extras=html
+    - poetry env info
+    # This will patch Sphinx to a later version than is in poetry.lock so that
+    # we build with a more up to date Sphinx. This should be eliminated when
+    # possible in favor of having a more up to date Sphinx in poetry.lock.
+    - pip install -r devtools/requirements-rtd.txt
 
-python:
-  install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - docs
 
 sphinx:
   fail_on_warning: true

.var/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

Dockerfile.devcontainer

@@ -41,17 +41,14 @@ RUN \
 
 RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
 
-COPY requirements*.txt /var/tmp/
-COPY docs/sphinx-requirements.txt /var/tmp/docs/sphinx-requirements.txt
+COPY devtools/requirements-poetry.in /var/tmp/
 
 RUN \
-    cd /var/tmp/ && \
-    grep -H . requirements*.txt docs/sphinx-requirements.txt && \
     python -m pip install --upgrade \
-        -r requirements.txt \
-        -r requirements.dev.txt \
-        -r requirements.dev-extra.txt \
-        -r docs/sphinx-requirements.txt \
         tox \
+        pre-commit \
+        -r /var/tmp/requirements-poetry.in \
         && \
     true
+
+RUN git config --system --add safe.directory /srv/workspace

README.md

@@ -75,7 +75,7 @@ or
 
 or from your locally cloned repository you can install it with one of the following options:
 
-    $ python setup.py install
+    $ poetry install  # installs into a poetry-managed venv
 
 or
 
@@ -182,21 +182,22 @@ Multiple other projects are contained within the RDFlib "family", see <https://g
 
 Run the test suite with `pytest`.
 ```shell
-pytest
+poetry install
+poetry run pytest
 ```
 
 ### Running test coverage on the host with coverage report
 
 Run the test suite and generate a HTML coverage report with `pytest` and `pytest-cov`.
 ```shell
-pytest --cov
+poetry run pytest --cov
 ```
 
 ### Viewing test coverage
 
 Once tests have produced HTML output of the coverage report, view it by running:
 ```shell
-pytest --cov --cov-report term --cov-report html
+poetry run pytest --cov --cov-report term --cov-report html
 python -m http.server --directory=htmlcov
 ```