checkpoint

checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint
RDFLib · Jan 10, 2023 · 752f61a · 752f61a
1 parent 7341453
commit 752f61a
Show file tree

Hide file tree

Showing 20 changed files with 496 additions and 190 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -27,6 +27,6 @@ updates:
     schedule:
       interval: weekly
   - package-ecosystem: pip
-    directory: docker/unstable/
+    directory: devtools/
     schedule:
       interval: weekly
diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
@@ -8,6 +8,10 @@ on:
 
 env:
   FORCE_COLOR: 1
+  XDG_CACHE_HOME: ${{ github.workspace }}/.cache
+  POETRY_CACHE_DIR: ${{ github.workspace }}/.cache/pypoetry
+  PIP_CACHE_DIR: ${{ github.workspace }}/.cache/pip
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -40,25 +44,32 @@ jobs:
             os: ubuntu-latest
             TOX_EXTRA_COMMAND: "flake8 --exit-zero rdflib"
             TOXENV_SUFFIX: "-docs"
+          - python-version: "3.11"
+            os: ubuntu-latest
+            TOXENV_SUFFIX: "-docs"
     steps:
       - uses: actions/checkout@v3
+      - name: Cache XDG_CACHE_HOME
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.XDG_CACHE_HOME }}
+          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
+          restore-keys: |
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Python Poetry Action
+        uses: abatilo/actions-poetry@v2.1.6
+        with:
+          poetry-version: 1.3.1
       - uses: actions/setup-java@v3
         if: ${{ matrix.extensive-tests }}
         with:
           distribution: "temurin"
           java-version: "17"
-      - name: Cache XDG_CACHE_HOME
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache
-          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
-          restore-keys: |
-            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
-            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Install Task
         uses: arduino/setup-task@v1
         with:
@@ -98,12 +109,6 @@ jobs:
             python-version: 3.8
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{env.DEFAULT_PYTHON}}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Python Poetry Action
-        uses: snok/install-poetry@v1
       - name: Cache XDG_CACHE_HOME
         uses: actions/cache@v3
         with:
@@ -112,6 +117,14 @@ jobs:
           restore-keys: |
             ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
             ${{ github.job }}-xdg-v1-${{ matrix.os }}-
+      - name: Set up Python ${{env.DEFAULT_PYTHON}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Python Poetry Action
+        uses: abatilo/actions-poetry@v2.1.6
+        with:
+          poetry-version: 1.3.1
       - name: Install Task
         uses: arduino/setup-task@v1
         with:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,7 +3,10 @@ ci:
   # https://pre-commit.ci/#configuration
   autoupdate_schedule: weekly
   autofix_prs: false
+  skip:
+    - pip-compile
 
+# https://pre-commit.com/#adding-pre-commit-plugins-to-your-project
 repos:
   - repo: https://github.com/pycqa/isort
     rev: 5.11.4
@@ -23,3 +26,24 @@ repos:
       pass_filenames: false
       require_serial: true
       args: ["."]
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.3.1
+    hooks:
+    - id: poetry-check
+    - id: poetry-lock
+      # sadly `--no-update` does not work on pre-commit.ci
+      args: ["--check"]
+  - repo: https://github.com/jazzband/pip-tools
+    rev: 6.12.0
+    hooks:
+    - id: pip-compile
+      name: pip-compile requirements-poetry.in
+      args:
+        [
+          --quiet,
+          --annotate,
+          --emit-options,
+          --resolver=backtracking,
+          devtools/requirements-poetry.in,
+        ]
+      files: ^devtools/requirements-poetry\.(in|txt)$
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -15,13 +15,20 @@ build:
     # Using 3.9 as earlier versions have trouble generating documentation for
     # `@typing.overload`` with type aliases.
     python: "3.9"
+  jobs:
+    post_create_environment:
+    # Using requirements-poetry.in as requirements-poetry.txt has conflicts with
+    # the readthedocs environment.
+    - pip install -r devtools/requirements-poetry.in
+    post_install:
+    - poetry config virtualenvs.create false
+    - poetry install --only=main --only=docs --extras=html
+    - poetry env info
+    # This will patch Sphinx to a later version than is in poetry.lock so that
+    # we build with a more up to date Sphinx. This should be eliminated when
+    # possible in favor of having a more up to date Sphinx in poetry.lock.
+    - pip install -r devtools/requirements-rtd.txt
 
-python:
-  install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - docs
 
 sphinx:
   fail_on_warning: true
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -259,7 +259,7 @@ tasks:
     cmds:
       - task: poetry:configure
         vars:
-          .CLI_ARGS: --no-root --only=flake8
+          CLI_ARGS: --no-root --only=flake8
       - task: flake8
 
   cmd:rdfpipe:
@@ -269,12 +269,25 @@ tasks:
 
   pip-compile:
     cmds:
-      - cmd: "{{.PIP_COMPILE}} --quiet --annotate --emit-options {{.CLI_ARGS}}"
+      - cmd: "{{.PIP_COMPILE}} --quiet --annotate --emit-options --resolver=backtracking {{.CLI_ARGS}}"
+
+  pip-compile:all:
+    desc: run pip-compile on all requirements.in files
+    cmds:
+      - cmd: |
+          shopt -s globstar;
+          set -x;
+          for file in ./**/requirements*.in
+          do
+            {{.PIP_COMPILE}} --quiet --annotate --emit-options \
+              --resolver backtracking \
+              "${file}"
+          done
+
   docker:prepare:
     cmds:
       - task: pip-compile
-        vars:
-          { CLI_ARGS: "-o docker/unstable/requirements.txt requirements.txt" }
+        vars: { CLI_ARGS: "docker/unstable/requirements.poetry.in" }
       - task: pip-compile
         vars: { CLI_ARGS: "docker/latest/requirements.in" }
   docker:unstable:
@@ -285,7 +298,8 @@ tasks:
           {{.DOCKER}} image pull {{.OCI_REFERENCE}}:unstable || :
 
           set -eo pipefail
-
+          mkdir -vp var
+          pipx run --spec=poetry poetry export -o var/requirements.txt
           pipx run --spec=build build --wheel
           {{.DOCKER}} buildx build \
             --cache-to=type=inline \

diff --git a/devtools/README.md b/devtools/README.md
@@ -0,0 +1,3 @@
+# devtools
+
+This directory contains development related scripts and files.
diff --git a/devtools/requirements-poetry.in b/devtools/requirements-poetry.in
@@ -0,0 +1,3 @@
+# Fixing this here as readthedocs can't use the compiled requirements-poetry.txt
+# due to conflicts.
+poetry==1.3.1
diff --git a/devtools/requirements-poetry.txt b/devtools/requirements-poetry.txt
@@ -0,0 +1,112 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --resolver=backtracking devtools/requirements-poetry.in
+#
+attrs==22.2.0
+    # via jsonschema
+cachecontrol[filecache]==0.12.11
+    # via poetry
+certifi==2022.12.7
+    # via requests
+cffi==1.15.1
+    # via cryptography
+charset-normalizer==2.1.1
+    # via requests
+cleo==2.0.1
+    # via poetry
+crashtest==0.4.1
+    # via
+    #   cleo
+    #   poetry
+cryptography==39.0.0
+    # via secretstorage
+distlib==0.3.6
+    # via virtualenv
+dulwich==0.20.50
+    # via poetry
+filelock==3.9.0
+    # via
+    #   poetry
+    #   virtualenv
+html5lib==1.1
+    # via poetry
+idna==3.4
+    # via requests
+importlib-metadata==6.0.0
+    # via keyring
+jaraco-classes==3.2.3
+    # via keyring
+jeepney==0.8.0
+    # via
+    #   keyring
+    #   secretstorage
+jsonschema==4.17.3
+    # via poetry
+keyring==23.13.1
+    # via poetry
+lockfile==0.12.2
+    # via
+    #   cachecontrol
+    #   poetry
+more-itertools==9.0.0
+    # via jaraco-classes
+msgpack==1.0.4
+    # via cachecontrol
+packaging==22.0
+    # via poetry
+pexpect==4.8.0
+    # via poetry
+pkginfo==1.9.3
+    # via poetry
+platformdirs==2.6.2
+    # via
+    #   poetry
+    #   virtualenv
+poetry==1.3.1
+    # via
+    #   -r devtools/requirements-poetry.in
+    #   poetry-plugin-export
+poetry-core==1.4.0
+    # via
+    #   poetry
+    #   poetry-plugin-export
+poetry-plugin-export==1.2.0
+    # via poetry
+ptyprocess==0.7.0
+    # via pexpect
+pycparser==2.21
+    # via cffi
+pyrsistent==0.19.3
+    # via jsonschema
+rapidfuzz==2.13.7
+    # via cleo
+requests==2.28.1
+    # via
+    #   cachecontrol
+    #   poetry
+    #   requests-toolbelt
+requests-toolbelt==0.10.1
+    # via poetry
+secretstorage==3.3.3
+    # via keyring
+shellingham==1.5.0.post1
+    # via poetry
+six==1.16.0
+    # via html5lib
+tomlkit==0.11.6
+    # via poetry
+trove-classifiers==2022.12.22
+    # via poetry
+urllib3==1.26.13
+    # via
+    #   dulwich
+    #   poetry
+    #   requests
+virtualenv==20.17.1
+    # via poetry
+webencodings==0.5.1
+    # via html5lib
+zipp==3.11.0
+    # via importlib-metadata
diff --git a/devtools/requirements-rtd.txt b/devtools/requirements-rtd.txt
@@ -0,0 +1,5 @@
+# This file contains requirements that get patched into readthedocs so that we
+# can build with a more up to date Sphinx than is in `poetry.lock`. This file
+# should be eliminated once we can get a more up-to-date version of sphinx in
+# poetry.lock. It is kept here so that dependabot can update it.
+Sphinx==5.3.0
diff --git a/docker/latest/requirements.in b/docker/latest/requirements.in
@@ -1,2 +1,4 @@
+# This file is used for building a docker image of hte latest rdflib release. It
+# will be updated by dependabot when new releases are made.
 rdflib==6.2.0
 html5lib
diff --git a/docker/latest/requirements.txt b/docker/latest/requirements.txt
@@ -1,22 +1,23 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
-#    pip-compile docker/latest/requirements.in
+#    pip-compile --resolver=backtracking ./docker/latest/requirements.in
 #
 html5lib==1.1
-    # via -r docker/latest/requirements.in
+    # via -r ./docker/latest/requirements.in
 isodate==0.6.1
     # via rdflib
 pyparsing==3.0.9
     # via rdflib
 rdflib==6.2.0
-    # via -r docker/latest/requirements.in
+    # via -r ./docker/latest/requirements.in
 six==1.16.0
-    # via html5lib
+    # via
+    #   html5lib
+    #   isodate
 webencodings==0.5.1
     # via html5lib
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
-
diff --git a/docker/unstable/Dockerfile b/docker/unstable/Dockerfile
@@ -1,6 +1,7 @@
 FROM docker.io/library/python:3.11.1-slim@sha256:8f3997244336ad0f067ad693f85906263a21a7f571b6577a64134aba889bc95e
 
-COPY docker/unstable/requirements.txt /var/tmp/build/
+# This file is generated from docker:unstable in Taskfile.yml
+COPY var/requirements.txt /var/tmp/build/
 
 RUN \
     pip install -r /var/tmp/build/requirements.txt

diff --git a/docker/unstable/requirements.txt b/docker/unstable/requirements.txt