Modernise tooling and fix CI failing (#335)

* Move from setup.py to pyproject.toml

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>

* Ensure unit tests run

IDK we sometimes have to invoke trial explicitly like this

* Use ruff instead of flake8

And remove py2-style type annotations, which ruff doesn't recognise

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>

* Bump GHA actions

part of matrix-org/synapse#14203

* Run black

Presumably this fixes complaints from #333 and #334

* Move isort config to pyproject

* Also lint stubs

* Bump GHA actions

part of matrix-org/synapse#14203

* Fix an unintended rename in #333

Needed for unit tests to be happy

* Changelog

---------

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>

.github/workflows/changelog_check.yml

@@ -6,11 +6,11 @@ jobs:
     if: ${{ github.base_ref == 'main'  || contains(github.base_ref, 'release-') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           ref: ${{github.event.pull_request.head.sha}}
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
       - run: python -m pip install towncrier

.github/workflows/docker_check.yml

@@ -33,10 +33,10 @@ jobs:
 
       # we explicitly check out the repository (and use `context: .` in buildx)
       # because we need to preserve the git metadata so that setuptools_scm
-      # (as part of Sygnal's setup.py) can deduce the package version.
+      # (part of build system config in pyproject.toml) can deduce the package version.
       # See: https://github.com/marketplace/actions/build-and-push-docker-images#path-context
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Build all platforms
         uses: docker/build-push-action@v2

.github/workflows/docker_push.yml

@@ -43,10 +43,10 @@ jobs:
 
       # we explicitly check out the repository (and use `context: .` in buildx)
       # because we need to preserve the git metadata so that setuptools_scm
-      # (as part of Sygnal's setup.py) can deduce the package version.
+      # (part of build system config in pyproject.toml) can deduce the package version.
       # See: https://github.com/marketplace/actions/build-and-push-docker-images#path-context
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Build and push all platforms
         uses: docker/build-push-action@v2

.github/workflows/pipeline.yml

@@ -8,8 +8,8 @@ jobs:
   check-code-style:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
       - run: python -m pip install tox
@@ -18,8 +18,8 @@ jobs:
   check-types-mypy:
     runs-on: ubuntu-latest
     steps: 
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
       - run: python -m pip install tox  
@@ -30,24 +30,27 @@ jobs:
     needs: [check-code-style, check-types-mypy]
     runs-on: ubuntu-latest
     steps: 
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
       - run: python -m pip install -e .
-      - run: trial tests
+      - run: python -m twisted.trial tests
 
   run-unit-tests-olddeps:
     name: Unit tests (old dependencies)
     needs: [ check-code-style, check-types-mypy ]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
-      - name: Install old dependencies
-        run: pip install -r <(./scripts-dev/old_deps.py)
+      - name: Patch pyproject.toml to require oldest dependencies
+        run: |
+          # Ugly. Could use something like https://pyproject-parser.readthedocs.io/en/latest/cli.html#info in the future.
+          sed --in-place=.bak -e 's/>=/==/g' pyproject.toml
+          diff pyproject.toml.bak pyproject.toml || true # diff returns 1 if there is a change
       - name: Install Sygnal
         run: python -m pip install -e .
-      - run: trial tests
+      - run: python -m twisted.trial tests

changelog.d/335.misc

@@ -0,0 +1 @@
+Move from setup.py to pyproject.toml.

pyproject.toml

@@ -33,3 +33,83 @@
         directory = "misc"
         name = "Internal Changes"
         showcontent = true
+
+[tool.isort]
+line_length = 88
+sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,TESTS,LOCALFOLDER"
+default_section = "THIRDPARTY"
+known_first_party = "sygnal"
+known_tests = "tests"
+multi_line_output = 3
+include_trailing_comma = true
+combine_as_imports = true
+
+
+[tool.ruff]
+line-length = 88
+ignore = [
+    "E501", # https://beta.ruff.rs/docs/rules/line-too-long/. Black enforces this for us.
+]
+
+[build-system]
+build-backend = "setuptools.build_meta"
+# Setuptools 64 adds support for PEP 660, which makes `pip install -e .` work.
+requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2"]
+
+[tool.setuptools_scm]
+# Note: including this section without any keys still has meaning, see
+# https://github.com/pypa/setuptools_scm/blob/51b3566170be25582b5c3216a54b024caf3d431f/README.rst?plain=1#L63-L71
+
+[tool.setuptools.packages.find]
+exclude = ["tests", "tests.*"]
+
+[project]
+name = "matrix-sygnal"
+dynamic = ["version"]
+description = "Reference Push Gateway for Matrix Notifications"
+readme = {file = "README.md", content-type = "text/markdown"}
+requires-python = "~=3.7"
+license = {file = "LICENSE"}
+authors = [
+    {name = "Matrix.org Team and contributors", email = "packages@matrix.org"}
+]
+dependencies = [
+    "aioapns>=1.10,<2.1",
+    "attrs>=19.2.0",
+    "cryptography>=2.6.1",
+    "idna>=2.8",
+    'importlib_metadata;python_version<"3.8"',
+    "jaeger-client>=4.0.0",
+    "matrix-common==1.0.0",
+    "opentracing>=2.2.0",
+    "prometheus_client>=0.7.0,<0.8",
+    "py-vapid>=1.7.0",
+    "pyOpenSSL>=17.5.0",
+    "pywebpush>=1.13.0",
+    "pyyaml>=5.1.1",
+    "sentry-sdk>=0.10.2",
+    "service_identity>=18.1.0",
+    "Twisted>=19.7",
+    'typing-extensions>=3.7.4;python_version<"3.8"',
+    "zope.interface>=4.6.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "black==22.3.0",
+    "coverage~=5.5",
+    "ruff==0.0.262",
+    "isort~=5.0",
+    "mypy==0.812",
+    "mypy-zope==0.3.0",
+    "towncrier",
+    "tox",
+    "types-opentracing>=2.4.2",
+    "typing-extensions>=3.7.4",
+]
+
+[project.urls]
+homepage = "https://github.com/matrix-org/sygnal"
+documentation = "https://github.com/matrix-org/sygnal/tree/main/docs"
+repository = "https://github.com/matrix-org/sygnal.git"
+changelog = "https://github.com/matrix-org/sygnal/blob/main/CHANGELOG.md"

scripts-dev/lint.sh

@@ -3,7 +3,8 @@
 # Runs linting scripts over the local Sygnal checkout
 # isort - sorts import statements
 # black - opinionated code formatter
-# flake8 - lints and finds mistakes
+# ruff - lints and finds mistakes
+# mypy - type checker
 
 set -e
 
@@ -84,8 +85,7 @@ else
     files=(
       "sygnal"
       "tests"
-      "scripts-dev"
-      "setup.py"
+      "stubs"
     )
   fi
 fi
@@ -98,5 +98,5 @@ set -x
 
 isort "${files[@]}"
 python3 -m black "${files[@]}"
-flake8 "${files[@]}"
+ruff --quiet --fix "${files[@]}"
 mypy "${files[@]}"

stubs/twisted/web/http.pyi

@@ -6,7 +6,7 @@ from twisted.internet.interfaces import IAddress, ITCPTransport
 from twisted.logger import Logger
 from twisted.protocols.basic import LineReceiver
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IRequest, IAccessLogFormatter
+from twisted.web.iweb import IAccessLogFormatter, IRequest
 from zope.interface import implementer, provider
 
 class HTTPChannel: ...

sygnal/apnspushkin.py

@@ -246,7 +246,6 @@ async def _dispatch_request(
         )
 
         try:
-
             with ACTIVE_REQUESTS_GAUGE.track_inprogress():
                 with SEND_TIME_HISTOGRAM.time():
                     response = await self._send_notification(request)