Add test coverage for the oldest supported Ruff version (#270)

Adds test coverage for the oldest supported Ruff version by adding the
Ruff version to the CI test matrix.

The Ruff versions in the matrix are generated by parsing the lower pin
from our `pyproject.toml` and the latest release from GitHub. I opted
for this approach because it's nice to see the version being tested in
the CI job names and it does not require manual updates.

The `just check` job was split out from tests because it requires `ruff
format` which is not available in old releases.

The tests did not pass on the old release of Ruff due missing `format`
support and the new documentation domain. I've added toggles in the test
to address those cases based on the Ruff version being tested.

Motivated by a desire for test coverage in
#266

.github/workflows/ci.yaml

@@ -7,12 +7,68 @@ on:
     branches: [main]
 
 jobs:
-  ci:
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: extractions/setup-just@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: 'pip'
+      - name: Install dependencies
+        run: just install
+      - name: Run checks
+        run: just check
+  ruff-versions:
+    name: Generate test versions
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install dependencies
+        run: |
+          sudo apt-get install -y ripgrep
+      - name: Determine test versions
+        id: set-versions
+        run: |
+          # Get the latest release version from GitHub
+          LATEST=$( \
+            curl -L \
+              -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              https://api.github.com/repos/astral-sh/ruff/releases/latest \
+              | jq '.tag_name' --raw-output \
+              | cut -c2- \
+          )
+          # Get the oldest supported version from the pyproject.toml
+          OLDEST=$(rg -No '"ruff>=(.*)"' -r '$1' pyproject.toml)
+
+          echo "::set-output name=versions::[\"$OLDEST\", \"$LATEST\"]"
+          echo "::set-output name=oldest::$OLDEST
+          echo "::set-output name=latest::$LATEST
+    outputs:
+      versions: ${{ steps.set-versions.outputs.versions }}
+      oldest: ${{ steps.set-versions.outputs.oldest }}
+      latest: ${{ steps.set-versions.outputs.latest }}
+  test:
+    name: Test (python-${{ matrix.python-version }}, ruff-${{ matrix.ruff-version }}, ${{ matrix.os }})
+    needs: ruff-versions
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12-dev"]
+        ruff-version: ${{ fromJson(needs.ruff-versions.outputs.versions) }}
         os: [ubuntu-latest, macos-latest, windows-latest]
+
+        exclude:
+          - os: windows-latest
+            ruff-version: ${{ needs.ruff-versions.outputs.oldest }}
+          - os: macos-latest
+            ruff-version: ${{ needs.ruff-versions.outputs.oldest }}
+
     runs-on: ${{ matrix.os }}
     steps:
       - uses: extractions/setup-just@v1
@@ -22,9 +78,10 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - name: Install dependencies
         run: just install
-      - name: Run checks
-        run: just check
+      - name: Install test Ruff version
+        run: pip install ruff==${{ matrix.ruff-version }}
       - name: Run tests
         run: just test

ruff_lsp/server.py

@@ -1053,7 +1053,7 @@ class Executable(NamedTuple):
 
 
 def _find_ruff_binary(
-    settings: WorkspaceSettings, version_requirement: SpecifierSet
+    settings: WorkspaceSettings, version_requirement: SpecifierSet | None
 ) -> Executable:
     """Returns the executable along with its version.
 
@@ -1063,7 +1063,9 @@ def _find_ruff_binary(
     path = _find_ruff_binary_path(settings)
 
     version = _executable_version(path)
-    if not version_requirement.contains(version, prereleases=True):
+    if version_requirement and not version_requirement.contains(
+        version, prereleases=True
+    ):
         message = f"Ruff {version_requirement} required, but found {version} at {path}"
         show_error(message)
         raise RuntimeError(message)

tests/conftest.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+import pytest
+from packaging.version import Version
+
+from ruff_lsp.server import _find_ruff_binary, _get_global_defaults, uris
+from ruff_lsp.settings import WorkspaceSettings
+
+
+@pytest.fixture(scope="session")
+def ruff_version() -> Version:
+    # Use the ruff-lsp directory as the workspace
+    workspace_path = str(Path(__file__).parent.parent)
+
+    settings = WorkspaceSettings(
+        **_get_global_defaults(),  # type: ignore[misc]
+        cwd=None,
+        workspacePath=workspace_path,
+        workspace=uris.from_fs_path(workspace_path),
+    )
+
+    return _find_ruff_binary(settings, version_requirement=None).version

tests/test_format.py

@@ -1,9 +1,15 @@
 from __future__ import annotations
 
+from contextlib import nullcontext
+
 import pytest
+from packaging.version import Version
 from pygls.workspace import Workspace
 
-from ruff_lsp.server import _format_document_impl
+from ruff_lsp.server import (
+    VERSION_REQUIREMENT_FORMATTER,
+    _format_document_impl,
+)
 from tests.client import utils
 
 original = """
@@ -15,14 +21,21 @@
 
 
 @pytest.mark.asyncio
-async def test_format(tmp_path):
+async def test_format(tmp_path, ruff_version: Version):
     test_file = tmp_path.joinpath("main.py")
     test_file.write_text(original)
     uri = utils.as_uri(str(test_file))
 
     workspace = Workspace(str(tmp_path))
     document = workspace.get_text_document(uri)
 
-    result = await _format_document_impl(document)
-    [edit] = result
-    assert edit.new_text == expected
+    handle_unsupported = (
+        pytest.raises(RuntimeError, match=f"Ruff .* required, but found {ruff_version}")
+        if not VERSION_REQUIREMENT_FORMATTER.contains(ruff_version)
+        else nullcontext()
+    )
+
+    with handle_unsupported:
+        result = await _format_document_impl(document)
+        [edit] = result
+        assert edit.new_text == expected

tests/test_server.py

@@ -3,9 +3,10 @@
 
 import os
 import tempfile
-import unittest
 from threading import Event
 
+from packaging.version import Version
+
 from tests.client import defaults, session, utils
 
 # Increase this if you want to attach a debugger
@@ -16,11 +17,19 @@
 print(x)
 """
 
+VERSION_REQUIREMENT_ASTRAL_DOCS = Version("0.0.291")
+
 
-class TestServer(unittest.TestCase):
+class TestServer:
     maxDiff = None
 
-    def test_linting_example(self) -> None:
+    def test_linting_example(self, ruff_version: Version) -> None:
+        expected_docs_url = (
+            "https://docs.astral.sh/ruff/"
+            if ruff_version >= VERSION_REQUIREMENT_ASTRAL_DOCS
+            else "https://beta.ruff.rs/docs/"
+        )
+
         with tempfile.NamedTemporaryFile(suffix=".py") as fp:
             fp.write(CONTENTS.encode())
             fp.flush()
@@ -60,7 +69,7 @@ def _handler(params):
                         {
                             "code": "F401",
                             "codeDescription": {
-                                "href": "https://docs.astral.sh/ruff/rules/unused-import"
+                                "href": expected_docs_url + "rules/unused-import"
                             },
                             "data": {
                                 "fix": {
@@ -88,7 +97,7 @@ def _handler(params):
                         {
                             "code": "F821",
                             "codeDescription": {
-                                "href": "https://docs.astral.sh/ruff/rules/undefined-name"
+                                "href": expected_docs_url + "rules/undefined-name"
                             },
                             "data": {"fix": None, "noqa_row": 3},
                             "message": "Undefined name `x`",
@@ -102,9 +111,15 @@ def _handler(params):
                     ],
                     "uri": uri,
                 }
-            self.assertEqual(expected, actual)
+            assert expected == actual
+
+    def test_no_initialization_options(self, ruff_version: Version) -> None:
+        expected_docs_url = (
+            "https://docs.astral.sh/ruff/"
+            if ruff_version >= VERSION_REQUIREMENT_ASTRAL_DOCS
+            else "https://beta.ruff.rs/docs/"
+        )
 
-    def test_no_initialization_options(self) -> None:
         with tempfile.NamedTemporaryFile(suffix=".py") as fp:
             fp.write(CONTENTS.encode())
             fp.flush()
@@ -149,7 +164,7 @@ def _handler(params):
                         {
                             "code": "F401",
                             "codeDescription": {
-                                "href": "https://docs.astral.sh/ruff/rules/unused-import"
+                                "href": expected_docs_url + "rules/unused-import"
                             },
                             "data": {
                                 "fix": {
@@ -177,7 +192,7 @@ def _handler(params):
                         {
                             "code": "F821",
                             "codeDescription": {
-                                "href": "https://docs.astral.sh/ruff/rules/undefined-name"
+                                "href": expected_docs_url + "rules/undefined-name"
                             },
                             "data": {"fix": None, "noqa_row": 3},
                             "message": "Undefined name `x`",
@@ -191,4 +206,4 @@ def _handler(params):
                     ],
                     "uri": uri,
                 }
-            self.assertEqual(expected, actual)
+            assert expected == actual