Merge pull request #63 from scrapy/update-tools

Update tool versions and typing

.github/workflows/checks.yml

@@ -22,6 +22,9 @@ jobs:
         - python-version: 3.12
           env:
             TOXENV: typing
+        - python-version: 3.12
+          env:
+            TOXENV: twinecheck
 
     steps:
     - uses: actions/checkout@v4

.github/workflows/publish.yml

@@ -15,11 +15,12 @@ jobs:
       with:
         python-version: 3.12
 
+    - name: Build
+      run: | 
+        pip install --upgrade build twine
+        python -m build
+
     - name: Publish to PyPI
-      run: |
-        pip install --upgrade pip
-        pip install --upgrade setuptools wheel twine
-        python setup.py sdist bdist_wheel
-        export TWINE_USERNAME=__token__
-        export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
-        twine upload dist/*
+      uses: pypa/gh-action-pypi-publish@v1.8.10
+      with:
+        password: ${{ secrets.PYPI_TOKEN }}

pyproject.toml

@@ -1,3 +1,9 @@
 [tool.black]
 line-length = 119
 include = '\.py$'
+
+[[tool.mypy.overrides]]
+module = "queuelib.tests.*"
+allow_untyped_defs = true
+allow_untyped_calls = true
+check_untyped_defs = false

queuelib/queue.py

@@ -6,18 +6,18 @@
 from abc import abstractmethod
 from collections import deque
 from contextlib import suppress
-from typing import Any, Optional
+from typing import Any, Optional, Deque, BinaryIO, Literal, cast, Dict
 
 
 class _BaseQueueMeta(type):
     """
     Metaclass to check queue classes against the necessary interface
     """
 
-    def __instancecheck__(cls, instance):
+    def __instancecheck__(cls, instance: Any) -> bool:
         return cls.__subclasscheck__(type(instance))  # pylint: disable=no-value-for-parameter
 
-    def __subclasscheck__(cls, subclass):
+    def __subclasscheck__(cls, subclass: Any) -> bool:
         return (
             hasattr(subclass, "push")
             and callable(subclass.push)
@@ -46,7 +46,7 @@ def peek(self) -> Optional[Any]:
         raise NotImplementedError()
 
     @abstractmethod
-    def __len__(self):
+    def __len__(self) -> int:
         raise NotImplementedError()
 
     def close(self) -> None:
@@ -57,7 +57,7 @@ class FifoMemoryQueue:
     """In-memory FIFO queue, API compliant with FifoDiskQueue."""
 
     def __init__(self) -> None:
-        self.q = deque()  # type: deque
+        self.q: Deque[Any] = deque()
 
     def push(self, obj: Any) -> None:
         self.q.append(obj)
@@ -71,7 +71,7 @@ def peek(self) -> Optional[Any]:
     def close(self) -> None:
         pass
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.q)
 
 
@@ -116,7 +116,7 @@ def push(self, string: bytes) -> None:
         self.info["size"] += 1
         self.info["head"] = [hnum, hpos]
 
-    def _openchunk(self, number: int, mode: str = "rb"):
+    def _openchunk(self, number: int, mode: Literal["rb", "ab+"] = "rb") -> BinaryIO:
         return open(os.path.join(self.path, f"q{number:05d}"), mode)
 
     def pop(self) -> Optional[bytes]:
@@ -163,13 +163,13 @@ def close(self) -> None:
             self._cleanup()
 
     def __len__(self) -> int:
-        return self.info["size"]
+        return cast(int, self.info["size"])
 
-    def _loadinfo(self, chunksize: int) -> dict:
+    def _loadinfo(self, chunksize: int) -> Dict[str, Any]:
         infopath = self._infopath()
         if os.path.exists(infopath):
             with open(infopath) as f:
-                info = json.load(f)
+                info = cast(Dict[str, Any], json.load(f))
         else:
             info = {
                 "chunksize": chunksize,
@@ -179,7 +179,7 @@ def _loadinfo(self, chunksize: int) -> dict:
             }
         return info
 
-    def _saveinfo(self, info: dict) -> None:
+    def _saveinfo(self, info: Dict[str, Any]) -> None:
         with open(self._infopath(), "w") as f:
             json.dump(info, f)
 
@@ -201,6 +201,7 @@ class LifoDiskQueue:
     SIZE_SIZE = struct.calcsize(SIZE_FORMAT)
 
     def __init__(self, path: str) -> None:
+        self.size: int
         self.path = path
         if os.path.exists(path):
             self.f = open(path, "rb+")
@@ -277,13 +278,13 @@ def pop(self) -> Optional[bytes]:
         with self._db as conn:
             for id_, item in conn.execute(self._sql_pop):
                 conn.execute(self._sql_del, (id_,))
-                return item
+                return cast(bytes, item)
         return None
 
     def peek(self) -> Optional[bytes]:
         with self._db as conn:
             for _, item in conn.execute(self._sql_pop):
-                return item
+                return cast(bytes, item)
         return None
 
     def close(self) -> None:
@@ -294,7 +295,7 @@ def close(self) -> None:
 
     def __len__(self) -> int:
         with self._db as conn:
-            return next(conn.execute(self._sql_size))[0]
+            return cast(int, next(conn.execute(self._sql_size))[0])
 
 
 class LifoSQLiteQueue(FifoSQLiteQueue):

queuelib/tests/test_queue.py

@@ -2,7 +2,7 @@
 import glob
 from abc import abstractmethod
 from unittest import mock
-from typing import Any, Optional
+from typing import Any, Optional, List
 
 import pytest
 
@@ -20,7 +20,7 @@
 
 class DummyQueue:
     def __init__(self) -> None:
-        self.q: list = []
+        self.q: List[Any] = []
 
     def push(self, obj: Any) -> None:
         self.q.append(obj)
@@ -213,7 +213,6 @@ def test_peek_lifo(self):
 
 
 class PersistentTestMixin:
-
     chunksize = 100000
 
     @pytest.mark.xfail(reason="Reenable once Scrapy.squeues stops extending from this testsuite")

setup.py

@@ -10,6 +10,9 @@
     author_email="info@scrapy.org",
     url="https://github.com/scrapy/queuelib",
     packages=find_packages(),
+    package_data={
+        "queuelib": ["py.typed"],
+    },
     platforms=["Any"],
     python_requires=">=3.8",
     classifiers=[

tox.ini

@@ -4,22 +4,20 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py, pypy3, black, flake8, pylint, security, typing
+envlist = py, pypy3, black, flake8, pylint, security, typing, twinecheck
 
 [testenv]
 deps =
-    pytest !=3.1.1, !=3.1.2
-    pytest-cov==2.11.1
+    pytest
+    pytest-cov
     setuptools
 commands =
     py.test --cov=queuelib --cov-report=xml --cov-report=term --cov-report=html {posargs:queuelib}
 
 [testenv:black]
 basepython = python3
 deps =
-    black==21.4b0
-    # 8.1.0 breaks black < 22.3.0
-    click==8.0.2
+    black==23.11.0
 commands =
     black --check {posargs:queuelib setup.py}
 
@@ -34,20 +32,30 @@ commands =
 basepython = python3
 deps =
     {[testenv]deps}
-    pylint==3.0.0
+    pylint==3.0.2
 commands =
     pylint {posargs:queuelib setup.py}
 
 [testenv:security]
 basepython = python3
 deps =
-    bandit==1.7.0
+    bandit==1.7.5
 commands =
     bandit -r -c .bandit.yml {posargs:queuelib setup.py}
 
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.0.1
+    mypy==1.7.0
+    pytest==7.4.3
 commands =
-    mypy --show-error-codes --ignore-missing-imports --follow-imports=skip {posargs:queuelib setup.py}
+    mypy --strict {posargs:queuelib}
+
+[testenv:twinecheck]
+basepython = python3
+deps =
+    twine==4.0.2
+    build==1.0.3
+commands =
+    python -m build --sdist
+    twine check dist/*