Add checker for PyPI packages

README.md

@@ -206,6 +206,21 @@ and add a template for source download URL:
 
 for git type sources, instead of `url-template`, set `tag-template` to derive git tag from version.
 
+### PyPI checker ###
+
+Check for Python package updates on PyPI.
+
+```json
+"x-checker-data": {
+    "type": "pypi",
+    "name": "Pillow"
+}
+```
+
+By default it will check for source package (`sdist` package type).
+To check for binary package instead, set `packagetype` to `bdist_wheel`
+(only noarch wheels are supported currently).
+
 ### JetBrains checker
 
 Special checker that will check for available updates

src/checkers/__init__.py

@@ -7,6 +7,7 @@
 from .anityachecker import AnityaChecker
 from .rustchecker import RustChecker
 from .gitchecker import GitChecker
+from .pypichecker import PyPIChecker
 
 
 # For each ExternalData, checkers are run in the order listed here, stopping once data.state is
@@ -19,6 +20,7 @@
     AnityaChecker,
     RustChecker,
     JSONChecker,
+    PyPIChecker,
     GitChecker,  # leave this last but one
     URLChecker,  # leave this last
 ]

src/checkers/pypichecker.py

@@ -0,0 +1,92 @@
+import logging
+from datetime import datetime
+from distutils.version import StrictVersion, LooseVersion
+import operator
+import re
+import typing as t
+
+import requests
+
+from ..lib.externaldata import Checker, ExternalFile
+
+log = logging.getLogger(__name__)
+
+PYPI_INDEX = "https://pypi.org/pypi"
+OPERATORS = {
+    "<": operator.lt,
+    "<=": operator.le,
+    ">": operator.gt,
+    ">=": operator.ge,
+    "==": operator.eq,
+    "!=": operator.ne,
+}
+BDIST_RE = re.compile(r"^(\S+)-(\d[\d\.\w]*\d)-(\S+)-(\S+)-(\S+).whl$")
+
+
+def _version_matches(version: str, constraints: t.List[t.Tuple[str, str]]):
+    if not constraints:
+        return True
+    for ver_oper, ver_limit in constraints:
+        oper = OPERATORS[ver_oper]
+        try:
+            matches = oper(StrictVersion(version), StrictVersion(ver_limit))
+        except ValueError:
+            matches = oper(LooseVersion(version), LooseVersion(ver_limit))
+        return matches
+    return None
+
+
+def _filter_downloads(
+    pypy_releases: t.Dict[str, t.List[t.Dict]],
+    constraints: t.List[t.Tuple[str, str]],
+    packagetype: str,
+) -> t.Generator[t.Tuple[str, t.Dict, datetime], None, None]:
+    for pypi_version, pypi_downloads in pypy_releases.items():
+        if not _version_matches(pypi_version, constraints):
+            continue
+        for download in pypi_downloads:
+            if download["packagetype"] != packagetype:
+                continue
+            if download["python_version"] not in ["source", "py3", "py2.py3"]:
+                continue
+            date = datetime.fromisoformat(download["upload_time_iso_8601"].rstrip("Z"))
+            yield (pypi_version, download, date)
+
+
+class PyPIChecker(Checker):
+    CHECKER_DATA_TYPE = "pypi"
+
+    def __init__(self):
+        self.session = requests.Session()
+
+    def check(self, external_data):
+        package_name = external_data.checker_data["name"]
+        package_type = external_data.checker_data.get("packagetype", "sdist")
+        if "versions" in external_data.checker_data:
+            constraints = [(o, l) for o, l in external_data.checker_data["versions"]]
+        else:
+            constraints = []
+
+        with self.session.get(f"{PYPI_INDEX}/{package_name}/json") as response:
+            response.raise_for_status()
+            pypi_data = response.json()
+
+        downloads = sorted(
+            _filter_downloads(pypi_data["releases"], constraints, package_type),
+            key=lambda r: r[2],
+        )
+
+        try:
+            pypi_version, pypi_download, pypi_date = downloads[-1]
+        except IndexError:
+            log.error("Couldn't find %s for package %s", package_type, package_name)
+            return
+
+        new_version = ExternalFile(
+            url=pypi_download["url"],
+            checksum=pypi_download["digests"]["sha256"],
+            size=pypi_download["size"],
+            version=pypi_version,
+            timestamp=pypi_date,
+        )
+        external_data.set_new_version(new_version)

tests/com.valvesoftware.Steam.yml

@@ -0,0 +1,40 @@
+id: com.valvesoftware.Steam
+modules:
+
+  - name: python-modules
+    sources:
+
+      - type: file
+        url: https://files.pythonhosted.org/packages/6d/38/c21ef5034684ffc0412deefbb07d66678332290c14bb5269c85145fbd55e/setuptools-50.3.2-py3-none-any.whl
+        sha256: 2c242a0856fbad7efbe560df4a7add9324f340cf48df43651e9604924466794a
+        x-checker-data:
+          type: pypi
+          name: setuptools
+          packagetype: bdist_wheel
+
+      - type: file
+        url: https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz
+        sha256: b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d
+        x-checker-data:
+          type: pypi
+          name: PyYAML
+          packagetype: sdist
+
+      - type: file
+        url: https://files.pythonhosted.org/packages/7a/c2/bf87cef932c45cb7b7a79a0a954e3307fcff209c7639182a2b9ae0127959/vdf-3.1-py2.py3-none-any.whl
+        sha256: a5da182b3ef888d45f39862725bc7bb2836515c9fc329843001e506e73bb5cd4
+        x-checker-data:
+          type: pypi
+          name: vdf
+          versions:
+            - ["==", "3.2"]
+          packagetype: bdist_wheel
+
+
+      - type: file
+        url: "https://files.pythonhosted.org/packages/3e/02/b09732ca4b14405ff159c470a612979acfc6e8645dc32f83ea0129709f7a/Pillow-7.2.0.tar.gz"
+        sha256: "97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626"
+        x-checker-data:
+          type: pypi
+          name: Pillow
+          packagetype: bdist_wheel

tests/test_pypichecker.py

@@ -0,0 +1,48 @@
+import os
+import unittest
+
+from src.checker import ManifestChecker
+from src.lib.utils import init_logging
+
+TEST_MANIFEST = os.path.join(os.path.dirname(__file__), "com.valvesoftware.Steam.yml")
+
+
+class TestPyPIChecker(unittest.TestCase):
+    def setUp(self):
+        init_logging()
+
+    def test_check(self):
+        checker = ManifestChecker(TEST_MANIFEST)
+        ext_data = checker.check()
+
+        self.assertEqual(len(ext_data), 4)
+        for data in ext_data:
+            if data.filename != "Pillow-7.2.0.tar.gz":
+                self.assertIsNotNone(data.new_version)
+                self.assertIsNotNone(data.new_version.url)
+                self.assertIsNotNone(data.new_version.checksum)
+                self.assertIsNotNone(data.new_version.version)
+                self.assertNotEqual(data.new_version.url, data.current_version.url)
+                self.assertNotEqual(
+                    data.new_version.checksum, data.current_version.checksum
+                )
+            if data.filename == "setuptools-50.3.2-py3-none-any.whl":
+                self.assertRegex(
+                    data.new_version.url,
+                    r"https://files.pythonhosted.org/packages/[a-f0-9/]+/setuptools-[\d\.]+-[\S\.]+-none-any.whl",
+                )
+            elif data.filename == "PyYAML-5.3.1.tar.gz":
+                self.assertRegex(
+                    data.new_version.url,
+                    r"https://files.pythonhosted.org/packages/[a-f0-9/]+/PyYAML-[\d\.]+.(tar.(gz|xz|bz2)|zip)",
+                )
+            elif data.filename == "vdf-3.1-py2.py3-none-any.whl":
+                self.assertRegex(
+                    data.new_version.url,
+                    r"https://files.pythonhosted.org/packages/[a-f0-9/]+/vdf-[\d\.]+-[\S\.]+-none-any.whl",
+                )
+                self.assertEqual(data.new_version.version, "3.2")
+            elif data.filename == "Pillow-7.2.0.tar.gz":
+                self.assertIsNone(data.new_version)
+            else:
+                self.fail(f"Unknown data {data.filename}")