From 841d1b95e42be40b7f4bbdc839d8d20e77b4815e Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Mon, 26 Oct 2020 13:38:11 +0000 Subject: [PATCH] support python3.9 and pydantic 1.7 (#214) * support python3.9 and pydantic 1.7 * linting * uprev mypy --- .github/workflows/ci.yml | 2 +- arq/connections.py | 30 +++++++++++++++++++++++++++--- docs/requirements.txt | 2 +- requirements.txt | 2 ++ setup.py | 1 + tests/requirements.txt | 2 +- tests/test_utils.py | 35 +++++++++++++++++++++++++++++++++-- 7 files changed, 66 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01414a69..055ff4d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: os: [ubuntu] - python-version: ['3.6', '3.7', '3.8'] + python-version: ['3.6', '3.7', '3.8', '3.9'] env: PYTHON: ${{ matrix.python-version }} diff --git a/arq/connections.py b/arq/connections.py index 9d0d3ca5..fa62a058 100644 --- a/arq/connections.py +++ b/arq/connections.py @@ -1,15 +1,17 @@ import asyncio import functools import logging +import ssl from dataclasses import dataclass from datetime import datetime, timedelta from operator import attrgetter -from ssl import SSLContext -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, Generator, List, Optional, Tuple, Union +from urllib.parse import urlparse from uuid import uuid4 import aioredis from aioredis import MultiExecError, Redis +from pydantic.validators import make_arbitrary_type_validator from .constants import default_queue_name, job_key_prefix, result_key_prefix from .jobs import Deserializer, Job, JobDef, JobResult, Serializer, deserialize_job, serialize_job @@ -18,6 +20,16 @@ logger = logging.getLogger('arq.connections') +class SSLContext(ssl.SSLContext): + """ + Required to avoid problems with + """ + + @classmethod + def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]: + yield make_arbitrary_type_validator(ssl.SSLContext) + + @dataclass class RedisSettings: """ @@ -38,8 +50,20 @@ class RedisSettings: sentinel: bool = False sentinel_master: str = 'mymaster' + @classmethod + def from_dsn(cls, dsn: str) -> 'RedisSettings': + conf = urlparse(dsn) + assert conf.scheme in {'redis', 'rediss'}, 'invalid DSN scheme' + return RedisSettings( + host=conf.hostname or 'localhost', + port=conf.port or 6379, + ssl=conf.scheme == 'rediss', + password=conf.password, + database=int((conf.path or '0').strip('/')), + ) + def __repr__(self) -> str: - return ''.format(' '.join(f'{k}={v}' for k, v in self.__dict__.items())) + return 'RedisSettings({})'.format(', '.join(f'{k}={v!r}' for k, v in self.__dict__.items())) # extra time after the job is expected to start when the job key should expire, 1 day in ms diff --git a/docs/requirements.txt b/docs/requirements.txt index ff77b744..96cc2f67 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ docutils==0.14 -Pygments==2.3.1 +Pygments==2.7.2 Sphinx==2.0.1 sphinxcontrib-websupport==1.1.0 diff --git a/requirements.txt b/requirements.txt index e4ab17d8..07dd334b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ -r docs/requirements.txt -r tests/requirements.txt + +pydantic==1.7 diff --git a/setup.py b/setup.py index a3104b61..3d110c7f 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Clustering', diff --git a/tests/requirements.txt b/tests/requirements.txt index 6662f535..fa2269ba 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -4,7 +4,7 @@ flake8==3.7.9 flake8-quotes==3 isort==4.3.21 msgpack==0.6.1 -mypy==0.770 +mypy==0.790 pycodestyle==2.5.0 pyflakes==2.1.1 pytest==5.3.5 diff --git a/tests/test_utils.py b/tests/test_utils.py index 3c8d86b5..9f30fb09 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,7 @@ from datetime import timedelta import pytest +from pydantic import BaseModel, validator import arq.typing import arq.utils @@ -13,8 +14,8 @@ def test_settings_changed(): settings = RedisSettings(port=123) assert settings.port == 123 assert ( - '' + "RedisSettings(host='localhost', port=123, database=0, password=None, ssl=None, conn_timeout=1, " + "conn_retries=5, conn_retry_delay=1, sentinel=False, sentinel_master='mymaster')" ) == str(settings) @@ -94,3 +95,33 @@ def test_to_seconds(input, output): def test_typing(): assert 'OptionType' in arq.typing.__all__ + + +def test_redis_settings_validation(): + class Settings(BaseModel): + redis_settings: RedisSettings + + @validator('redis_settings', always=True, pre=True) + def parse_redis_settings(cls, v): + if isinstance(v, str): + return RedisSettings.from_dsn(v) + else: + return v + + s1 = Settings(redis_settings='redis://foobar:123/4') + assert s1.redis_settings.host == 'foobar' + assert s1.redis_settings.host == 'foobar' + assert s1.redis_settings.port == 123 + assert s1.redis_settings.database == 4 + assert s1.redis_settings.ssl is False + + s2 = Settings(redis_settings={'host': 'testing.com'}) + assert s2.redis_settings.host == 'testing.com' + assert s2.redis_settings.port == 6379 + + with pytest.raises(ValueError, match='instance of SSLContext expected'): + Settings(redis_settings={'ssl': 123}) + + s3 = Settings(redis_settings={'ssl': True}) + assert s3.redis_settings.host == 'localhost' + assert s3.redis_settings.ssl is True