From 949921a2333913576f0751a06a97c648082ba901 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 10 Jun 2024 13:21:03 +0300 Subject: [PATCH 1/6] Use Hatch for packaging --- MANIFEST.in | 1 - pyproject.toml | 35 +++++++++++++++++++++++++++++++---- setup.cfg | 28 ---------------------------- 3 files changed, 31 insertions(+), 33 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index b9a6ef3..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include laituri py.typed diff --git a/pyproject.toml b/pyproject.toml index aa4cece..d8f4bf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,36 @@ [build-system] -requires = [ - "setuptools>=42", - "wheel" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "laituri" +dynamic = ["version"] +description = "Docker Toolkit for Python" +readme = "README.md" +license = "MIT" +requires-python = ">=3.6" +authors = [ + { name = "Valohai", email = "hait@valohai.com" }, +] +classifiers = [ + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", +] +dependencies = [ + "requests>=2.23,<3", +] + +[project.urls] +Homepage = "https://github.com/valohai/laituri" + +[tool.hatch.version] +path = "laituri/__init__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/laituri", ] -build-backend = "setuptools.build_meta" [tool.mypy] strict = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d45ecf6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,28 +0,0 @@ -[metadata] -name = laituri -description = Docker Toolkit for Python -long_description = file:README.md -long_description_content_type = text/markdown -version = attr:laituri.__version__ -author = Valohai -author_email = hait@valohai.com -maintainer = Ruksi Laine -maintainer_email = me@ruk.si -url = https://github.com/valohai/laituri -license = MIT -classifiers = - Programming Language :: Python :: 3 - License :: OSI Approved :: MIT License - Operating System :: OS Independent - -[options] -include_package_data = True -python_requires = >=3.7 -packages = find: -install_requires = - requests>=2.23,<3 - -[options.packages.find] -exclude = - laituri_tests - laituri_tests.* From dd8ddb49f3c765588ebd4bf4212d6e8b941af89e Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 10 Jun 2024 13:23:24 +0300 Subject: [PATCH 2/6] Update lint tools; format with Ruff --- .github/workflows/ci.yml | 2 +- .pre-commit-config.yaml | 9 +++++---- laituri/docker/credential_manager/docker_v1.py | 18 ++++++++++++------ .../credential_manager/ecr_with_role_v1.py | 2 ++ laituri/utils/retry.py | 2 +- laituri_tests/test_docker_v1.py | 4 ++-- .../test_registry_credentials_callback_v1.py | 4 ++-- laituri_tests/test_retry.py | 1 - pyproject.toml | 5 +++++ 9 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 550b654..06f61ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,4 +34,4 @@ jobs: with: python-version: '3.10' - uses: actions/checkout@v3 - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38d6cb6..18b433b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,23 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: debug-statements - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.256 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.8 hooks: - id: ruff args: - --fix - --exit-non-zero-on-fix + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 + rev: v1.10.0 hooks: - id: mypy exclude: laituri_tests/test_.* diff --git a/laituri/docker/credential_manager/docker_v1.py b/laituri/docker/credential_manager/docker_v1.py index cf4c626..0fd8d5c 100644 --- a/laituri/docker/credential_manager/docker_v1.py +++ b/laituri/docker/credential_manager/docker_v1.py @@ -50,7 +50,8 @@ def docker_login(domain: str, username: str, password: str) -> bool: args = [ get_docker_command(), 'login', - '--username', username, + '--username', + username, '--password-stdin', domain, ] @@ -76,11 +77,16 @@ def docker_logout(domain: str) -> None: try: log.debug(f'Running `docker logout {domain}`') - subprocess.check_call([ - get_docker_command(), - 'logout', - domain, - ], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, timeout=settings.DOCKER_TIMEOUT) + subprocess.check_call( + [ + get_docker_command(), + 'logout', + domain, + ], + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + timeout=settings.DOCKER_TIMEOUT, + ) except subprocess.CalledProcessError as cpe: message = cpe.stdout.decode('utf-8', errors='ignore') log.warning(f'Failed `docker logout {domain}`: {message}') diff --git a/laituri/docker/credential_manager/ecr_with_role_v1.py b/laituri/docker/credential_manager/ecr_with_role_v1.py index 77215c3..ce4dd7e 100644 --- a/laituri/docker/credential_manager/ecr_with_role_v1.py +++ b/laituri/docker/credential_manager/ecr_with_role_v1.py @@ -16,6 +16,7 @@ def ecr_with_role_v1_credential_manager( auth_tries: int, ) -> Iterator[None]: from boto3 import Session + role_name = registry_credentials['role_name'] creds = get_role_credentials_from_instance_metadata(role_name) try: @@ -47,6 +48,7 @@ def ecr_with_role_v1_credential_manager( def get_role_credentials_from_instance_metadata(role_name: str) -> Dict[str, str]: from botocore.utils import InstanceMetadataFetcher + fetcher = InstanceMetadataFetcher() creds: Dict[str, str] = fetcher.retrieve_iam_role_credentials() if creds.get('role_name') != role_name: diff --git a/laituri/utils/retry.py b/laituri/utils/retry.py index 9df6d75..7c953b8 100644 --- a/laituri/utils/retry.py +++ b/laituri/utils/retry.py @@ -28,7 +28,7 @@ def wrapped_func(*args, **kwargs): # type: ignore try: return func(*args, **kwargs) except Exception: - delay = (2 ** (attempt - 1)) # 1, 2, 4, 8, 16, 32... + delay = 2 ** (attempt - 1) # 1, 2, 4, 8, 16, 32... delay += random.random() # a tiny bit of random for desynchronizing multiple potential users delay = min(delay, max_delay) time.sleep(delay) diff --git a/laituri_tests/test_docker_v1.py b/laituri_tests/test_docker_v1.py index 0cd30f4..90e1de6 100644 --- a/laituri_tests/test_docker_v1.py +++ b/laituri_tests/test_docker_v1.py @@ -12,7 +12,7 @@ 'type': 'docker', 'version': '1', 'username': 'edward', - 'password': 'scissors123' + 'password': 'scissors123', } @@ -83,7 +83,7 @@ def test_fallback_with_invalid_credential_configuration(mocker, missing_value): with get_credential_manager( image=image, registry_credentials=VALID_DOCKER_CREDENTIALS, - log_status=my_logging_callback + log_status=my_logging_callback, ): my_action() diff --git a/laituri_tests/test_registry_credentials_callback_v1.py b/laituri_tests/test_registry_credentials_callback_v1.py index 147f99a..6f4215c 100644 --- a/laituri_tests/test_registry_credentials_callback_v1.py +++ b/laituri_tests/test_registry_credentials_callback_v1.py @@ -33,8 +33,8 @@ def test_callback_retry(mocker, requests_mock, with_header: bool, image: str): {'status_code': 500}, {'exc': requests.exceptions.ConnectTimeout}, {'status_code': 200}, # no JSON - {'status_code': 200, 'json': VALID_CALLBACK_RESPONSE} - ] + {'status_code': 200, 'json': VALID_CALLBACK_RESPONSE}, + ], ) mock_popen = mocker.patch('subprocess.Popen', new_callable=create_mock_popen) mocker.patch('time.sleep') # removes retry delays for testing diff --git a/laituri_tests/test_retry.py b/laituri_tests/test_retry.py index d7dee6c..02ee7ae 100644 --- a/laituri_tests/test_retry.py +++ b/laituri_tests/test_retry.py @@ -4,7 +4,6 @@ class TestRetry: - @pytest.fixture(autouse=True) def disable_sleep(self, mocker): return mocker.patch('time.sleep') # removes retry delays for testing diff --git a/pyproject.toml b/pyproject.toml index d8f4bf4..953e00b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,8 @@ exclude = "laituri_tests/test_.*" [tool.ruff] target-version = "py37" line-length = 120 + +[tool.ruff.lint] mccabe.max-complexity = 10 flake8-tidy-imports.ban-relative-imports = "all" select = [ @@ -52,3 +54,6 @@ select = [ "W", # pycodestyle ] ignore = [] + +[tool.ruff.format] +quote-style = "preserve" From 880750fbc58911ed3dbd4100272c9a8a5027a323 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 10 Jun 2024 13:29:30 +0300 Subject: [PATCH 3/6] Use dev extra instead of requirements-dev.txt --- .github/workflows/ci.yml | 2 +- README.md | 2 +- pyproject.toml | 8 ++++++ requirements-dev.in | 5 ---- requirements-dev.txt | 55 ---------------------------------------- 5 files changed, 10 insertions(+), 62 deletions(-) delete mode 100644 requirements-dev.in delete mode 100644 requirements-dev.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06f61ec..36de3f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: with: python-version: '${{ matrix.python-version }}' - uses: actions/checkout@v3 - - run: 'pip install -e . -r requirements-dev.txt' + - run: 'pip install -e .[dev]' - run: pytest -vvv --cov . - uses: codecov/codecov-action@v2 diff --git a/README.md b/README.md index 93d09f6..7839360 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Installing editable library version in the current virtual environment. ```bash # install this package and all development dependencies -pip install -e . -r requirements-dev.txt pip-tools pre-commit && pre-commit install +pip install -e .[dev] pre-commit && pre-commit install # manually run lint and type checks pre-commit run --all-files diff --git a/pyproject.toml b/pyproject.toml index 953e00b..5ff1c95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,14 @@ dependencies = [ "requests>=2.23,<3", ] +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "pytest-mock", + "requests-mock", +] + [project.urls] Homepage = "https://github.com/valohai/laituri" diff --git a/requirements-dev.in b/requirements-dev.in deleted file mode 100644 index 2aecd1d..0000000 --- a/requirements-dev.in +++ /dev/null @@ -1,5 +0,0 @@ -pytest -pytest-cov -pytest-mock -requests-mock -requests<2.26.0 diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 8057b39..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,55 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.7 -# by the following command: -# -# pip-compile requirements-dev.in -# -attrs==22.2.0 - # via pytest -certifi==2022.12.7 - # via requests -chardet==4.0.0 - # via requests -coverage[toml]==7.2.2 - # via pytest-cov -exceptiongroup==1.1.1 - # via pytest -idna==2.10 - # via requests -importlib-metadata==6.0.0 - # via - # pluggy - # pytest -iniconfig==2.0.0 - # via pytest -packaging==23.0 - # via pytest -pluggy==1.0.0 - # via pytest -pytest==7.2.2 - # via - # -r requirements-dev.in - # pytest-cov - # pytest-mock -pytest-cov==4.0.0 - # via -r requirements-dev.in -pytest-mock==3.10.0 - # via -r requirements-dev.in -requests==2.25.1 - # via - # -r requirements-dev.in - # requests-mock -requests-mock==1.10.0 - # via -r requirements-dev.in -six==1.16.0 - # via requests-mock -tomli==2.0.1 - # via - # coverage - # pytest -typing-extensions==4.5.0 - # via importlib-metadata -urllib3==1.26.15 - # via requests -zipp==3.15.0 - # via importlib-metadata From 5224f89a3a34e2a2869ed9f9c5b5593ba27d6ee4 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 10 Jun 2024 13:31:47 +0300 Subject: [PATCH 4/6] Renovate GitHub Actions configuration --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/publish.yml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36de3f7..80b511a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,19 +19,19 @@ jobs: - '3.10' steps: - name: 'Set up Python ${{ matrix.python-version }}' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '${{ matrix.python-version }}' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: 'pip install -e .[dev]' - run: pytest -vvv --cov . - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v4 Lint: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6296df3..9d935a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-20.04 if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: pip install -U pip setuptools build twine - run: python -m build . - run: twine upload --verbose dist/* From 6365e39fccf4a374fccf245fb214b76981ddb87d Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 10 Jun 2024 13:39:06 +0300 Subject: [PATCH 5/6] Remove outdated CodeQL configuration --- .github/workflows/codeql-analysis.yml | 40 --------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 30a1c87..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: [master] - pull_request: - branches: [master] - schedule: - - cron: '0 13 * * 4' - -jobs: - - Analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - language: ['python'] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 From 7a192963d27073370a3d3fe56c2f0d18a7eacbfb Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 10 Jun 2024 13:42:28 +0300 Subject: [PATCH 6/6] Drop support for Pythons older than 3.8 --- .github/workflows/ci.yml | 6 +++--- README.md | 2 +- laituri/docker/credential_manager/docker_v1.py | 2 +- pyproject.toml | 5 +++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80b511a..fe917f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,8 @@ jobs: strategy: matrix: python-version: - - '3.7' - - '3.10' + - '3.8' + - '3.12' steps: - name: 'Set up Python ${{ matrix.python-version }}' uses: actions/setup-python@v5 @@ -32,6 +32,6 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.1 diff --git a/README.md b/README.md index 7839360..41b43b4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![codecov](https://codecov.io/gh/valohai/laituri/branch/master/graph/badge.svg)](https://codecov.io/gh/valohai/laituri) `laituri` is a set of Docker-related Python snippets used at [Valohai](https://valohai.com/). -You can use it with Python >= 3.7. +You can use it with Python >= 3.8. ## Usage diff --git a/laituri/docker/credential_manager/docker_v1.py b/laituri/docker/credential_manager/docker_v1.py index 0fd8d5c..afa88de 100644 --- a/laituri/docker/credential_manager/docker_v1.py +++ b/laituri/docker/credential_manager/docker_v1.py @@ -58,7 +58,7 @@ def docker_login(domain: str, username: str, password: str) -> bool: log.debug(f"Running `{' '.join(args)}`") proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) try: - cmd_input = (password + '\n').encode('utf-8') + cmd_input = f"{password}\n".encode() stdout, _ = proc.communicate(input=cmd_input, timeout=settings.DOCKER_TIMEOUT) except subprocess.TimeoutExpired as te: raise DockerLoginFailed('timed out') from te diff --git a/pyproject.toml b/pyproject.toml index 5ff1c95..4bca66c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "Docker Toolkit for Python" readme = "README.md" license = "MIT" -requires-python = ">=3.6" +requires-python = ">=3.8" authors = [ { name = "Valohai", email = "hait@valohai.com" }, ] @@ -45,7 +45,7 @@ strict = true exclude = "laituri_tests/test_.*" [tool.ruff] -target-version = "py37" +target-version = "py38" line-length = 120 [tool.ruff.lint] @@ -59,6 +59,7 @@ select = [ "I", # isort "T", # debugger and print "TID", # flake8-tidy-imports + "UP", # pyupgrade "W", # pycodestyle ] ignore = []