diff --git a/.cruft.json b/.cruft.json index 1bbcd13..d7732d6 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/iterative/py-template", - "commit": "efb0a22625c1d83e687f2bd284f3e96a20d16711", + "commit": "e4ec95f4cfd03d4af0a8604d462ee11d07d63b42", "context": { "cookiecutter": { "project_name": "dvc-task", diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 56f6f98..393a944 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,12 +7,6 @@ updates: interval: "weekly" labels: - "maintenance" - # Update via cruft - ignore: - - dependency-name: "mkdocs*" - - dependency-name: "pytest*" - - dependency-name: "pylint" - - dependency-name: "mypy" - directory: "/" package-ecosystem: "github-actions" @@ -20,10 +14,3 @@ updates: interval: "weekly" labels: - "maintenance" - # Update via cruft - ignore: - - dependency-name: "actions/checkout" - - dependency-name: "actions/setup-python" - - dependency-name: "pypa/gh-action-pypi-publish" - - dependency-name: "codecov/codecov-action" - - dependency-name: "peter-evans/create-pull-request" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a05bc14..02cf79f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -15,7 +15,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20e499d..c0e3d29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,25 +16,25 @@ jobs: id-token: write runs-on: ubuntu-latest steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' - - name: Upgrade pip and nox - run: | - pip install --upgrade pip nox - pip --version - nox --version + - name: Upgrade pip and nox + run: | + pip install --upgrade pip nox + pip --version + nox --version - - name: Build package - run: nox -s build + - name: Build package + run: nox -s build - - name: Upload package - if: github.event_name == 'release' - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Upload package + if: github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 54f7f6e..719b149 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,38 +21,38 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, windows-latest, macos-latest] - pyv: ['3.8', '3.9', '3.10', '3.11'] + pyv: ['3.8', '3.9', '3.10', '3.11', '3.12'] include: - - {os: ubuntu-latest, pyv: 'pypy3.8'} + - {os: ubuntu-latest, pyv: 'pypy3.8'} steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Set up Python ${{ matrix.pyv }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.pyv }} + - name: Set up Python ${{ matrix.pyv }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.pyv }} - - name: Upgrade pip and nox - run: | - python -m pip install --upgrade pip nox - pip --version - nox --version + - name: Upgrade pip and nox + run: | + python -m pip install --upgrade pip nox + pip --version + nox --version - - name: Lint code and check dependencies - run: nox -s lint safety + - name: Lint code and check dependencies + run: nox -s lint safety - - name: Run tests - run: nox -s tests-${{ matrix.nox_pyv || matrix.pyv }} -- --cov-report=xml + - name: Run tests + run: nox -s tests-${{ matrix.nox_pyv || matrix.pyv }} -- --cov-report=xml - - name: Upload coverage report - uses: codecov/codecov-action@v3.1.4 + - name: Upload coverage report + uses: codecov/codecov-action@v3 - - name: Build package - run: nox -s build + - name: Build package + run: nox -s build - - name: Build docs - run: nox -s docs + - name: Build docs + run: nox -s docs diff --git a/.github/workflows/update-template.yaml b/.github/workflows/update-template.yaml index 59f0b2e..0b91f76 100644 --- a/.github/workflows/update-template.yaml +++ b/.github/workflows/update-template.yaml @@ -9,8 +9,8 @@ jobs: update: runs-on: ubuntu-latest steps: - - name: Check out the repository - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@v4 - - name: Update template - uses: iterative/py-template@main + - name: Update template + uses: iterative/py-template@main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d2e9be..445c7f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,8 @@ default_language_version: python: python3 repos: - - repo: https://github.com/psf/black - rev: 23.9.1 - hooks: - - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -17,38 +13,26 @@ repos: args: ['--assume-in-merge'] - id: check-toml - id: check-yaml - - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending args: ['--fix=lf'] - id: sort-simple-yaml - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.3.4' + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell additional_dependencies: ["tomli"] - - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 - hooks: - - id: pyupgrade - args: [--py38-plus] - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==23.7.10 - - flake8-comprehensions==3.14.0 - - flake8-debugger==4.1.2 - - flake8-string-format==0.3.0 - - repo: https://github.com/pycqa/bandit - rev: 1.7.5 + - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.12.0 hooks: - - id: bandit - args: [-c, pyproject.toml] - additional_dependencies: ["bandit[toml]"] + - id: pretty-format-toml + args: [--autofix, --no-sort] + - id: pretty-format-yaml + args: [--autofix, --indent, '2', '--offset', '2', --preserve-quotes] diff --git a/mkdocs.yml b/mkdocs.yml index e0d2d5c..9d31012 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,15 +19,15 @@ extra: link: https://github.com/iterative/ plugins: -- search -- gen-files: - scripts: - - docs/gen_ref_pages.py -- section-index -- mkdocstrings: - handlers: - python: - rendering: - show_submodules: no - watch: - - src/dvc_task + - search + - gen-files: + scripts: + - docs/gen_ref_pages.py + - section-index + - mkdocstrings: + handlers: + python: + rendering: + show_submodules: no + watch: + - src/dvc_task diff --git a/noxfile.py b/noxfile.py index ff610f4..8adc39c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,4 +1,5 @@ """Automation using nox.""" + import glob import os @@ -35,7 +36,6 @@ def lint(session: nox.Session) -> None: args = *(session.posargs or ("--show-diff-on-failure",)), "--all-files" session.run("pre-commit", "run", *args) session.run("python", "-m", "mypy") - session.run("python", "-m", "pylint", *locations) @nox.session diff --git a/pyproject.toml b/pyproject.toml index 72b5ec8..148415c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,30 +2,59 @@ requires = ["setuptools>=48", "setuptools_scm[toml]>=6.3.1"] build-backend = "setuptools.build_meta" +[project] +name = "dvc-task" +description = "Extensible task queue used in DVC." +readme = "README.rst" +license = {text = "Apache-2.0"} +authors = [{name = "Iterative", email = "support@dvc.org"}] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Development Status :: 4 - Beta" +] +requires-python = ">=3.8" +dynamic = ["version"] +dependencies = [ + "celery>=5.3.0,<6", + "funcy>=1.17", + "kombu>=5.3.0,<6", + "pywin32>=225; sys_platform == 'win32'", + "shortuuid>=1.0.8" +] + +[project.urls] +Issues = "https://github.com/iterative/dvc-task/issues" +Source = "https://github.com/iterative/dvc-task" + +[project.optional-dependencies] +docs = [ + "mkdocs>=1.5.2,<2", + "mkdocs-gen-files>=0.5.0,<1", + "mkdocs-material>=9.3.1,<10", + "mkdocs-section-index>=0.3.6,<1", + "mkdocstrings-python>=1.6.3,<2" +] +tests = [ + "pytest>=7,<9", + "pytest-cov>=4.1.0", + "pytest-mock", + "pytest-sugar" +] +dev = [ + "dvc-task[tests,docs]", + "mypy==1.9.0" +] + [tool.setuptools_scm] -[tool.black] -line-length = 88 -include = '\.pyi?$' -exclude = ''' -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' - -[tool.isort] -profile = "black" -known_first_party = ["dvc_task"] -line_length = 88 +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false [tool.pytest.ini_options] addopts = "-ra" @@ -40,13 +69,13 @@ source = ["src", "*/site-packages"] [tool.coverage.report] show_missing = true exclude_lines = [ - "pragma: no cover", - "if __name__ == .__main__.:", - "if typing.TYPE_CHECKING:", - "if TYPE_CHECKING:", - "raise NotImplementedError", - "raise AssertionError", - "@overload", + "pragma: no cover", + "if __name__ == .__main__.:", + "if typing.TYPE_CHECKING:", + "if TYPE_CHECKING:", + "raise NotImplementedError", + "raise AssertionError", + "@overload" ] [tool.mypy] @@ -65,26 +94,86 @@ files = ["src", "tests"] [[tool.mypy.overrides]] module = [ - "celery.*", - "funcy.*", - "shortuuid", - "kombu.*", + "celery.*", + "funcy.*", + "shortuuid", + "kombu.*" ] ignore_missing_imports = true -[tool.pylint.format] -max-line-length = 88 +[tool.codespell] +ignore-words-list = " " + +[tool.ruff] +output-format = "full" +show-fixes = true + +[tool.ruff.lint] +ignore = [ + "ISC001", # single-line-implicit-string-concatenation + "PLR2004", # magic-value-comparison + "PLW2901", # redefined-loop-name + "RET501", # unnecessary-return-none + "RET502", # implicit-return-value + "RET503", # implicit-return + "S101", # assert + "SIM105", # suppressible-exception + "SIM108", # if-else-block-instead-of-if-exp + "SIM117" # multiple-with-statements +] +select = [ + "A", # flake8-buitlins + "ASYNC", # flake8-async + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "C90", # mccabe + "DTZ", # flake8-datetimez + "E", # pycodestyle - Error + "EXE", # flake8-executable + "F", # pyflakes + "FLY", # flynt-rules + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "INP", # flake8-no-pep420 + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + "PERF101", # perflint + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "Q", # flae8-quotes + "RET", # flake8-return + "RSE", # flake8-raise + "RUF", # ruff + "S", # flake8-bandit + "SIM", # flake8-simplify + "SLOT", # flake8-slots + "T10", # flake8-debugger + "T20", # flake8-print + "TCH", # flake8-type-checking + "TCH", # flake8-type-checking + "TID", # flake8-tidy-imports + "UP", # pyupgrade + "W", # pycodestyle - Warning + "YTT" # flake8-2020 +] -[tool.pylint.message_control] -enable = ["c-extension-no-member", "no-else-return"] +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false +parametrize-names-type = "csv" -[tool.pylint.variables] -dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" -ignored-argument-names = "_.*|^ignored_|^unused_|args|kwargs" +[tool.ruff.lint.per-file-ignores] +"noxfile.py" = ["D", "PTH"] +"tests/**" = ["S", "ARG001", "ARG002", "ANN"] +"docs/**" = ["INP"] -[tool.codespell] -ignore-words-list = " " +[tool.ruff.lint.flake8-type-checking] +strict = true -[tool.bandit] -exclude_dirs = ["tests"] -skips = ["B101"] +[tool.ruff.lint.isort] +known-first-party = ["dvc_task"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index afed862..0000000 --- a/setup.cfg +++ /dev/null @@ -1,79 +0,0 @@ -[metadata] -description = Extensible task queue used in DVC. -name = dvc-task -long_description = file: README.rst -long_description_content_type = text/x-rst -license = Apache-2.0 -license_file = LICENSE -url = https://github.com/iterative/dvc-task -platforms=any -authors = Iterative -maintainer_email = support@dvc.org -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Development Status :: 4 - Beta - -[options] -python_requires = >=3.8 -zip_safe = False -package_dir= - =src -packages = find: -install_requires= - celery>=5.3.0,<6 - kombu>=5.3.0,<6 - funcy>=1.17 - shortuuid>=1.0.8 - pywin32>=225; sys_platform == 'win32' - -[options.extras_require] -docs = - mkdocs==1.5.2 - mkdocs-gen-files==0.5.0 - mkdocs-material==9.3.1 - mkdocs-section-index==0.3.6 - mkdocstrings-python==1.6.3 -tests = - pytest==7.2.0 - pytest-sugar==0.9.6 - pytest-cov==3.0.0 - pytest-mock==3.8.2 - pylint==2.15.0 - mypy==0.971 - flaky==3.7.0 - pytest-test-utils>=0.0.6 - pytest-celery - celery-types==0.15.0 -dev = - %(tests)s - %(docs)s - -[options.packages.find] -exclude = - tests - tests.* -where=src - -[options.package_data] -dvc_task = py.typed - -[flake8] -ignore= - # Whitespace before ':' - E203 - # Too many leading '#' for block comment - E266 - # Line break occurred before a binary operator - W503 - # unindexed parameters in the str.format, see: - # https://pypi.org/project/flake8-string-format/ - P1 -max_line_length = 88 -max-complexity = 15 -select = B,C,E,F,W,T4,B902,T,P -show_source = true -count = true diff --git a/src/dvc_task/app/__init__.py b/src/dvc_task/app/__init__.py index c5a382e..b244ff7 100644 --- a/src/dvc_task/app/__init__.py +++ b/src/dvc_task/app/__init__.py @@ -1,4 +1,5 @@ """DVC Task app factories.""" + from .filesystem import FSApp __all__ = ["FSApp"] diff --git a/src/dvc_task/app/filesystem.py b/src/dvc_task/app/filesystem.py index a0e6c4b..135089e 100644 --- a/src/dvc_task/app/filesystem.py +++ b/src/dvc_task/app/filesystem.py @@ -1,4 +1,5 @@ """(Local) filesystem based Celery application.""" + import logging import os from datetime import datetime @@ -10,7 +11,7 @@ from kombu.utils.encoding import bytes_to_str from kombu.utils.json import loads -from ..utils import makedirs, remove, unc_path +from dvc_task.utils import makedirs, remove, unc_path logger = logging.getLogger(__name__) @@ -254,7 +255,7 @@ def _delete_expired( pass queues = set(exclude) if exclude else set() - now = datetime.now().timestamp() + now = datetime.now().timestamp() # noqa: DTZ005 for msg in self._iter_data_folder(): _delete_expired(msg, queues, now, self._queued_msg_path_cache) for msg in self._iter_processed_folder(): diff --git a/src/dvc_task/proc/__init__.py b/src/dvc_task/proc/__init__.py index c807eec..f36845f 100644 --- a/src/dvc_task/proc/__init__.py +++ b/src/dvc_task/proc/__init__.py @@ -1,4 +1,5 @@ """Process management module.""" + from .manager import ProcessManager from .process import ManagedProcess, ProcessInfo diff --git a/src/dvc_task/proc/exceptions.py b/src/dvc_task/proc/exceptions.py index a6a5c88..c87b68e 100644 --- a/src/dvc_task/proc/exceptions.py +++ b/src/dvc_task/proc/exceptions.py @@ -1,5 +1,6 @@ """Process exceptions.""" -from ..exceptions import DvcTaskError + +from dvc_task.exceptions import DvcTaskError class ProcessNotTerminatedError(DvcTaskError): @@ -16,7 +17,7 @@ def __init__(self, name): super().__init__(f"Managed process '{name}' does not exist.") -class TimeoutExpired(DvcTaskError): +class TimeoutExpired(DvcTaskError): # noqa: N818 """Process timeout expired.""" def __init__(self, cmd, timeout): diff --git a/src/dvc_task/proc/manager.py b/src/dvc_task/proc/manager.py index 2cfffba..0252bda 100644 --- a/src/dvc_task/proc/manager.py +++ b/src/dvc_task/proc/manager.py @@ -8,11 +8,12 @@ import time from typing import Dict, Generator, List, Optional, Tuple, Union -from celery import Signature, signature # pylint: disable=no-name-in-module +from celery import Signature, signature from funcy.flow import reraise from shortuuid import uuid -from ..utils import remove +from dvc_task.utils import remove + from .exceptions import ProcessNotTerminatedError, UnsupportedSignalError from .process import ProcessInfo @@ -77,7 +78,7 @@ def processes(self) -> Generator[Tuple[str, "ProcessInfo"], None, None]: except KeyError: continue - def run_signature( + def run_signature( # noqa: PLR0913 self, args: Union[str, List[str]], name: Optional[str] = None, @@ -111,7 +112,7 @@ def run_signature( immutable=immutable, ) - def send_signal(self, name: str, sig: int, group: bool = False): + def send_signal(self, name: str, sig: int, group: bool = False): # noqa: C901, PLR0912 """Send `signal` to the specified named process.""" try: process_info = self[name] @@ -133,8 +134,8 @@ def handle_closed_process(): if process_info.returncode is None: try: if sys.platform != "win32" and group: - pgid = os.getpgid(process_info.pid) # pylint: disable=no-member - os.killpg(pgid, sig) # pylint: disable=no-member + pgid = os.getpgid(process_info.pid) + os.killpg(pgid, sig) else: os.kill(process_info.pid, sig) except ProcessLookupError: @@ -153,7 +154,9 @@ def interrupt(self, name: str, group: bool = True): """Send interrupt signal to specified named process""" if sys.platform == "win32": self.send_signal( - name, signal.CTRL_C_EVENT, group # pylint: disable=no-member + name, + signal.CTRL_C_EVENT, + group, ) else: self.send_signal(name, signal.SIGINT, group) @@ -167,7 +170,7 @@ def kill(self, name: str, group: bool = False): if sys.platform == "win32": self.send_signal(name, signal.SIGTERM, group) else: - self.send_signal(name, signal.SIGKILL, group) # pylint: disable=no-member + self.send_signal(name, signal.SIGKILL, group) def remove(self, name: str, force: bool = False): """Remove the specified named process from this manager. diff --git a/src/dvc_task/proc/process.py b/src/dvc_task/proc/process.py index 5f20f23..ecd817d 100644 --- a/src/dvc_task/proc/process.py +++ b/src/dvc_task/proc/process.py @@ -1,10 +1,11 @@ """Managed process module.""" + import json import logging import multiprocessing as mp import os import shlex -import subprocess # nosec B404 +import subprocess import tempfile from contextlib import AbstractContextManager, ExitStack from dataclasses import asdict, dataclass @@ -13,7 +14,8 @@ from funcy import cached_property from shortuuid import uuid -from ..utils import makedirs +from dvc_task.utils import makedirs + from .exceptions import TimeoutExpired logger = logging.getLogger(__name__) @@ -165,16 +167,15 @@ def run(self): "Appending output to '%s'", self.stdout_path, ) - stdout = self._fd_stack.enter_context(open(self.stdout_path, "ab")) + stdout = self._fd_stack.enter_context(open(self.stdout_path, "ab")) # noqa: SIM115 try: - # pylint: disable=consider-using-with - self._proc = subprocess.Popen( # nosec B603 + self._proc = subprocess.Popen( self.args, stdin=subprocess.DEVNULL, stdout=stdout, stderr=subprocess.STDOUT, close_fds=True, - shell=False, + shell=False, # noqa: S603 env=self.env, ) self._dump() @@ -218,7 +219,6 @@ def spawn(cls, *args, **kwargs) -> Optional[int]: ) proc.start() # Do not terminate the child daemon when the main process exits - # pylint: disable=protected-access mp.process._children.discard(proc) # type: ignore[attr-defined] return proc.pid @@ -231,5 +231,5 @@ def _spawn(cls, *args, **kwargs): class _DaemonProcess(mp.Process): def run(self): if os.name != "nt": - os.setpgid(0, 0) # pylint: disable=no-member + os.setpgid(0, 0) super().run() diff --git a/src/dvc_task/proc/tasks.py b/src/dvc_task/proc/tasks.py index b0f91de..aa1f347 100644 --- a/src/dvc_task/proc/tasks.py +++ b/src/dvc_task/proc/tasks.py @@ -1,4 +1,5 @@ """Celery tasks.""" + from typing import Any, Dict from celery import shared_task @@ -7,9 +8,7 @@ @shared_task(bind=True) -def run( # pylint: disable=unused-argument - self, *args: Any, **kwargs: Any -) -> Dict[str, Any]: +def run(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: """Run a command inside a celery task. Accepts the same arguments as `proc.process.ManagedProcess`. diff --git a/src/dvc_task/utils.py b/src/dvc_task/utils.py index 3667f4e..7a72fbb 100644 --- a/src/dvc_task/utils.py +++ b/src/dvc_task/utils.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) -def _chmod(func: Callable, path: str, excinfo: Any): # pylint: disable=unused-argument +def _chmod(func: Callable, path: str, excinfo: Any): perm = os.lstat(path).st_mode perm |= stat.S_IWRITE diff --git a/src/dvc_task/worker/temporary.py b/src/dvc_task/worker/temporary.py index 28e4e77..500102d 100644 --- a/src/dvc_task/worker/temporary.py +++ b/src/dvc_task/worker/temporary.py @@ -1,4 +1,5 @@ """Temporary worker module.""" + import logging import os import threading @@ -16,7 +17,7 @@ class TemporaryWorker: """Temporary worker that automatically shuts down when queue is empty.""" - def __init__( # pylint: disable=too-many-arguments + def __init__( self, app: Celery, timeout: int = 60, diff --git a/tests/app/test_filesystem.py b/tests/app/test_filesystem.py index d8e0197..73f11f8 100644 --- a/tests/app/test_filesystem.py +++ b/tests/app/test_filesystem.py @@ -1,15 +1,18 @@ """Filesystem app tests.""" + import json -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional import pytest from celery.backends.filesystem import FilesystemBackend from funcy import first -from kombu.message import Message from pytest_test_utils import TmpDir from dvc_task.app.filesystem import FSApp, _get_fs_config +if TYPE_CHECKING: + from kombu.message import Message + TEST_MSG: Dict[str, Any] = { "body": "", "content-encoding": "utf-8", @@ -58,7 +61,7 @@ def test_config(tmp_dir: TmpDir): - """Should return a filesystem broker/resut config.""" + """Should return a filesystem broker/result config.""" config = _get_fs_config(str(tmp_dir), mkdir=True) assert (tmp_dir / "broker" / "control").is_dir() assert (tmp_dir / "broker" / "in").is_dir() @@ -128,7 +131,7 @@ def test_reject(tmp_dir: TmpDir): app.reject(msg.delivery_tag) assert not (tmp_dir / "broker" / "in" / "foo.msg").exists() - with pytest.raises(ValueError): + with pytest.raises(ValueError): # noqa: PT011 app.reject(TEST_MSG["properties"]["delivery_tag"]) @@ -146,7 +149,7 @@ def test_purge(tmp_dir: TmpDir): app.purge(msg.delivery_tag) assert not (tmp_dir / "broker" / "processed" / "foo.msg").exists() - with pytest.raises(ValueError): + with pytest.raises(ValueError): # noqa: PT011 app.purge(TEST_MSG["properties"]["delivery_tag"]) @@ -170,7 +173,7 @@ def test_gc(tmp_dir: TmpDir): } ) - app._gc() # pylint: disable=protected-access + app._gc() assert not (tmp_dir / "broker" / "in" / "expired.msg").exists() assert (tmp_dir / "broker" / "in" / "unexpired.msg").exists() assert (tmp_dir / "broker" / "in" / "ticket.msg").exists() @@ -199,7 +202,7 @@ def test_gc_exclude(tmp_dir: TmpDir): } ) - app._gc(exclude=["celery"]) # pylint: disable=protected-access + app._gc(exclude=["celery"]) assert (tmp_dir / "broker" / "in" / "expired.msg").exists() assert (tmp_dir / "broker" / "in" / "unexpired.msg").exists() assert (tmp_dir / "broker" / "in" / "ticket.msg").exists() diff --git a/tests/proc/conftest.py b/tests/proc/conftest.py index 9ed7ad9..1ca5f22 100644 --- a/tests/proc/conftest.py +++ b/tests/proc/conftest.py @@ -1,4 +1,5 @@ """Process test fixtures.""" + import json import os from typing import Optional @@ -28,7 +29,7 @@ def popen_pid(mocker: MockerFixture) -> int: @pytest.fixture(name="process_manager") def fixture_process_manager(tmp_dir: TmpDir) -> ProcessManager: """Return a process manager which uses tmp_dir as the working dir.""" - return ProcessManager(tmp_dir) + return ProcessManager(os.fspath(tmp_dir)) def create_process( @@ -38,7 +39,7 @@ def create_process( returncode: Optional[int] = None, ): """Create a test process info directory.""" - info_path = manager._get_info_path(name) # pylint: disable=protected-access + info_path = manager._get_info_path(name) os.makedirs(os.path.dirname(info_path)) process_info = ProcessInfo( pid=pid, diff --git a/tests/proc/test_manager.py b/tests/proc/test_manager.py index 3aa1b37..face9c4 100644 --- a/tests/proc/test_manager.py +++ b/tests/proc/test_manager.py @@ -1,4 +1,5 @@ """Process manager tests.""" + import builtins import signal import sys @@ -35,11 +36,12 @@ def test_send_signal( mock_kill.reset_mock() process_manager.send_signal( running_process, - signal.CTRL_C_EVENT, # pylint: disable=no-member + signal.CTRL_C_EVENT, True, ) mock_kill.assert_called_once_with( - PID_RUNNING, signal.CTRL_C_EVENT # pylint: disable=no-member + PID_RUNNING, + signal.CTRL_C_EVENT, ) mock_kill.reset_mock() @@ -64,7 +66,7 @@ def side_effect(*args): err = OSError() err.winerror = 87 raise err - raise ProcessLookupError() + raise ProcessLookupError mocker.patch("os.kill", side_effect=side_effect) with pytest.raises(ProcessLookupError): @@ -77,9 +79,9 @@ def side_effect(*args): if sys.platform == "win32": SIGKILL = signal.SIGTERM - SIGINT = signal.CTRL_C_EVENT # pylint: disable=no-member + SIGINT = signal.CTRL_C_EVENT else: - SIGKILL = signal.SIGKILL # pylint: disable=no-member + SIGKILL = signal.SIGKILL SIGINT = signal.SIGINT @@ -125,7 +127,7 @@ def test_remove( @pytest.mark.parametrize("force", [True, False]) -def test_cleanup( # pylint: disable=too-many-arguments +def test_cleanup( # noqa: PLR0913 mocker: MockerFixture, tmp_dir: TmpDir, process_manager: ProcessManager, diff --git a/tests/proc/test_process.py b/tests/proc/test_process.py index e0dba19..5d37fa3 100644 --- a/tests/proc/test_process.py +++ b/tests/proc/test_process.py @@ -1,4 +1,5 @@ """Process tests.""" + import json import subprocess from typing import List, Union @@ -41,7 +42,6 @@ def test_run(popen_pid: int): def test_wait(mocker: MockerFixture): """Wait should block while process is running and incomplete.""" with ManagedProcess("/bin/foo") as proc: - # pylint: disable=protected-access proc._proc.wait = mocker.Mock( side_effect=subprocess.TimeoutExpired("/bin/foo", 5) ) @@ -50,11 +50,11 @@ def test_wait(mocker: MockerFixture): proc._proc.wait = mocker.Mock(return_value=None) proc._proc.returncode = 0 - assert 0 == proc.wait() + assert proc.wait() == 0 proc._proc.wait.assert_called_once() # once subprocess return code is set, future ManagedProcess.wait() # calls should not block proc._proc.wait.reset_mock() - assert 0 == proc.wait() + assert proc.wait() == 0 proc._proc.wait.assert_not_called() diff --git a/tests/proc/test_tasks.py b/tests/proc/test_tasks.py index 71d5be2..73e8862 100644 --- a/tests/proc/test_tasks.py +++ b/tests/proc/test_tasks.py @@ -1,4 +1,5 @@ """Process task tests.""" + from typing import Any, Dict import pytest diff --git a/tests/test_utils.py b/tests/test_utils.py index e1206b1..bd15495 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ """Utils tests.""" + import os import pytest @@ -9,5 +10,5 @@ @pytest.mark.skipif(os.name != "nt", reason="Windows only") def test_unc_path(): """Windows paths should be converted to UNC paths.""" - assert r"\\?\c:\foo" == unc_path(r"c:\foo") - assert r"\\foo\bar" == unc_path(r"\\foo\bar") + assert unc_path(r"c:\foo") == r"\\?\c:\foo" + assert unc_path(r"\\foo\bar") == r"\\foo\bar" diff --git a/tests/worker/test_temporary.py b/tests/worker/test_temporary.py index 2e35e7b..66dd518 100644 --- a/tests/worker/test_temporary.py +++ b/tests/worker/test_temporary.py @@ -1,4 +1,5 @@ """Temporary Worker tests.""" + import sys import pytest