diff --git a/.github/workflows/cheatsheet.yml b/.github/workflows/cheatsheet.yml index 85adc0b158..0836dae7ac 100644 --- a/.github/workflows/cheatsheet.yml +++ b/.github/workflows/cheatsheet.yml @@ -41,7 +41,7 @@ jobs: python -m pip install --upgrade pip python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.cache.outputs.cache-hit == 'true' run: | diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml index fe3e573c88..351b297017 100644 --- a/.github/workflows/test_deploy.yml +++ b/.github/workflows/test_deploy.yml @@ -71,7 +71,7 @@ jobs: python -m pip install coveralls poetry poetry-dynamic-versioning[plugin] wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs python -c "import pydocstyle;print(pydocstyle.__file__)" echo "${{ env.pythonLocation }}" - name: Install renku into cache @@ -79,7 +79,7 @@ jobs: env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -123,14 +123,14 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | poetry self add "poetry-dynamic-versioning[plugin]" - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -168,13 +168,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -209,13 +209,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Build Package env: POETRY_VIRTUALENVS_CREATE: false @@ -258,13 +258,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -329,13 +329,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -406,13 +406,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -483,13 +483,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -546,7 +546,7 @@ jobs: python -m pip install wheel poetry poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs git config --global --add user.name "Renku Bot" git config --global --add user.email "renku@datascience.ch" - name: Test with pytest @@ -584,7 +584,7 @@ jobs: python -m pip install wheel poetry poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs git config --global --add user.name "Renku Bot" git config --global --add user.email "renku@datascience.ch" - name: Test with pytest @@ -629,7 +629,7 @@ jobs: python -m pip install wheel poetry poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs git config --global --add user.name "Renku Bot" git config --global --add user.email "renku@datascience.ch" - name: Test with pytest @@ -673,7 +673,7 @@ jobs: python -m pip install wheel poetry poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs git config --global --add user.name "Renku Bot" git config --global --add user.email "renku@datascience.ch" - name: Test with pytest @@ -719,13 +719,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -816,13 +816,13 @@ jobs: python -m pip install coveralls poetry wheel twine poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Install renku into cache if: steps.dependency-cache.outputs.cache-hit == 'true' env: POETRY_VIRTUALENVS_CREATE: false run: | - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs - name: Set Git config run: | git config --global --add user.name "Renku Bot" @@ -901,7 +901,7 @@ jobs: python -m pip install wheel poetry poetry self add "poetry-dynamic-versioning[plugin]" make download-templates - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs git config --global --add user.name "Renku Bot" git config --global --add user.email "renku@datascience.ch" brew install rclone osxfuse @@ -952,7 +952,7 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry poetry-lock-package twine poetry self add "poetry-dynamic-versioning[plugin]" - poetry install -E toil -E service --with dev,tests,docs + poetry install -E service --with dev,tests,docs git config --global --add user.name "Renku Bot" git config --global --add user.email "renku@datascience.ch" - name: Tag if necessary diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2168da94da..96f1cbb6e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,6 +41,8 @@ repos: - id: pydocstyle args: - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413 + additional_dependencies: + - toml - repo: https://github.com/koalaman/shellcheck-precommit rev: v0.8.0 hooks: diff --git a/Dockerfile b/Dockerfile index b69a1ca6cf..8d27d83a74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ RUN if [ -n "${BUILD_CORE_SERVICE}" ]; then export EXT_BUILD="-E service" ; fi & /opt/poetry/bin/poetry config virtualenvs.options.no-setuptools true && \ /opt/poetry/bin/poetry config virtualenvs.options.no-pip true && \ /opt/poetry/bin/poetry self add "poetry-dynamic-versioning[plugin]" && \ - /opt/poetry/bin/poetry install ${EXT_BUILD} -E toil + /opt/poetry/bin/poetry install ${EXT_BUILD} FROM base ARG BUILD_CORE_SERVICE diff --git a/pyproject.toml b/pyproject.toml index 6dc293bf5a..7b3813af1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ requests = ">=2.23.0,<2.28.2" rich = ">=9.3.0,<12.6.0" shellingham = "1.5.0" tabulate = ">=0.7.7,<0.8.11" -toil = { version = "==5.7.1", optional = true } +toil = "==5.7.1" tqdm = "<4.62.4,>=4.48.1" werkzeug = ">=1.0.0,<2.2.3" yagup = ">=0.1.1" @@ -175,7 +175,6 @@ sphinx-tabs = "==3.2.0" sphinxcontrib-spelling = "7.*" [tool.poetry.extras] -toil = ["toil"] service = [ "apispec", "apispec-webframeworks", @@ -222,7 +221,7 @@ exclude = ''' add_ignore = ["D105", "D107", "D202", "D401"] [tool.bandit] -skips = ["B101", "B603", "B404"] +skips = ["B101", "B603", "B607", "B404"] [tool.isort] multi_line_output = 3 @@ -281,7 +280,6 @@ show_column_numbers = true show_error_codes = true show_error_context = true warn_unreachable = true -implicit_optional = true check_untyped_defs = true [[tool.mypy.overrides]] diff --git a/renku/command/command_builder/command.py b/renku/command/command_builder/command.py index 9db5a11409..3f7f7a9a38 100644 --- a/renku/command/command_builder/command.py +++ b/renku/command/command_builder/command.py @@ -469,7 +469,7 @@ def with_communicator(self, communicator: CommunicationCallback) -> "Command": return Communicator(self, communicator) @check_finalized - def with_database(self, write: bool = False, path: str = None, create: bool = False) -> "Command": + def with_database(self, write: bool = False, path: Optional[str] = None, create: bool = False) -> "Command": """Provide an object database connection. Args: diff --git a/renku/core/dataset/dataset.py b/renku/core/dataset/dataset.py index 15f078179e..906e3269c3 100644 --- a/renku/core/dataset/dataset.py +++ b/renku/core/dataset/dataset.py @@ -401,7 +401,7 @@ def export_dataset(name: str, provider_name: str, tag: Optional[str], **kwargs): provider_name = provider_name.lower() # TODO: all these callbacks are ugly, improve in #737 - config_key_secret = "access_token" + config_key_secret = "access_token" # nosec dataset: Optional[Dataset] = datasets_provenance.get_by_name(name, strict=True, immutable=True) @@ -615,7 +615,7 @@ def update_datasets( elif (include or exclude) and update_all: raise errors.ParameterError("Cannot specify include and exclude filters when updating all datasets") elif (include or exclude) and names and any(d for d in imported_datasets if d.name in names): - raise errors.IncompatibleParametersError(a="--include/--exclude", b="imported datasets") + raise errors.IncompatibleParametersError(first_param="--include/--exclude", second_param="imported datasets") names = names or [d.name for d in all_datasets] @@ -1166,12 +1166,11 @@ def should_include(filepath: Path) -> bool: return True + creators_set = set() if isinstance(creators, str): creators_set = set(creators.split(",")) elif isinstance(creators, list) or isinstance(creators, tuple): creators_set = set(creators) - else: - creators_set = set() records = [] unused_names = set(names) if names is not None else set() diff --git a/renku/core/dataset/datasets_provenance.py b/renku/core/dataset/datasets_provenance.py index b7ee893dfe..3e657e183c 100644 --- a/renku/core/dataset/datasets_provenance.py +++ b/renku/core/dataset/datasets_provenance.py @@ -63,13 +63,11 @@ def get_by_id(self, id: str, immutable: bool = False) -> Optional["Dataset"]: return None @overload - def get_by_name( - self, name: str, *, immutable: bool = False, strict: Literal[False] = False - ) -> Optional["Dataset"]: # noqa: D102 + def get_by_name(self, name: str, *, immutable: bool = False, strict: Literal[False] = False) -> Optional["Dataset"]: ... @overload - def get_by_name(self, name: str, *, immutable: bool = False, strict: Literal[True]) -> "Dataset": # noqa: D102 + def get_by_name(self, name: str, *, immutable: bool = False, strict: Literal[True]) -> "Dataset": ... def get_by_name( @@ -136,7 +134,7 @@ def add_or_update(self, dataset: "Dataset", date: Optional[datetime] = None, cre self.dataset_gateway.add_or_remove(dataset) - def remove(self, dataset, date: datetime = None, creator: "Person" = None): + def remove(self, dataset, date: Optional[datetime] = None, creator: Optional["Person"] = None): """Remove a dataset.""" from renku.domain_model.dataset import Dataset diff --git a/renku/core/dataset/pointer_file.py b/renku/core/dataset/pointer_file.py index 79c3aea71c..fd4dede380 100644 --- a/renku/core/dataset/pointer_file.py +++ b/renku/core/dataset/pointer_file.py @@ -28,7 +28,7 @@ from renku.infrastructure.repository import Repository -def create_pointer_file(target: Union[str, Path], checksum: str = None): +def create_pointer_file(target: Union[str, Path], checksum: Optional[str] = None): """Create a new pointer file.""" target = Path(target).resolve() @@ -87,7 +87,7 @@ def update_external_file(path: Union[Path, str], checksum: Optional[str]): create_external_file(target=target, path=absolute_path, checksum=checksum) -def create_external_file(target: Path, path: Union[Path, str], checksum: str = None): +def create_external_file(target: Path, path: Union[Path, str], checksum: Optional[str] = None): """Create a new external file.""" try: pointer_file = create_pointer_file(target=target, checksum=checksum) diff --git a/renku/core/dataset/providers/dataverse.py b/renku/core/dataset/providers/dataverse.py index 4afdf15a0a..5f3daacda4 100644 --- a/renku/core/dataset/providers/dataverse.py +++ b/renku/core/dataset/providers/dataverse.py @@ -145,8 +145,8 @@ def get_exporter( dataset: "Dataset", *, tag: Optional["DatasetTag"], - dataverse_server: str = None, - dataverse_name: str = None, + dataverse_server: Optional[str] = None, + dataverse_name: Optional[str] = None, publish: bool = False, **kwargs, ) -> "ExporterApi": diff --git a/renku/core/dataset/providers/olos.py b/renku/core/dataset/providers/olos.py index c7242896f5..439737af20 100644 --- a/renku/core/dataset/providers/olos.py +++ b/renku/core/dataset/providers/olos.py @@ -60,7 +60,7 @@ def get_export_parameters() -> List["ProviderParameter"]: return [ProviderParameter("dlcm-server", help="DLCM server base url.", type=str)] def get_exporter( - self, dataset: "Dataset", *, tag: Optional["DatasetTag"], dlcm_server: str = None, **kwargs + self, dataset: "Dataset", *, tag: Optional["DatasetTag"], dlcm_server: Optional[str] = None, **kwargs ) -> "OLOSExporter": """Create export manager for given dataset.""" diff --git a/renku/core/dataset/providers/web.py b/renku/core/dataset/providers/web.py index 0d151acc55..e63268c60d 100644 --- a/renku/core/dataset/providers/web.py +++ b/renku/core/dataset/providers/web.py @@ -21,7 +21,7 @@ import os import urllib from pathlib import Path -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple from urllib.parse import urlparse from renku.core import errors @@ -55,7 +55,7 @@ def add( destination: Path, *, extract: bool = False, - filename: str = None, + filename: Optional[str] = None, multiple: bool = False, **kwargs, ) -> List["DatasetAddMetadata"]: @@ -109,7 +109,7 @@ def download_file( uri: str, destination: Path, extract: bool = False, - filename: str = None, + filename: Optional[str] = None, multiple: bool = False, delay: float = 0, ) -> List["DatasetAddMetadata"]: diff --git a/renku/core/errors.py b/renku/core/errors.py index 613693abcc..4d33dcdf79 100644 --- a/renku/core/errors.py +++ b/renku/core/errors.py @@ -18,7 +18,7 @@ """Renku exceptions.""" from pathlib import Path -from typing import List, Union +from typing import List, Optional, Union import click from packaging.version import Version @@ -84,9 +84,13 @@ class ParseError(RenkuException): class IncompatibleParametersError(ParameterError): """Raise in case of incompatible parameters/flags.""" - def __init__(self, a: str = None, b: str = None): + def __init__(self, first_param: Optional[str] = None, second_param: Optional[str] = None): """Build a custom message.""" - message = f"{a} is incompatible with {b}" if a is not None and b is not None else "Incompatible parameters" + message = ( + f"{first_param} is incompatible with {second_param}" + if first_param is not None and second_param is not None + else "Incompatible parameters" + ) super().__init__(message) @@ -622,7 +626,7 @@ def __init__(self, reason: str): class GraphCycleError(RenkuException): """Raised when a parameter reference cannot be resolved to a parameter.""" - def __init__(self, cycles: List[List[str]], message: str = None): + def __init__(self, cycles: List[List[str]], message: Optional[str] = None): """Embed exception and build a custom message.""" if message: super().__init__(message) @@ -699,7 +703,7 @@ def __init__(self, current_version: Version, minimum_version: Version) -> None: class DatasetProviderNotFound(DatasetException, ParameterError): """Raised when a dataset provider cannot be found based on a URI or a provider name.""" - def __init__(self, *, name: str = None, uri: str = None, message: str = None): + def __init__(self, *, name: Optional[str] = None, uri: Optional[str] = None, message: Optional[str] = None): if message is None: if name: message = f"Provider '{name}' not found" @@ -725,7 +729,7 @@ class RCloneException(DatasetException): class StorageObjectNotFound(RCloneException): """Raised when a file or directory cannot be found in the remote storage.""" - def __init__(self, error: str = None): + def __init__(self, error: Optional[str] = None): message = "Cannot find file/directory" if error: message = f"{message}: {error}" diff --git a/renku/core/interface/plan_gateway.py b/renku/core/interface/plan_gateway.py index 21c6f9bd5d..687b157185 100644 --- a/renku/core/interface/plan_gateway.py +++ b/renku/core/interface/plan_gateway.py @@ -38,7 +38,7 @@ def get_by_name_or_id(self, name_or_id: str) -> AbstractPlan: """Get a plan by name or id.""" raise NotImplementedError() - def list_by_name(self, starts_with: str, ends_with: str = None) -> List[str]: + def list_by_name(self, starts_with: str, ends_with: Optional[str] = None) -> List[str]: """Search plans by name.""" raise NotImplementedError diff --git a/renku/core/util/os.py b/renku/core/util/os.py index 3264f6abc9..ccc97a7a8b 100644 --- a/renku/core/util/os.py +++ b/renku/core/util/os.py @@ -38,7 +38,9 @@ def get_relative_path_to_cwd(path: Union[Path, str]) -> str: return os.path.relpath(absolute_path, os.getcwd()) -def get_absolute_path(path: Union[Path, str], base: Union[Path, str] = None, resolve_symlinks: bool = False) -> str: +def get_absolute_path( + path: Union[Path, str], base: Optional[Union[Path, str]] = None, resolve_symlinks: bool = False +) -> str: """Return absolute normalized path.""" if base is not None: base = Path(base).resolve() if resolve_symlinks else os.path.abspath(base) @@ -251,7 +253,7 @@ def hash_file_descriptor(file: BinaryIO, hash_type: str = "sha256") -> str: hash_type = hash_type.lower() assert hash_type in ("sha256", "md5") - hash_value = hashlib.sha256() if hash_type == "sha256" else hashlib.md5() + hash_value = hashlib.sha256() if hash_type == "sha256" else hashlib.md5() # nosec for byte_block in iter(lambda: file.read(BLOCK_SIZE), b""): hash_value.update(byte_block) diff --git a/renku/core/workflow/model/workflow_file.py b/renku/core/workflow/model/workflow_file.py index 70079008ec..05a6a8470d 100644 --- a/renku/core/workflow/model/workflow_file.py +++ b/renku/core/workflow/model/workflow_file.py @@ -168,12 +168,12 @@ def __init__( command: str, date_created: Optional[datetime] = None, description: Optional[str] = None, - inputs: List[Input] = None, + inputs: Optional[List[Input]] = None, keywords: Optional[List[str]] = None, name: str, original_command: Optional[str] = None, - outputs: List[Output] = None, - parameters: List[Parameter] = None, + outputs: Optional[List[Output]] = None, + parameters: Optional[List[Parameter]] = None, path: Union[Path, str], success_codes: Optional[List[int]] = None, workflow_file_name: str, diff --git a/renku/core/workflow/plan_factory.py b/renku/core/workflow/plan_factory.py index df5cf8a72b..a4c8627fbd 100644 --- a/renku/core/workflow/plan_factory.py +++ b/renku/core/workflow/plan_factory.py @@ -148,7 +148,7 @@ def split_command_and_args(self): return cmd, args @staticmethod - def _is_ignored_path(candidate: Union[Path, str], ignored_list: Set[str] = None) -> bool: + def _is_ignored_path(candidate: Union[Path, str], ignored_list: Optional[Set[str]] = None) -> bool: """Return True if the path is in ignored list.""" return ignored_list is not None and str(candidate) in ignored_list @@ -358,7 +358,7 @@ def _get_mimetype(file: Path) -> List[str]: # TODO: specify the actual mime-type of the file return ["application/octet-stream"] - def guess_type(self, value: Union[Path, str], ignore_filenames: Set[str] = None) -> Tuple[Any, str]: + def guess_type(self, value: Union[Path, str], ignore_filenames: Optional[Set[str]] = None) -> Tuple[Any, str]: """Return new value and CWL parameter type.""" if not self._is_ignored_path(value, ignore_filenames) and all(value != v for v, _ in self.explicit_parameters): candidate = self._resolve_existing_subpath(value) @@ -417,9 +417,9 @@ def add_command_output( prefix: Optional[str] = None, position: Optional[int] = None, postfix: Optional[str] = None, - encoding_format: List[str] = None, + encoding_format: Optional[List[str]] = None, name: Optional[str] = None, - id: str = None, + id: Optional[str] = None, mapped_to: Optional[MappedIOStream] = None, ): """Create a CommandOutput.""" diff --git a/renku/domain_model/dataset.py b/renku/domain_model/dataset.py index 345399bede..d9ab01008d 100644 --- a/renku/domain_model/dataset.py +++ b/renku/domain_model/dataset.py @@ -49,7 +49,7 @@ def is_dataset_name_valid(name: str) -> bool: return name is not None and name == get_slug(name, lowercase=False) -def generate_default_name(title: str, version: str = None) -> str: +def generate_default_name(title: str, version: Optional[str] = None) -> str: """Get dataset name.""" max_length = 24 # For compatibility with older versions use title as name if it is valid; otherwise, use encoded title @@ -202,7 +202,7 @@ class RemoteEntity(Slots): __slots__ = ("checksum", "id", "path", "url") - def __init__(self, *, checksum: str, id: str = None, path: Union[Path, str], url: str): + def __init__(self, *, checksum: str, id: Optional[str] = None, path: Union[Path, str], url: str): super().__init__() self.checksum: str = checksum self.id: str = id or RemoteEntity.generate_id(checksum=checksum, path=path, url=url) @@ -309,7 +309,7 @@ def is_equal_to(self, other: "DatasetFile"): and self.source == other.source ) - def remove(self, date: datetime = None): + def remove(self, date: Optional[datetime] = None): """Create a new instance and mark it as removed.""" date_removed = fix_datetime(date) or local_now() self.date_removed = date_removed @@ -465,7 +465,7 @@ def copy(self) -> "Dataset": dataset.keywords = list(dataset.keywords or []) return dataset - def replace_identifier(self, identifier: str = None): + def replace_identifier(self, identifier: Optional[str] = None): """Replace dataset's identifier and update relevant fields. NOTE: Call this only for newly-created/-imported datasets that don't have a mutability chain because it sets @@ -480,7 +480,11 @@ def replace_identifier(self, identifier: str = None): # NOTE: Do not unset `same_as` because it can be set for imported datasets def derive_from( - self, dataset: "Dataset", creator: Optional["Person"], identifier: str = None, date_created: datetime = None + self, + dataset: "Dataset", + creator: Optional["Person"], + identifier: Optional[str] = None, + date_created: Optional[datetime] = None, ): """Make `self` a derivative of `dataset` and update related fields.""" assert dataset is not None, "Cannot derive from None" @@ -507,7 +511,7 @@ def _assign_new_identifier(self, identifier: Optional[str]): # NOTE: We also need to re-assign the _p_oid since identifier has changed self.reassign_oid() - def remove(self, date: datetime = None): + def remove(self, date: Optional[datetime] = None): """Mark the dataset as removed.""" self.date_removed = fix_datetime(date) or local_now() diff --git a/renku/domain_model/provenance/activity.py b/renku/domain_model/provenance/activity.py index e635f53085..4f6a74fcff 100644 --- a/renku/domain_model/provenance/activity.py +++ b/renku/domain_model/provenance/activity.py @@ -144,7 +144,7 @@ def from_plan( project_gateway: IProjectGateway, started_at_time: datetime, ended_at_time: datetime, - annotations: List[Annotation] = None, + annotations: Optional[List[Annotation]] = None, id: Optional[str] = None, ): """Convert a ``Plan`` to a ``Activity``.""" @@ -284,7 +284,7 @@ def delete(self, when: datetime = local_now()): class ActivityCollection(Persistent): """Represent a list of activities.""" - def __init__(self, *, activities: List[Activity], id: str = None): + def __init__(self, *, activities: List[Activity], id: Optional[str] = None): self.activities: List[Activity] = activities or [] self.id: str = id or self.generate_id() @@ -304,7 +304,7 @@ def __init__( agents: List[Union[Person, SoftwareAgent]], association: Association, ended_at_time: datetime, - id: str = None, + id: Optional[str] = None, invalidated_at: Optional[datetime] = None, project_id: Optional[str] = None, started_at_time: datetime, diff --git a/renku/domain_model/workflow/parameter.py b/renku/domain_model/workflow/parameter.py index d67b39841f..bce882a892 100644 --- a/renku/domain_model/workflow/parameter.py +++ b/renku/domain_model/workflow/parameter.py @@ -44,7 +44,7 @@ class MappedIOStream: STREAMS = ["stdin", "stdout", "stderr"] - def __init__(self, *, id: str = None, stream_type: str): + def __init__(self, *, id: Optional[str] = None, stream_type: str): assert stream_type in MappedIOStream.STREAMS self.id: str = id or MappedIOStream.generate_id(stream_type) @@ -199,14 +199,14 @@ def __init__( self, *, default_value: Any = None, - description: str = None, + description: Optional[str] = None, id: str, - name: str = None, + name: Optional[str] = None, name_set_by_user: bool = False, position: Optional[int] = None, - prefix: str = None, - derived_from: str = None, - postfix: str = None, + prefix: Optional[str] = None, + derived_from: Optional[str] = None, + postfix: Optional[str] = None, ): super().__init__( default_value=default_value, @@ -365,7 +365,7 @@ def __init__( @staticmethod def generate_id( - plan_id: str, position: Optional[int] = None, postfix: str = None, name: Optional[str] = None + plan_id: str, position: Optional[int] = None, postfix: Optional[str] = None, name: Optional[str] = None ) -> str: """Generate an id for CommandOutput.""" return CommandParameterBase._generate_id( diff --git a/renku/domain_model/workflow/plan.py b/renku/domain_model/workflow/plan.py index c7d2dad59c..e6f46f07cd 100644 --- a/renku/domain_model/workflow/plan.py +++ b/renku/domain_model/workflow/plan.py @@ -182,7 +182,7 @@ def __init__( date_modified: Optional[datetime] = None, derived_from: Optional[str] = None, description: Optional[str] = None, - hidden_inputs: List[HiddenInput] = None, + hidden_inputs: Optional[List[HiddenInput]] = None, id: str, inputs: Optional[List[CommandInput]] = None, date_removed: Optional[datetime] = None, diff --git a/renku/domain_model/workflow/workflow_file.py b/renku/domain_model/workflow/workflow_file.py index 9b028f7a49..5cd5641ac2 100644 --- a/renku/domain_model/workflow/workflow_file.py +++ b/renku/domain_model/workflow/workflow_file.py @@ -39,7 +39,7 @@ def __init__(self, *, path: Union[Path, str], **kwargs): self.path: str = str(path) @staticmethod - def generate_id(path: Union[Path, str] = None, sequence: Optional[int] = None, **_) -> str: + def generate_id(path: Optional[Union[Path, str]] = None, sequence: Optional[int] = None, **_) -> str: """Generate an identifier for Plan.""" assert path, "Path is needed to generate id for WorkflowFileCompositePlan" @@ -47,7 +47,7 @@ def generate_id(path: Union[Path, str] = None, sequence: Optional[int] = None, * # changed later if the plan is a derivative key = f"{path}" if sequence is None else f"{path}::{sequence}" key_bytes = key.encode("utf-8") - return CompositePlan.generate_id(uuid=hashlib.md5(key_bytes).hexdigest()[:32]) + return CompositePlan.generate_id(uuid=hashlib.md5(key_bytes).hexdigest()[:32]) # nosec def assign_new_id(self, *, sequence: Optional[int] = None, **_) -> str: """Assign a new UUID or a deterministic.""" @@ -68,14 +68,16 @@ def __init__(self, *, path: Union[Path, str], **kwargs): self.path: str = str(path) @staticmethod - def generate_id(path: Union[Path, str] = None, name: str = None, sequence: Optional[int] = None, **_) -> str: + def generate_id( + path: Optional[Union[Path, str]] = None, name: Optional[str] = None, sequence: Optional[int] = None, **_ + ) -> str: """Generate an identifier for Plan.""" assert path, "Path is needed to generate id for WorkflowFilePlan" assert name, "Name is needed to generate id for WorkflowFilePlan" key = f"{path}::{name}" if sequence is None else f"{path}::{name}::{sequence}" key_bytes = key.encode("utf-8") - return Plan.generate_id(uuid=hashlib.md5(key_bytes).hexdigest()[:32]) + return Plan.generate_id(uuid=hashlib.md5(key_bytes).hexdigest()[:32]) # nosec @staticmethod def validate_name(name: str): diff --git a/renku/infrastructure/database.py b/renku/infrastructure/database.py index de125611ff..a03fd4a53d 100644 --- a/renku/infrastructure/database.py +++ b/renku/infrastructure/database.py @@ -251,7 +251,9 @@ def _initialize_root(self): self._root._p_oid = Database.ROOT_OID self.register(self._root) - def add_index(self, name: str, object_type: type, attribute: str = None, key_type: type = None) -> "Index": + def add_index( + self, name: str, object_type: type, attribute: Optional[str] = None, key_type: Optional[type] = None + ) -> "Index": """Add an index. Args: @@ -286,7 +288,7 @@ def add_root_object(self, name: str, obj: Persistent): self._root[name] = obj - def add(self, object: persistent.Persistent, oid: OID_TYPE = None): + def add(self, object: persistent.Persistent, oid: OID_TYPE): """Add a new object to the database. NOTE: Normally, we add objects to indexes but this method adds objects directly to Dataset's root. Use it only diff --git a/renku/infrastructure/gateway/plan_gateway.py b/renku/infrastructure/gateway/plan_gateway.py index e9fd6c352a..64347c248f 100644 --- a/renku/infrastructure/gateway/plan_gateway.py +++ b/renku/infrastructure/gateway/plan_gateway.py @@ -44,7 +44,7 @@ def get_by_name_or_id(self, name_or_id: str) -> AbstractPlan: raise errors.WorkflowNotFoundError(name_or_id) return workflow - def list_by_name(self, starts_with: str, ends_with: str = None) -> List[str]: + def list_by_name(self, starts_with: str, ends_with: Optional[str] = None) -> List[str]: """Search plans by name.""" return [ name diff --git a/renku/infrastructure/repository.py b/renku/infrastructure/repository.py index 7f76d9fb86..8de51970bc 100644 --- a/renku/infrastructure/repository.py +++ b/renku/infrastructure/repository.py @@ -242,8 +242,8 @@ def commit( message: str, *, amend: bool = False, - author: "Actor" = None, - committer: "Actor" = None, + author: Optional["Actor"] = None, + committer: Optional["Actor"] = None, no_verify: bool = False, no_edit: bool = False, paths: Optional[List[Union[Path, str]]] = None, @@ -280,12 +280,12 @@ def clean(self, paths: Optional[Sequence[Union[Path, str]]] = None): def fetch( self, - remote: Union["Remote", str] = None, - refspec: Union["Branch", str] = None, + remote: Optional[Union["Remote", str]] = None, + refspec: Optional[Union["Branch", str]] = None, all: bool = False, tags: bool = False, unshallow: bool = False, - depth: int = None, + depth: Optional[int] = None, ): """Update a remote branches.""" if all: @@ -308,14 +308,14 @@ def move(self, *sources: Union[Path, str], destination: Union[Path, str], force: """Move source files to the destination.""" self.run_git_command("mv", *sources, destination, force=force) - def pull(self, remote: Union["Remote", str] = None, refspec: Union["Branch", str] = None): + def pull(self, remote: Optional[Union["Remote", str]] = None, refspec: Optional[Union["Branch", str]] = None): """Update changes from remotes.""" self.run_git_command("pull", _to_string(remote), _to_string(refspec)) def push( self, - remote: Union["Remote", str] = None, - refspec: Union["Branch", str] = None, + remote: Optional[Union["Remote", str]] = None, + refspec: Optional[Union["Branch", str]] = None, *, no_verify: bool = False, set_upstream: bool = False, @@ -344,7 +344,7 @@ def remove( """Remove paths from repository or index.""" self.run_git_command("rm", "--", *paths, cached=index, ignore_unmatch=not_exists_ok, r=recursive, force=force) - def reset(self, reference: Union["Branch", "Commit", "Reference", str] = None, hard: bool = False): + def reset(self, reference: Optional[Union["Branch", "Commit", "Reference", str]] = None, hard: bool = False): """Reset a git repository to a given reference.""" self.run_git_command("reset", _to_string(reference), hard=hard) @@ -365,6 +365,7 @@ def create_worktree( Args: path(Path): Target folder. reference(Union[Branch, Commit, Reference, str]): the reference to base the tree on. + branch(str, optional): Optional new branch to create in the worktree. checkout(bool, optional): Whether to perform a checkout of the reference (Default value = False). detach(bool, optional): Whether to detach HEAD in worktree (Default value = False). """ @@ -862,7 +863,7 @@ def _get_user_from_configuration(configuration: "Configuration") -> "Actor": return Actor(name=name, email=email) - def get_configuration(self, writable=False, scope: str = None) -> "Configuration": + def get_configuration(self, writable=False, scope: Optional[str] = None) -> "Configuration": """Return git configuration. NOTE: Scope can be "global" or "local". @@ -875,7 +876,7 @@ def get_global_configuration(writable: bool = False) -> "Configuration": return Configuration(repository=None, writable=writable) def get_existing_paths_in_revision( - self, paths: Union[List[Union[Path, str]], Set[Union[Path, str]]] = None, revision: str = "HEAD" + self, paths: Union[List[Union[Path, str]], Optional[Set[Union[Path, str]]]] = None, revision: str = "HEAD" ) -> List[str]: """List all paths that exist in a revision.""" @@ -950,14 +951,17 @@ def hash_string(content: str) -> str: """Calculate the object-hash for a blob with specified content.""" content_bytes = content.encode("utf-8") data = f"blob {len(content_bytes)}\0".encode("utf-8") + content_bytes - return hashlib.sha1(data).hexdigest() + return hashlib.sha1(data).hexdigest() # nosec class Repository(BaseRepository): """Abstract Base repository.""" def __init__( - self, path: Union[Path, str] = ".", search_parent_directories: bool = False, repository: git.Repo = None + self, + path: Union[Path, str] = ".", + search_parent_directories: bool = False, + repository: Optional[git.Repo] = None, ): repo = repository or _create_repository(path, search_parent_directories) @@ -1002,7 +1006,7 @@ def clone_from( return cls(path=path, repository=repository) @classmethod - def initialize(cls, path: Union[Path, str], *, bare: bool = False, branch: str = None) -> "Repository": + def initialize(cls, path: Union[Path, str], *, bare: bool = False, branch: Optional[str] = None) -> "Repository": """Initialize a git repository.""" try: Path(path).mkdir(parents=True, exist_ok=True) @@ -1340,7 +1344,9 @@ def tree(self) -> Dict[str, Object]: return {o.path: Object.from_object(o) for o in self._commit.tree.traverse()} def get_changes( - self, paths: Union[Path, str, List[Union[Path, str]], None] = None, commit: Union[str, "Commit"] = None + self, + paths: Union[Path, str, List[Union[Path, str]], None] = None, + commit: Optional[Union[str, "Commit"]] = None, ) -> List[Diff]: """Return list of changes in a commit. @@ -1706,7 +1712,7 @@ def remove(self, tag: Union[Tag, str]): class Configuration: """Git configuration manager.""" - def __init__(self, repository: git.Repo = None, scope: str = None, writable: bool = True): + def __init__(self, repository: Optional[git.Repo] = None, scope: Optional[str] = None, writable: bool = True): assert scope is None or scope in ("global", "local"), f"Invalid scope: '{scope}'" self._read_only = not writable diff --git a/renku/ui/api/models/activity.py b/renku/ui/api/models/activity.py index 47863fd11f..51d2160672 100644 --- a/renku/ui/api/models/activity.py +++ b/renku/ui/api/models/activity.py @@ -137,9 +137,9 @@ def list() -> List["Activity"]: @staticmethod def filter( *, - inputs: Union[Path, str, Iterable[Union[Path, str]], Callable[[str], bool]] = None, - outputs: Union[Path, str, Iterable[Union[Path, str]], Callable[[str], bool]] = None, - parameters: Union[str, Iterable[str], Callable[[str], bool]] = None, + inputs: Optional[Union[Path, str, Iterable[Union[Path, str]], Callable[[str], bool]]] = None, + outputs: Optional[Union[Path, str, Iterable[Union[Path, str]], Callable[[str], bool]]] = None, + parameters: Optional[Union[str, Iterable[str], Callable[[str], bool]]] = None, values: Union[Any, Iterable[Any], Callable[[Any], bool]] = None, ) -> List["Activity"]: """Return a filtered list of activities in a project. @@ -240,8 +240,8 @@ def _filter_by_path( @staticmethod def filter_by_parameter( - name: Union[str, Iterable[str], Callable[[str], bool]] = None, - value: Union[Any, Iterable[Any], Callable[[Any], bool]] = None, + name: Optional[Union[str, Iterable[str], Callable[[str], bool]]] = None, + value: Optional[Union[Any, Iterable[Any], Callable[[Any], bool]]] = None, ) -> List["Activity"]: """Return a filtered list of activities based on parameters and their values. @@ -260,8 +260,8 @@ def filter_by_parameter( @staticmethod def _filter_by_parameter( - name: Union[str, Iterable[str], Callable[[str], bool]] = None, - value: Union[Any, Iterable[Any], Callable[[Any], bool]] = None, + name: Optional[Union[str, Iterable[str], Callable[[str], bool]]] = None, + value: Optional[Union[Any, Iterable[Any], Callable[[Any], bool]]] = None, activity_gateway: Optional["ActivityGateway"] = None, ) -> Set[core_activity.Activity]: if activity_gateway is None: @@ -395,7 +395,7 @@ def __repr__(self): return f"" -def get_activities(plan_id: str = None) -> List[Activity]: +def get_activities(plan_id: Optional[str] = None) -> List[Activity]: """Return list of activities that use a plan or one of its predecessor plans.""" activity_gateway = get_activity_gateway() plan_gateway = get_plan_gateway() diff --git a/renku/ui/api/models/parameter.py b/renku/ui/api/models/parameter.py index 990463bd54..954ade7d97 100644 --- a/renku/ui/api/models/parameter.py +++ b/renku/ui/api/models/parameter.py @@ -251,9 +251,9 @@ def __init__( self, name: str, value: Union[Path, str, bool, int, float], - default_value: Union[Path, str, bool, int, float] = None, + default_value: Optional[Union[Path, str, bool, int, float]] = None, description: Optional[str] = None, - parameters: List[Parameter] = None, + parameters: Optional[List[Parameter]] = None, ): super().__init__(name=name, value=value) self.default_value: Union[Path, str, bool, int, float] = default_value if default_value is not None else value diff --git a/renku/ui/api/models/plan.py b/renku/ui/api/models/plan.py index 0673f499d6..52bfe6e326 100644 --- a/renku/ui/api/models/plan.py +++ b/renku/ui/api/models/plan.py @@ -52,11 +52,11 @@ def __init__( deleted: bool = False, description: Optional[str] = None, id: str, - inputs: List[Input] = None, + inputs: Optional[List[Input]] = None, keywords: Optional[List[str]] = None, name: Optional[str] = None, - outputs: List[Output] = None, - parameters: List[Parameter] = None, + outputs: Optional[List[Output]] = None, + parameters: Optional[List[Parameter]] = None, success_codes: Optional[List[int]] = None, ): self.command: str = command @@ -133,10 +133,10 @@ def __init__( description: Optional[str] = None, id: str, keywords: Optional[List[str]] = None, - links: List[Link] = None, - mappings: List[Mapping] = None, + links: Optional[List[Link]] = None, + mappings: Optional[List[Mapping]] = None, name: Optional[str] = None, - plans: List[Union["CompositePlan", "Plan"]] = None, + plans: Optional[List[Union["CompositePlan", "Plan"]]] = None, ): self.date_created: Optional[datetime] = date_created self.deleted: bool = deleted diff --git a/renku/ui/service/views/api_versions.py b/renku/ui/service/views/api_versions.py index 69829c7987..2959907fcb 100644 --- a/renku/ui/service/views/api_versions.py +++ b/renku/ui/service/views/api_versions.py @@ -38,7 +38,7 @@ def add_url_rule( endpoint: Optional[str] = None, view_func: Optional[Callable] = None, provide_automatic_options: Optional[bool] = None, - versions: List[ApiVersion] = None, + versions: Optional[List[ApiVersion]] = None, **options: Any, ) -> None: """Overwrite Blueprint add_url_rule to support versioning."""