chore: remove implicit_optional=true from mypy (#3272)

.github/workflows/cheatsheet.yml

@@ -41,7 +41,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.cache.outputs.cache-hit == 'true'
         run: |

.github/workflows/test_deploy.yml

@@ -71,15 +71,15 @@ jobs:
           python -m pip install coveralls poetry poetry-dynamic-versioning[plugin] wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
           python -c "import pydocstyle;print(pydocstyle.__file__)"
           echo "${{ env.pythonLocation }}"
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -123,14 +123,14 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
           poetry self add "poetry-dynamic-versioning[plugin]"
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -168,13 +168,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -209,13 +209,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Build Package
         env:
           POETRY_VIRTUALENVS_CREATE: false
@@ -258,13 +258,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -329,13 +329,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -406,13 +406,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -483,13 +483,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -546,7 +546,7 @@ jobs:
           python -m pip install wheel poetry
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
           git config --global --add user.name "Renku Bot"
           git config --global --add user.email "renku@datascience.ch"
       - name: Test with pytest
@@ -584,7 +584,7 @@ jobs:
           python -m pip install wheel poetry
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
           git config --global --add user.name "Renku Bot"
           git config --global --add user.email "renku@datascience.ch"
       - name: Test with pytest
@@ -629,7 +629,7 @@ jobs:
           python -m pip install wheel poetry
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
           git config --global --add user.name "Renku Bot"
           git config --global --add user.email "renku@datascience.ch"
       - name: Test with pytest
@@ -673,7 +673,7 @@ jobs:
           python -m pip install wheel poetry
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
           git config --global --add user.name "Renku Bot"
           git config --global --add user.email "renku@datascience.ch"
       - name: Test with pytest
@@ -719,13 +719,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -816,13 +816,13 @@ jobs:
           python -m pip install coveralls poetry wheel twine
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Install renku into cache
         if: steps.dependency-cache.outputs.cache-hit == 'true'
         env:
           POETRY_VIRTUALENVS_CREATE: false
         run: |
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
       - name: Set Git config
         run: |
           git config --global --add user.name "Renku Bot"
@@ -901,7 +901,7 @@ jobs:
           python -m pip install wheel poetry
           poetry self add "poetry-dynamic-versioning[plugin]"
           make download-templates
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
           git config --global --add user.name "Renku Bot"
           git config --global --add user.email "renku@datascience.ch"
           brew install rclone osxfuse
@@ -952,7 +952,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install poetry poetry-lock-package twine
           poetry self add "poetry-dynamic-versioning[plugin]"
-          poetry install -E toil -E service --with dev,tests,docs
+          poetry install -E service --with dev,tests,docs
           git config --global --add user.name "Renku Bot"
           git config --global --add user.email "renku@datascience.ch"
       - name: Tag if necessary

.pre-commit-config.yaml

@@ -41,6 +41,8 @@ repos:
       - id: pydocstyle
         args:
           - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
+        additional_dependencies:
+          - toml
   - repo: https://github.com/koalaman/shellcheck-precommit
     rev: v0.8.0
     hooks:

Dockerfile

@@ -36,7 +36,7 @@ RUN if [ -n "${BUILD_CORE_SERVICE}" ]; then export EXT_BUILD="-E service" ; fi &
     /opt/poetry/bin/poetry config virtualenvs.options.no-setuptools true && \
     /opt/poetry/bin/poetry config virtualenvs.options.no-pip true  && \
     /opt/poetry/bin/poetry self add "poetry-dynamic-versioning[plugin]" && \
-    /opt/poetry/bin/poetry install ${EXT_BUILD} -E toil
+    /opt/poetry/bin/poetry install ${EXT_BUILD}
 
 FROM base
 ARG BUILD_CORE_SERVICE

pyproject.toml

@@ -95,7 +95,7 @@ requests = ">=2.23.0,<2.28.2"
 rich = ">=9.3.0,<12.6.0"
 shellingham = "1.5.0"
 tabulate = ">=0.7.7,<0.8.11"
-toil = { version = "==5.7.1", optional = true }
+toil = "==5.7.1"
 tqdm = "<4.62.4,>=4.48.1"
 werkzeug = ">=1.0.0,<2.2.3"
 yagup = ">=0.1.1"
@@ -175,7 +175,6 @@ sphinx-tabs = "==3.2.0"
 sphinxcontrib-spelling = "7.*"
 
 [tool.poetry.extras]
-toil = ["toil"]
 service = [
     "apispec",
     "apispec-webframeworks",
@@ -222,7 +221,7 @@ exclude = '''
 add_ignore = ["D105", "D107", "D202", "D401"]
 
 [tool.bandit]
-skips = ["B101", "B603", "B404"]
+skips = ["B101", "B603", "B607", "B404"]
 
 [tool.isort]
 multi_line_output = 3
@@ -281,7 +280,6 @@ show_column_numbers = true
 show_error_codes = true
 show_error_context = true
 warn_unreachable = true
-implicit_optional = true
 check_untyped_defs = true
 
 [[tool.mypy.overrides]]

renku/command/command_builder/command.py

@@ -469,7 +469,7 @@ def with_communicator(self, communicator: CommunicationCallback) -> "Command":
         return Communicator(self, communicator)
 
     @check_finalized
-    def with_database(self, write: bool = False, path: str = None, create: bool = False) -> "Command":
+    def with_database(self, write: bool = False, path: Optional[str] = None, create: bool = False) -> "Command":
         """Provide an object database connection.
 
         Args:

renku/core/dataset/dataset.py

@@ -401,7 +401,7 @@ def export_dataset(name: str, provider_name: str, tag: Optional[str], **kwargs):
     provider_name = provider_name.lower()
 
     # TODO: all these callbacks are ugly, improve in #737
-    config_key_secret = "access_token"
+    config_key_secret = "access_token"  # nosec
 
     dataset: Optional[Dataset] = datasets_provenance.get_by_name(name, strict=True, immutable=True)
 
@@ -615,7 +615,7 @@ def update_datasets(
     elif (include or exclude) and update_all:
         raise errors.ParameterError("Cannot specify include and exclude filters when updating all datasets")
     elif (include or exclude) and names and any(d for d in imported_datasets if d.name in names):
-        raise errors.IncompatibleParametersError(a="--include/--exclude", b="imported datasets")
+        raise errors.IncompatibleParametersError(first_param="--include/--exclude", second_param="imported datasets")
 
     names = names or [d.name for d in all_datasets]
 
@@ -1166,12 +1166,11 @@ def should_include(filepath: Path) -> bool:
 
         return True
 
+    creators_set = set()
     if isinstance(creators, str):
         creators_set = set(creators.split(","))
     elif isinstance(creators, list) or isinstance(creators, tuple):
         creators_set = set(creators)
-    else:
-        creators_set = set()
 
     records = []
     unused_names = set(names) if names is not None else set()

renku/core/dataset/datasets_provenance.py

@@ -63,13 +63,11 @@ def get_by_id(self, id: str, immutable: bool = False) -> Optional["Dataset"]:
         return None
 
     @overload
-    def get_by_name(
-        self, name: str, *, immutable: bool = False, strict: Literal[False] = False
-    ) -> Optional["Dataset"]:  # noqa: D102
+    def get_by_name(self, name: str, *, immutable: bool = False, strict: Literal[False] = False) -> Optional["Dataset"]:
         ...
 
     @overload
-    def get_by_name(self, name: str, *, immutable: bool = False, strict: Literal[True]) -> "Dataset":  # noqa: D102
+    def get_by_name(self, name: str, *, immutable: bool = False, strict: Literal[True]) -> "Dataset":
         ...
 
     def get_by_name(
@@ -136,7 +134,7 @@ def add_or_update(self, dataset: "Dataset", date: Optional[datetime] = None, cre
 
         self.dataset_gateway.add_or_remove(dataset)
 
-    def remove(self, dataset, date: datetime = None, creator: "Person" = None):
+    def remove(self, dataset, date: Optional[datetime] = None, creator: Optional["Person"] = None):
         """Remove a dataset."""
         from renku.domain_model.dataset import Dataset
 

renku/core/dataset/pointer_file.py

@@ -28,7 +28,7 @@
 from renku.infrastructure.repository import Repository
 
 
-def create_pointer_file(target: Union[str, Path], checksum: str = None):
+def create_pointer_file(target: Union[str, Path], checksum: Optional[str] = None):
     """Create a new pointer file."""
     target = Path(target).resolve()
 
@@ -87,7 +87,7 @@ def update_external_file(path: Union[Path, str], checksum: Optional[str]):
     create_external_file(target=target, path=absolute_path, checksum=checksum)
 
 
-def create_external_file(target: Path, path: Union[Path, str], checksum: str = None):
+def create_external_file(target: Path, path: Union[Path, str], checksum: Optional[str] = None):
     """Create a new external file."""
     try:
         pointer_file = create_pointer_file(target=target, checksum=checksum)

renku/core/dataset/providers/dataverse.py

@@ -145,8 +145,8 @@ def get_exporter(
         dataset: "Dataset",
         *,
         tag: Optional["DatasetTag"],
-        dataverse_server: str = None,
-        dataverse_name: str = None,
+        dataverse_server: Optional[str] = None,
+        dataverse_name: Optional[str] = None,
         publish: bool = False,
         **kwargs,
     ) -> "ExporterApi":

renku/core/dataset/providers/olos.py

@@ -60,7 +60,7 @@ def get_export_parameters() -> List["ProviderParameter"]:
         return [ProviderParameter("dlcm-server", help="DLCM server base url.", type=str)]
 
     def get_exporter(
-        self, dataset: "Dataset", *, tag: Optional["DatasetTag"], dlcm_server: str = None, **kwargs
+        self, dataset: "Dataset", *, tag: Optional["DatasetTag"], dlcm_server: Optional[str] = None, **kwargs
     ) -> "OLOSExporter":
         """Create export manager for given dataset."""