squashme: address comments

renku/core/dataset/providers/dataverse.py

@@ -74,7 +74,7 @@
 ]
 
 
-class DataverseProvider(ProviderApi):
+class DataverseProvider(ProviderApi, IDatasetProviderPlugin):
     """Dataverse API provider."""
 
     priority = ProviderPriority.HIGH
@@ -185,6 +185,12 @@ def set_export_parameters(client_dispatcher: IClientDispatcher):
         set_export_parameters()
         return DataverseExporter(dataset=dataset, server_url=self._server_url, dataverse_name=self._dataverse_name)
 
+    @classmethod
+    @hookimpl
+    def dataset_provider(cls) -> "Type[DataverseProvider]":
+        """The defintion of the provider."""
+        return cls
+
 
 class DataverseImporter(RepositoryImporter):
     """Dataverse record serializer."""
@@ -582,12 +588,3 @@ def make_file_url(file_id, base_url):
     args_dict = {"persistentId": file_id}
     url_parts[4] = urllib.parse.urlencode(args_dict)
     return urllib.parse.urlunparse(url_parts)
-
-
-class DataverseProviderPlugin(IDatasetProviderPlugin):
-    """Dataverse provider plugin."""
-
-    @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
-        """The defintion of the provider."""
-        return DataverseProvider

renku/core/dataset/providers/doi.py

@@ -30,7 +30,7 @@
 DOI_BASE_URL = "https://dx.doi.org"
 
 
-class DOIProvider(ProviderApi):
+class DOIProvider(ProviderApi, IDatasetProviderPlugin):
     """`doi.org <http://doi.org>`_ registry API provider."""
 
     priority = ProviderPriority.HIGHER
@@ -73,6 +73,12 @@ def serialize(response):
         query_response = query(uri)
         return serialize(query_response)
 
+    @classmethod
+    @hookimpl
+    def dataset_provider(cls) -> "Type[DOIProvider]":
+        """The defintion of the provider."""
+        return cls
+
 
 class DOIImporter(ImporterApi):
     """Response from `doi.org <http://doi.org>`_ for DOI metadata."""
@@ -151,12 +157,3 @@ def make_doi_url(doi):
         parsed_url = parsed_url._replace(scheme="")
         doi = parsed_url.geturl()
     return urllib.parse.urljoin(DOI_BASE_URL, doi)
-
-
-class DOIProviderPlugin(IDatasetProviderPlugin):
-    """DOI provider plugin."""
-
-    @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
-        """The defintion of the provider."""
-        return DOIProvider

renku/core/dataset/providers/factory.py

@@ -17,13 +17,13 @@
 # limitations under the License.
 """A factory to get various data providers."""
 
-from typing import TYPE_CHECKING, Type, List
+from typing import TYPE_CHECKING
 from urllib.parse import urlparse
 
 from renku.core import errors
+from renku.core.plugin.dataset_provider import get_supported_dataset_providers
 from renku.core.util import communication
 from renku.core.util.doi import is_doi
-from renku.core.plugin.dataset_provider import get_supported_dataset_providers
 
 if TYPE_CHECKING:
     from renku.core.dataset.providers.api import ProviderApi
@@ -33,7 +33,7 @@ class ProviderFactory:
     """Create a provider type from URI."""
 
     @staticmethod
-    def get_providers() -> "List[Type[ProviderApi]]":
+    def get_providers():
         """Return a list of providers sorted based on their priorities (higher priority providers come first)."""
         providers = get_supported_dataset_providers()
         return sorted(providers, key=lambda p: p.priority)

renku/core/dataset/providers/git.py

@@ -39,7 +39,7 @@
     from renku.core.management.client import LocalClient
 
 
-class GitProvider(ProviderApi):
+class GitProvider(ProviderApi, IDatasetProviderPlugin):
     """Git provider."""
 
     priority = ProviderPriority.NORMAL
@@ -189,11 +189,8 @@ def get_metadata(src: Path, dst: Path) -> Optional["DatasetAddMetadata"]:
 
         return results
 
-
-class GitProviderPlugin(IDatasetProviderPlugin):
-    """Git provider plugin."""
-
+    @classmethod
     @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
+    def dataset_provider(cls) -> "Type[GitProvider]":
         """The defintion of the provider."""
-        return GitProvider
+        return cls

renku/core/dataset/providers/local.py

@@ -37,7 +37,7 @@
     from renku.domain_model.dataset import Dataset, DatasetTag
 
 
-class FilesystemProvider(ProviderApi):
+class FilesystemProvider(ProviderApi, IDatasetProviderPlugin):
     """Local filesystem provider."""
 
     priority = ProviderPriority.LOW
@@ -103,7 +103,7 @@ def add(
         absolute_dataset_data_dir = (client.path / get_dataset_data_dir(client, dataset_name)).resolve()
         source_root = Path(get_absolute_path(path))
         is_within_repo = is_subpath(path=source_root, base=client.path)
-        warnings = []
+        warnings: List[str] = []
 
         def check_recursive_addition(src: Path):
             if src.resolve() == absolute_dataset_data_dir:
@@ -183,6 +183,12 @@ def get_importer(self, uri, **kwargs):
         """Get import manager."""
         raise NotImplementedError
 
+    @classmethod
+    @hookimpl
+    def dataset_provider(cls) -> "Type[FilesystemProvider]":
+        """The defintion of the provider."""
+        return cls
+
 
 class LocalExporter(ExporterApi):
     """Local export manager."""
@@ -247,12 +253,3 @@ def export(self, client=None, **kwargs) -> str:
 
         communication.echo(f"Dataset metadata was copied to {metadata_path}")
         return str(dst_root)
-
-
-class FilesystemProviderPlugin(IDatasetProviderPlugin):
-    """Filesystem provider plugin."""
-
-    @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
-        """The defintion of the provider."""
-        return FilesystemProvider

renku/core/dataset/providers/olos.py

@@ -37,7 +37,7 @@
     from renku.domain_model.dataset import Dataset, DatasetTag
 
 
-class OLOSProvider(ProviderApi):
+class OLOSProvider(ProviderApi, IDatasetProviderPlugin):
     """Provider for OLOS integration."""
 
     priority = ProviderPriority.HIGH
@@ -89,6 +89,12 @@ def set_export_parameters(client_dispatcher: IClientDispatcher):
         set_export_parameters()
         return OLOSExporter(dataset=dataset, server_url=self._server_url)
 
+    @classmethod
+    @hookimpl
+    def dataset_provider(cls) -> "Type[OLOSProvider]":
+        """The defintion of the provider."""
+        return cls
+
 
 class OLOSExporter(ExporterApi):
     """OLOS export manager."""
@@ -284,12 +290,3 @@ def _check_response(response):
                     response.status_code, json_res["message"] if "message" in json_res else json_res["status"]
                 )
             )
-
-
-class OLOSProviderPlugin(IDatasetProviderPlugin):
-    """OLOS provider plugin."""
-
-    @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
-        """The defintion of the provider."""
-        return OLOSProvider

renku/core/dataset/providers/renku.py

@@ -22,7 +22,7 @@
 import urllib
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Type
 
 from renku.command.command_builder.command import inject
 from renku.command.login import read_renku_token
@@ -45,7 +45,7 @@
     from renku.domain_model.dataset import Dataset
 
 
-class RenkuProvider(ProviderApi):
+class RenkuProvider(ProviderApi, IDatasetProviderPlugin):
     """Renku API provider."""
 
     priority = ProviderPriority.HIGH
@@ -226,6 +226,12 @@ def _prepare_auth(self, uri):
 
         self._authorization_header = {"Authorization": f"Bearer {token}"} if token else {}
 
+    @classmethod
+    @hookimpl
+    def dataset_provider(cls) -> "Type[RenkuProvider]":
+        """The defintion of the provider."""
+        return cls
+
 
 class RenkuImporter(ImporterApi):
     """Renku record serializer."""
@@ -550,12 +556,3 @@ def _migrate_project():
             )
         finally:
             communication.enable()
-
-
-class RenkuProviderPlugin(IDatasetProviderPlugin):
-    """Renku provider plugin."""
-
-    @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
-        """The defintion of the provider."""
-        return RenkuProvider

renku/core/dataset/providers/web.py

@@ -38,7 +38,7 @@
     from renku.core.management.client import LocalClient
 
 
-class WebProvider(ProviderApi):
+class WebProvider(ProviderApi, IDatasetProviderPlugin):
     """A provider for downloading data from web URLs."""
 
     priority = ProviderPriority.LOWEST
@@ -71,6 +71,12 @@ def add(
             client=client, uri=uri, destination=destination, extract=extract, filename=filename, multiple=multiple
         )
 
+    @classmethod
+    @hookimpl
+    def dataset_provider(cls) -> "Type[WebProvider]":
+        """The defintion of the provider."""
+        return cls
+
 
 def _ensure_dropbox(url):
     """Ensure dropbox url is set for file download."""
@@ -193,12 +199,3 @@ def subscribe_communication_listeners(function, **kwargs):
             files.extend(future.result())
 
     return files
-
-
-class WebProviderPlugin(IDatasetProviderPlugin):
-    """Web provider plugin."""
-
-    @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
-        """The defintion of the provider."""
-        return WebProvider

renku/core/dataset/providers/zenodo.py

@@ -54,7 +54,7 @@
 ZENODO_NEW_DEPOSIT_URL = "depositions"
 
 
-class ZenodoProvider(ProviderApi):
+class ZenodoProvider(ProviderApi, IDatasetProviderPlugin):
     """Zenodo registry API provider."""
 
     priority = ProviderPriority.HIGH
@@ -122,6 +122,12 @@ def get_exporter(
         self._publish = publish
         return ZenodoExporter(dataset=dataset, publish=self._publish, tag=tag)
 
+    @classmethod
+    @hookimpl
+    def dataset_provider(cls) -> "Type[ZenodoProvider]":
+        """The defintion of the provider."""
+        return cls
+
 
 class ZenodoImporter(RepositoryImporter):
     """Zenodo importer."""
@@ -559,12 +565,3 @@ def make_records_url(record_id):
         str: Full URL for the record.
     """
     return urllib.parse.urljoin(ZENODO_BASE_URL, pathlib.posixpath.join(ZENODO_API_PATH, "records", record_id))
-
-
-class ZenodoProviderPlugin(IDatasetProviderPlugin):
-    """Zenodo provider plugin."""
-
-    @hookimpl
-    def dataset_provider(self) -> "Type[ProviderApi]":
-        """The defintion of the provider."""
-        return ZenodoProvider

renku/core/plugin/dataset_provider.py

@@ -16,11 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Plugin hooks for renku workflow customization."""
-from typing import List, Type
+from typing import TYPE_CHECKING, List, Type
 
 import pluggy
 
-from renku.core.dataset.providers.api import ProviderApi
+if TYPE_CHECKING:
+    from renku.core.dataset.providers.api import ProviderApi
 
 hookspec = pluggy.HookspecMarker("renku")
 

renku/core/plugin/implementations/__init__.py

@@ -19,37 +19,37 @@
 
 from typing import TYPE_CHECKING, List, Type
 
+from renku.core.dataset.providers.dataverse import DataverseProvider
+from renku.core.dataset.providers.git import GitProvider
+from renku.core.dataset.providers.local import FilesystemProvider
+from renku.core.dataset.providers.olos import OLOSProvider
+from renku.core.dataset.providers.renku import RenkuProvider
+from renku.core.dataset.providers.web import WebProvider
+from renku.core.dataset.providers.zenodo import ZenodoProvider
 from renku.core.session.docker import DockerSessionProvider
 from renku.core.session.renkulab import RenkulabSessionProvider
 from renku.core.workflow.converters.cwl import CWLExporter
 from renku.core.workflow.providers.cwltool import CWLToolProvider
-from renku.core.dataset.providers.dataverse import DataverseProviderPlugin
-from renku.core.dataset.providers.git import GitProviderPlugin
-from renku.core.dataset.providers.local import FilesystemProviderPlugin
-from renku.core.dataset.providers.olos import OLOSProviderPlugin
-from renku.core.dataset.providers.renku import RenkuProviderPlugin
-from renku.core.dataset.providers.web import WebProviderPlugin
-from renku.core.dataset.providers.zenodo import ZenodoProviderPlugin
 
 if TYPE_CHECKING:
+    from renku.core.dataset.providers.api import ProviderApi
     from renku.domain_model.session import ISessionProvider
     from renku.domain_model.workflow.converters import IWorkflowConverter
     from renku.domain_model.workflow.provider import IWorkflowProvider
-    from renku.core.dataset.providers.api import ProviderApi
 
 __all__: List[str] = []
 
 session_providers: "List[Type[ISessionProvider]]" = [DockerSessionProvider, RenkulabSessionProvider]
 workflow_exporters: "List[Type[IWorkflowConverter]]" = [CWLExporter]
 workflow_providers: "List[Type[IWorkflowProvider]]" = [CWLToolProvider]
 dataset_providers: "List[Type[ProviderApi]]" = [
-    DataverseProviderPlugin,
-    GitProviderPlugin,
-    FilesystemProviderPlugin,
-    OLOSProviderPlugin,
-    RenkuProviderPlugin,
-    WebProviderPlugin,
-    ZenodoProviderPlugin,
+    DataverseProvider,
+    GitProvider,
+    FilesystemProvider,
+    OLOSProvider,
+    RenkuProvider,
+    WebProvider,
+    ZenodoProvider,
 ]
 
 try: