feat: dataverse import (#626)

* feat: add DataverseProvider for importing datasets

* feat: add support for arbitrary dataverse repos

* Removes leftover url_lower variable

* adds error handling for http errors
moves provider detection to individual providers
fixes dataset file property casing
improves doi uri sanitizing

* Adds unit tests

* Moves unit tests to integration tests

* Fix formatting

* Fixes bug where provider resolution depended on the output order of dict.items()

* Removes is_doi variable and adds dynamic message for supported providers

* Fixes minor formatting  and style issues for PR

* Fixes import sorting

Author: Panaetius

conftest.py

@@ -17,10 +17,14 @@
 # limitations under the License.
 """Pytest configuration."""
 
+import json
 import os
+import pathlib
+import re
 import shutil
 import tempfile
 import time
+import urllib
 
 import pytest
 import responses
@@ -299,6 +303,84 @@ def zenodo_sandbox(client):
     )
 
 
+@pytest.fixture
+def doi_responses():
+    """Responses for doi.org requests."""
+    from renku.cli._providers.doi import DOI_BASE_URL
+    from renku.cli._providers.dataverse import (
+        DATAVERSE_API_PATH, DATAVERSE_VERSION_API
+    )
+
+    with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
+
+        def doi_callback(request):
+            response_url = (
+                'https://dataverse.harvard.edu/citation'
+                '?persistentId=doi:10.11588/data/yyxx1122'
+            )
+            if 'zenodo' in request.url:
+                response_url = 'https://zenodo.org/record/3363060'
+            return (
+                200, {
+                    'Content-Type': 'application/json'
+                },
+                json.dumps({
+                    'type': 'dataset',
+                    'id': request.url,
+                    'author': [{
+                        'family': 'Doe',
+                        'given': 'John'
+                    }],
+                    'contributor': [{
+                        'contributorType': 'ContactPerson',
+                        'family': 'Doe',
+                        'given': 'John'
+                    }],
+                    'issued': {
+                        'date-parts': [[2019]]
+                    },
+                    'abstract': 'Test Dataset',
+                    'DOI': '10.11588/data/yyxx1122',
+                    'publisher': 'heiDATA',
+                    'title': 'dataset',
+                    'URL': response_url
+                })
+            )
+
+        rsps.add_callback(
+            method='GET',
+            url=re.compile('{base_url}/.*'.format(base_url=DOI_BASE_URL)),
+            callback=doi_callback
+        )
+
+        def version_callback(request):
+            return (
+                200, {
+                    'Content-Type': 'application/json'
+                },
+                json.dumps({
+                    'status': 'OK',
+                    'data': {
+                        'version': '4.1.3',
+                        'build': 'abcdefg'
+                    }
+                })
+            )
+
+        base_url = 'https://dataverse.harvard.edu'
+
+        url_parts = list(urllib.parse.urlparse(base_url))
+        url_parts[2] = pathlib.posixpath.join(
+            DATAVERSE_API_PATH, DATAVERSE_VERSION_API
+        )
+        pattern = '{url}.*'.format(url=urllib.parse.urlunparse(url_parts))
+
+        rsps.add_callback(
+            method='GET', url=re.compile(pattern), callback=version_callback
+        )
+        yield rsps
+
+
 @pytest.fixture
 def cli(client, run):
     """Return a callable Renku CLI.

renku/cli/_providers/__init__.py

@@ -19,35 +19,54 @@
 from urllib.parse import urlparse
 
 from renku.cli._providers.zenodo import ZenodoProvider
+from renku.cli._providers.dataverse import DataverseProvider
 from renku.utils.doi import is_doi
 
 
 class ProviderFactory:
     """Create a provider type from URI."""
 
-    PROVIDERS = {'zenodo': ZenodoProvider}
+    PROVIDERS = {'dataverse': DataverseProvider, 'zenodo': ZenodoProvider}
 
     @staticmethod
     def from_uri(uri):
         """Get provider type based on uri."""
         is_doi_ = is_doi(uri)
-        if is_doi_ is False:
+        if is_doi_ is None:
             url = urlparse(uri)
             if bool(url.scheme and url.netloc and url.params == '') is False:
                 return None, 'Cannot parse URL.'
 
         provider = None
-        if 'zenodo' in uri:
-            provider = ZenodoProvider(is_doi=is_doi_)
+        warning = ''
+
+        for _, potential_provider in ProviderFactory.PROVIDERS.items():
+            try:
+                if potential_provider.supports(uri):
+                    provider = potential_provider
+                    break
+            except (Exception, BaseException) as e:
+                warning += 'Couldn\'t test provider {prov}: {err}\n'.format(
+                    prov=potential_provider, err=e
+                )
+
+        supported_providers = ', '.join(ProviderFactory.PROVIDERS.keys())
 
         if is_doi_ and provider is None:
             return None, (
-                'Provider {} not found. '.format(
+                warning + 'Provider {} not found. '.format(
                     uri.split('/')[1].split('.')[0]  # Get DOI provider name.
-                ) + 'Currently supporting following providers: (Zenodo, )'
+                ) + 'Currently supporting following providers: {}'.
+                format(supported_providers)
             )
-
-        return provider, None
+        elif provider is None:
+            return None, (
+                warning + 'Provider not found for {}. '.format(uri) +
+                'Currently supporting following providers: {}'.
+                format(supported_providers)
+            )
+        else:
+            return provider(is_doi=is_doi_), warning
 
     @staticmethod
     def from_id(provider_id):

renku/cli/_providers/api.py

@@ -30,6 +30,12 @@ def get_exporter(self, dataset, secret):
         """Get export manager."""
         pass
 
+    @staticmethod
+    @abc.abstractmethod
+    def supports(uri):
+        """Whether or not this provider supports a given uri."""
+        pass
+
 
 class ExporterApi(abc.ABC):
     """Interface defining exporter methods."""