feat(datasets): import data from zenodo (#509)

Author: jsam

.gitignore

@@ -6,6 +6,9 @@ __pycache__/
 .pytest_cache/
 *.py[cod]
 
+# VSCode
+.vscode/
+
 # Idea software family
 .idea/
 

renku/cli/_providers/__init__.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Third party data registry integration."""
+from urllib.parse import urlparse
+
+from renku.cli._providers.doi import DOIProvider
+from renku.cli._providers.zenodo import ZenodoProvider
+
+
+class ProviderFactory:
+    """Create a provider type from URI."""
+
+    @staticmethod
+    def from_uri(uri):
+        """Get provider type based on uri."""
+        is_doi = DOIProvider.is_doi(uri)
+        if is_doi is False:
+            url = urlparse(uri)
+            if bool(url.scheme and url.netloc and url.params == '') is False:
+                return None, 'Cannot parse URL.'
+
+        provider = None
+        if 'zenodo' in uri:
+            provider = ZenodoProvider(is_doi=is_doi)
+
+        if is_doi and provider is None:
+            return None, (
+                'Provider {} not found. '.format(
+                    uri.split('/')[1].split('.')[0]  # Get DOI provider name.
+                ) + 'Currently supporting following providers: (Zenodo, )'
+            )
+
+        return provider, None

renku/cli/_providers/api.py

@@ -0,0 +1,26 @@
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""API for providers."""
+import abc
+
+
+class ProviderApi(abc.ABC):
+    """Interface defining provider methods."""
+
+    @abc.abstractmethod
+    def find_record(self, uri):
+        """Find record by uri."""
+        pass

renku/cli/_providers/doi.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DOI API integration."""
+import re
+import urllib
+
+import attr
+import requests
+
+from renku.cli._providers.api import ProviderApi
+
+doi_regexp = re.compile(
+    r'(doi:\s*|(?:https?://)?(?:dx\.)?doi\.org/)?(10\.\d+(.\d+)*/.+)$',
+    flags=re.I
+)
+"""See http://en.wikipedia.org/wiki/Digital_object_identifier."""
+
+DOI_BASE_URL = 'https://dx.doi.org'
+
+
+def make_doi_url(doi):
+    """Create URL to access DOI metadata."""
+    return urllib.parse.urljoin(DOI_BASE_URL, doi)
+
+
+@attr.s
+class DOIMetadata:
+    """Response from doi.org for DOI metadata."""
+
+    id = attr.ib(kw_only=True)
+    DOI = attr.ib(kw_only=True)
+    URL = attr.ib(kw_only=True)
+    type = attr.ib(kw_only=True, default=None)
+    categories = attr.ib(kw_only=True, default=None)
+    author = attr.ib(kw_only=True, default=None)
+    version = attr.ib(kw_only=True, default=None)
+    issued = attr.ib(kw_only=True, default=None)
+    title = attr.ib(kw_only=True, default=None)
+    abstract = attr.ib(kw_only=True, default=None)
+    language = attr.ib(kw_only=True, default=None)
+    publisher = attr.ib(kw_only=True, default=None)
+
+
+@attr.s
+class DOIProvider(ProviderApi):
+    """doi.org registry API provider."""
+
+    headers = attr.ib(
+        default={'accept': 'application/vnd.citationstyles.csl+json'}
+    )
+    timeout = attr.ib(default=3)
+
+    @staticmethod
+    def is_doi(uri):
+        """Check if uri is DOI."""
+        return doi_regexp.match(uri)
+
+    def _query(self, doi):
+        """Retrieve metadata for given doi."""
+        url = doi
+        if doi.startswith('http') is False:
+            url = make_doi_url(doi)
+
+        response = requests.get(url, headers=self.headers)
+        if response.status_code != 200:
+            raise LookupError('record not found')
+
+        return response
+
+    def find_record(self, uri):
+        """Finds DOI record."""
+        response = self._query(uri).json()
+        return DOIMetadata(**response)

renku/cli/_providers/zenodo.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Zenodo API integration."""
+import pathlib
+import re
+import urllib
+from urllib.parse import urlparse
+
+import attr
+import requests
+
+from renku.cli._providers.api import ProviderApi
+from renku.cli._providers.doi import DOIProvider
+
+ZENODO_BASE_URL = 'https://zenodo.org'
+ZENODO_BASE_PATH = 'api'
+
+
+def make_records_url(record_id):
+    """Create URL to access record by ID."""
+    return urllib.parse.urljoin(
+        ZENODO_BASE_URL,
+        pathlib.posixpath.join(ZENODO_BASE_PATH, 'records', record_id)
+    )
+
+
+@attr.s
+class ZenodoFile:
+    """Zenodo record file."""
+
+    checksum = attr.ib()
+    links = attr.ib()
+    bucket = attr.ib()
+    key = attr.ib()
+    size = attr.ib()
+    type = attr.ib()
+
+    @property
+    def remote_url(self):
+        """Get remote URL as ``urllib.ParseResult``."""
+        return urllib.parse.urlparse(self.links['self'])
+
+    @property
+    def name(self):
+        """Get file name."""
+        return self.remote_url.path.split('/')[-1]
+
+
+@attr.s
+class ZenodoRecord:
+    """Zenodo record."""
+
+    id = attr.ib()
+    conceptrecid = attr.ib()
+
+    doi = attr.ib()
+    files = attr.ib()
+    links = attr.ib()
+    metadata = attr.ib()
+    owners = attr.ib()
+    revision = attr.ib()
+    stats = attr.ib()
+
+    created = attr.ib()
+    updated = attr.ib()
+
+    conceptdoi = attr.ib(default=None)
+    _zenodo = attr.ib(kw_only=True, default=None)
+
+    @property
+    def last_version(self):
+        """Check if record is at last possible version."""
+        return self.version['is_last']
+
+    @property
+    def version(self):
+        """Get record version."""
+        return self.metadata['relations']['version'][0]
+
+    @property
+    def display_version(self):
+        """Get display version."""
+        return 'v{0}'.format(self.version['index'])
+
+    @property
+    def display_name(self):
+        """Get record display name."""
+        return '{0}_{1}'.format(
+            re.sub(r'\W+', '', self.metadata['title']).lower()[:16],
+            self.display_version
+        )
+
+    def get_files(self):
+        """Get Zenodo files metadata as ``ZenodoFile``."""
+        if len(self.files) == 0:
+            raise LookupError('no files have been found')
+
+        return [ZenodoFile(**file_) for file_ in self.files]
+
+
+@attr.s
+class ZenodoProvider(ProviderApi):
+    """zenodo.org registry API provider."""
+
+    is_doi = attr.ib(default=False)
+
+    @staticmethod
+    def record_id(uri):
+        """Extract record id from uri."""
+        return urlparse(uri).path.split('/')[-1]
+
+    def find_record(self, uri):
+        """Retrieves a record from Zenodo.
+
+        :raises: ``LookupError``
+        :param uri: DOI or URL
+        :return: ``ZenodoRecord``
+        """
+        if self.is_doi:
+            return self.find_record_by_doi(uri)
+
+        return self.get_record(uri)
+
+    def find_record_by_doi(self, doi):
+        """Resolve the DOI and make a record for the retrieved record id."""
+        doi = DOIProvider().find_record(doi)
+        return self.get_record(ZenodoProvider.record_id(doi.URL))
+
+    def get_record(self, uri):
+        """Retrieve record metadata and return ``ZenodoRecord``."""
+        record_id = ZenodoProvider.record_id(uri)
+        response = requests.get(make_records_url(record_id))
+        if response.status_code != 200:
+            raise LookupError('record not found')
+
+        return ZenodoRecord(**response.json(), zenodo=self)