feat: integrate metadata from zenodo (#545)

Author: jsam

.travis.yml

@@ -34,12 +34,19 @@ env:
   - REQUIREMENTS=lowest
   - REQUIREMENTS=release
 
+dist: xenial
+
 python:
   - "3.5"
   - "3.6"
+  - "3.7"
 
 stages:
   - name: test
+  - name: integration
+    if: branch = master
+  - name: test OSX
+    if: branch = master
   - name: publish 🐍  🐳
     if: type = push AND (branch = master OR tag IS present)
   - name: brew 🍺
@@ -84,7 +91,51 @@ after_success:
 
 jobs:
   include:
-    - stage: test
+    - stage: integration
+      os: linux
+      dist: xenial
+      language: python
+      env:
+        - REQUIREMENTS=release
+      script: pytest -m integration -v
+    - stage: integration
+      python: "3.6"
+      os: linux
+      dist: xenial
+      language: python
+      env:
+        - REQUIREMENTS=release
+    - stage: integration
+      python: "3.7"
+      os: linux
+      dist: xenial
+      language: python
+      env:
+        - REQUIREMENTS=release
+    - stage: integration
+      os: linux
+      dist: xenial
+      language: python
+      env:
+        - REQUIREMENTS=lowest
+      script: pytest -m integration -v
+    - stage: integration
+      python: "3.6"
+      os: linux
+      dist: xenial
+      language: python
+      env:
+        - REQUIREMENTS=lowest
+    - stage: integration
+      python: "3.7"
+      os: linux
+      dist: xenial
+      language: python
+      env:
+        - REQUIREMENTS=lowest
+
+    - stage: test OSX
+      if: branch = master
       language: generic
       sudo: true
       os: osx
@@ -93,12 +144,6 @@ jobs:
       sudo: true
       os: osx
       osx_image: xcode9.2
-    - python: "3.7"
-      os: linux
-      dist: xenial
-      language: python
-      env:
-        - REQUIREMENTS=release
 
     - stage: publish 🐍  🐳
       python: 3.6
@@ -110,6 +155,7 @@ jobs:
           distributions: "sdist bdist_wheel"
           on:
             all_branches: true
+
     - # stage: publish
       sudo: required
       services:
@@ -121,7 +167,7 @@ jobs:
         - sudo apt-get -y install docker-ce
       script: make docker-login docker-push
 
-    - stage: brew 🍺
+    - # stage: brew 🍺
       language: generic
       sudo: true
       os: osx

docs/models/datasets.rst

@@ -39,9 +39,9 @@ Manage files in the dataset.
    :inherited-members:
 
 
-Author
-------
+Creator
+-------
 
-.. autoclass:: renku.models.datasets.Author
+.. autoclass:: renku.models.datasets.Creator
    :members:
    :inherited-members:

renku/api/datasets.py

@@ -33,7 +33,7 @@
 from renku import errors
 from renku._compat import Path
 from renku.models._git import GitURL
-from renku.models.datasets import Author, Dataset, DatasetFile, NoneType
+from renku.models.datasets import Creator, Dataset, DatasetFile, NoneType
 
 
 @attr.s
@@ -249,7 +249,7 @@ def _add_from_url(self, dataset, path, url, link=False, **kwargs):
                 DatasetFile(
                     path=result,
                     url=url,
-                    author=dataset.author,
+                    creator=dataset.creator,
                     dataset=dataset.name,
                 )
         }
@@ -290,7 +290,7 @@ def _add_from_git(self, dataset, path, url, target, **kwargs):
                         DatasetFile(
                             path=result,
                             url=url,
-                            author=dataset.author,
+                            creator=dataset.creator,
                             dataset=dataset.name,
                         )
                 }
@@ -377,13 +377,13 @@ def _add_from_git(self, dataset, path, url, target, **kwargs):
 
         os.symlink(os.path.relpath(str(src), str(dst.parent)), str(dst))
 
-        # grab all the authors from the commit history
+        # grab all the creators from the commit history
         git_repo = Repo(str(submodule_path.absolute()))
-        authors = []
+        creators = []
         for commit in git_repo.iter_commits(paths=target):
-            author = Author.from_commit(commit)
-            if author not in authors:
-                authors.append(author)
+            creator = Creator.from_commit(commit)
+            if creator not in creators:
+                creators.append(creator)
 
         dataset_path = self.renku_datasets_path / dataset.name
         result = os.path.relpath(str(dst), start=str(dataset_path))
@@ -398,7 +398,7 @@ def _add_from_git(self, dataset, path, url, target, **kwargs):
                 DatasetFile(
                     path=result,
                     url=url,
-                    author=authors,
+                    creator=creators,
                     dataset=dataset.name,  # TODO detect original dataset
                 )
         }

renku/api/repository.py

@@ -350,9 +350,9 @@ def init_repository(self, name=None, force=False):
 
         self.repo.description = name or path.name
 
-        # Check that an author can be determined from Git.
-        from renku.models.datasets import Author
-        Author.from_git(self.repo)
+        # Check that an creator can be determined from Git.
+        from renku.models.datasets import Creator
+        Creator.from_git(self.repo)
 
         # TODO read existing gitignore and create a unique set of rules
         import pkg_resources

renku/cli/_exc.py

@@ -117,9 +117,9 @@ def _handle_sentry(self):
             with capture_internal_exceptions():
                 from git import Repo
                 from renku.cli._git import get_git_home
-                from renku.models.datasets import Author
+                from renku.models.datasets import Creator
 
-                user = Author.from_git(Repo(get_git_home()))
+                user = Creator.from_git(Repo(get_git_home()))
 
                 scope.user = {'name': user.name, 'email': user.email}
 

renku/cli/_format/dataset_files.py

@@ -35,7 +35,7 @@ def tabular(client, records):
             records,
             headers=OrderedDict((
                 ('added', None),
-                ('authors_csv', 'authors'),
+                ('creators_csv', 'creators'),
                 ('dataset', None),
                 ('full_path', 'path'),
             )),

renku/cli/_format/datasets.py

@@ -32,9 +32,10 @@ def tabular(client, datasets):
             datasets,
             headers=OrderedDict((
                 ('short_id', 'id'),
-                ('name', None),
+                ('display_name', None),
+                ('version', None),
                 ('created', None),
-                ('authors_csv', 'authors'),
+                ('creators_csv', 'creators'),
             )),
         )
     )

renku/cli/_providers/__init__.py

@@ -18,8 +18,8 @@
 """Third party data registry integration."""
 from urllib.parse import urlparse
 
-from renku.cli._providers.doi import DOIProvider
 from renku.cli._providers.zenodo import ZenodoProvider
+from renku.utils.doi import is_doi
 
 
 class ProviderFactory:
@@ -28,17 +28,17 @@ class ProviderFactory:
     @staticmethod
     def from_uri(uri):
         """Get provider type based on uri."""
-        is_doi = DOIProvider.is_doi(uri)
-        if is_doi is False:
+        is_doi_ = is_doi(uri)
+        if is_doi_ is False:
             url = urlparse(uri)
             if bool(url.scheme and url.netloc and url.params == '') is False:
                 return None, 'Cannot parse URL.'
 
         provider = None
         if 'zenodo' in uri:
-            provider = ZenodoProvider(is_doi=is_doi)
+            provider = ZenodoProvider(is_doi=is_doi_)
 
-        if is_doi and provider is None:
+        if is_doi_ and provider is None:
             return None, (
                 'Provider {} not found. '.format(
                     uri.split('/')[1].split('.')[0]  # Get DOI provider name.

renku/cli/_providers/doi.py

@@ -16,20 +16,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """DOI API integration."""
-import re
 import urllib
 
 import attr
 import requests
 
 from renku.cli._providers.api import ProviderApi
 
-doi_regexp = re.compile(
-    r'(doi:\s*|(?:https?://)?(?:dx\.)?doi\.org/)?(10\.\d+(.\d+)*/.+)$',
-    flags=re.I
-)
-"""See http://en.wikipedia.org/wiki/Digital_object_identifier."""
-
 DOI_BASE_URL = 'https://dx.doi.org'
 
 
@@ -39,22 +32,35 @@ def make_doi_url(doi):
 
 
 @attr.s
-class DOIMetadata:
+class DOIMetadataSerializer:
     """Response from doi.org for DOI metadata."""
 
     id = attr.ib(kw_only=True)
-    DOI = attr.ib(kw_only=True)
-    URL = attr.ib(kw_only=True)
+
+    doi = attr.ib(kw_only=True)
+
+    url = attr.ib(kw_only=True)
+
     type = attr.ib(kw_only=True, default=None)
+
     categories = attr.ib(kw_only=True, default=None)
+
     author = attr.ib(kw_only=True, default=None)
+
     version = attr.ib(kw_only=True, default=None)
+
     issued = attr.ib(kw_only=True, default=None)
+
     title = attr.ib(kw_only=True, default=None)
+
     abstract = attr.ib(kw_only=True, default=None)
+
     language = attr.ib(kw_only=True, default=None)
+
     publisher = attr.ib(kw_only=True, default=None)
 
+    container_title = attr.ib(kw_only=True, default=None)
+
 
 @attr.s
 class DOIProvider(ProviderApi):
@@ -66,9 +72,14 @@ class DOIProvider(ProviderApi):
     timeout = attr.ib(default=3)
 
     @staticmethod
-    def is_doi(uri):
-        """Check if uri is DOI."""
-        return doi_regexp.match(uri)
+    def _serialize(response):
+        """Serialize HTTP response for DOI."""
+        return DOIMetadataSerializer(
+            **{
+                key.replace('-', '_').lower(): value
+                for key, value in response.items()
+            }
+        )
 
     def _query(self, doi):
         """Retrieve metadata for given doi."""
@@ -85,4 +96,4 @@ def _query(self, doi):
     def find_record(self, uri):
         """Finds DOI record."""
         response = self._query(uri).json()
-        return DOIMetadata(**response)
+        return DOIProvider._serialize(response)