serializers: OpenAIRE JSON

Signed-off-by: Alexander Ioannidis <a.ioannidis@cern.ch>

tests/unit/records/test_schemas_openaire_json.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Zenodo.
+# Copyright (C) 2017 CERN.
+#
+# Zenodo is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Zenodo is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Zenodo; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""Zenodo CSL mapping test."""
+
+from __future__ import absolute_import, print_function
+
+import pytest
+from invenio_records.api import Record
+
+from zenodo.modules.records.serializers import openaire_json_v1
+
+
+@pytest.fixture()
+def minimal_oai_record(minimal_record):
+    """Minimal OAI record."""
+    minimal_record['_oai'] = {
+        'id': 'oai:zenodo.org:{}'.format(minimal_record['recid'])
+    }
+    return minimal_record
+
+
+def test_minimal(app, db, minimal_oai_record, recid_pid):
+    """Test minimal record."""
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj == {
+        'originalId': 'oai:zenodo.org:123',
+        'type': 'publication',
+        'resourceType': '0001',
+        'title': 'Test',
+        'licenseCode': 'OPEN',
+        'url': 'https://zenodo.org/record/123',
+        'authors': ['Test'],
+        'description': 'My description',
+        'pids': [{'type': 'oai', 'value': 'oai:zenodo.org:123'}],
+        'hostedById': 'opendoar____::2659',
+        'collectedFromId': 'opendoar____::2659',
+    }
+
+
+def test_resource_types(app, db, minimal_oai_record, recid_pid):
+    """"Test resource types."""
+    minimal_oai_record['doi'] = '10.1234/foo'
+
+    minimal_oai_record.update({'resource_type': {'type': 'dataset'}})
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    # Datasets use the DOI
+    assert obj['originalId'] == '10.1234/foo'
+    assert obj['collectedFromId'] == 're3data_____::r3d100010468'
+    assert obj['hostedById'] == 're3data_____::r3d100010468'
+    assert obj['resourceType'] == '0021'
+    assert obj['type'] == 'dataset'
+
+    minimal_oai_record.update({'resource_type': {'type': 'poster'}})
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['originalId'] == 'oai:zenodo.org:123'
+    assert obj['collectedFromId'] == 'opendoar____::2659'
+    assert obj['hostedById'] == 'opendoar____::2659'
+    assert obj['resourceType'] == '0001'
+    assert obj['type'] == 'publication'
+
+
+def test_grants(app, db, minimal_oai_record, recid_pid):
+    """"Test grants."""
+    minimal_oai_record['grants'] = [
+        {
+            'acronym': 'WorkAble',
+            'identifiers': {
+                'eurepo': 'info:eu-repo/grantAgreement/EC/FP7/244909/'
+            },
+        }
+    ]
+
+    minimal_oai_record.update({'resource_type': {'type': 'dataset'}})
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['linksToProjects'] == [
+        'info:eu-repo/grantAgreement/EC/FP7/244909///WorkAble'
+    ]
+
+
+def test_pids(app, db, minimal_oai_record, recid_pid):
+    """"Test PIDs."""
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert sorted(obj['pids']) == \
+        sorted([{'value': 'oai:zenodo.org:123', 'type': 'oai'}])
+
+    minimal_oai_record['doi'] = '10.1234/foo'
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert sorted(obj['pids']) == \
+        sorted([{'value': 'oai:zenodo.org:123', 'type': 'oai'},
+                {'value': '10.1234/foo', 'type': 'doi'}])
+
+
+def test_publisher(app, db, minimal_oai_record, recid_pid):
+    """Test publisher."""
+    minimal_oai_record['doi'] = '10.5281/12345'
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['publisher'] == 'Zenodo'
+
+    minimal_oai_record['part_of'] = {'publisher': 'The Good Publisher'}
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['publisher'] == 'The Good Publisher'
+
+    minimal_oai_record['imprint'] = {'publisher': 'The Bad Publisher'}
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['publisher'] == 'The Bad Publisher'
+
+
+def test_license_code(app, db, minimal_oai_record, recid_pid):
+    """Test license code."""
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['licenseCode'] == 'OPEN'
+
+    minimal_oai_record['access_right'] = 'restricted'
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['licenseCode'] == 'RESTRICTED'
+
+    minimal_oai_record['access_right'] = 'embargoed'
+    minimal_oai_record['embargo_date'] = '2017-04-22'
+    obj = openaire_json_v1.transform_record(
+        recid_pid, Record(minimal_oai_record))
+    assert obj['licenseCode'] == 'EMBARGO'
+    assert obj['embargoEndDate'] == '2017-04-22'

zenodo/config.py

@@ -275,6 +275,20 @@ def _(x):
 OPENAIRE_SCHEMAS_HOST = 'zenodo.org'
 #: Hostname for OpenAIRE's grant resolver.
 OPENAIRE_JSONRESOLVER_GRANTS_HOST = 'dx.zenodo.org'
+#: OpenAIRE Zenodo IDs
+OPENAIRE_ZENODO_IDS = {
+    'publication': 'opendoar____::2659',
+    'dataset': 're3data_____::r3d100010468',
+}
+#: OpenAIRE Zenodo namespace prefixes
+OPENAIRE_NAMESPACE_PREFIXES = {
+    'publication': 'od______2659',
+    'dataset': 'r37b0ad08687',
+}
+#: OpenAIRE API endpoint.
+OPENAIRE_API_URL = 'https://beta.services.openaire.eu/is/mvc'
+# TODO: Check if we are to use Dev or Beta endpoint...
+# OPENAIRE_API_URL = 'http://dev.openaire.research-infrastructures.eu/is/mvc'
 
 # OpenDefinition
 # ==============

zenodo/modules/openaire/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Zenodo.
+# Copyright (C) 2017 CERN.
+#
+# Zenodo is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Zenodo is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Zenodo; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""Zenodo OpenAIRE module."""
+
+from __future__ import absolute_import, print_function

zenodo/modules/openaire/schema.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Zenodo.
+# Copyright (C) 2017 CERN.
+#
+# Zenodo is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Zenodo is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Zenodo; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""Zenodo OpenaAIRE-JSON schema."""
+
+from __future__ import absolute_import, print_function
+
+from collections import namedtuple
+
+from flask import current_app
+from marshmallow import Schema, fields, missing
+
+from zenodo.modules.records.models import ObjectType
+from zenodo.modules.records.serializers.fields import DateString
+
+OpenAIREType = namedtuple('OpenAIREType', ('type', 'resource_type'))
+
+
+class RecordSchemaOpenAIREJSON(Schema):
+    """Schema for records in OpenAIRE-JSON.
+
+    OpenAIRE Schema: https://www.openaire.eu/schema/1.0/oaf-result-1.0.xsd
+    OpenAIRE Vocabularies: http://api.openaire.eu/vocabularies
+    """
+
+    originalId = fields.Method('get_original_id', required=True)
+    title = fields.Str(attribute='metadata.title', required=True)
+    description = fields.Str(attribute='metadata.description')
+    url = fields.Method('get_url', required=True)
+
+    authors = fields.List(fields.Str(attribute='name'),
+                          attribute='metadata.creators')
+
+    type = fields.Method('get_type')
+    resourceType = fields.Method('get_resource_type', required=True)
+    language = fields.Str(attribute='metadata.language')
+
+    licenseCode = fields.Method('get_license_code', required=True)
+    embargoEndDate = DateString(attribute='metadata.embargo_date')
+
+    publisher = fields.Method('get_publisher')
+    collectedFromId = fields.Method('get_openaire_id', required=True)
+    hostedById = fields.Method('get_openaire_id')
+
+    linksToProjects = fields.Method('get_links_to_projects')
+    pids = fields.Method('get_pids')
+
+    def _resolve_openaire_type(self, obj):
+        # TODO: Move to utils.py?
+        metadata = obj.get('metadata')
+        obj_type = ObjectType.get_by_dict(metadata.get('resource_type'))
+        if obj_type['internal_id'] == 'dataset':
+            return OpenAIREType('dataset', '0021')
+        else:
+            return OpenAIREType('publication', '0001')
+
+    def get_original_id(self, obj):
+        """Get Original Id."""
+        openaire_type = self._resolve_openaire_type(obj)
+        if openaire_type.type == 'publication':
+            return obj.get('metadata', {}).get('_oai', {}).get('id')
+        if openaire_type.type == 'dataset':
+            return obj.get('metadata', {}).get('doi')
+
+    def get_type(self, obj):
+        """Get record type."""
+        return self._resolve_openaire_type(obj).type
+
+    def get_resource_type(self, obj):
+        """Get resource type."""
+        return self._resolve_openaire_type(obj).resource_type
+
+    def get_openaire_id(self, obj):
+        """Get OpenAIRE Zenodo ID."""
+        # TODO: Move to utils.py?
+        openaire_type = self._resolve_openaire_type(obj).type
+        return current_app.config['OPENAIRE_ZENODO_IDS'].get(openaire_type)
+
+    # Mapped from: http://api.openaire.eu/vocabularies/dnet:access_modes
+    LICENSE_MAPPING = {
+        'open': 'OPEN',
+        'embargoed': 'EMBARGO',
+        'restricted': 'RESTRICTED',
+        'closed': 'CLOSED',
+    }
+
+    def get_license_code(self, obj):
+        """Get license code."""
+        metadata = obj.get('metadata')
+        return self.LICENSE_MAPPING.get(
+            metadata.get('access_right'), 'UNKNOWN')
+
+    def get_links_to_projects(self, obj):
+        """Get project/grant links."""
+        metadata = obj.get('metadata')
+        grants = metadata.get('grants', [])
+        links = []
+        for grant in grants:
+            # Add grant acronynm to the link:
+            #   [info:eu-repo/grantAgreement/EC/FP6/027819/] + [//ICEA]
+            eurepo = grant.get('identifiers', {}).get('eurepo')
+            links.append('{eurepo}//{acronym}'.format(
+                eurepo=eurepo, acronym=grant.get('acronym', '')))
+        return links or missing
+
+    def get_pids(self, obj):
+        """Get record PIDs."""
+        metadata = obj.get('metadata')
+        pids = [{'type': 'oai', 'value': metadata['_oai']['id']}]
+        if 'doi' in metadata:
+            pids.append({'type': 'doi', 'value': metadata['doi']})
+        return pids
+
+    def get_url(self, obj):
+        """Get record URL."""
+        # TODO: Zenodo or DOI URL? ("zenodo.org/..." or "doi.org/...")
+        return current_app.config['ZENODO_RECORDS_UI_LINKS_FORMAT'].format(
+            recid=obj['metadata']['recid'])
+
+    def get_publisher(self, obj):
+        """Get publisher."""
+        m = obj['metadata']
+        imprint_publisher = m.get('imprint', {}).get('publisher')
+        if imprint_publisher:
+            return imprint_publisher
+        part_publisher = m.get('part_of', {}).get('publisher')
+        if part_publisher:
+            return part_publisher
+        if m.get('doi', '').startswith('10.5281/'):
+            return 'Zenodo'
+        return missing

zenodo/modules/records/serializers/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Zenodo.
-# Copyright (C) 2016 CERN.
+# Copyright (C) 2016, 2017 CERN.
 #
 # Zenodo is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -47,6 +47,7 @@
     ZenodoDataCite31Serializer
 from zenodo.modules.records.serializers.marc21 import ZenodoMARCXMLSerializer
 from zenodo.modules.records.serializers.dc import ZenodoDublinCoreSerializer
+from zenodo.modules.openaire.schema import RecordSchemaOpenAIREJSON
 
 # Serializers
 # ===========
@@ -84,6 +85,8 @@
 csl_v1 = JSONSerializer(RecordSchemaCSLJSON, replace_refs=True)
 #: CSL Citation Formatter serializer
 citeproc_v1 = CiteprocSerializer(csl_v1)
+#: OpenAIRE JSON serializer
+openaire_json_v1 = JSONSerializer(RecordSchemaOpenAIREJSON, replace_refs=True)
 
 
 # Records-REST serializers
@@ -106,6 +109,9 @@
     csl_v1, 'application/vnd.citationstyles.csl+json')
 #: CSL Citation Formatter serializer for individual records.
 citeproc_v1_response = record_responsify(citeproc_v1, 'text/x-bibliography')
+#: OpenAIRE JSON serializer for individual records.
+openaire_json_v1_response = record_responsify(openaire_json_v1,
+                                              'application/x-openaire+json')
 
 
 #: JSON record serializer for search results.