Merge branch 'feature/files-on-anything' of https://github.com/Center…

…ForOpenScience/osf.io into feature/node-preprint

CHANGELOG

@@ -2,8 +2,22 @@
 Changelog
 *********
 
+0.146.0 (2018-07-02)
+====================
+
+- Migrate from EZID to Datacite (for nodes) and Crossref (for preprints) for creating DOIs.
+
+0.145.0 (2018-06-27)
+====================
+
+- Add more metadata and relationships to API requests in support of Analytics page updates
+- Allow PATCHing of View Only Links in API v2
+- Create a scopes endpoint for User PAT Settings updates
+- Improve /v2/ route to allow for fewer requests by spaces
+
 0.144.0 (2018-06-13)
 ====================
+
 - Allow deleting child components from component widget dropdown
 - Make preprints checkable for spam
 - Add fields required for preprint withdrawal

addons/wiki/static/wikiPage.js

@@ -86,7 +86,13 @@ function ViewWidget(visible, version, viewText, rendered, contentURL, allowMathj
                 request.done(function (resp) {
                     if(self.visible()) {
                         var $markdownElement = $('#wikiViewRender');
-                        var rawContent = resp.wiki_content || '*Add important information, links, or images here to describe your project.*';
+                        if (resp.wiki_content){
+                            var rawContent = resp.wiki_content
+                        } else if(window.contextVars.currentUser.canEdit) {
+                            var rawContent = '*Add important information, links, or images here to describe your project.*';
+                        } else {
+                            var rawContent = '*No wiki content.*';
+                        }
                         if (resp.rendered_before_update) {
                             // Use old md renderer. Don't mathjaxify
                             self.allowMathjaxification(false);

admin/nodes/views.py

@@ -419,7 +419,7 @@ def get_context_data(self, **kwargs):
     def delete(self, request, *args, **kwargs):
         node = self.get_object()
         node.confirm_ham(save=True)
-        osf_admin_change_status_identifier(node, 'public')
+        osf_admin_change_status_identifier(node)
         update_admin_log(
             user_id=self.request.user.id,
             object_id=node._id,

api/base/serializers.py

@@ -136,6 +136,13 @@ def should_show(self, instance):
         return request and request.user == instance
 
 
+class ShowIfAdminScopeOrAnonymous(ConditionalField):
+
+    def should_show(self, instance):
+        request = self.context.get('request')
+        return request and (request.user.is_anonymous or utils.has_admin_scope(request))
+
+
 class HideIfRegistration(ConditionalField):
     """
     If node is a registration, this field will return None.

api/base/urls.py

@@ -16,6 +16,7 @@
                 url(r'^', include('waffle.urls')),
                 url(r'^wb/', include('api.wb.urls', namespace='wb')),
                 url(r'^banners/', include('api.banners.urls', namespace='banners')),
+                url(r'^crossref/', include('api.crossref.urls', namespace='crossref')),
             ],
         )
         ),
@@ -44,6 +45,7 @@
                 url(r'^providers/', include('api.providers.urls', namespace='providers')),
                 url(r'^registrations/', include('api.registrations.urls', namespace='registrations')),
                 url(r'^requests/', include('api.requests.urls', namespace='requests')),
+                url(r'^scopes/', include('api.scopes.urls', namespace='scopes')),
                 url(r'^search/', include('api.search.urls', namespace='search')),
                 url(r'^subscriptions/', include('api.subscriptions.urls', namespace='subscriptions')),
                 url(r'^taxonomies/', include('api.taxonomies.urls', namespace='taxonomies')),

api/base/views.py

@@ -34,7 +34,8 @@
 from api.users.serializers import UserSerializer
 from framework.auth.oauth_scopes import CoreScopes
 from osf.models import Contributor, MaintenanceState, BaseFileNode
-
+from waffle.models import Flag
+from waffle import flag_is_active
 
 class JSONAPIBaseView(generics.GenericAPIView):
 
@@ -394,18 +395,21 @@ def create(self, *args, **kwargs):
 def root(request, format=None, **kwargs):
     """
     The documentation for the Open Science Framework API can be found at [developer.osf.io](https://developer.osf.io).
+    The contents of this endpoint are variable and subject to change without notification.
     """
     if request.user and not request.user.is_anonymous:
         user = request.user
         current_user = UserSerializer(user, context={'request': request}).data
     else:
         current_user = None
+    flags = [name for name in Flag.objects.values_list('name', flat=True) if flag_is_active(request, name)]
     kwargs = request.parser_context['kwargs']
     return_val = {
         'meta': {
             'message': 'Welcome to the OSF API.',
             'version': request.version,
             'current_user': current_user,
+            'active_flags': flags,
         },
         'links': {
             'nodes': utils.absolute_reverse('nodes:node-list', kwargs=kwargs),

api/citations/utils.py

@@ -181,7 +181,7 @@ def remove_extra_period_after_right_quotation(cit):
 
 def chicago_reformat(node, cit):
     cit = remove_extra_period_after_right_quotation(cit)
-    new_csl = cit.split(str(node.csl['issued']['date-parts'][0][0]))
+    new_csl = cit.split(str(node.csl['issued']['date-parts'][0][0]), 1)
     contributors_list = list(node.visible_contributors)
     contributors_list_length = len(contributors_list)
     # throw error if there is no visible contributor

api/crossref/permissions.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+import hmac
+import hashlib
+
+from rest_framework import permissions
+from rest_framework import exceptions
+
+from framework import sentry
+from website import settings
+
+
+class RequestComesFromMailgun(permissions.BasePermission):
+    """Verify that request comes from Mailgun.
+    Adapted here from conferences/message.py
+    Signature comparisons as recomended from mailgun docs:
+    https://documentation.mailgun.com/en/latest/user_manual.html#webhooks
+    """
+    def has_permission(self, request, view):
+        if request.method != 'POST':
+            raise exceptions.MethodNotAllowed(method=request.method)
+        data = request.data
+        if not data:
+            raise exceptions.ParseError('Request body is empty')
+        if not settings.MAILGUN_API_KEY:
+            return False
+        signature = hmac.new(
+            key=settings.MAILGUN_API_KEY,
+            msg='{}{}'.format(
+                data['timestamp'],
+                data['token'],
+            ),
+            digestmod=hashlib.sha256,
+        ).hexdigest()
+        if 'signature' not in data:
+            error_message = 'Signature required in request body'
+            sentry.log_message(error_message)
+            raise exceptions.ParseError(error_message)
+        if not hmac.compare_digest(unicode(signature), unicode(data['signature'])):
+            raise exceptions.ParseError('Invalid signature')
+        return True

api/crossref/urls.py

@@ -0,0 +1,9 @@
+from django.conf.urls import url
+
+from api.crossref import views
+
+app_name = 'osf'
+
+urlpatterns = [
+    url(r'^email/$', views.ParseCrossRefConfirmation.as_view(), name=views.ParseCrossRefConfirmation.view_name),
+]

api/crossref/views.py

@@ -0,0 +1,86 @@
+import logging
+
+import lxml.etree
+from django.http import HttpResponse
+from django.views.decorators.csrf import csrf_exempt
+from rest_framework.views import APIView
+
+from api.crossref.permissions import RequestComesFromMailgun
+from framework.auth.views import mails
+from osf.models import Preprint
+from website import settings
+
+logger = logging.getLogger(__name__)
+
+
+class ParseCrossRefConfirmation(APIView):
+    # This view goes under the _/ namespace
+    view_name = 'parse_crossref_confirmation_email'
+    view_category = 'identifiers'
+
+    permission_classes = (
+        RequestComesFromMailgun,
+    )
+
+    @csrf_exempt
+    def dispatch(self, request, *args, **kwargs):
+        return super(ParseCrossRefConfirmation, self).dispatch(request, *args, **kwargs)
+
+    def get_serializer_class(self):
+        return None
+
+    def post(self, request):
+        crossref_email_content = lxml.etree.fromstring(str(request.POST['body-plain']))
+        status = crossref_email_content.get('status').lower()  # from element doi_batch_diagnostic
+        record_count = int(crossref_email_content.find('batch_data/record_count').text)
+        records = crossref_email_content.xpath('//record_diagnostic')
+        dois_processed = 0
+
+        if status == 'completed':
+            guids = []
+            # Keep track of errors recieved, ignore those that are handled
+            unexpected_errors = False
+            for record in records:
+                doi = getattr(record.find('doi'), 'text', None)
+                guid = doi.split('/')[-1] if doi else None
+                guids.append(guid)
+                preprint = Preprint.load(guid) if guid else None
+                if record.get('status').lower() == 'success' and doi:
+                    msg = record.find('msg').text
+                    created = bool(msg == 'Successfully added')
+                    legacy_doi = preprint.get_identifier(category='legacy_doi')
+                    if created or legacy_doi:
+                        # Sets preprint_doi_created and saves the preprint
+                        preprint.set_identifier_values(doi=doi, save=True)
+                    # Double records returned when possible matching DOI is found in crossref
+                    elif 'possible preprint/vor pair' not in msg.lower():
+                        # Directly updates the identifier
+                        preprint.set_identifier_value(category='doi', value=doi)
+
+                    dois_processed += 1
+
+                    # Mark legacy DOIs overwritten by newly batch confirmed crossref DOIs
+                    if legacy_doi:
+                        legacy_doi.remove()
+
+                elif record.get('status').lower() == 'failure':
+                    if 'Relation target DOI does not exist' in record.find('msg').text:
+                        logger.warn('Related publication DOI does not exist, sending metadata again without it...')
+                        client = preprint.get_doi_client()
+                        client.create_identifier(preprint, category='doi', include_relation=False)
+                    else:
+                        unexpected_errors = True
+            logger.info('Creation success email received from CrossRef for preprints: {}'.format(guids))
+
+        if dois_processed != record_count or status != 'completed':
+            if unexpected_errors:
+                batch_id = crossref_email_content.find('batch_id').text
+                mails.send_mail(
+                    to_addr=settings.OSF_SUPPORT_EMAIL,
+                    mail=mails.CROSSREF_ERROR,
+                    batch_id=batch_id,
+                    email_content=request.POST['body-plain'],
+                )
+                logger.error('Error submitting metadata for batch_id {} with CrossRef, email sent to help desk'.format(batch_id))
+
+        return HttpResponse('Mail received', status=200)

api/identifiers/serializers.py

@@ -5,7 +5,7 @@
 
 class RegistrationIdentifierSerializer(JSONAPISerializer):
 
-    category = ser.CharField(read_only=True)
+    category = ser.SerializerMethodField()
 
     filterable_fields = frozenset(['category'])
 
@@ -23,6 +23,11 @@ class RegistrationIdentifierSerializer(JSONAPISerializer):
     class Meta:
         type_ = 'identifiers'
 
+    def get_category(self, obj):
+        if obj.category == 'legacy_doi':
+            return 'doi'
+        return obj.category
+
     def get_absolute_url(self, obj):
         return obj.absolute_api_v2_url
 

api/licenses/serializers.py

@@ -16,6 +16,7 @@ class LicenseSerializer(JSONAPISerializer):
     type = TypeField()
     name = ser.CharField(required=True, help_text='License name')
     text = ser.CharField(required=True, help_text='Full text of the license')
+    url = ser.URLField(required=False, help_text='URL for the license')
     required_fields = ser.ListField(source='properties', read_only=True,
                                     help_text='Fields required for this license (provided to help front-end validators)')
     links = LinksField({'self': 'get_absolute_url'})

api/nodes/serializers.py

@@ -9,7 +9,7 @@
                                   NodeFileHyperLinkField, RelationshipField,
                                   ShowIfVersion, TargetTypeField, TypeField,
                                   WaterbutlerLink, relationship_diff, BaseAPISerializer,
-                                  HideIfWikiDisabled)
+                                  HideIfWikiDisabled, ShowIfAdminScopeOrAnonymous)
 from api.base.settings import ADDONS_FOLDER_CONFIGURABLE
 from api.base.utils import (absolute_reverse, get_object_or_error,
                             get_user_auth, is_truthy)
@@ -24,7 +24,6 @@
 from rest_framework import exceptions
 from addons.base.exceptions import InvalidAuthError, InvalidFolderError
 from website.exceptions import NodeStateError
-from osf.exceptions import PreprintStateError
 from osf.models import (Comment, DraftRegistration, Institution,
                         MetaSchema, AbstractNode, PrivateLink)
 from osf.models.external import ExternalAccount
@@ -212,14 +211,14 @@ class NodeSerializer(TaxonomizableSerializerMixin, JSONAPISerializer):
     tags = ValuesListField(attr_name='name', child=ser.CharField(), required=False)
     access_requests_enabled = ser.BooleanField(read_only=False, required=False)
     node_license = NodeLicenseSerializer(required=False, source='license')
+    analytics_key = ShowIfAdminScopeOrAnonymous(ser.CharField(read_only=True, source='keenio_read_key'))
     template_from = ser.CharField(required=False, allow_blank=False, allow_null=False,
                                   help_text='Specify a node id for a node you would like to use as a template for the '
                                             'new node. Templating is like forking, except that you do not copy the '
                                             'files, only the project structure. Some information is changed on the top '
                                             'level project by submitting the appropriate fields in the request body, '
                                             'and some information will not change. By default, the description will '
                                             'be cleared and the project will be made private.')
-
     current_user_can_comment = ser.SerializerMethodField(help_text='Whether the current user is allowed to post comments')
     current_user_permissions = ser.SerializerMethodField(help_text='List of strings representing the permissions '
                                                                    'for the current user on this node.')
@@ -288,7 +287,8 @@ class NodeSerializer(TaxonomizableSerializerMixin, JSONAPISerializer):
 
     forks = RelationshipField(
         related_view='nodes:node-forks',
-        related_view_kwargs={'node_id': '<_id>'}
+        related_view_kwargs={'node_id': '<_id>'},
+        related_meta={'count': 'get_forks_count'}
     )
 
     node_links = ShowIfVersion(RelationshipField(
@@ -298,6 +298,18 @@ class NodeSerializer(TaxonomizableSerializerMixin, JSONAPISerializer):
         help_text='This feature is deprecated as of version 2.1. Use linked_nodes instead.'
     ), min_version='2.0', max_version='2.0')
 
+    linked_by_nodes = RelationshipField(
+        related_view='nodes:node-linked-by-nodes',
+        related_view_kwargs={'node_id': '<_id>'},
+        related_meta={'count': 'get_linked_by_nodes_count'},
+    )
+
+    linked_by_registrations = RelationshipField(
+        related_view='nodes:node-linked-by-registrations',
+        related_view_kwargs={'node_id': '<_id>'},
+        related_meta={'count': 'get_linked_by_registrations_count'},
+    )
+
     parent = RelationshipField(
         related_view='nodes:node-detail',
         related_view_kwargs={'node_id': '<parent_node._id>'},
@@ -460,6 +472,15 @@ def get_registration_links_count(self, obj):
                 count += 1
         return count
 
+    def get_linked_by_nodes_count(self, obj):
+        return obj._parents.filter(is_node_link=True, parent__is_deleted=False, parent__type='osf.node').count()
+
+    def get_linked_by_registrations_count(self, obj):
+        return obj._parents.filter(is_node_link=True, parent__type='osf.registration', parent__retraction__isnull=True).count()
+
+    def get_forks_count(self, obj):
+        return obj.forks.exclude(type='osf.registration').exclude(is_deleted=True).count()
+
     def get_unread_comments_count(self, obj):
         user = get_user_auth(self.context['request']).user
         node_comments = Comment.find_n_unread(user=user, node=obj, page='node')
@@ -969,10 +990,7 @@ def update(self, instance, validated_data):
             if index is not None:
                 node.move_contributor(instance.user, auth, index, save=True)
             node.update_contributor(instance.user, permission, bibliographic, auth, save=True)
-        except NodeStateError as e:
-            raise exceptions.ValidationError(detail=e.message)
-        except PreprintStateError as e:
-            # For use in PreprintContributorDetailSerializer
+        except node.state_error as e:
             raise exceptions.ValidationError(detail=e.message)
         except ValueError as e:
             raise exceptions.ValidationError(detail=e.message)

api/nodes/urls.py

@@ -34,6 +34,8 @@
     url(r'^(?P<node_id>\w+)/logs/$', views.NodeLogList.as_view(), name=views.NodeLogList.view_name),
     url(r'^(?P<node_id>\w+)/node_links/$', views.NodeLinksList.as_view(), name=views.NodeLinksList.view_name),
     url(r'^(?P<node_id>\w+)/node_links/(?P<node_link_id>\w+)/', views.NodeLinksDetail.as_view(), name=views.NodeLinksDetail.view_name),
+    url(r'^(?P<node_id>\w+)/linked_by_nodes/$', views.NodeLinkedByNodesList.as_view(), name=views.NodeLinkedByNodesList.view_name),
+    url(r'^(?P<node_id>\w+)/linked_by_registrations/$', views.NodeLinkedByRegistrationsList.as_view(), name=views.NodeLinkedByRegistrationsList.view_name),
     url(r'^(?P<node_id>\w+)/preprints/$', views.NodePreprintsList.as_view(), name=views.NodePreprintsList.view_name),
     url(r'^(?P<node_id>\w+)/registrations/$', views.NodeRegistrationsList.as_view(), name=views.NodeRegistrationsList.view_name),
     url(r'^(?P<node_id>\w+)/relationships/institutions/$', views.NodeInstitutionsRelationship.as_view(), name=views.NodeInstitutionsRelationship.view_name),