Merge branch 'Research-Resource-CSV-Fix' into duplicate-orcid-affilia…

…tion-records

orcid_hub/__init__.py

@@ -135,6 +135,7 @@ def default(self, o):
         return super().default(o)
 
 
+app.config["JSON_AS_ASCII"] = False
 app.json_encoder = JSONEncoder
 
 

orcid_hub/apis.py

@@ -19,7 +19,7 @@
 from flask_swagger import swagger
 from rq import get_current_job
 
-from . import api, app, models, oauth, rq, schemas
+from . import api, app, models, oauth, rq, schemas, utils
 from .login_provider import roles_required
 from .models import (ORCID_ID_REGEX, AffiliationRecord, AsyncOrcidResponse, Client, FundingRecord,
                      OrcidToken, PeerReviewRecord, PropertyRecord, ResourceRecord, Role, Task,
@@ -2559,7 +2559,6 @@ class ResourceListAPI(TaskResource):
 
     def load_from_json(self, task=None):
         """Load records form the JSON upload."""
-        # breakpoint()
         return ResourceRecord.load(
             request.data.decode("utf-8"), filename=self.filename, task=task)
 
@@ -3450,15 +3449,18 @@ def exeute_orcid_call_async(method, url, data, headers):
     ar.save()
 
 
-@app.route("/api/v1/<string:orcid>/webhook", methods=["PUT", "DELETE"])
 @app.route("/api/v1/<string:orcid>/webhook/<path:callback_url>", methods=["PUT", "DELETE"])
+@app.route("/api/v1/<string:orcid>/webhook", methods=["PUT", "DELETE"])
+@app.route("/api/v1/webhook/<path:callback_url>", methods=["PUT", "DELETE"])
+@app.route("/api/v1/webhook", methods=["PUT", "POST", "PATCH"])
 @oauth.require_oauth()
-def register_webhook(orcid, callback_url=None):
+def register_webhook(orcid=None, callback_url=None):
     """Handle webhook registration for an individual user with direct client call-back."""
-    try:
-        validate_orcid_id(orcid)
-    except Exception as ex:
-        return jsonify({"error": "Missing or invalid ORCID iD.", "message": str(ex)}), 415
+    if orcid:
+        try:
+            validate_orcid_id(orcid)
+        except Exception as ex:
+            return jsonify({"error": "Missing or invalid ORCID iD.", "message": str(ex)}), 415
     if callback_url == "undefined":
         callback_url = None
     if callback_url:
@@ -3469,17 +3471,62 @@ def register_webhook(orcid, callback_url=None):
                 "message": f"Invalid call-back URL: {callback_url}"
             }), 415
 
-    try:
-        user = User.get(orcid=orcid)
-    except User.DoesNotExist:
-        return jsonify({
-            "error": "Invalid ORCID iD.",
-            "message": f"User with given ORCID ID '{orcid}' doesn't exist."
-        }), 404
+    if orcid:
+        try:
+            user = User.get(orcid=orcid)
+        except User.DoesNotExist:
+            return jsonify({
+                "error": "Invalid ORCID iD.",
+                "message": f"User with given ORCID ID '{orcid}' doesn't exist."
+            }), 404
 
-    orcid_resp = register_orcid_webhook(user, callback_url, delete=request.method == "DELETE")
-    resp = make_response('', orcid_resp.status_code if orcid_resp else 204)
-    if orcid_resp and "Location" in orcid_resp.headers:
-        resp.headers["Location"] = orcid_resp.headers["Location"]
+        orcid_resp = register_orcid_webhook(user, callback_url, delete=request.method == "DELETE")
+        resp = make_response('', orcid_resp.status_code if orcid_resp else 204)
+        if orcid_resp and "Location" in orcid_resp.headers:
+            resp.headers["Location"] = orcid_resp.headers["Location"]
+        return resp
 
-    return resp
+    else:
+        org = current_user.organisation
+        was_enabled = org.webhook_enabled
+
+        if request.method == "DELETE":
+            if org.webhook_enabled:
+                job = utils.disable_org_webhook.queue(org)
+                return jsonify({"job-id": job.id}), 200
+            return '', 204
+        else:
+            data = request.json or {}
+            enabled = data.get("enabled")
+            url = data.get("url", callback_url)
+            append_orcid = data.get("append-orcid")
+            apikey = data.get("apikey")
+            email_notifications_enabled = data.get("email-notifications-enabled")
+            notification_email = data.get("notification-email")
+
+            if url is not None:
+                org.webhook_url = url
+            if append_orcid is not None:
+                org.webhook_append_orcid = bool(append_orcid)
+            if apikey is not None:
+                org.webhook_apikey = apikey
+            if email_notifications_enabled is not None:
+                org.email_notifications_enabled = bool(email_notifications_enabled)
+            if notification_email is not None:
+                org.notification_email = notification_email
+            org.save()
+
+            data = {k: v for (k, v) in [
+                ("enabled", org.webhook_enabled),
+                ("url", org.webhook_url),
+                ("append-orcid", org.webhook_append_orcid),
+                ("apikey", org.webhook_apikey),
+                ("email-notifications-enabled", org.email_notifications_enabled),
+                ("notification-email", org.notification_email),
+            ] if v is not None}
+
+            if enabled or email_notifications_enabled:
+                job = utils.enable_org_webhook.queue(org)
+                data["job-id"] = job.id
+
+            return jsonify(data), 201 if (not was_enabled and enabled) else 200

orcid_hub/models.py

@@ -1002,16 +1002,11 @@ class UserOrg(AuditedModel):
     user = ForeignKeyField(User, on_delete="CASCADE", index=True, backref="user_orgs")
     org = ForeignKeyField(
         Organisation, on_delete="CASCADE", index=True, verbose_name="Organisation", backref="user_orgs")
-
     is_admin = BooleanField(
         null=True, default=False, help_text="User is an administrator for the organisation")
 
     # Affiliation bit-map:
     affiliations = SmallIntegerField(default=0, null=True, verbose_name="EDU Person Affiliations")
-    # created_by = ForeignKeyField(
-    #     User, on_delete="SET NULL", null=True, backref="created_user_orgs")
-    # updated_by = ForeignKeyField(
-    #     User, on_delete="SET NULL", null=True, backref="updated_user_orgs")
 
     # TODO: the access token should be either here or in a separate list
     # access_token = CharField(max_length=120, unique=True, null=True)
@@ -3503,7 +3498,7 @@ class Meta:  # noqa: D101,D106
 class Invitee(BaseModel):
     """Common model bits of the invitees records."""
 
-    identifier = CharField(max_length=120, verbose_name="Local Identifier", null=True)
+    identifier = CharField(max_length=120, null=True, verbose_name="Local Identifier")
     email = CharField(max_length=120, null=True)
     orcid = OrcidIdField(null=True)
     first_name = CharField(max_length=120, null=True)
@@ -4038,7 +4033,7 @@ def index(ex):
 
         def val(row, column, default=None):
             idx = idxs.get(column)
-            if not idx or idx < 0 or idx >= len(row):
+            if idx is None or idx < 0 or idx >= len(row):
                 return default
             return row[idx].strip() or default
 

orcid_hub/templates/header.html

@@ -80,14 +80,16 @@
                                                                                                        title="Import a works batch file">Works</a></li>
                     <li id="load_peer_review"><a href="{{ url_for('load_researcher_peer_review')}}"  data-toggle="tooltip"
                                                                                         title="Import a peer reviews batch file">Peer Reviews</a></li>
-                    <li id="load_researcher_urls"><a href="{{ url_for('load_researcher_urls')}}"  data-toggle="tooltip"
+                    <li id="load_properties"><a href="{{ url_for('load_properties')}}"  data-toggle="tooltip"
+                                                                    title="Import a researcher properties batch file (url, other names, keywords, and/or countries)">Researcher Properties</a></li>
+                    <!-- li id="load_researcher_urls"><a href="{{ url_for('load_researcher_urls')}}"  data-toggle="tooltip"
                                                                     title="Import a researcher urls batch file">Researcher Urls</a></li>
                     <li id="load_other_names"><a href="{{ url_for('load_other_names')}}"  data-toggle="tooltip"
                                                                     title="Import researcher other names batch file">Researcher Other Names</a></li>
                     <li id="load_keyword"><a href="{{ url_for('load_keyword')}}"  data-toggle="tooltip"
                                                                                         title="Import researcher keyword batch file">Researcher Keyword</a></li>
                     <li id="load_country"><a href="{{ url_for('load_country')}}"  data-toggle="tooltip"
-                                                                                        title="Import researcher Country batch file">Researcher Country</a></li>
+                                                                                        title="Import researcher Country batch file">Researcher Country</a></li -->
                     <li id="load_other_ids"><a href="{{ url_for('load_other_ids')}}"  data-toggle="tooltip"
                                                                                         title="Import researcher Other IDs batch file">Researcher Other IDs</a></li>
                     <li id="_load_task_RESOURCE"><a href="{{ url_for('load_task', task_type='RESOURCE')}}" data-toggle="tooltip" title="Import a research resource batch file">Resources</a></li>

orcid_hub/utils.py

@@ -27,11 +27,11 @@
 
 from . import app, db, orcid_client, rq
 from .models import (AFFILIATION_TYPES, Affiliation, AffiliationRecord, Delegate, FundingInvitee,
-                     FundingRecord, Log, MailLog, MessageRecord, NestedDict, Invitee, OtherIdRecord,
-                     OrcidToken, Organisation, OrgInvitation, PartialDate, PeerReviewExternalId,
-                     PeerReviewInvitee, PeerReviewRecord, PropertyRecord, ResourceRecord, Role,
-                     Task, TaskType, User, UserInvitation, UserOrg, WorkInvitee, WorkRecord,
-                     get_val, readup_file)
+                     FundingRecord, Log, MailLog, MessageRecord, NestedDict, Invitee,
+                     OtherIdRecord, OrcidToken, Organisation, OrgInvitation, PartialDate,
+                     PeerReviewExternalId, PeerReviewInvitee, PeerReviewRecord, PropertyRecord,
+                     RecordInvitee, ResourceRecord, Role, Task, TaskType, User, UserInvitation,
+                     UserOrg, WorkInvitee, WorkRecord, get_val, readup_file)
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -525,10 +525,10 @@ def create_or_update_resources(user, org_id, records, *args, **kwargs):
         OrcidToken.user_id == user.id, OrcidToken.org_id == org.id,
         OrcidToken.scopes.contains("/activities/update")).first()
     api = orcid_client.MemberAPIV3(org, user, access_token=token.access_token)
-    resources = api.get_resources().get("group")
+    resources = api.get_resources()
 
     if resources:
-
+        resources = resources.get("group")
         resources = [
             r for r in resources if any(
                 rr.get("source", "source-client-id", "path") == org.orcid_client_id
@@ -552,7 +552,6 @@ def match_record(records, record):
                     # if all(eid.get("external-id-value") != record.external_id_value
                     #         for eid in rr.get("proposal", "external-ids", "external-id")):
                     #     continue
-
                     if put_code in taken_put_codes:
                         continue
 
@@ -581,11 +580,18 @@ def match_record(records, record):
                     resp = api.post("research-resource", rr.orcid_research_resource)
 
                 if resp.status == 201:
+                    orcid, put_code = resp.headers["Location"].split("/")[-3::2]
                     rr.add_status_line("ORCID record was created.")
                 else:
+                    orcid = user.orcid
                     rr.add_status_line("ORCID record was updated.")
-                if not put_code:
-                    rr.put_code = resp.headers["Location"].split('/')[-1]
+                if not rr.put_code and put_code:
+                    rr.put_code = int(put_code)
+                if not rr.orcid and orcid:
+                    rr.orcid = orcid
+                visibility = json.loads(resp.data).get("visibility") if hasattr(resp, "data") else None
+                if rr.visibility != visibility:
+                    rr.visibility = visibility
 
             except ApiException as ex:
                 if ex.status == 404:
@@ -2019,6 +2025,7 @@ def process_resource_records(max_rows=20, record_id=None):
             tasks = tasks.where(ResourceRecord.id.in_(record_id))
         else:
             tasks = tasks.where(ResourceRecord.id == record_id)
+
     for (task_id, org_id, user), tasks_by_user in groupby(tasks, lambda t: (
             t.id,
             t.org_id,
@@ -2304,7 +2311,6 @@ def register_orcid_webhook(user, callback_url=None, delete=False):
     If URL is given, it will be used for as call-back URL.
     """
     local_handler = (callback_url is None)
-
     # Don't delete the webhook if there is anyther organisation with enabled webhook:
     if local_handler and delete and user.organisations.where(Organisation.webhook_enabled).count() > 0:
         return
@@ -2341,6 +2347,7 @@ def notify_about_update(user, event_type="UPDATED"):
                                          user.orcid,
                                          user.created_at or user.updated_at,
                                          user.updated_at or user.created_at,
+                                         apikey=org.webhook_apikey,
                                          event_type=event_type,
                                          append_orcid=org.webhook_append_orcid)
 
@@ -2358,7 +2365,7 @@ def notify_about_update(user, event_type="UPDATED"):
 
 @rq.job(timeout=300)
 def invoke_webhook_handler(webhook_url=None, orcid=None, created_at=None, updated_at=None, message=None,
-                           event_type="UPDATED", url=None, attempts=5, append_orcid=False):
+                           apikey=None, event_type="UPDATED", url=None, attempts=5, append_orcid=False):
     """Propagate 'updated' event to the organisation event handler URL."""
     if not message:
         url = app.config["ORCID_BASE_URL"] + orcid
@@ -2386,7 +2393,10 @@ def invoke_webhook_handler(webhook_url=None, orcid=None, created_at=None, update
             url += orcid
 
     try:
-        resp = requests.post(url, json=message)
+        if apikey:
+            resp = requests.post(url, json=message, headers=dict(apikey=apikey))
+        else:
+            resp = requests.post(url, json=message)
     except:
         if attempts == 1:
             raise
@@ -2396,6 +2406,7 @@ def invoke_webhook_handler(webhook_url=None, orcid=None, created_at=None, update
             invoke_webhook_handler.schedule(timedelta(minutes=5 *
                                                       (6 - attempts) if attempts < 6 else 5),
                                             message=message,
+                                            apikey=apikey,
                                             url=url,
                                             attempts=attempts - 1)
         else:
@@ -2408,7 +2419,7 @@ def enable_org_webhook(org):
     """Enable Organisation Webhook."""
     org.webhook_enabled = True
     org.save()
-    for u in org.users.where(User.webhook_enabled.NOT()):
+    for u in org.users.where(User.webhook_enabled.NOT(), User.orcid.is_null(False)):
         register_orcid_webhook.queue(u)
 
 
@@ -2417,7 +2428,7 @@ def disable_org_webhook(org):
     """Disable Organisation Webhook."""
     org.webhook_enabled = False
     org.save()
-    for u in org.users.where(User.webhook_enabled):
+    for u in org.users.where(User.webhook_enabled, User.orcid.is_null(False)):
         register_orcid_webhook.queue(u, delete=True)
 
 
@@ -2518,7 +2529,7 @@ def dump_yaml(data):
     """Dump the objects into YAML representation."""
     yaml.add_representer(datetime, SafeRepresenterWithISODate.represent_datetime, Dumper=Dumper)
     yaml.add_representer(defaultdict, SafeRepresenter.represent_dict)
-    return yaml.dump(data)
+    return yaml.dump(data, allow_unicode=True)
 
 
 def enqueue_user_records(user):
@@ -2540,6 +2551,11 @@ def enqueue_user_records(user):
             records = records.join(WorkInvitee).where(
                 (WorkInvitee.email.is_null() | (WorkInvitee.email == user.email)),
                 (WorkInvitee.orcid.is_null() | (WorkInvitee.orcid == user.orcid)))
+        elif task.task_type == TaskType.RESOURCE and task.is_raw:
+            invitee_model = task.record_model.invitees.rel_model
+            records = records.join(RecordInvitee).join(Invitee).where(
+                (invitee_model.email.is_null() | (invitee_model.email == user.email)),
+                (invitee_model.orcid.is_null() | (invitee_model.orcid == user.orcid)))
         else:
             records = records.where(
                 (task.record_model.email.is_null() | (task.record_model.email == user.email)),