import JSON/YAML files and tests

orcid_hub/config.py

@@ -139,3 +139,6 @@
 # rq-dashboard config:
 RQ_POLL_INTERVAL = 5000  #: Web interface poll period for updates in ms
 WEB_BACKGROUND = "gray"
+
+# Other flask settings
+JSONIFY_PRETTYPRINT_REGULAR = True

orcid_hub/models.py

@@ -1161,6 +1161,8 @@ def records(self):
         """Get all task record query."""
         if not self.task_type or self.task_type in [TaskType.SYNC, TaskType.NONE]:
             return None
+        if self.is_raw:
+            return MessageRecord.select().where(MessageRecord.task == self)
         return getattr(self, self.task_type.name.lower() + "_records")
 
     @lazy_property
@@ -4094,20 +4096,107 @@ def orcid_research_resource(self):
     def to_export_dict(self):
         """Map the funding record to dict for export into JSON/YAML."""
         d = self.orcid_research_resource
-        d["invitee"] = Invitee.to_export_dict(self)
+        d["invitees"] = [Invitee.to_export_dict(self), ]
         return d
 
-
-class Record(RecordModel):
+    @classmethod
+    def load(
+            cls, data, task=None, task_id=None, filename=None, override=True,
+            skip_schema_validation=False, org=None):
+        """Load ORCID message record task form JSON/YAML."""
+        return MessageRecord.load(data,
+                                  filename=filename,
+                                  override=True,
+                                  skip_schema_validation=True,
+                                  org=org or current_user.organisation,
+                                  task_type=TaskType.RESOURCE,
+                                  version="3.0")
+
+
+class MessageRecord(RecordModel):
     """ORCID message loaded from structured batch task file."""
 
+    task = ForeignKeyField(Task, backref="message_records", on_delete="CASCADE")
     version = CharField(null=True)
-    type = CharField()
+    # type = CharField()
     message = TextField()
     invitees = ManyToManyField(Invitee, backref="records")
 
+    @classmethod
+    def load(cls, data, task=None, task_id=None, filename=None, override=True,
+             skip_schema_validation=False, org=None, task_type=None, version="3.0"):
+        """Load ORCID message record task form JSON/YAML."""
+        if isinstance(data, str):
+            data = json.loads(data) if filename.lower().endswith(".json") else yaml.load(data)
+        if org is None:
+            org = current_user.organisation if current_user else None
+        if not skip_schema_validation:
+            jsonschema.validate(data, schemas.affiliation_task)
+        if not task and task_id:
+            task = Task.select().where(Task.id == task_id).first()
+        if not task and "id" in data and override and task_type:
+            task_id = int(data["id"])
+            task = Task.select().where(
+                    Task.id == task_id,
+                    Task.task_type == task_type,
+                    Task.is_raw).first()
+        if not filename:
+            filename = (data.get("filename") or datetime.utcnow().isoformat(timespec="seconds"))
+        if task and not task_type:
+            task_type = task.task_type
+        if not task_type and isinstance(data, dict) and "type" in data:
+            task_type = TaskType[data["type"].upper()]
+
+        if isinstance(data, dict):
+            records = data.get("records")
+        else:
+            records = data
+
+        with db.atomic() as transaction:
+            try:
+                if not task:
+                    task = Task.create(
+                            org=org, filename=filename, task_type=task_type, is_raw=True)
+                elif override:
+                    task.record_model.delete().where(task.record_model.task == task).execute()
+
+                is_enqueue = False
+
+                for r in records:
+                    invitees = r.get("invitees")
+                    if not invitees:
+                        raise ModelException(f"Missing invitees, expected to have at lease one: {r}")
+                    del(r["invitees"])
+
+                    message = json.dumps(r, indent=2)
+                    if "id" in r and not override:
+                        rec = cls.get(int(r["id"]))
+                        if rec.message != message:
+                            rec.message = message
+                        if rec.version != version:
+                            rec.version = version
+                    else:
+                        rec = cls.create(task=task, version=version, message=message)
+                    rec.invitees.add([
+                        Invitee.create(
+                            orcid=i.get("ORCID-iD"),
+                            email=i.get("email"),
+                            first_name=i.get("first-name"),
+                            last_name=i.get("last-name"),
+                            put_code=i.get("put-code"),
+                            visibility=i.get("visibility")) for i in invitees])
+
+                if is_enqueue:
+                    from .utils import enqueue_task_records
+                    enqueue_task_records(task)
+            except:
+                transaction.rollback()
+                app.logger.exception("Failed to load affiliation record task file.")
+                raise
+        return task
+
 
-RecordInvitee = Record.invitees.get_through_model()
+RecordInvitee = MessageRecord.invitees.get_through_model()
 
 
 class Delegate(BaseModel):
@@ -4377,7 +4466,7 @@ def create_tables(safe=True, drop=False):
             Token,
             Delegate,
             AsyncOrcidResponse,
-            Record,
+            MessageRecord,
             Invitee,
             RecordInvitee,
             ResourceRecord,

orcid_hub/templates/view_tasks.html

@@ -30,10 +30,16 @@
     <a class="icon" href="{{ url_for('propertyrecord.index_view', task_id=row.id, url=request.url) }}" title="View Details">
       <span class="fa fa-eye glyphicon glyphicon-eye-open"></span>
     </a>
-{% elif row.task_type == 9 %}
+  {% elif row.task_type == 9 %}
+    {% if row.is_raw %}
     <a class="icon" href="{{ url_for('resourcerecord.index_view', task_id=row.id, url=request.url) }}" title="View Details">
       <span class="fa fa-eye glyphicon glyphicon-eye-open"></span>
     </a>
+    {% else %}
+    <a class="icon" href="{{ url_for('messagerecord.index_view', task_id=row.id, url=request.url) }}" title="View Details">
+      <span class="fa fa-eye glyphicon glyphicon-eye-open"></span>
+    </a>
+    {% endif %}
 {% elif row.task_type == 11 %}
     <a class="icon" href="{{ url_for('sync_profiles', task_id=row.id, url=request.url) }}" title="View Details">
       <span class="fa fa-eye glyphicon glyphicon-eye-open"></span>

orcid_hub/views.py

@@ -2119,6 +2119,7 @@ def export(self, export_type):
 admin.add_view(ProfilePropertyRecordAdmin(PropertyRecord))
 admin.add_view(ProfilePropertyRecordAdmin(OtherIdRecord))
 admin.add_view(ResourceRecordAdmin())
+admin.add_view(ResourceRecordAdmin(models.MessageRecord))
 admin.add_view(ViewMembersAdmin(name="viewmembers", endpoint="viewmembers"))
 
 admin.add_view(UserOrgAmin(UserOrg))
@@ -2856,7 +2857,8 @@ def load_task(task_type):
                 task = record_model.load(content, filename=filename)
 
             flash(f"Successfully loaded {task.record_count} rows.")
-            return redirect(url_for(task_type.name.lower() + "record.index_view", task_id=task.id))
+            task_view = ("message" if task.is_raw else task_type.name.lower()) + "record.index_view"
+            return redirect(url_for(task_view, task_id=task.id))
         except (
                 ValueError,
                 ModelException,

tests/test_views.py

@@ -4490,16 +4490,22 @@ def test_research_resources(client, mocker):
     for export_type in ["csv", "tsv", "json", "yaml"]:
         resp = client.get(f"/admin/resourcerecord/export/{export_type}/?task_id={task.id}")
         file_name = f"resources_reimport.{export_type}"
-        if export_type not in ["json", "yaml"]:
-            resp = client.post(
-                "/load/task/RESOURCE",
-                data={"file_": (BytesIO(resp.data), file_name)},
-                follow_redirects=True)
-            assert resp.status_code == 200
-            assert b"SYN-18-UOA-001" in resp.data
+        resp = client.post(
+            "/load/task/RESOURCE",
+            data={"file_": (BytesIO(resp.data), file_name)},
+            follow_redirects=True)
+        assert resp.status_code == 200
+        assert b"SYN-18-UOA-001" in resp.data
+
+        t = Task.get(filename=file_name)
+        assert t.records.count() == 2
+        if export_type in ["json", "yaml"]:
+            assert t.is_raw
+            assert t.record_model == models.MessageRecord
+        else:
             assert b"invitee1@mailinator.com" in resp.data
-            t = Task.get(filename=file_name)
-            assert t.records.count() == 2
+            assert not t.is_raw
+            assert t.record_model == models.ResourceRecord
 
     send = mocker.patch("emails.message.MessageSendMixin.send")