Merge c641de5 into fb21317

.gitignore

@@ -67,3 +67,5 @@ docs/_build/
 local_settings.py
 settings.yaml
 *.sq3
+#IDEs
+.idea

.secrets.baseline

@@ -391,9 +391,9 @@
         "filename": "tests/test_drs.py",
         "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
         "is_verified": false,
-        "line_number": 31
+        "line_number": 37
       }
     ]
   },
-  "generated_at": "2023-03-23T20:46:39Z"
+  "generated_at": "2023-04-10T21:30:00Z"
 }

indexd/drs/blueprint.py

@@ -111,7 +111,7 @@ def indexd_to_drs(record, expand=False):
 
     name = record["file_name"] if "file_name" in record else record["name"]
 
-    created_time = (
+    index_created_time = (
         record["created_date"] if "created_date" in record else record["created_time"]
     )
 
@@ -123,10 +123,14 @@ def indexd_to_drs(record, expand=False):
         else ""
     )
 
-    updated_date = (
+    index_updated_time = (
         record["updated_date"] if "updated_date" in record else record["updated_time"]
     )
 
+    created_time = record.get("content_created_date", "")
+
+    updated_time = record.get("content_updated_date", "")
+
     form = record["form"] if "form" in record else "bundle"
 
     description = record["description"] if "description" in record else None
@@ -141,11 +145,12 @@ def indexd_to_drs(record, expand=False):
 
     drs_object = {
         "id": did,
-        "description": "",
         "mime_type": "application/json",
         "name": name,
+        "index_created_time": index_created_time,
+        "index_updated_time": index_updated_time,
         "created_time": created_time,
-        "updated_time": updated_date,
+        "updated_time": updated_time,
         "size": record["size"],
         "aliases": alias,
         "self_uri": self_uri,

indexd/index/blueprint.py

@@ -407,6 +407,19 @@ def post_index_record():
     version = flask.request.json.get("version")
     baseid = flask.request.json.get("baseid")
     uploader = flask.request.json.get("uploader")
+    description = flask.request.json.get("description")
+    created_time = flask.request.json.get("created_time")
+    updated_time = flask.request.json.get("updated_time")
+
+    if updated_time is None:
+        updated_time = created_time
+
+    if updated_time is not None and created_time is None:
+        raise UserError("Cannot set updated_time without created_time")
+
+    if updated_time is not None and created_time is not None:
+        if updated_time < created_time:
+            raise UserError("updated_time cannot come before created_date")
 
     did, rev, baseid = blueprint.index_driver.add(
         form,
@@ -422,6 +435,9 @@ def post_index_record():
         hashes=hashes,
         baseid=baseid,
         uploader=uploader,
+        description=description,
+        created_time=created_time,
+        updated_time=updated_time,
     )
 
     ret = {"did": did, "rev": rev, "baseid": baseid}
@@ -508,9 +524,13 @@ def put_index_record(record):
         raise UserError(err)
 
     rev = flask.request.args.get("rev")
+    json = flask.request.json
+    if "updated_time" in json and "created_time" in json:
+        if json["updated_time"] < json["created_time"]:
+            raise UserError("updated_time cannot come before created_date")
 
     # authorize done in update
-    did, baseid, rev = blueprint.index_driver.update(record, rev, flask.request.json)
+    did, baseid, rev = blueprint.index_driver.update(record, rev, json)
 
     ret = {"did": did, "baseid": baseid, "rev": rev}
 
@@ -553,6 +573,16 @@ def add_index_record_version(record):
     metadata = flask.request.json.get("metadata")
     urls_metadata = flask.request.json.get("urls_metadata")
     version = flask.request.json.get("version")
+    description = flask.request.json.get("description")
+    created_time = flask.request.json.get("created_time")
+    updated_time = flask.request.json.get("updated_time")
+
+    if updated_time is None:
+        updated_time = created_time
+
+    if updated_time is not None and created_time is not None:
+        if updated_time < created_time:
+            raise UserError("updated_time cannot come before created_date")
 
     # authorize done in add_version for both the old and new authz
     did, baseid, rev = blueprint.index_driver.add_version(
@@ -568,6 +598,9 @@ def add_index_record_version(record):
         urls_metadata=urls_metadata,
         version=version,
         hashes=hashes,
+        description=description,
+        created_time=created_time,
+        updated_time=updated_time,
     )
 
     ret = {"did": did, "baseid": baseid, "rev": rev}

indexd/index/driver.py

@@ -58,6 +58,9 @@ def add(
         hashes=None,
         baseid=None,
         uploader=None,
+        description=None,
+        created_date=None,
+        updated_date=None,
     ):
         """
         Creates record for given data.
@@ -100,6 +103,9 @@ def add_version(
         acl=None,
         authz=None,
         hashes=None,
+        description=None,
+        created_time=None,
+        updated_time=None,
     ):
         """
         Add a record version given did

indexd/index/drivers/alchemy.py

@@ -80,6 +80,9 @@ class IndexRecord(Base):
     file_name = Column(String, index=True)
     version = Column(String, index=True)
     uploader = Column(String, index=True)
+    description = Column(String)
+    content_created_date = Column(DateTime, default=datetime.datetime.fromtimestamp(0))
+    content_updated_date = Column(DateTime, default=datetime.datetime.fromtimestamp(0))
 
     urls = relationship(
         "IndexRecordUrl", backref="index_record", cascade="all, delete-orphan"
@@ -120,6 +123,16 @@ def to_document_dict(self):
         }
         created_date = self.created_date.isoformat()
         updated_date = self.updated_date.isoformat()
+        content_created_date = (
+            self.content_created_date.isoformat()
+            if self.content_created_date != datetime.datetime.fromtimestamp(0)
+            else ""
+        )
+        content_updated_date = (
+            self.content_updated_date.isoformat()
+            if self.content_created_date != datetime.datetime.fromtimestamp(0)
+            else ""
+        )
 
         return {
             "did": self.did,
@@ -138,6 +151,9 @@ def to_document_dict(self):
             "form": self.form,
             "created_date": created_date,
             "updated_date": updated_date,
+            "description": self.description,
+            "content_created_date": content_created_date,
+            "content_updated_date": content_updated_date,
         }
 
 
@@ -675,6 +691,9 @@ def add(
         hashes=None,
         baseid=None,
         uploader=None,
+        description=None,
+        created_time=None,
+        updated_time=None,
     ):
         """
         Creates a new record given size, urls, acl, authz, hashes, metadata,
@@ -733,6 +752,19 @@ def add(
                 IndexRecordMetadata(did=record.did, key=m_key, value=m_value)
                 for m_key, m_value in metadata.items()
             ]
+
+            record.description = description
+
+            if created_time is not None:
+                record.content_created_date = datetime.datetime.fromisoformat(
+                    created_time
+                )
+                record.content_updated_date = (
+                    datetime.datetime.fromisoformat(updated_time)
+                    if updated_time is not None
+                    else record.content_created_date
+                )
+
             session.merge(base_version)
 
             try:
@@ -1137,7 +1169,15 @@ def update(self, did, rev, changing_fields):
         """
         authz_err_msg = "Auth error when attempting to update a record. User must have '{}' access on '{}' for service 'indexd'."
 
-        composite_fields = ["urls", "acl", "authz", "metadata", "urls_metadata"]
+        composite_fields = [
+            "urls",
+            "acl",
+            "authz",
+            "metadata",
+            "urls_metadata",
+            "created_time",
+            "updated_time",
+        ]
 
         with self.session as session:
             query = session.query(IndexRecord).filter(IndexRecord.did == did)
@@ -1209,6 +1249,19 @@ def update(self, did, rev, changing_fields):
 
                 create_urls_metadata(changing_fields["urls_metadata"], record, session)
 
+            if "created_time" in changing_fields:
+                record.content_created_date = datetime.datetime.fromisoformat(
+                    changing_fields["created_time"]
+                )
+            if "updated_time" in changing_fields:
+                if record.content_created_date == datetime.datetime.fromtimestamp(0):
+                    raise UserError(
+                        "Cannot set updated_time on record that does not have a created_time"
+                    )
+                record.content_updated_date = datetime.datetime.fromisoformat(
+                    changing_fields["updated_time"]
+                )
+
             for key, value in changing_fields.items():
                 if key not in composite_fields:
                     # No special logic needed for other updates.
@@ -1259,6 +1312,9 @@ def add_version(
         acl=None,
         authz=None,
         hashes=None,
+        description=None,
+        created_time=None,
+        updated_time=None,
     ):
         """
         Add a record version given did
@@ -1297,6 +1353,9 @@ def add_version(
             record.size = size
             record.file_name = file_name
             record.version = version
+            record.description = description
+            record.content_created_date = created_time
+            record.content_updated_date = updated_time
 
             record.urls = [IndexRecordUrl(did=record.did, url=url) for url in urls]
 

indexd/index/schema.py

@@ -31,6 +31,10 @@
             "description": "optional version string of the object",
             "type": "string",
         },
+        "description": {
+            "description": "optional description string of the object",
+            "type": "string",
+        },
         "uploader": {
             "description": "optional uploader of the object",
             "type": "string",
@@ -46,6 +50,16 @@
             "type": "string",
             "pattern": "^.*[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
         },
+        "created_time": {
+            "description": "Timestamp of content creation. Refers to the underyling content, not the JSON object.",
+            "type": "string",
+            "format": "date-time",
+        },
+        "updated_time": {
+            "description": "Timestamp of content update, identical to created_time in systems that do not support updates. Refers to the underyling content, not the JSON object.",
+            "type": "string",
+            "format": "date-time",
+        },
         "hashes": {
             "type": "object",
             "properties": {
@@ -82,6 +96,9 @@
         "uploader": {"type": ["string", "null"]},
         "metadata": {"type": "object"},
         "urls_metadata": {"type": "object"},
+        "description": {"type": ["string", "null"]},
+        "created_time": {"type": ["string", "null"], "format": "date-time"},
+        "updated_time": {"type": ["string", "null"], "format": "date-time"},
     },
 }
 

migrations/versions/15f2e9345ade_create_tables.py

@@ -66,6 +66,9 @@ def upgrade() -> None:
         sa.Column("file_name", sa.VARCHAR(), nullable=True),
         sa.Column("version", sa.VARCHAR(), nullable=True),
         sa.Column("uploader", sa.VARCHAR(), nullable=True),
+        sa.Column("description", sa.VARCHAR(), nullable=True),
+        sa.Column("content_created_date", sa.DateTime, nullable=True),
+        sa.Column("content_updated_date", sa.DateTime, nullable=True),
         sa.ForeignKeyConstraint(
             ["baseid"],
             ["base_version.baseid"],

tests/postgres/migrations/test_15f2e9345ade_create_tables.py

@@ -62,8 +62,12 @@ def test_upgrade(postgres_driver):
         ("file_name", "character varying"),
         ("version", "character varying"),
         ("uploader", "character varying"),
+        ("description", "character varying"),
+        ("content_created_date", "timestamp without time zone"),
+        ("content_updated_date", "timestamp without time zone"),
     ]
-    assert sorted(expected_schema) == sorted([i for i in cols])
+    actual_schema = sorted([i for i in cols])
+    assert sorted(expected_schema) == actual_schema
 
 
 def test_downgrade(postgres_driver):