Merge pull request #348 from uc-cdis/feat/PXP-6421-Indexd-support-des…

…c-version-fields

feat(PXP-6421): DRS description and version support added

.gitignore

@@ -67,3 +67,9 @@ docs/_build/
 local_settings.py
 settings.yaml
 *.sq3
+
+# IDE
+.idea
+
+# pyenv
+.python-version

indexd/drs/blueprint.py

@@ -129,7 +129,7 @@ def indexd_to_drs(record, expand=False):
 
     form = record["form"] if "form" in record else "bundle"
 
-    description = record["description"] if "description" in record else None
+    description = record["description"] if "description" in record else ""
 
     alias = (
         record["alias"]
@@ -141,7 +141,6 @@ def indexd_to_drs(record, expand=False):
 
     drs_object = {
         "id": did,
-        "description": "",
         "mime_type": "application/json",
         "name": name,
         "created_time": created_time,
@@ -155,9 +154,6 @@ def indexd_to_drs(record, expand=False):
         "description": description,
     }
 
-    if "description" in record:
-        drs_object["description"] = record["description"]
-
     if "bundle_data" in record:
         drs_object["contents"] = []
         for bundle in record["bundle_data"]:

indexd/index/blueprint.py

@@ -7,7 +7,7 @@
 
 from indexd import auth
 
-from indexd.errors import AuthError, AuthzError
+from indexd.errors import AuthError, AuthzError, IndexdUnexpectedError
 from indexd.errors import UserError
 
 from .schema import PUT_RECORD_SCHEMA
@@ -243,7 +243,7 @@ def get_urls():
 
 # NOTE: /index/<record>/deeper-route methods are above /index/<record> so that routing
 # prefers these first. Without this ordering, newer versions of the web framework
-# were interpretting index/e383a3aa-316e-4a51-975d-d699eff41bd2/aliases/ as routing
+# were interpreting index/e383a3aa-316e-4a51-975d-d699eff41bd2/aliases/ as routing
 # to /index/<record> where <record> was "e383a3aa-316e-4a51-975d-d699eff41bd2/aliases/"
 
 
@@ -407,6 +407,7 @@ def post_index_record():
     version = flask.request.json.get("version")
     baseid = flask.request.json.get("baseid")
     uploader = flask.request.json.get("uploader")
+    description = flask.request.json.get("description")
 
     did, rev, baseid = blueprint.index_driver.add(
         form,
@@ -422,6 +423,7 @@ def post_index_record():
         hashes=hashes,
         baseid=baseid,
         uploader=uploader,
+        description=description,
     )
 
     ret = {"did": did, "rev": rev, "baseid": baseid}
@@ -553,6 +555,7 @@ def add_index_record_version(record):
     metadata = flask.request.json.get("metadata")
     urls_metadata = flask.request.json.get("urls_metadata")
     version = flask.request.json.get("version")
+    description = flask.request.json.get("description")
 
     # authorize done in add_version for both the old and new authz
     did, baseid, rev = blueprint.index_driver.add_version(
@@ -568,6 +571,7 @@ def add_index_record_version(record):
         urls_metadata=urls_metadata,
         version=version,
         hashes=hashes,
+        description=description,
     )
 
     ret = {"did": did, "baseid": baseid, "rev": rev}

indexd/index/drivers/alchemy.py

@@ -77,6 +77,7 @@ class IndexRecord(Base):
     file_name = Column(String, index=True)
     version = Column(String, index=True)
     uploader = Column(String, index=True)
+    description = Column(String)
 
     urls = relationship(
         "IndexRecordUrl", backref="index_record", cascade="all, delete-orphan"
@@ -135,6 +136,7 @@ def to_document_dict(self):
             "form": self.form,
             "created_date": created_date,
             "updated_date": updated_date,
+            "description": self.description,
         }
 
 
@@ -682,6 +684,7 @@ def add(
         hashes=None,
         baseid=None,
         uploader=None,
+        description=None,
     ):
         """
         Creates a new record given size, urls, acl, authz, hashes, metadata,
@@ -741,6 +744,9 @@ def add(
                 IndexRecordMetadata(did=record.did, key=m_key, value=m_value)
                 for m_key, m_value in metadata.items()
             ]
+
+            record.description = description
+
             session.merge(base_version)
 
             try:
@@ -1267,6 +1273,7 @@ def add_version(
         acl=None,
         authz=None,
         hashes=None,
+        description=None,
     ):
         """
         Add a record version given did
@@ -1305,6 +1312,7 @@ def add_version(
             record.size = size
             record.file_name = file_name
             record.version = version
+            record.description = description
 
             record.urls = [IndexRecordUrl(did=record.did, url=url) for url in urls]
 
@@ -1433,7 +1441,6 @@ def get_all_versions(self, did):
             )
 
             for idx, record in enumerate(records):
-
                 ret[idx] = record.to_document_dict()
 
         return ret
@@ -1562,7 +1569,6 @@ def len(self):
         Number of unique records stored by backend.
         """
         with self.session as session:
-
             return session.execute(
                 select([func.count()]).select_from(IndexRecord)
             ).scalar()

indexd/index/schema.py

@@ -31,6 +31,10 @@
             "description": "optional version string of the object",
             "type": "string",
         },
+        "description": {
+            "description": "optional description string of the object",
+            "type": "string",
+        },
         "uploader": {
             "description": "optional uploader of the object",
             "type": "string",

openapis/swagger.yaml

@@ -1449,6 +1449,8 @@ definitions:
         type: object
       version:
         type: string
+      description:
+        type: string
       urls:
         type: array
         items:
@@ -2350,7 +2352,11 @@ definitions:
         type: array
         items:
           $ref: '#/definitions/Checksum'
-
+      description:
+        type: string
+      version:
+        type: string
+        description: optional version string of the object
       bundles:
         type: array
         items:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexd"
-version = "3.5.0"
+version = "4.1.0"
 description = "Gen3 Indexing Service"
 authors = ["CTDS UChicago <cdis@uchicago.edu>"]
 license = "Apache-2.0"
@@ -18,7 +18,7 @@ doiclient = {git = "https://github.com/uc-cdis/doiclient", rev = "1.0.0"}
 dosclient = {git = "https://github.com/uc-cdis/dosclient", rev = "1.1.0"}
 gen3authz = "^1.0.4"
 hsclient = {git = "https://github.com/uc-cdis/hsclient", rev = "1.0.0"}
-indexclient = "^2.1.0"
+indexclient = {git = "https://github.com/uc-cdis/indexclient", branch="feat/PXP-6421-Update-indexclient-to-have-description"}
 jsonschema = "^3.2"
 flask = "^2.0.1"
 psycopg2 = "^2.7"

tests/test_drs.py

@@ -1,4 +1,6 @@
 import json
+import sqlite3
+
 import tests.conftest
 import requests
 import responses
@@ -23,7 +25,7 @@ def generate_presigned_url_response(did, status=200, **query_params):
     return presigned_url
 
 
-def get_doc(has_version=True, urls=list(), drs_list=0):
+def get_doc(has_version=True, urls=list(), has_description=True):
     doc = {
         "form": "object",
         "size": 123,
@@ -34,14 +36,25 @@ def get_doc(has_version=True, urls=list(), drs_list=0):
         doc["version"] = "1"
     if urls:
         doc["urls"] = urls
-    # if drs_list > 0:
-    #     ret = {"drs_objects": []}
-    #     for _ in range(drs_list):
-    #         ret["drs_objects"].append(doc)
-    #     return ret
+    if has_description:
+        doc["description"] = "A description"
     return doc
 
 
+def get_bundle(client, user, has_description=True):
+    docs = [get_doc(), get_doc()]
+    dids = []
+    for doc in docs:
+        res = client.post("/index/", json=doc, headers=user)
+        assert res.status_code == 200
+        dids.append(res.json["did"])
+    bundle = get_bundle_doc(bundles=dids)
+    if has_description:
+        bundle["description"] = "A description"
+
+    return bundle
+
+
 def test_drs_get(client, user):
     data = get_doc()
     res_1 = client.post("/index/", json=data, headers=user)
@@ -55,12 +68,43 @@ def test_drs_get(client, user):
     for k in data["hashes"]:
         assert rec_2["checksums"][0]["checksum"] == data["hashes"][k]
         assert rec_2["checksums"][0]["type"] == k
-    assert rec_2["version"]
+    assert rec_2["description"] == data["description"]
     assert rec_2["self_uri"] == "drs://testprefix:" + rec_1["did"].split(":")[1]
     # according to ga4gh DRS blobs objects are NOT supposed to have contents. Only DRS Bundle objects should include the contetnts field
     assert "contents" not in rec_2
 
 
+def test_drs_get_no_description(client, user):
+    data = get_doc(has_description=False)
+    res_1 = client.post("/index/", json=data, headers=user)
+    assert res_1.status_code == 200
+    rec_1 = res_1.json
+    res_2 = client.get("/ga4gh/drs/v1/objects/" + rec_1["did"])
+    assert res_2.status_code == 200
+    rec_2 = res_2.json
+    assert rec_2["description"] is None
+
+
+def test_drs_get_bundle(client, user):
+    bundle = get_bundle(client, user)
+    bundle_res = client.post("/bundle/", json=bundle, headers=user)
+    assert bundle_res.status_code == 200
+    bundle_id = bundle_res.json["bundle_id"]
+    drs_res = client.get(f"/ga4gh/drs/v1/objects/{bundle_id}", headers=user)
+    assert drs_res.status_code == 200
+    assert drs_res.json["description"] == bundle["description"]
+
+
+def test_drs_get_bundle_no_description(client, user):
+    bundle = get_bundle(client, user, has_description=False)
+    bundle_res = client.post("/bundle/", json=bundle, headers=user)
+    assert bundle_res.status_code == 200
+    bundle_id = bundle_res.json["bundle_id"]
+    drs_res = client.get(f"/ga4gh/drs/v1/objects/{bundle_id}", headers=user)
+    assert drs_res.status_code == 200
+    assert drs_res.json["description"] is ""
+
+
 def test_drs_get_no_default(client, user):
     # Change default index driver settings to use no prefix
     settings["config"]["INDEX"]["driver"].config["DEFAULT_PREFIX"] = None
@@ -112,7 +156,7 @@ def test_drs_list(client, user):
         submitted_guids.append(did)
         bundle_data = get_bundle_doc(bundles=[did])
         res2 = client.post("/bundle/", json=bundle_data, headers=user)
-        assert res_1.status_code == 200
+        assert res2.status_code == 200
 
     res_2 = client.get("/ga4gh/drs/v1/objects")
     assert res_2.status_code == 200

tests/test_setup.py

@@ -24,6 +24,7 @@
         (7, "file_name", "VARCHAR", 0, None, 0),
         (8, "version", "VARCHAR", 0, None, 0),
         (9, "uploader", "VARCHAR", 0, None, 0),
+        (10, "description", "VARCHAR", 0, None, 0),
     ],
     "index_record_hash": [
         (0, "did", "VARCHAR", 1, None, 1),