Merge 68a517e into db40aec

indexd/bundle/blueprint.py

@@ -0,0 +1,128 @@
+import re
+import json
+import flask
+import jsonschema
+import os.path
+import subprocess
+import uuid
+import hashlib
+from indexd.index.version_data import VERSION, COMMIT
+
+from indexd.auth import authorize
+
+from indexd.errors import AuthError
+from indexd.errors import UserError
+
+from indexd.index.schema import BUNDLE_SCHEMA
+
+from indexd.index.errors import NoRecordFound
+from indexd.index.errors import MultipleRecordsFound
+from indexd.index.errors import UnhealthyCheck
+
+from cdislogging import get_logger
+
+logger = get_logger("indexd/index blueprint", log_level="info")
+
+blueprint = flask.Blueprint("index", __name__)
+
+blueprint.config = dict()
+blueprint.index_driver = None
+
+ACCEPTABLE_HASHES = {
+    "md5": re.compile(r"^[0-9a-f]{32}$").match,
+    "sha1": re.compile(r"^[0-9a-f]{40}$").match,
+    "sha256": re.compile(r"^[0-9a-f]{64}$").match,
+    "sha512": re.compile(r"^[0-9a-f]{128}$").match,
+    "crc": re.compile(r"^[0-9a-f]{8}$").match,
+    "etag": re.compile(r"^[0-9a-f]{32}(-\d+)?$").match,
+}
+
+
+def validate_hashes(**hashes):
+    """
+    Validate hashes against known and valid hashing algorithms.
+    """
+    if not all(h in ACCEPTABLE_HASHES for h in hashes):
+        raise UserError("invalid hash types specified")
+
+    if not all(ACCEPTABLE_HASHES[h](v) for h, v in hashes.items()):
+        raise UserError("invalid hash values specified")
+
+
+# @blueprint.route("/bundle/", methods=["GET"])
+# def get_bundle():
+
+
+# @blueprint.route("/bundle/<path:record>", methods=["GET"])
+# def get_bundle_record(record):
+#     """
+#     Returns a record.
+#     """
+
+#     ret = blueprint.index_driver.get_bundle(record)
+
+#     return flask.jsonify(ret), 200
+
+# @blueprint.route("/bundle/", methods=["POST"])
+# @authorize
+# def post_bundle():
+#     """
+#     Create a new bundle
+#     """
+#     print(" ")
+#     print("BUNDLE POST!!!!")
+#     print(" ")
+
+#     try:
+#         jsonschema.validate(flask.request.json, BUNDLE_SCHEMA)
+#     except jsonschema.ValidationError as err:
+#         raise UserError(err)
+
+#     name = flask.request.json.get("name")
+#     bundle = flask.request.json.get("bundles")
+
+#     bundle_id = str(uuid.uuid4())
+#     checksum = str(hashlib.md5(bundle_id))
+
+#     bundle_data = bundle
+
+#     ret = blueprint.index_driver.add_bundle(
+#         bundle_id=bundle_id,
+#         name=name,
+#         checksum=checksum,
+#         size=1,
+#         bundle_data=bundle_data,
+#     )
+
+#     return flask.jsonify(ret), 200
+
+
+@blueprint.errorhandler(NoRecordFound)
+def handle_no_record_error(err):
+    return flask.jsonify(error=str(err)), 404
+
+
+@blueprint.errorhandler(MultipleRecordsFound)
+def handle_multiple_records_error(err):
+    return flask.jsonify(error=str(err)), 409
+
+
+@blueprint.errorhandler(UserError)
+def handle_user_error(err):
+    return flask.jsonify(error=str(err)), 400
+
+
+@blueprint.errorhandler(AuthError)
+def handle_auth_error(err):
+    return flask.jsonify(error=str(err)), 403
+
+
+@blueprint.errorhandler(UnhealthyCheck)
+def handle_unhealthy_check(err):
+    return "Unhealthy", 500
+
+
+@blueprint.record
+def get_config(setup_state):
+    config = setup_state.app.config["INDEX"]
+    blueprint.index_driver = config["driver"]

indexd/drs/blueprint.py

@@ -3,6 +3,8 @@
 from indexd.errors import UserError
 from indexd.index.errors import NoRecordFound as IndexNoRecordFound
 from indexd.errors import UnexpectedError
+
+# from indexd.index.blueprint import get_index
 from indexd.index.blueprint import get_index
 
 blueprint = flask.Blueprint("drs", __name__)
@@ -16,9 +18,17 @@ def get_drs_object(object_id):
     """
     Returns a specific DRSobject with object_id
     """
+    expand = True if flask.request.args.get("expand") == "true" else False
+
     ret = blueprint.index_driver.get(object_id)
 
-    return flask.jsonify(indexd_to_drs(ret)), 200
+    data = (
+        indexd_to_drs(ret, expand=False, list_drs=False)
+        if not flask.request.args.get("expand")
+        else indexd_to_drs(ret, expand=expand, list_drs=False)
+    )
+
+    return flask.jsonify(data), 200
 
 
 @blueprint.route("/ga4gh/drs/v1/objects", methods=["GET"])
@@ -49,30 +59,49 @@ def get_signed_url(object_id, access_id):
     return res, 200
 
 
-def indexd_to_drs(record, list_drs=False):
+def indexd_to_drs(record, expand, list_drs=False):
+
     bearer_token = flask.request.headers.get("AUTHORIZATION")
-    self_uri = "drs://" + flask.current_app.hostname + "/" + record["did"]
+
+    did = record["did"] if "did" in record else record["bundle_id"]
+
+    self_uri = "drs://" + flask.current_app.hostname + "/" + did
+
+    name = record["file_name"] if "file_name" in record else record["name"]
+
+    created_time = (
+        record["created_date"] if "created_date" in record else record["created_time"]
+    )
+
+    version = record["rev"] if "rev" in record else ""
+
+    updated_date = (
+        record["updated_date"] if "updated_date" in record else record["updated_time"]
+    )
+
     drs_object = {
-        "id": record["did"],
+        "id": did,
         "description": "",
         "mime_type": "application/json",
-        "name": record["file_name"],
-        "created_time": record["created_date"],
-        "updated_time": record["updated_date"],
+        "name": name,
+        "created_time": created_time,
+        "updated_time": updated_date,
         "size": record["size"],
         "aliases": [],
         "contents": [],
         "self_uri": self_uri,
-        "version": record["rev"],
+        "version": version,
     }
 
     if "description" in record:
         drs_object["description"] = record["description"]
     if "alias" in record:
         drs_object["aliases"].append(record["alias"])
 
-    if "contents" in record:
-        drs_object["contents"] = record["contents"]
+    if "bundle_data" in record:
+        bundle_data = record["bundle_data"]
+        for bundle in bundle_data:
+            drs_object["contents"].append(bundle_to_drs(bundle, expand=expand))
 
     # access_methods mapping
     if "urls" in record:
@@ -94,12 +123,42 @@ def indexd_to_drs(record, list_drs=False):
                     "region": "",
                 }
             )
-        print(drs_object)
 
     # parse out checksums
     drs_object["checksums"] = []
-    for k in record["hashes"]:
-        drs_object["checksums"].append({"checksum": record["hashes"][k], "type": k})
+    if "hashes" in record:
+        for k in record["hashes"]:
+            drs_object["checksums"].append({"checksum": record["hashes"][k], "type": k})
+    else:
+        drs_object["checksums"].append({"checksum": record["checksum"], "type": "md5"})
+
+    return drs_object
+
+
+def bundle_to_drs(record, expand):
+    did = record["id"] if "id" in record else record["bundle_id"]
+
+    self_uri = "drs://" + flask.current_app.hostname + "/" + did
+
+    name = record["file_name"] if "file_name" in record else record["name"]
+
+    drs_object = {
+        "id": did,
+        "name": name,
+        "self_uri": self_uri,
+        "contents": [],
+    }
+    contents = (
+        record["contents"]
+        if "contents" in record
+        else record["bundle_data"]
+        if "bundle_data" in record
+        else None
+    )
+
+    if expand and contents != None:
+        for content in contents:
+            drs_object["contents"].append(bundle_to_drs(content, expand=True))
 
     return drs_object
 

indexd/index/blueprint.py

@@ -1,6 +1,8 @@
 import re
 import json
 import flask
+import hashlib
+import uuid
 import jsonschema
 import os.path
 import subprocess
@@ -14,6 +16,7 @@
 from .schema import PUT_RECORD_SCHEMA
 from .schema import POST_RECORD_SCHEMA
 from .schema import RECORD_ALIAS_SCHEMA
+from .schema import BUNDLE_SCHEMA
 
 from .errors import NoRecordFound
 from .errors import MultipleRecordsFound
@@ -523,6 +526,84 @@ def version():
     return flask.jsonify(base), 200
 
 
+@blueprint.route("/bundle/", methods=["POST"])
+def post_bundle():
+    """
+    Create a new bundle
+    """
+    from indexd.drs.blueprint import indexd_to_drs, get_drs_object
+
+    try:
+        authorize("create", ["/services/indexd/bundles"])
+    except:
+        raise AuthError("Invalid Token.")
+    try:
+        jsonschema.validate(flask.request.json, BUNDLE_SCHEMA)
+    except jsonschema.ValidationError as err:
+        raise UserError(err)
+
+    name = flask.request.json.get("name")
+    bundles = flask.request.json.get("bundles")
+    bundle_id = flask.request.json.get("bundle_id")
+
+    if len(bundles) == 0:
+        raise UserError("Bundle data required.")
+    if not name:
+        raise UserError("Bundle name required.")
+
+    bundle_data = []
+    size = 0
+    for bundle in bundles:
+        data = get_index_record(bundle)[0]
+        data = data.json
+        size += data["size"]
+        if "bundle_data" not in data:
+            # check if its a bundle or an object
+            data = indexd_to_drs(data, list_drs=True, expand=True)
+        bundle_data.append(data)
+    ret = blueprint.index_driver.add_bundle(
+        bundle_id=bundle_id, name=name, size=size, bundle_data=str(bundle_data),
+    )
+
+    return flask.jsonify(ret), 200
+
+
+@blueprint.route("/bundle/", methods=["GET"])
+def get_bundle_record_list():
+    """
+    Returns a list of bundle records.
+    """
+
+    ret = blueprint.index_driver.get_bundle_list()
+
+    return flask.jsonify(ret), 200
+
+
+@blueprint.route("/bundle/<path:bundle_id>", methods=["GET"])
+def get_bundle_record_with_id(bundle_id):
+    """
+    Returns a record given bundle_id
+    """
+    expand = False if flask.request.args.get("expand") == "false" else True
+    ret = blueprint.index_driver.get(bundle_id, expand)
+
+    return flask.jsonify(ret), 200
+
+
+@blueprint.route("/bundle/<path:bundle_id>", methods=["DELETE"])
+def delete_bundle_record(bundle_id):
+    """
+    Delete bundle record given bundle_id
+    """
+    try:
+        authorize("delete", ["/services/indexd/bundles"])
+    except:
+        raise AuthError("Invalid Token.")
+    blueprint.index_driver.delete_bundle(bundle_id)
+
+    return "", 200
+
+
 @blueprint.errorhandler(NoRecordFound)
 def handle_no_record_error(err):
     return flask.jsonify(error=str(err)), 404