Drs endpoints (#261)

* (feat) DRS endpoints

* presigned_url endpoint

* added requirements

* fence_url fix

* removed print statements

* added more test/second endpt working

* presigned url not created while listing

* Review Changes

* Fixed Swagger

* Fixed self_uri and added more tests

* added self uri and version

* quick fix for self_uri not working on env

* better error handling

* protocol fix+base_url change+swagger edit

* remove unused imports

* test fixes

* test fixes

Co-authored-by: BinamB <binam@binam-XPS-13-9360.cdis>

indexd/app.py

@@ -3,8 +3,10 @@
 from .index.blueprint import blueprint as indexd_index_blueprint
 from .alias.blueprint import blueprint as indexd_alias_blueprint
 from .dos.blueprint import blueprint as indexd_dos_blueprint
+from .drs.blueprint import blueprint as indexd_drs_blueprint
 from .blueprint import blueprint as cross_blueprint
 
+from indexd.fence_client import FenceClient
 from indexd.urls.blueprint import blueprint as index_urls_blueprint
 
 import os
@@ -19,10 +21,16 @@ def app_init(app, settings=None):
         from .default_settings import settings
     app.config.update(settings["config"])
     app.auth = settings["auth"]
+    app.fence_client = FenceClient(
+        url=os.environ.get("PRESIGNED_FENCE_URL")
+        or "http://presigned-url-fence-service"
+    )
+    app.hostname = os.environ.get("HOSTNAME") or "http://example.io"
     app.register_blueprint(indexd_bulk_blueprint)
     app.register_blueprint(indexd_index_blueprint)
     app.register_blueprint(indexd_alias_blueprint)
     app.register_blueprint(indexd_dos_blueprint)
+    app.register_blueprint(indexd_drs_blueprint)
     app.register_blueprint(cross_blueprint)
     app.register_blueprint(index_urls_blueprint, url_prefix="/_query/urls")
 

indexd/auth/__init__.py

@@ -16,7 +16,7 @@ def authorize(*p):
     not present, or fallback to the previous check.
     """
     if len(p) == 1:
-        f, = p
+        (f,) = p
 
         @wraps(f)
         def check_auth(*args, **kwargs):

indexd/default_settings.py

@@ -32,6 +32,7 @@
     )
 }
 
+
 CONFIG["DIST"] = [
     {
         "name": "Other IndexD",
@@ -46,6 +47,12 @@
         "hints": [],
         "type": "dos",
     },
+    {
+        "name": "DRS System",
+        "host": "https://example.com/api/ga4gh/drs/v1/",
+        "hints": [],
+        "type": "drs",
+    },
 ]
 
 AUTH = SQLAlchemyAuthDriver("sqlite:///auth.sq3")

indexd/drs/blueprint.py

@@ -0,0 +1,140 @@
+import flask
+from indexd.errors import AuthError
+from indexd.errors import UserError
+from indexd.index.errors import NoRecordFound as IndexNoRecordFound
+from indexd.errors import UnexpectedError
+from indexd.index.blueprint import get_index
+
+blueprint = flask.Blueprint("drs", __name__)
+
+blueprint.config = dict()
+blueprint.index_driver = None
+
+
+@blueprint.route("/ga4gh/drs/v1/objects/<path:object_id>", methods=["GET"])
+def get_drs_object(object_id):
+    """
+    Returns a specific DRSobject with object_id
+    """
+    ret = blueprint.index_driver.get(object_id)
+
+    return flask.jsonify(indexd_to_drs(ret)), 200
+
+
+@blueprint.route("/ga4gh/drs/v1/objects", methods=["GET"])
+def list_drs_records():
+    records = get_index()[0].json["records"]
+    ret = {"drs_objects": [indexd_to_drs(record, True) for record in records]}
+
+    return flask.jsonify(ret), 200
+
+
+@blueprint.route(
+    "/ga4gh/drs/v1/objects/<path:object_id>/access",
+    defaults={"access_id": None},
+    methods=["GET"],
+)
+@blueprint.route(
+    "/ga4gh/drs/v1/objects/<path:object_id>/access/<path:access_id>", methods=["GET"]
+)
+def get_signed_url(object_id, access_id):
+    if not access_id:
+        raise (UserError("Access ID/Protocol is required."))
+    res = flask.current_app.fence_client.get_signed_url_for_object(
+        object_id=object_id, access_id=access_id
+    )
+    if not res:
+        raise IndexNoRecordFound("No signed url found")
+
+    return res, 200
+
+
+def indexd_to_drs(record, list_drs=False):
+    bearer_token = flask.request.headers.get("AUTHORIZATION")
+    self_uri = "drs://" + flask.current_app.hostname + "/" + record["did"]
+    drs_object = {
+        "id": record["did"],
+        "description": "",
+        "mime_type": "application/json",
+        "name": record["file_name"],
+        "created_time": record["created_date"],
+        "updated_time": record["updated_date"],
+        "size": record["size"],
+        "aliases": [],
+        "contents": [],
+        "self_uri": self_uri,
+        "version": record["rev"],
+    }
+
+    if "description" in record:
+        drs_object["description"] = record["description"]
+    if "alias" in record:
+        drs_object["aliases"].append(record["alias"])
+
+    if "contents" in record:
+        drs_object["contents"] = record["contents"]
+
+    # access_methods mapping
+    if "urls" in record:
+        drs_object["access_methods"] = []
+        for location in record["urls"]:
+            location_type = location.split(":")[
+                0
+            ]  # (s3, gs, ftp, gsiftp, globus, htsget, https, file)
+
+            drs_object["access_methods"].append(
+                {
+                    "type": location_type,
+                    "access_url": flask.current_app.fence_client.get_signed_url_for_object(
+                        record["did"], ""
+                    )
+                    if bearer_token and not list_drs
+                    else {"url": location},
+                    "access_id": location_type,
+                    "region": "",
+                }
+            )
+        print(drs_object)
+
+    # parse out checksums
+    drs_object["checksums"] = []
+    for k in record["hashes"]:
+        drs_object["checksums"].append({"checksum": record["hashes"][k], "type": k})
+
+    return drs_object
+
+
+@blueprint.errorhandler(UserError)
+def handle_user_error(err):
+    ret = {"msg": str(err), "status_code": 400}
+    return flask.jsonify(ret), 400
+
+
+@blueprint.errorhandler(AuthError)
+def handle_auth_error(err):
+    ret = {"msg": str(err), "status_code": 401}
+    return flask.jsonify(ret), 401
+
+
+@blueprint.errorhandler(AuthError)
+def handle_requester_auth_error(err):
+    ret = {"msg": str(err), "status_code": 403}
+    return flask.jsonify(ret), 403
+
+
+@blueprint.errorhandler(IndexNoRecordFound)
+def handle_no_index_record_error(err):
+    ret = {"msg": str(err), "status_code": 404}
+    return flask.jsonify(ret), 404
+
+
+@blueprint.errorhandler(UnexpectedError)
+def handle_unexpected_error(err):
+    ret = {"msg": str(err), "status_code": 500}
+    return flask.jsonify(ret), 500
+
+
+@blueprint.record
+def get_config(setup_state):
+    index_config = setup_state.app.config["INDEX"]
+    blueprint.index_driver = index_config["driver"]

indexd/errors.py

@@ -11,3 +11,9 @@ class ConfigurationError(Exception):
     """
     Configuration error.
     """
+
+
+class UnexpectedError(Exception):
+    """
+    Unexpected Error
+    """

indexd/fence_client.py

@@ -0,0 +1,45 @@
+from cdislogging import get_logger
+
+import flask
+import requests
+
+
+from indexd.index.errors import NoRecordFound as IndexNoRecordFound
+from indexd.errors import UnexpectedError
+from indexd.auth.errors import AuthError
+
+
+logger = get_logger(__name__)
+
+
+class FenceClient(object):
+    def __init__(self, url):
+        self.url = url
+
+    def get_signed_url_for_object(self, object_id, access_id):
+        fence_server = self.url
+        api_url = fence_server.rstrip("/") + "/data/download/"
+        url = api_url + object_id
+        headers = flask.request.headers
+        if "AUTHORIZATION" not in headers:
+            logger.error("Bearer Token not available.")
+            raise AuthError("Not Authorized. Access Token Required.")
+        if access_id:
+            url += "?protocol=" + access_id
+        try:
+            req = requests.get(url, headers=headers)
+        except Exception as e:
+            logger.error("failed to reach fence at {0}: {1}".format(url + object_id, e))
+            raise UnexpectedError("Failed to retrieve access url")
+        if req.status_code == 404:
+            logger.error(
+                "Not found. Fence could not find {}: {} with access id: {}".format(
+                    url + object_id, req.text, access_id
+                )
+            )
+            raise IndexNoRecordFound(
+                "No document with id:{} with access_id:{}".format(object_id, access_id)
+            )
+        elif req.status_code != 200:
+            raise UnexpectedError(req.text)
+        return req.json()