Skip to content
Permalink
Browse files
AIRAVATA-3420 Implementing get_download_url, moving download view to sdk
  • Loading branch information
machristie committed Apr 26, 2021
1 parent d785896 commit c80aa136889125717d299920082cd27192e5731f
Show file tree
Hide file tree
Showing 6 changed files with 142 additions and 16 deletions.
@@ -1,9 +1,19 @@
import warnings
from urllib.parse import urlencode

from django.urls import reverse
from django.urls import path, reverse

from . import views


def get_download_url(data_product_uri):
"""Get URL for downloading data product identified by data_product_uri."""
return (reverse("django_airavata_api:download_file") + "?" +
"""(Deprecated) Get URL for downloading data product identified by data_product_uri."""
warnings.warn("Use user_storage.get_download_url instead.", DeprecationWarning)
return (reverse("airavata_django_portal_sdk:download_file") + "?" +
urlencode({"data-product-uri": data_product_uri}))


app_name = 'airavata_django_portal_sdk'
urlpatterns = [
path('download', views.download_file, name='download_file'),
]
@@ -7,6 +7,7 @@
dir_exists,
exists,
experiment_dir_exists,
get_download_url,
get_experiment_dir,
get_file,
get_file_metadata,
@@ -33,6 +34,7 @@
'dir_exists',
'exists',
'experiment_dir_exists',
'get_download_url',
'get_experiment_dir',
'get_file',
'get_file_metadata',
@@ -7,7 +7,7 @@
import os
import warnings
from http import HTTPStatus
from urllib.parse import quote, unquote, urlparse
from urllib.parse import quote, unquote, urlencode, urlparse

import requests
from airavata.model.data.replica.ttypes import (
@@ -21,6 +21,8 @@
from django.core.exceptions import ObjectDoesNotExist

from ..util import convert_iso8601_to_datetime
from django.urls import reverse
from airavata_django_portal_sdk.user_storage.backends.base import ProvidesDownloadUrl

logger = logging.getLogger(__name__)

@@ -181,6 +183,25 @@ def move_input_file(request, data_product=None, path=None, data_product_uri=None
return move(request, data_product=data_product, path=path, data_product_uri=data_product_uri, storage_resource_id=storage_resource_id)


def get_download_url(request, data_product=None, data_product_uri=None):
if data_product is None:
data_product = _get_data_product(request, data_product_uri)
if _is_remote_api():
raise NotImplementedError()
else:
storage_resource_id, path = _get_replica_resource_id_and_filepath(data_product)
backend = get_user_storage_provider(request,
owner_username=data_product.ownerName,
storage_resource_id=storage_resource_id)
if isinstance(backend, ProvidesDownloadUrl):
return backend.get_download_url(path)
else:
# if backend doesn't provide a download url, then use default one
# that uses backend to read the file
return (reverse("airavata_django_portal_sdk:download_file") + "?" +
urlencode({"data-product-uri": data_product.productUri}))


def open_file(request, data_product=None, data_product_uri=None):
"""
Return file object for replica if it exists in user storage. One of
@@ -435,6 +456,8 @@ def listdir(request, path, storage_resource_id=None):
mime_type = data_product.productMetadata['mime-type']
file['data-product-uri'] = data_product_uri
file['mime_type'] = mime_type
# TODO: remove this, there's no need for hidden files
file['hidden'] = False
return directories, files


@@ -1,4 +1,16 @@

class ProvidesDownloadUrl:
"""Mixin for UserStorageProvider that provides download url."""
def get_download_url(self, resource_path):
raise NotImplementedError()


class ProvidesUploadUrl:
"""Mixin for UserStorageProvider that provides upload url."""
def get_upload_url(self, resource_path):
raise NotImplementedError()


class UserStorageProvider:
def __init__(self, authz_token, resource_id, context=None, **kwargs):
self.authz_token = authz_token
@@ -13,15 +25,9 @@ def save(self, resource_path, file, name=None, content_type=None):
"""
raise NotImplementedError()

def get_upload_url(self, resource_path):
raise NotImplementedError()

def open(self, resource_path):
raise NotImplementedError()

def get_download_url(self, resource_path):
raise NotImplementedError()

def exists(self, resource_path):
raise NotImplementedError()

@@ -1,16 +1,18 @@
import io
import logging
import os
from datetime import datetime

import grpc
import requests

from . import CredCommon_pb2, MFTApi_pb2, MFTApi_pb2_grpc
from .base import UserStorageProvider
from .base import ProvidesDownloadUrl, UserStorageProvider

logger = logging.getLogger(__name__)


class MFTUserStorageProvider(UserStorageProvider):
class MFTUserStorageProvider(UserStorageProvider, ProvidesDownloadUrl):

def __init__(self, authz_token, resource_id, context=None, resource_token=None, mft_api_endpoint=None, mft_api_secure=False, resource_per_gateway=False, **kwargs):
super().__init__(authz_token, resource_id, context=context, **kwargs)
@@ -100,9 +102,6 @@ def get_metadata(self, resource_path):
"resource_path": d.resourcePath,
"created_time": created_time,
"size": size,
# TODO how to handle hidden directories or directories for
# staging input file uploads
"hidden": False
}
)
files_data = []
@@ -127,7 +126,6 @@ def get_metadata(self, resource_path):
"resource_path": f.resourcePath,
"created_time": created_time,
"size": size,
"hidden": False,
}
)
return directories_data, files_data
@@ -182,6 +180,36 @@ def is_dir(self, resource_path):
logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}")
return False

def get_download_url(self, resource_path):
with grpc.insecure_channel(self.mft_api_endpoint) as channel:
child_path = self._get_child_path(resource_path)
stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
download_request = MFTApi_pb2.HttpDownloadApiRequest(
# sourceResourceId=self.resource_id,
# FIXME: just hacking in something to force it to work
sourceResourceId="remote-ssh-resource",
# sourcePath=child_path,
sourceToken=self.resource_token,
sourceType="SCP",
targetAgent="agent0",
mftAuthorizationToken=self.auth_token,
)
try:
response = stub.submitHttpDownload(download_request)
logger.debug(f"Download request for {self.resource_id}:{child_path}. Response = {response}")
return response.url
except Exception as e:
logger.error(f"submitHttpDownload request {download_request} failed.")
raise Exception(f"Failed to get download url for {resource_path}") from e

def open(self, resource_path):
download_url = self.get_download_url(resource_path)
r = requests.get(download_url)
r.raise_for_status()
file = io.BytesIO(r.content)
file.name = os.path.basename(resource_path)
return file

def _get_child_path(self, resource_path):
"""Convert possibly relative child path into absolute path."""
if not resource_path.startswith("/"):
@@ -0,0 +1,57 @@
import logging
import os
from urllib.parse import urlparse

from django.contrib.auth.decorators import login_required
from django.core.exceptions import ObjectDoesNotExist
from django.http import FileResponse, Http404
from django.shortcuts import redirect
from django.urls import reverse

from airavata_django_portal_sdk import user_storage

logger = logging.getLogger(__name__)


# TODO: moving this view out of REST API means losing access token based authentication
@login_required
def download_file(request):

data_product_uri = request.GET.get('data-product-uri', '')
download_url = user_storage.get_download_url(request, data_product_uri=data_product_uri)
# If the download_url resolves to this view, then handle it directly
if urlparse(download_url).path == reverse('airavata_django_portal_sdk:download_file'):
return _internal_download_file(request)
else:
return redirect(download_url)


def _internal_download_file(request):
data_product_uri = request.GET.get('data-product-uri', '')
force_download = 'download' in request.GET
data_product = None
try:
data_product = request.airavata_client.getDataProduct(
request.authz_token, data_product_uri)
mime_type = "application/octet-stream" # default mime-type
if (data_product.productMetadata and
'mime-type' in data_product.productMetadata):
mime_type = data_product.productMetadata['mime-type']
# 'mime-type' url parameter overrides
mime_type = request.GET.get('mime-type', mime_type)
except Exception as e:
logger.warning("Failed to load DataProduct for {}"
.format(data_product_uri), exc_info=True)
raise Http404("data product does not exist") from e
try:
data_file = user_storage.open_file(request, data_product)
response = FileResponse(data_file, content_type=mime_type)
file_name = os.path.basename(data_file.name)
if mime_type == 'application/octet-stream' or force_download:
response['Content-Disposition'] = ('attachment; filename="{}"'
.format(file_name))
else:
response['Content-Disposition'] = f'inline; filename="{file_name}"'
return response
except ObjectDoesNotExist as e:
raise Http404(str(e)) from e

0 comments on commit c80aa13

Please sign in to comment.