Skip to content
Permalink
Browse files
AIRAVATA-3542 include and exclude patterns for multi experiment download
  • Loading branch information
machristie committed Nov 19, 2021
1 parent 28ccd98 commit 5a658fbac19cc623caf713664e7ad3b4e75f6455
Showing 2 changed files with 39 additions and 11 deletions.
@@ -1,12 +1,19 @@
from rest_framework import serializers

# class DownloadIncludeSerializer(serializers.Serializer):
# pattern = serializers.CharField()

class FilenamePatternSerializer(serializers.Serializer):
pattern = serializers.CharField()


class IncludeFilenamePatternSerializer(FilenamePatternSerializer):
rename = serializers.CharField(required=False)


class ExperimentDownloadSerializer(serializers.Serializer):
experiment_id = serializers.CharField()
# includes = DownloadIncludeSerializer(many=True)
path = serializers.CharField(default="")
includes = IncludeFilenamePatternSerializer(many=True, required=False, default=None)
excludes = FilenamePatternSerializer(many=True, required=False, default=None)


class MultiExperimentDownloadSerializer(serializers.Serializer):
@@ -1,3 +1,4 @@
import fnmatch
import logging
import os
import tempfile
@@ -121,9 +122,12 @@ def download_experiments(request, download_id=None):
for experiment in experiments:
experiment_id = experiment['experiment_id']
# Load experiment to make sure user has access to experiment
experiment = request.airavata_client.getExperiment(request.authz_token, experiment_id)
_add_experiment_directory_to_zipfile(request, zf, experiment_id, path="",
zipfile_prefix=get_valid_filename(experiment.experimentName))
experiment_model = request.airavata_client.getExperiment(request.authz_token, experiment_id)
path = experiment['path']
_add_experiment_directory_to_zipfile(
request, zf, experiment_id, path,
zipfile_prefix=os.path.join(get_valid_filename(experiment_model.experimentName), path),
includes=experiment['includes'], excludes=experiment['excludes'])

filename = "experiments.zip"
fp.seek(0)
@@ -146,14 +150,31 @@ def _add_directory_to_zipfile(request, zf, path, directory=""):
_add_directory_to_zipfile(request, zf, path, os.path.join(directory, d['name']))


def _add_experiment_directory_to_zipfile(request, zf, experiment_id, path, directory="", zipfile_prefix=""):
def _add_experiment_directory_to_zipfile(request, zf, experiment_id, path, directory="", zipfile_prefix="", includes=None, excludes=None):
directories, files = user_storage.list_experiment_dir(request, experiment_id, os.path.join(path, directory))
for file in files:
o = user_storage.open_file(request, data_product_uri=file['data-product-uri'])
zf.writestr(os.path.join(zipfile_prefix, directory, file['name']), o.read())
if os.path.getsize(zf.filename) > MAX_DOWNLOAD_ZIPFILE_SIZE:
raise Exception(f"Zip file size exceeds max of {MAX_DOWNLOAD_ZIPFILE_SIZE} bytes")
matches = _matches_filters(file['name'], includes=includes, excludes=excludes)
if matches:
o = user_storage.open_file(request, data_product_uri=file['data-product-uri'])
zf.writestr(os.path.join(zipfile_prefix, directory, file['name']), o.read())
if os.path.getsize(zf.filename) > MAX_DOWNLOAD_ZIPFILE_SIZE:
raise Exception(f"Zip file size exceeds max of {MAX_DOWNLOAD_ZIPFILE_SIZE} bytes")
for d in directories:
_add_experiment_directory_to_zipfile(request, zf, experiment_id, path,
directory=os.path.join(directory, d['name']),
zipfile_prefix=zipfile_prefix)


def _matches_filters(filename, includes=None, excludes=None):
# excludes take precedence
if excludes is not None and len(excludes) > 0:
for exclude in excludes:
if fnmatch.fnmatch(filename, exclude['pattern']):
return False
# if there are no include patterns, default to include all
if includes is None or len(includes) == 0:
return True
for include in includes:
if fnmatch.fnmatch(filename, include['pattern']):
return True
return False

0 comments on commit 5a658fb

Please sign in to comment.