Skip to content

Commit

Permalink
Doc style updates; copy-to return dict with copied datasets
Browse files Browse the repository at this point in the history
  • Loading branch information
@afgane
afgane committed Jul 13, 2018
1 parent 7e8be2f commit b3f1a7c
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 31 deletions.
36 changes: 21 additions & 15 deletions lib/galaxy/managers/cloud.py
View file
Expand Up @@ -228,29 +228,32 @@ def copy_to(self, trans, history_id, provider, bucket, credentials, dataset_ids=
:param history_id: the (encoded) id of history from which the object should be copied.
:type provider: string
:param provider: the name of cloud-based resource provided. A list of supported providers is given in
`SUPPORTED_PROVIDERS` variable.
:param provider: the name of cloud-based resource provided. A list of supported providers
is given in `SUPPORTED_PROVIDERS` variable.
:type bucket: string
:param bucket: the name of a bucket to which data should be copied (e.g., a bucket name on AWS S3).
:param bucket: the name of a bucket to which data should be copied (e.g., a bucket
name on AWS S3).
:type credentials: dict
:param credentials: a dictionary containing all the credentials required to authenticated to the
specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}).
:param credentials: a dictionary containing all the credentials required to authenticated
to the specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN,
"access_key": YOUR_AWS_ACCESS_TOKEN}).
:type dataset_ids: set
:param dataset_ids: [Optional] The list of (decoded) dataset ID(s) belonging to the given history which
should be copied to the given provider. If not provided, Galaxy copies all the datasets belonging to the
given history.
:param dataset_ids: [Optional] The list of (decoded) dataset ID(s) belonging to the given
history which should be copied to the given provider. If not provided,
Galaxy copies all the datasets belonging to the given history.
:type overwrite_existing: boolean
:param overwrite_existing: [Optional] If set to "True", and an object with same name of the dataset
to be copied already exist in the bucket, Galaxy replaces the existing object with the dataset to
be copied. If set to "False", Galaxy appends datatime to the dataset name to prevent overwriting
existing, if any, object.
:rtype: void
:return: void
:param overwrite_existing: [Optional] If set to "True", and an object with same name of the
dataset to be copied already exist in the bucket, Galaxy replaces
the existing object with the dataset to be copied. If set to
"False", Galaxy appends datetime to the dataset name to prevent
overwriting the existing object.
:rtype: list
:return: A list of labels for the objects that were uploaded.
"""
if CloudProviderFactory is None:
raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
Expand All @@ -261,10 +264,13 @@ def copy_to(self, trans, history_id, provider, bucket, credentials, dataset_ids=
raise ObjectNotFound("Could not find the specified bucket `{}`.".format(bucket))

history = trans.sa_session.query(trans.app.model.History).get(history_id)
uploaded = []
for hda in history.datasets:
if dataset_ids is None or hda.dataset.id in dataset_ids:
object_label = hda.name
if overwrite_existing is False and bucket_obj.get(object_label) is not None:
object_label += "-" + datetime.datetime.now().strftime("%y-%m-%d-%H-%M-%S")
created_obj = bucket_obj.create_object(object_label)
created_obj.upload_from_file(hda.dataset.get_file_name())
uploaded.append(object_label)
return uploaded
34 changes: 18 additions & 16 deletions lib/galaxy/webapps/galaxy/api/cloud.py
View file
Expand Up @@ -116,28 +116,29 @@ def copy_to(self, trans, payload, **kwargs):
:type trans: galaxy.web.framework.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type payload: dict
:type payload: dictionary
:param payload: A dictionary structure containing the following keys:
* history_id the (encoded) id of history from which the object should be copied.
* provider: the name of a cloud-based resource provided (e.g., `aws`, `azure`, or `openstack`).
* provider: the name of a cloud-based resource provider (e.g., `aws`, `azure`, or `openstack`).
* bucket: the name of a bucket to which data should be copied (e.g., a bucket name on AWS S3).
* credentials: a dictionary containing all the credentials required to authenticated to the
specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN,
"access_key": YOUR_AWS_ACCESS_TOKEN}).
* dataset_ids: [Optional; default: None]
A list of encoded dataset IDs belonging to the specified history,
which should be copied to the given bucket. If not provided, Galaxy copies
A list of encoded dataset IDs belonging to the specified history
that should be copied to the given bucket. If not provided, Galaxy copies
all the datasets belonging the specified history.
* overwrite_existing: [Optional; default: False]
A boolean value. If set to "True", and an object with same name of the dataset
to be copied already exist in the bucket, Galaxy replaces the existing object
with the dataset to be copied. If set to "False", Galaxy appends datatime
to the dataset name to prevent overwriting existing, if any, object.
with the dataset to be copied. If set to "False", Galaxy appends datetime
to the dataset name to prevent overwriting an existing object.
:param kwargs:
:rtype: string
:return: a message confirming successful copy from Galaxy to a cloud-based storage.
:rtype: dictionary
:return: Information about the copied datasets, including uploaded_dataset_labels
and destination bucket name.
"""
missing_arguments = []
encoded_history_id = payload.get("history_id", None)
Expand Down Expand Up @@ -179,11 +180,12 @@ def copy_to(self, trans, payload, **kwargs):
raise ActionInputError("The following provided dataset IDs are invalid, please correct them and retry. "
"{}".format(invalid_dataset_ids))

self.cloud_manager.upload(trans=trans,
history_id=history_id,
provider=provider,
bucket=bucket,
credentials=credentials,
dataset_ids=dataset_ids,
overwrite_existing=payload.get("overwrite_existing", False))
return 'The selected dataset(s) are uploaded successfully!'
uploaded = self.cloud_manager.copy_to(trans=trans,
history_id=history_id,
provider=provider,
bucket=bucket,
credentials=credentials,
dataset_ids=dataset_ids,
overwrite_existing=payload.get("overwrite_existing", False))
return {'uploaded_dataset_labels': uploaded,
'bucket_name': bucket}

0 comments on commit b3f1a7c

Please sign in to comment.