From e3b63e16d08fe08556bc690eaf5eec3e00b8aaae Mon Sep 17 00:00:00 2001 From: vjalili Date: Wed, 11 Apr 2018 12:09:13 -0700 Subject: [PATCH 01/76] Add migration script for cloudauthz table. --- .../versions/0142_add_cloudauthz_tables.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py diff --git a/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py b/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py new file mode 100644 index 000000000000..904850f3af00 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py @@ -0,0 +1,44 @@ +""" +Migration script to add a new tables for CloudAuthz (tokens required to access cloud-based resources). +""" +from __future__ import print_function + +import logging + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table + +from galaxy.model.custom_types import JSONType + +log = logging.getLogger(__name__) +metadata = MetaData() + +cloudauthz = Table( + "cloudauthz", metadata, + Column('id', Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), + Column('provider', String(255)), + Column('config', JSONType), + Column('tokens', JSONType), + Column('last_update', DateTime), + Column('last_activity', DateTime)) + + +def upgrade(migrate_engine): + print(__doc__) + metadata.bind = migrate_engine + metadata.reflect() + + try: + cloudauthz.create() + except Exception as e: + log.exception("Failed to create cloudauthz table: {}".format(str(e))) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + + try: + cloudauthz.drop() + except Exception as e: + log.exception("Failed to drop cloudauthz table: {}".format(str(e))) From c25e69253f105b9f20ed1b114e5bfdeecad60310 Mon Sep 17 00:00:00 2001 From: vjalili Date: Wed, 11 Apr 2018 15:15:58 -0700 Subject: [PATCH 02/76] Add CloudAuthz class to model. --- lib/galaxy/model/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index daae3897dc45..a85a586cc3ea 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4687,6 +4687,16 @@ def create_social_auth(cls, user, uid, provider): return instance +class CloudAuthz(object): + def __init__(self, user, provider, config): + self.user = user + self.provider = provider + self.config = config + self.tokens = None + self.last_update = datetime.now() + self.last_activity = datetime.now() + + class Page(Dictifiable): dict_element_visible_keys = ['id', 'title', 'latest_revision_id', 'slug', 'published', 'importable', 'deleted'] From dd9e8a5e6258ab14c7481691c3b5e6ef54c00836 Mon Sep 17 00:00:00 2001 From: vjalili Date: Wed, 11 Apr 2018 15:16:42 -0700 Subject: [PATCH 03/76] Add CloudAuthz to mapping. --- lib/galaxy/model/mapping.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index a46a0af3d6a6..6f82ab225af4 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -133,6 +133,16 @@ Column('lifetime', Integer), Column('assoc_type', VARCHAR(64))) +model.CloudAuthz.table = Table( + "cloudauthz", metadata, + Column('id', Integer, primary_key=True), + Column('user_id', Integer, ForeignKey("galaxy_user.id"), index=True), + Column('provider', String(255)), + Column('config', JSONType), + Column('tokens', JSONType), + Column('last_update', DateTime), + Column('last_activity', DateTime)) + model.PasswordResetToken.table = Table( "password_reset_token", metadata, Column("token", String(32), primary_key=True, unique=True, index=True), @@ -1477,6 +1487,12 @@ def simple_mapping(model, **kwds): backref='social_auth') )) +mapper(model.CloudAuthz, model.CloudAuthz.table, properties=dict( + user=relation(model.User, + primaryjoin=(model.CloudAuthz.table.c.user_id == model.User.table.c.id), + backref='cloudauthz') +)) + mapper(model.ValidationError, model.ValidationError.table) simple_mapping(model.HistoryDatasetAssociation, From 14288a2b150d49e46577c2e8238e93d927548202 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 16 Apr 2018 13:55:51 -0700 Subject: [PATCH 04/76] Add authn_id and user_id to cloudauthz. --- lib/galaxy/model/__init__.py | 5 +++-- lib/galaxy/model/mapping.py | 6 +++++- .../model/migrate/versions/0142_add_cloudauthz_tables.py | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a85a586cc3ea..fb6e176d234d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4688,10 +4688,11 @@ def create_social_auth(cls, user, uid, provider): class CloudAuthz(object): - def __init__(self, user, provider, config): - self.user = user + def __init__(self, user_id, provider, config, authn_id): + self.user_id = user_id self.provider = provider self.config = config + self.authn_id = authn_id self.tokens = None self.last_update = datetime.now() self.last_activity = datetime.now() diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6f82ab225af4..8550cc300225 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -139,6 +139,7 @@ Column('user_id', Integer, ForeignKey("galaxy_user.id"), index=True), Column('provider', String(255)), Column('config', JSONType), + Column('authn_id', Integer, ForeignKey("oidc_user_authnz_tokens.id"), index=True), Column('tokens', JSONType), Column('last_update', DateTime), Column('last_activity', DateTime)) @@ -1490,7 +1491,10 @@ def simple_mapping(model, **kwds): mapper(model.CloudAuthz, model.CloudAuthz.table, properties=dict( user=relation(model.User, primaryjoin=(model.CloudAuthz.table.c.user_id == model.User.table.c.id), - backref='cloudauthz') + backref='cloudauthz'), + authn=relation(model.UserAuthnzToken, + primaryjoin=(model.CloudAuthz.table.c.authn_id == model.UserAuthnzToken.table.c.id), + backref='cloudauthz') )) mapper(model.ValidationError, model.ValidationError.table) diff --git a/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py b/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py index 904850f3af00..7f47e083cb79 100644 --- a/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py +++ b/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py @@ -18,6 +18,7 @@ Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column('provider', String(255)), Column('config', JSONType), + Column('authn_id', Integer, ForeignKey("oidc_user_authnz_tokens.id"), index=True), Column('tokens', JSONType), Column('last_update', DateTime), Column('last_activity', DateTime)) From 7f5a59d1347af33285b7e6df5955cedf9d456221 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 16 Apr 2018 14:00:07 -0700 Subject: [PATCH 05/76] Add cloudauthz api and manger, and define a route for API calls. --- lib/galaxy/managers/cloudauthzs.py | 20 ++++++ lib/galaxy/webapps/galaxy/api/cloudauthz.py | 80 +++++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 3 + 3 files changed, 103 insertions(+) create mode 100644 lib/galaxy/managers/cloudauthzs.py create mode 100644 lib/galaxy/webapps/galaxy/api/cloudauthz.py diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py new file mode 100644 index 000000000000..8be08f56a0c2 --- /dev/null +++ b/lib/galaxy/managers/cloudauthzs.py @@ -0,0 +1,20 @@ +""" +Manager and serializer for cloud authorizations (cloudauthzs). +""" + +import logging + +from galaxy import model +from galaxy.managers import sharable +from galaxy.managers import deletable + +og = logging.getLogger(__name__) + + +class CloudAuthzManager(sharable.SharableModelManager): + + model_class = model.CloudAuthz + foreign_key_name = 'cloudauthz' + + def __init__(self, app, *args, **kwargs): + super(CloudAuthzManager, self).__init__(app, *args, **kwargs) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py new file mode 100644 index 000000000000..002c3a0c8664 --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -0,0 +1,80 @@ +""" +API operations on defining cloud authorizations. +""" + +import logging + +from galaxy import web +from galaxy.managers import cloudauthzs +from galaxy.web.base.controller import BaseAPIController + +log = logging.getLogger(__name__) + + +class CloudAuthzController(BaseAPIController): + """ + RESTfull controller for defining cloud authorizations. + """ + + def __init__(self, app): + super(CloudAuthzController, self).__init__(app) + self.cloudauthzs_manager = cloudauthzs.CloudAuthzManager(app) + + @web.expose_api + def index(self, trans, **kwargs): + """ + + :param trans: + :param kwargs: + :return: + """ + pass + + @web.expose_api + def create(self, trans, payload, **kwargs): + """ + + :param trans: + :param payload: + :param kwargs: + :return: + """ + if not isinstance(payload, dict): + trans.response.status = 400 + return {'status': '400', + 'message': 'Invalid payload data type. The payload is expected to be a dictionary, ' + 'but received data of type `{}`.'.format(str(type(payload)))} + + missing_arguments = [] + provider = payload.get('provider', None) + if provider is None: + missing_arguments.append('provider') + + config = payload.get('config', None) + if config is None: + missing_arguments.append('config') + + authn_id = payload.get('authn_id', None) + if authn_id is None: + missing_arguments.append('authn_id') + + if len(missing_arguments) > 0: + trans.response.status = 400 + return {'status': '400', + 'message': 'The following required arguments are missing in the payload: {}'.format(missing_arguments)} + + try: + new_cloudauthz = self.cloudauthzs_manager.create( + user_id=trans.user.id, + provider=provider, + config=config, + authn_id=authn_id + ) + + log.debug('Created a new cloudauthz record for the user id `{}` '.format(str(trans.user.id))) + trans.response.status = '200' + return None + + except Exception as e: + log.exception('An unexpected error has occurred while responding to the create request of the cloudauthz API. ' + str(e)) + trans.response.status = '500 Internal Server Error' diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index a14865185b7a..7e24afcd56d3 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -268,6 +268,9 @@ def populate_api_routes(webapp, app): webapp.mapper.resource('group', 'groups', path_prefix='/api') webapp.mapper.resource_with_deleted('quota', 'quotas', path_prefix='/api') + webapp.mapper.connect('/api/cloud/authz/create', action='create', controller='cloudauthz') + webapp.mapper.resource('cloudauthz', 'cloudauthz', path_prefix='/api') + webapp.mapper.connect('get_custom_builds_metadata', '/api/histories/{id}/custom_builds_metadata', controller='histories', From c6af14ba28e1f36fef4008b57595e90e1282e5cd Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 16 Apr 2018 14:46:25 -0700 Subject: [PATCH 06/76] Add CloudAuthz summary view serializer. --- lib/galaxy/managers/cloudauthzs.py | 43 +++++++++++++++++++++ lib/galaxy/webapps/galaxy/api/cloudauthz.py | 4 +- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 8be08f56a0c2..70556e98043d 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -5,6 +5,7 @@ import logging from galaxy import model +from galaxy.managers import base from galaxy.managers import sharable from galaxy.managers import deletable @@ -18,3 +19,45 @@ class CloudAuthzManager(sharable.SharableModelManager): def __init__(self, app, *args, **kwargs): super(CloudAuthzManager, self).__init__(app, *args, **kwargs) + + +class CloudAuthzsSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): # sharable.SharableModelSerializer, deletable.PurgableSerializerMixin): + """ + Interface/service object for serializing cloud authorizations (cloudauthzs) into dictionaries. + """ + model_manager_class = CloudAuthzManager + + def __init__(self, app, **kwargs): + super(CloudAuthzsSerializer, self).__init__(app, **kwargs) + self.cloudauthzs_manager = self.manager + + self.default_view = 'summary' + self.add_view('summary', [ + 'id', + 'model_class', + 'user_id', + 'provider', + 'config', + 'authn_id', + 'last_update', + 'last_activity' + ]) + + def add_serializers(self): + super(CloudAuthzsSerializer, self).add_serializers() + deletable.PurgableSerializerMixin.add_serializers(self) + + # Arguments of the following lambda functions: + # i : an instance of galaxy.model.CloudAuthz. + # k : serialized dictionary key (e.g., 'model_class', 'provider'). + # **c: a dictionary containing 'trans' and 'user' objects. + self.serializers.update({ + 'id' : lambda i, k, **c: self.app.security.encode_id(i.id), + 'model_class' : lambda *a, **c: 'CloudAuthz', + 'user_id' : lambda i, k, **c: self.app.security.encode_id(i.user_id), + 'provider' : lambda i, k, **c: str(i.provider), + 'config' : lambda i, k, **c: str(i.config), + 'authn_id' : lambda i, k, **c: self.app.security.encode_id(i.authn_id), + 'last_update' : lambda i, k, **c: str(i.last_update), + 'last_activity': lambda i, k, **c: str(i.last_activity) + }) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 002c3a0c8664..4f72ecd0ad57 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -70,10 +70,10 @@ def create(self, trans, payload, **kwargs): config=config, authn_id=authn_id ) - + view = self.cloudauthzs_serializer.serialize_to_view(new_cloudauthz, trans=trans, **self._parse_serialization_params(kwargs, 'summary')) log.debug('Created a new cloudauthz record for the user id `{}` '.format(str(trans.user.id))) trans.response.status = '200' - return None + return view except Exception as e: log.exception('An unexpected error has occurred while responding to the create request of the cloudauthz API. ' + str(e)) From ee255be84748748bfd01e6862fed0c2970f55ac2 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 16 Apr 2018 14:47:03 -0700 Subject: [PATCH 07/76] Fix a bug in CloudAuthz summary view serializer. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 4f72ecd0ad57..2af84e1dbdcc 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -19,6 +19,7 @@ class CloudAuthzController(BaseAPIController): def __init__(self, app): super(CloudAuthzController, self).__init__(app) self.cloudauthzs_manager = cloudauthzs.CloudAuthzManager(app) + self.cloudauthzs_serializer = cloudauthzs.CloudAuthzsSerializer(app) @web.expose_api def index(self, trans, **kwargs): From 5bf003011a5efc8eb9d02d61170e23bbd1481c81 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 16 Apr 2018 15:27:12 -0700 Subject: [PATCH 08/76] Update CloudAuthz index; return all the cloudauthz records for a user. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 2af84e1dbdcc..93751de4b1be 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -29,7 +29,11 @@ def index(self, trans, **kwargs): :param kwargs: :return: """ - pass + rtv = [] + for cloudauthz in trans.user.cloudauthz: + rtv.append(self.cloudauthzs_serializer.serialize_to_view( + cloudauthz, user=trans.user, trans=trans, **self._parse_serialization_params(kwargs, 'summary'))) + return rtv @web.expose_api def create(self, trans, payload, **kwargs): From 6f043824978bccdc208929c84ac7ca83f00ca9f5 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 16 Apr 2018 15:34:12 -0700 Subject: [PATCH 09/76] Change router for cloudauthz index. --- lib/galaxy/webapps/galaxy/buildapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 7e24afcd56d3..74eff9e63276 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -268,8 +268,8 @@ def populate_api_routes(webapp, app): webapp.mapper.resource('group', 'groups', path_prefix='/api') webapp.mapper.resource_with_deleted('quota', 'quotas', path_prefix='/api') + webapp.mapper.connect('/api/cloud/authz/', action='index', controller='cloudauthz') webapp.mapper.connect('/api/cloud/authz/create', action='create', controller='cloudauthz') - webapp.mapper.resource('cloudauthz', 'cloudauthz', path_prefix='/api') webapp.mapper.connect('get_custom_builds_metadata', '/api/histories/{id}/custom_builds_metadata', From 8a980b1f7f7e2aa3b16ce66e1852be14e2d784c1 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 16 Apr 2018 16:06:40 -0700 Subject: [PATCH 10/76] Refactoring *thzs -> *thz; remove (s). --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 93751de4b1be..24c4651ed512 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -18,8 +18,8 @@ class CloudAuthzController(BaseAPIController): def __init__(self, app): super(CloudAuthzController, self).__init__(app) - self.cloudauthzs_manager = cloudauthzs.CloudAuthzManager(app) - self.cloudauthzs_serializer = cloudauthzs.CloudAuthzsSerializer(app) + self.cloudauthz_manager = cloudauthzs.CloudAuthzManager(app) + self.cloudauthz_serializer = cloudauthzs.CloudAuthzsSerializer(app) @web.expose_api def index(self, trans, **kwargs): @@ -31,7 +31,7 @@ def index(self, trans, **kwargs): """ rtv = [] for cloudauthz in trans.user.cloudauthz: - rtv.append(self.cloudauthzs_serializer.serialize_to_view( + rtv.append(self.cloudauthz_serializer.serialize_to_view( cloudauthz, user=trans.user, trans=trans, **self._parse_serialization_params(kwargs, 'summary'))) return rtv @@ -69,13 +69,13 @@ def create(self, trans, payload, **kwargs): 'message': 'The following required arguments are missing in the payload: {}'.format(missing_arguments)} try: - new_cloudauthz = self.cloudauthzs_manager.create( + new_cloudauthz = self.cloudauthz_manager.create( user_id=trans.user.id, provider=provider, config=config, authn_id=authn_id ) - view = self.cloudauthzs_serializer.serialize_to_view(new_cloudauthz, trans=trans, **self._parse_serialization_params(kwargs, 'summary')) + view = self.cloudauthz_serializer.serialize_to_view(new_cloudauthz, trans=trans, **self._parse_serialization_params(kwargs, 'summary')) log.debug('Created a new cloudauthz record for the user id `{}` '.format(str(trans.user.id))) trans.response.status = '200' return view From 7760c98953524e6e5355f7ca678759dd7f2977d9 Mon Sep 17 00:00:00 2001 From: vjalili Date: Tue, 17 Apr 2018 09:39:50 -0700 Subject: [PATCH 11/76] Assess cloudauthz config type in API before passing it to its manager. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 24c4651ed512..48d0ea8be9b8 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -68,6 +68,10 @@ def create(self, trans, payload, **kwargs): return {'status': '400', 'message': 'The following required arguments are missing in the payload: {}'.format(missing_arguments)} + if not isinstance(config, dict): + return {'status': '400', + 'message': 'Invalid type for the required `config` variable; expect `dict` but received `{}`.'.format(type(config))} + try: new_cloudauthz = self.cloudauthz_manager.create( user_id=trans.user.id, From 193b73f26ecb661a4fb180a64806a0c7f0c6a2ba Mon Sep 17 00:00:00 2001 From: vjalili Date: Tue, 17 Apr 2018 10:09:04 -0700 Subject: [PATCH 12/76] Fix a wrong order of imports. --- lib/galaxy/managers/cloudauthzs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 70556e98043d..3767c5adbf4e 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -6,8 +6,8 @@ from galaxy import model from galaxy.managers import base -from galaxy.managers import sharable from galaxy.managers import deletable +from galaxy.managers import sharable og = logging.getLogger(__name__) From 0585f73384221cc024a62ecb1857f53670ced091 Mon Sep 17 00:00:00 2001 From: vjalili Date: Tue, 17 Apr 2018 10:56:59 -0700 Subject: [PATCH 13/76] Update cloudauthz API documentation. --- lib/galaxy/managers/cloudauthzs.py | 2 +- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 44 +++++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 3767c5adbf4e..86a4dd905340 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -21,7 +21,7 @@ def __init__(self, app, *args, **kwargs): super(CloudAuthzManager, self).__init__(app, *args, **kwargs) -class CloudAuthzsSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): # sharable.SharableModelSerializer, deletable.PurgableSerializerMixin): +class CloudAuthzsSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): """ Interface/service object for serializing cloud authorizations (cloudauthzs) into dictionaries. """ diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 48d0ea8be9b8..5cfab023782a 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -1,5 +1,11 @@ """ API operations on defining cloud authorizations. + +Through means of cloud authorization a user is able to grant a Galaxy server a secure access to his/her +cloud-based resources without sharing his/her long-lasting credentials. + +User provides a provider-specific configuration, which Galaxy users to request temporary credentials +from the provider to access the user's resources. """ import logging @@ -24,10 +30,16 @@ def __init__(self, app): @web.expose_api def index(self, trans, **kwargs): """ + * GET /api/cloud/authz + Lists all the cloud authorizations user has defined. + + :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :param trans: Galaxy web transaction - :param trans: - :param kwargs: - :return: + :param kwargs: empty dict + + :rtype: list of dict + :return: a list of cloud authorizations (each represented in key-value pair format) defined for the user. """ rtv = [] for cloudauthz in trans.user.cloudauthz: @@ -38,11 +50,27 @@ def index(self, trans, **kwargs): @web.expose_api def create(self, trans, payload, **kwargs): """ - - :param trans: - :param payload: - :param kwargs: - :return: + * POST /api/cloud/authz/create + Request to store the payload as a cloudauthz (cloud authorization) configuration for a user. + + :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :param trans: Galaxy web transaction + + :type payload: dict + :param payload: A dictionary structure containing the following keys: + * provider: the cloud-based resource provider to which this configuration belongs to. + * config: a dictionary containing all the configuration required to request temporary credentials + from the provider. + * authn_id: the (encoded) ID of a third-party authentication of a user. To have this ID, user must + have logged-in to this Galaxy server using third-party identity (e.g., Google), or has + associated his/her Galaxy account with a third-party OIDC-based identity. See this page: + https://galaxyproject.org/admin/authentication/ + :param kwargs: empty dict + + :rtype: dict + :return: a dictionary with the following kvp: + * status: HTTP response code + * message: A message complementary to the response code. """ if not isinstance(payload, dict): trans.response.status = 400 From afa31d8b0978010e91724e86e2b08e9ffe698796 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 23 Apr 2018 15:47:06 -0700 Subject: [PATCH 14/76] Update exception handling in cloudauthz to leverage Galaxy.exceptions. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 26 ++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 5cfab023782a..19ddaac9a236 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -11,6 +11,12 @@ import logging from galaxy import web +from galaxy.exceptions import ( + ActionInputError, + InternalServerError, + RequestParameterInvalidException, + RequestParameterMissingException +) from galaxy.managers import cloudauthzs from galaxy.web.base.controller import BaseAPIController @@ -73,10 +79,8 @@ def create(self, trans, payload, **kwargs): * message: A message complementary to the response code. """ if not isinstance(payload, dict): - trans.response.status = 400 - return {'status': '400', - 'message': 'Invalid payload data type. The payload is expected to be a dictionary, ' - 'but received data of type `{}`.'.format(str(type(payload)))} + raise ActionInputError('Invalid payload data type. The payload is expected to be a dictionary, but ' + 'received data of type `{}`.'.format(str(type(payload)))) missing_arguments = [] provider = payload.get('provider', None) @@ -92,13 +96,12 @@ def create(self, trans, payload, **kwargs): missing_arguments.append('authn_id') if len(missing_arguments) > 0: - trans.response.status = 400 - return {'status': '400', - 'message': 'The following required arguments are missing in the payload: {}'.format(missing_arguments)} + raise RequestParameterMissingException('The following required arguments are missing in the payload: ' + '{}'.format(missing_arguments)) if not isinstance(config, dict): - return {'status': '400', - 'message': 'Invalid type for the required `config` variable; expect `dict` but received `{}`.'.format(type(config))} + raise RequestParameterInvalidException('Invalid type for the required `config` variable; expect `dict` ' + 'but received `{}`.'.format(type(config))) try: new_cloudauthz = self.cloudauthz_manager.create( @@ -113,5 +116,6 @@ def create(self, trans, payload, **kwargs): return view except Exception as e: - log.exception('An unexpected error has occurred while responding to the create request of the cloudauthz API. ' + str(e)) - trans.response.status = '500 Internal Server Error' + msg = 'An unexpected error has occurred while responding to the create request of the cloudauthz API.' + str(e) + log.exception(msg) + raise InternalServerError(msg) From bff2f9f9f72988ffec9b9f92b935f48e39b79423 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 27 Apr 2018 10:30:32 -0700 Subject: [PATCH 15/76] Decode and assert `authn_id` in cloudauthz API. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 19ddaac9a236..2c6d5a37c263 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -14,6 +14,7 @@ from galaxy.exceptions import ( ActionInputError, InternalServerError, + MalformedId, RequestParameterInvalidException, RequestParameterMissingException ) @@ -102,6 +103,10 @@ def create(self, trans, payload, **kwargs): if not isinstance(config, dict): raise RequestParameterInvalidException('Invalid type for the required `config` variable; expect `dict` ' 'but received `{}`.'.format(type(config))) + try: + authn_id = self.decode_id(authn_id) + except Exception: + raise MalformedId('Invalid `authn_id`!') try: new_cloudauthz = self.cloudauthz_manager.create( From 4a944cebe9bf94579cd06eb3a4f5d4a46004f2ab Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 27 Apr 2018 18:47:59 -0700 Subject: [PATCH 16/76] Check if an `authn_id` exist & if user it belongs to the logged-in user. --- lib/galaxy/managers/cloudauthzs.py | 10 ++++++++++ lib/galaxy/webapps/galaxy/api/cloudauthz.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 86a4dd905340..fee86a4af475 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -4,6 +4,7 @@ import logging +from galaxy import exceptions from galaxy import model from galaxy.managers import base from galaxy.managers import deletable @@ -20,6 +21,15 @@ class CloudAuthzManager(sharable.SharableModelManager): def __init__(self, app, *args, **kwargs): super(CloudAuthzManager, self).__init__(app, *args, **kwargs) + def can_user_assume_authn(self, trans, authn_id): + qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) + if qres is None: + raise exceptions.ObjectNotFound + if qres.user_id != trans.user.id: + og.critical('The user with ID:`{}` requested creation of a cloudauthz record and associating it with ' + 'the authnz record ID:`{}`, which belongs to another user.'.format(trans.user.id, authn_id)) + raise exceptions.ItemAccessibilityException + class CloudAuthzsSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): """ diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 2c6d5a37c263..d1addcaa250d 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -14,7 +14,9 @@ from galaxy.exceptions import ( ActionInputError, InternalServerError, + ItemAccessibilityException, MalformedId, + ObjectNotFound, RequestParameterInvalidException, RequestParameterMissingException ) @@ -108,6 +110,13 @@ def create(self, trans, payload, **kwargs): except Exception: raise MalformedId('Invalid `authn_id`!') + try: + self.cloudauthz_manager.can_user_assume_authn(trans, authn_id) + except ObjectNotFound: + raise ObjectNotFound('Authentication record with the given `authn_id` not found.') + except ItemAccessibilityException: + raise ItemAccessibilityException('The specified authentication with ID:`{}` belongs to another user.'.format(authn_id)) + try: new_cloudauthz = self.cloudauthz_manager.create( user_id=trans.user.id, From 900f88e87784a7a80901af1fa2d72b16f0a4e394 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 10 Aug 2018 12:03:40 -0700 Subject: [PATCH 17/76] Fix a typo: "og" -> "log" --- lib/galaxy/managers/cloudauthzs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index fee86a4af475..71c72b0b863e 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -10,7 +10,7 @@ from galaxy.managers import deletable from galaxy.managers import sharable -og = logging.getLogger(__name__) +log = logging.getLogger(__name__) class CloudAuthzManager(sharable.SharableModelManager): @@ -26,7 +26,7 @@ def can_user_assume_authn(self, trans, authn_id): if qres is None: raise exceptions.ObjectNotFound if qres.user_id != trans.user.id: - og.critical('The user with ID:`{}` requested creation of a cloudauthz record and associating it with ' + log.critical('The user with ID:`{}` requested creation of a cloudauthz record and associating it with ' 'the authnz record ID:`{}`, which belongs to another user.'.format(trans.user.id, authn_id)) raise exceptions.ItemAccessibilityException From fd269464a154aa03b70dc55cb1bd996c81218052 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 10 Aug 2018 12:59:26 -0700 Subject: [PATCH 18/76] Add a method to authnz manager to request CloudAuthz cloud access token. --- lib/galaxy/authnz/managers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 910b34c9bad4..a329a15010d2 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -1,4 +1,5 @@ +import cloudauthz import importlib import logging import xml.etree.ElementTree as ET @@ -143,3 +144,18 @@ def disconnect(self, provider, trans, disconnect_redirect_url=None): '{}'.format(provider, trans.user.username, str(e)) log.exception(msg) return False, msg, None + + def request_cloud_access_tokens(self, cloudauthz): + """ + + :param cloudauthz: + :return: + """ + ca = cloudauthz.CloudAuthz() + config = cloudauthz.config + config['id_token'] = cloudauthz.authn.get_id_token() + import ast + dicctt = ast.literal_eval(cloudauthz.authn.extra_data) + config['id_token'] = dicctt.get('id_token') + re = ca.authorize(cloudauthz.provider, config) + pass From 3c9b852faf3150692b954c5e3d52911346d4aa10 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 10 Aug 2018 14:57:09 -0700 Subject: [PATCH 19/76] Update the migration script number. --- ...142_add_cloudauthz_tables.py => 0143_add_cloudauthz_tables.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/galaxy/model/migrate/versions/{0142_add_cloudauthz_tables.py => 0143_add_cloudauthz_tables.py} (100%) diff --git a/lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py b/lib/galaxy/model/migrate/versions/0143_add_cloudauthz_tables.py similarity index 100% rename from lib/galaxy/model/migrate/versions/0142_add_cloudauthz_tables.py rename to lib/galaxy/model/migrate/versions/0143_add_cloudauthz_tables.py From e0e02c2e1d75790874c423d48fc4edfb325c20b3 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 10 Aug 2018 15:40:09 -0700 Subject: [PATCH 20/76] Add a method to authnz manager to get cloud access credentials. --- lib/galaxy/authnz/managers.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index a329a15010d2..97e5da541c3f 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -1,5 +1,5 @@ -import cloudauthz +from cloudauthz import CloudAuthz import importlib import logging import xml.etree.ElementTree as ET @@ -145,17 +145,14 @@ def disconnect(self, provider, trans, disconnect_redirect_url=None): log.exception(msg) return False, msg, None - def request_cloud_access_tokens(self, cloudauthz): + @staticmethod + def get_cloud_access_credentials(cloudauthz): """ :param cloudauthz: :return: """ - ca = cloudauthz.CloudAuthz() config = cloudauthz.config config['id_token'] = cloudauthz.authn.get_id_token() - import ast - dicctt = ast.literal_eval(cloudauthz.authn.extra_data) - config['id_token'] = dicctt.get('id_token') - re = ca.authorize(cloudauthz.provider, config) - pass + ca = CloudAuthz() + return ca.authorize(cloudauthz.provider, config) From e510f07528221ddf325dcced890db2432a4cb7db Mon Sep 17 00:00:00 2001 From: vjalili Date: Sun, 12 Aug 2018 11:41:06 -0700 Subject: [PATCH 21/76] Replace credentials with authz_id in the upload API of cloud storage. --- lib/galaxy/webapps/galaxy/api/cloud.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 05fc8356477e..9bc8d4210e25 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -105,9 +105,9 @@ def upload(self, trans, payload, **kwargs): if objects is None: missing_arguments.append("objects") - credentials = payload.get("credentials", None) - if credentials is None: - missing_arguments.append("credentials") + authz_id = payload.get("authz_id", None) + if authz_id is None: + missing_arguments.append("authz_id") if len(missing_arguments) > 0: raise ActionInputError("The following required arguments are missing in the payload: {}".format(missing_arguments)) @@ -126,7 +126,7 @@ def upload(self, trans, payload, **kwargs): provider=provider, bucket=bucket, objects=objects, - credentials=credentials, + authz_id=authz_id, input_args=payload.get("input_args", None)) rtv = [] for dataset in datasets: From 841f04400003ddc44465e5c2c1c0ef1a86cf6e18 Mon Sep 17 00:00:00 2001 From: vjalili Date: Sun, 12 Aug 2018 11:58:08 -0700 Subject: [PATCH 22/76] Update cloud upload manager to use authz id instead of credentials. --- lib/galaxy/managers/cloud.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 5f369474ce67..45eccabf249d 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -162,7 +162,7 @@ def _get_inputs(obj, key, input_args): 'files_0|url_paste': key.generate_url(expires_in=SINGED_URL_TTL), } - def upload(self, trans, history_id, provider, bucket, objects, credentials, input_args=None): + def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_args=None): """ Implements the logic of uploading a file from a cloud-based storage (e.g., Amazon S3) and persisting it as a Galaxy dataset. @@ -201,6 +201,8 @@ def upload(self, trans, history_id, provider, bucket, objects, credentials, inpu if input_args is None: input_args = {} + authz = trans.sa_session.query(trans.app.model.CloudAuthz).get(authz_id) + credentials = trans.app.authnz_manager.request_cloud_access_tokens(authz) connection = self._configure_provider(provider, credentials) try: bucket_obj = connection.object_store.get(bucket) From e1da0471604498ee207e7cae4f563f063d81b34b Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 11:39:42 -0700 Subject: [PATCH 23/76] Configure AWS using temporary credentials generated by CloudAuthz. --- lib/galaxy/managers/cloud.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 45eccabf249d..e795e510a2a9 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -62,16 +62,21 @@ def _configure_provider(provider, credentials): if provider == 'aws': access = credentials.get('access_key', None) if access is None: - missing_credentials.append('access_key') + access = credentials.get("AccessKeyId", None) + if access is None: + missing_credentials.append('access_key') secret = credentials.get('secret_key', None) if secret is None: - missing_credentials.append('secret_key') + secret = credentials.get("SecretAccessKey", None) + if secret is None: + missing_credentials.append('secret_key') if len(missing_credentials) > 0: raise RequestParameterMissingException("The following required key(s) are missing from the provided " "credentials object: {}".format(missing_credentials)) - + session_token = credentials.get("SessionToken") config = {'aws_access_key': access, - 'aws_secret_key': secret} + 'aws_secret_key': secret, + "aws_session_token": session_token} connection = CloudProviderFactory().create_provider(ProviderList.AWS, config) elif provider == "azure": subscription = credentials.get('subscription_id', None) From 64f3b3db526105e023c990a6a1ebf074e4df92c8 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 11:40:25 -0700 Subject: [PATCH 24/76] Fix a bug calling different function from authnz manager. --- lib/galaxy/managers/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index e795e510a2a9..bb39424f1981 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -207,7 +207,7 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a input_args = {} authz = trans.sa_session.query(trans.app.model.CloudAuthz).get(authz_id) - credentials = trans.app.authnz_manager.request_cloud_access_tokens(authz) + credentials = trans.app.authnz_manager.get_cloud_access_credentials(authz) connection = self._configure_provider(provider, credentials) try: bucket_obj = connection.object_store.get(bucket) From 123785ef2510e1ed185a28b4c6423de2cd018e0d Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 12:45:22 -0700 Subject: [PATCH 25/76] Update the upload manager to adhere with latest CB interface changes. --- lib/galaxy/managers/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index bb39424f1981..04d17de0d2b2 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -210,7 +210,7 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a credentials = trans.app.authnz_manager.get_cloud_access_credentials(authz) connection = self._configure_provider(provider, credentials) try: - bucket_obj = connection.object_store.get(bucket) + bucket_obj = connection.storage.buckets.get(bucket) if bucket_obj is None: raise RequestParameterInvalidException("The bucket `{}` not found.".format(bucket)) except Exception as e: @@ -219,7 +219,7 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a datasets = [] for obj in objects: try: - key = bucket_obj.get(obj) + key = bucket_obj.objects.get(obj) except Exception as e: raise MessageException("The following error occurred while getting the object {}: {}".format(obj, str(e))) if key is None: From e292b2b38a7f448de3a060a3ec4ac1ae5803cbc3 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 15:18:26 -0700 Subject: [PATCH 26/76] In cloud API: accept encoded authz id, and decode it. --- lib/galaxy/webapps/galaxy/api/cloud.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 9bc8d4210e25..84b2d32cb42e 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -105,8 +105,8 @@ def upload(self, trans, payload, **kwargs): if objects is None: missing_arguments.append("objects") - authz_id = payload.get("authz_id", None) - if authz_id is None: + encoded_authz_id = payload.get("authz_id", None) + if encoded_authz_id is None: missing_arguments.append("authz_id") if len(missing_arguments) > 0: @@ -117,6 +117,11 @@ def upload(self, trans, payload, **kwargs): except exceptions.MalformedId as e: raise ActionInputError('Invalid history ID. {}'.format(e)) + try: + authz_id = self.decode_id(encoded_authz_id) + except exceptions.MalformedId as e: + raise ActionInputError('Invalid authz ID. {}'.format(e)) + if not isinstance(objects, list): raise ActionInputError('The `objects` should be a list, but received an object of type {} instead.'.format( type(objects))) From e6b93a090be676501ab418eb7e59da201fcbc307 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 15:59:19 -0700 Subject: [PATCH 27/76] Assert if user can assume a specific authz. --- lib/galaxy/managers/cloud.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 04d17de0d2b2..9ebdc833b76f 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -167,6 +167,14 @@ def _get_inputs(obj, key, input_args): 'files_0|url_paste': key.generate_url(expires_in=SINGED_URL_TTL), } + @staticmethod + def _can_user_assume_authz(user, authz): + if user.id != authz.user_id: + msg = "The request authorization configuration (with ID:`{}`) is not accessible for user with " \ + "ID:`{}`.".format(authz.id, user.id) + log.warn(msg) + raise ItemAccessibilityException(msg) + def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_args=None): """ Implements the logic of uploading a file from a cloud-based storage (e.g., Amazon S3) @@ -207,6 +215,7 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a input_args = {} authz = trans.sa_session.query(trans.app.model.CloudAuthz).get(authz_id) + self._can_user_assume_authz(trans.user, authz) credentials = trans.app.authnz_manager.get_cloud_access_credentials(authz) connection = self._configure_provider(provider, credentials) try: From 8d9f86b7fb285c7543cf30021a81be62e10e3644 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 16:08:02 -0700 Subject: [PATCH 28/76] Handle item cloudauthn accessibility in its manager rather than API. --- lib/galaxy/managers/cloudauthzs.py | 11 +++++++---- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 7 +------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 71c72b0b863e..2a6d4f36e96e 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -24,11 +24,14 @@ def __init__(self, app, *args, **kwargs): def can_user_assume_authn(self, trans, authn_id): qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) if qres is None: - raise exceptions.ObjectNotFound + msg = "Authentication record with the given `authn_id` (`{}`) not found.".format(authn_id) + log.debug(msg) + raise exceptions.ObjectNotFound(msg) if qres.user_id != trans.user.id: - log.critical('The user with ID:`{}` requested creation of a cloudauthz record and associating it with ' - 'the authnz record ID:`{}`, which belongs to another user.'.format(trans.user.id, authn_id)) - raise exceptions.ItemAccessibilityException + msg = "The request authentication with ID `{}` is not accessible to user with ID " \ + "`{}`.".format(authn_id, trans.user.id) + log.warn(msg) + raise exceptions.ItemAccessibilityException(msg) class CloudAuthzsSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index d1addcaa250d..3910c6a5feca 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -110,12 +110,7 @@ def create(self, trans, payload, **kwargs): except Exception: raise MalformedId('Invalid `authn_id`!') - try: - self.cloudauthz_manager.can_user_assume_authn(trans, authn_id) - except ObjectNotFound: - raise ObjectNotFound('Authentication record with the given `authn_id` not found.') - except ItemAccessibilityException: - raise ItemAccessibilityException('The specified authentication with ID:`{}` belongs to another user.'.format(authn_id)) + self.cloudauthz_manager.can_user_assume_authn(trans, authn_id) try: new_cloudauthz = self.cloudauthz_manager.create( From be882841e15545923f1af63338951ff2ff572823 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 16:36:46 -0700 Subject: [PATCH 29/76] Check for accessibility and availability of authz config. --- lib/galaxy/managers/cloud.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 9ebdc833b76f..1eb9eaf48772 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -168,12 +168,16 @@ def _get_inputs(obj, key, input_args): } @staticmethod - def _can_user_assume_authz(user, authz): - if user.id != authz.user_id: + def _get_authz_config(trans, authz_id): + authz = trans.sa_session.query(trans.app.model.CloudAuthz).get(authz_id) + if authz is None: + raise ObjectNotFound("An authorization configuration with given ID not found.") + if trans.user.id != authz.user_id: msg = "The request authorization configuration (with ID:`{}`) is not accessible for user with " \ - "ID:`{}`.".format(authz.id, user.id) + "ID:`{}`.".format(authz.id, trans.user.id) log.warn(msg) raise ItemAccessibilityException(msg) + return authz def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_args=None): """ @@ -214,8 +218,7 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a if input_args is None: input_args = {} - authz = trans.sa_session.query(trans.app.model.CloudAuthz).get(authz_id) - self._can_user_assume_authz(trans.user, authz) + authz = self._get_authz_config(trans, authz_id) credentials = trans.app.authnz_manager.get_cloud_access_credentials(authz) connection = self._configure_provider(provider, credentials) try: From f36d5b549cb41547e816c7c7a37014cd5ae830f4 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 16:39:17 -0700 Subject: [PATCH 30/76] Make a func in cloudauthz manager static. --- lib/galaxy/managers/cloudauthzs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 2a6d4f36e96e..a05fb2d0ac9e 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -21,7 +21,8 @@ class CloudAuthzManager(sharable.SharableModelManager): def __init__(self, app, *args, **kwargs): super(CloudAuthzManager, self).__init__(app, *args, **kwargs) - def can_user_assume_authn(self, trans, authn_id): + @staticmethod + def can_user_assume_authn(trans, authn_id): qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) if qres is None: msg = "Authentication record with the given `authn_id` (`{}`) not found.".format(authn_id) From a77971bce9694ba307197cbbc51b40bd1cb4d49e Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 13 Aug 2018 16:59:21 -0700 Subject: [PATCH 31/76] Update exception messages to avoid exposing decoded IDs in Authz manager --- lib/galaxy/managers/cloudauthzs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index a05fb2d0ac9e..495dbbf86dee 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -25,12 +25,13 @@ def __init__(self, app, *args, **kwargs): def can_user_assume_authn(trans, authn_id): qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) if qres is None: - msg = "Authentication record with the given `authn_id` (`{}`) not found.".format(authn_id) + msg = "Authentication record with the given `authn_id` (`{}`) not found.".format( + trans.security.encode_id(authn_id)) log.debug(msg) raise exceptions.ObjectNotFound(msg) if qres.user_id != trans.user.id: msg = "The request authentication with ID `{}` is not accessible to user with ID " \ - "`{}`.".format(authn_id, trans.user.id) + "`{}`.".format(trans.security.encode_id(authn_id), trans.security.encode_id(trans.user.id)) log.warn(msg) raise exceptions.ItemAccessibilityException(msg) From 3f2f8fd3e04530d5ed76ae5122a0eb32550605e5 Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 11:31:26 -0700 Subject: [PATCH 32/76] Move the on_the_fly_config func to module level. --- lib/galaxy/authnz/psa_authnz.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index 2d3a2a6b5525..5103d531a20b 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -107,13 +107,6 @@ def _setup_google_backend(self, oidc_backend_config): if oidc_backend_config.get('prompt') is not None: self.config[setting_name('AUTH_EXTRA_ARGUMENTS')]['prompt'] = oidc_backend_config.get('prompt') - def _on_the_fly_config(self, trans): - trans.app.model.PSACode.trans = trans - trans.app.model.UserAuthnzToken.trans = trans - trans.app.model.PSANonce.trans = trans - trans.app.model.PSAPartial.trans = trans - trans.app.model.PSAAssociation.trans = trans - def _get_helper(self, name, do_import=False): this_config = self.config.get(setting_name(name), DEFAULTS.get(name, None)) return do_import and module_member(this_config) or this_config @@ -130,13 +123,13 @@ def _login_user(self, backend, user, social_user): self.config['user'] = user def authenticate(self, trans): - self._on_the_fly_config(trans) + on_the_fly_config(trans) strategy = Strategy(trans, Storage, self.config) backend = self._load_backend(strategy, self.config['redirect_uri']) return do_auth(backend) def callback(self, state_token, authz_code, trans, login_redirect_url): - self._on_the_fly_config(trans) + on_the_fly_config(trans) self.config[setting_name('LOGIN_REDIRECT_URL')] = login_redirect_url strategy = Strategy(trans, Storage, self.config) strategy.session_set(BACKENDS_NAME[self.config['provider']] + '_state', state_token) @@ -149,7 +142,7 @@ def callback(self, state_token, authz_code, trans, login_redirect_url): return redirect_url, self.config.get('user', None) def disconnect(self, provider, trans, disconnect_redirect_url=None, association_id=None): - self._on_the_fly_config(trans) + on_the_fly_config(trans) self.config[setting_name('DISCONNECT_REDIRECT_URL')] =\ disconnect_redirect_url if disconnect_redirect_url is not None else () strategy = Strategy(trans, Storage, self.config) @@ -242,6 +235,14 @@ def is_integrity_error(cls, exception): return exception.__class__ is IntegrityError +def on_the_fly_config(trans): + trans.app.model.PSACode.trans = trans + trans.app.model.UserAuthnzToken.trans = trans + trans.app.model.PSANonce.trans = trans + trans.app.model.PSAPartial.trans = trans + trans.app.model.PSAAssociation.trans = trans + + def contains_required_data(response=None, is_new=False, **kwargs): """ This function is called as part of authentication and authorization From 12ef351cd2821291a4d90ff7a882dde40e6140b4 Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 11:33:31 -0700 Subject: [PATCH 33/76] Pass strategy requesting ID token in get_cloud_access_credentials. --- lib/galaxy/authnz/managers.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 97e5da541c3f..3dbfd80d1f14 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -5,7 +5,7 @@ import xml.etree.ElementTree as ET from xml.etree.ElementTree import ParseError -from .psa_authnz import PSAAuthnz +from .psa_authnz import * log = logging.getLogger(__name__) @@ -90,8 +90,15 @@ def _parse_google_config(self, config_xml): rtv['prompt'] = config_xml.find('prompt').text return rtv + def _unify_provider_name(self, provider): + if provider in self.oidc_backends_config: + return provider.lower() + for k, v in BACKENDS_NAME.iteritems(): + if v == provider: + return k.lower() + def _get_authnz_backend(self, provider): - provider = provider.lower() + provider = self._unify_provider_name(provider) if provider in self.oidc_backends_config: try: return True, "", PSAAuthnz(provider, self.oidc_config, self.oidc_backends_config[provider]) @@ -145,14 +152,17 @@ def disconnect(self, provider, trans, disconnect_redirect_url=None): log.exception(msg) return False, msg, None - @staticmethod - def get_cloud_access_credentials(cloudauthz): + def get_cloud_access_credentials(self, trans, cloudauthz): """ + :param trans: :param cloudauthz: :return: """ config = cloudauthz.config - config['id_token'] = cloudauthz.authn.get_id_token() + success, message, backend = self._get_authnz_backend(cloudauthz.authn.provider) + strategy = Strategy(trans, Storage, backend.config) + on_the_fly_config(trans) + config['id_token'] = cloudauthz.authn.get_id_token(strategy) ca = CloudAuthz() return ca.authorize(cloudauthz.provider, config) From 34484e9e1eb92b8955943e95d8a36b63f983a7ea Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 11:34:02 -0700 Subject: [PATCH 34/76] pass trans when asking for cloud access credentials. --- lib/galaxy/managers/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 1eb9eaf48772..62eee8628306 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -219,7 +219,7 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a input_args = {} authz = self._get_authz_config(trans, authz_id) - credentials = trans.app.authnz_manager.get_cloud_access_credentials(authz) + credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz) connection = self._configure_provider(provider, credentials) try: bucket_obj = connection.storage.buckets.get(bucket) From e4ea4af05676d6c28e639d9404dc29e6a523ef90 Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 14:39:28 -0700 Subject: [PATCH 35/76] Define a mapping connecting a user instance with its cloud authzs. --- lib/galaxy/model/mapping.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 7d23da87a177..5d882b49f605 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1694,6 +1694,8 @@ def simple_mapping(model, **kwds): api_keys=relation(model.APIKeys, backref="user", order_by=desc(model.APIKeys.table.c.create_time)), + cloudauthzs=relation(model.CloudAuthz, + primaryjoin=model.CloudAuthz.table.c.user_id == model.User.table.c.id), )) mapper(model.PasswordResetToken, model.PasswordResetToken.table, From c97313e0eb951ed56de39b534ac4669dfc33d03f Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 14:40:32 -0700 Subject: [PATCH 36/76] Implement equals, eq & ne for CloudAuthz type. --- lib/galaxy/model/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1fa6cafcabf3..ccb75472bc90 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4886,6 +4886,21 @@ def __init__(self, user_id, provider, config, authn_id): self.last_update = datetime.now() self.last_activity = datetime.now() + def __eq__(self, other): + if not isinstance(other, CloudAuthz): + return False + return self.equals(other.user_id, other.provider, other.authn_id, other.config) + + def __ne__(self, other): + return not self.__eq__(other) + + def equals(self, user_id, provider, authn_id, config): + return (self.user_id == user_id and + self.provider == provider and + self.authn_id == authn_id and + len({k: self.config[k] for k in self.config if k in config and + self.config[k] == config[k]}) == len(self.config)) + class Page(Dictifiable): dict_element_visible_keys = ['id', 'title', 'latest_revision_id', 'slug', 'published', 'importable', 'deleted'] From d2fbff164b56a6249916d9ab36ae595c269982c1 Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 14:41:04 -0700 Subject: [PATCH 37/76] Avoid two cloud authz configuration with exact same config. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 3910c6a5feca..a9689310c4c2 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -112,6 +112,12 @@ def create(self, trans, payload, **kwargs): self.cloudauthz_manager.can_user_assume_authn(trans, authn_id) + # No two authorization configuration with + # exact same key/value should not exist. + for ca in trans.user.cloudauthzs: + if ca.equals(trans.user.id, provider, authn_id, config): + raise ActionInputError("A similar cloud authorization configuration is already defined.") + try: new_cloudauthz = self.cloudauthz_manager.create( user_id=trans.user.id, From 50876622fb7dc2503183dd8062846e8669fd4228 Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 15:18:54 -0700 Subject: [PATCH 38/76] Move a function from cloudauthzs to authnz manager. --- lib/galaxy/authnz/managers.py | 18 ++++++++++++++++++ lib/galaxy/managers/cloudauthzs.py | 15 --------------- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 3dbfd80d1f14..3d193ebb93b2 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -1,5 +1,9 @@ from cloudauthz import CloudAuthz + +from galaxy import exceptions +from galaxy import model + import importlib import logging import xml.etree.ElementTree as ET @@ -110,6 +114,20 @@ def _get_authnz_backend(self, provider): log.debug(msg) return False, msg, None + @staticmethod + def can_user_assume_authn(trans, authn_id): + qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) + if qres is None: + msg = "Authentication record with the given `authn_id` (`{}`) not found.".format( + trans.security.encode_id(authn_id)) + log.debug(msg) + raise exceptions.ObjectNotFound(msg) + if qres.user_id != trans.user.id: + msg = "The request authentication with ID `{}` is not accessible to user with ID " \ + "`{}`.".format(trans.security.encode_id(authn_id), trans.security.encode_id(trans.user.id)) + log.warn(msg) + raise exceptions.ItemAccessibilityException(msg) + def authenticate(self, provider, trans): """ :type provider: string diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 495dbbf86dee..095e777b000c 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -4,7 +4,6 @@ import logging -from galaxy import exceptions from galaxy import model from galaxy.managers import base from galaxy.managers import deletable @@ -21,20 +20,6 @@ class CloudAuthzManager(sharable.SharableModelManager): def __init__(self, app, *args, **kwargs): super(CloudAuthzManager, self).__init__(app, *args, **kwargs) - @staticmethod - def can_user_assume_authn(trans, authn_id): - qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) - if qres is None: - msg = "Authentication record with the given `authn_id` (`{}`) not found.".format( - trans.security.encode_id(authn_id)) - log.debug(msg) - raise exceptions.ObjectNotFound(msg) - if qres.user_id != trans.user.id: - msg = "The request authentication with ID `{}` is not accessible to user with ID " \ - "`{}`.".format(trans.security.encode_id(authn_id), trans.security.encode_id(trans.user.id)) - log.warn(msg) - raise exceptions.ItemAccessibilityException(msg) - class CloudAuthzsSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): """ diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index a9689310c4c2..73e4df5cacd3 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -110,7 +110,7 @@ def create(self, trans, payload, **kwargs): except Exception: raise MalformedId('Invalid `authn_id`!') - self.cloudauthz_manager.can_user_assume_authn(trans, authn_id) + trans.app.authnz_manager.can_user_assume_authn(trans, authn_id) # No two authorization configuration with # exact same key/value should not exist. From 7e68a992d3772ccd8ebbfb6a52f19fea6efbd78b Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 20:08:12 -0700 Subject: [PATCH 39/76] Move the logic of getting authnz config from manager to authnz/managers. --- lib/galaxy/authnz/managers.py | 12 ++++++++++++ lib/galaxy/managers/cloud.py | 10 +--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 3d193ebb93b2..7e945e79b6a0 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -128,6 +128,18 @@ def can_user_assume_authn(trans, authn_id): log.warn(msg) raise exceptions.ItemAccessibilityException(msg) + @staticmethod + def try_get_authz_config(trans, authz_id): + qres = trans.sa_session.query(model.CloudAuthz).get(authz_id) + if qres is None: + raise exceptions.ObjectNotFound("An authorization configuration with given ID not found.") + if trans.user.id != qres.user_id: + msg = "The request authorization configuration (with ID:`{}`) is not accessible for user with " \ + "ID:`{}`.".format(qres.id, trans.user.id) + log.warn(msg) + raise exceptions.ItemAccessibilityException(msg) + return qres + def authenticate(self, provider, trans): """ :type provider: string diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 62eee8628306..d52b78ff0534 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -169,15 +169,7 @@ def _get_inputs(obj, key, input_args): @staticmethod def _get_authz_config(trans, authz_id): - authz = trans.sa_session.query(trans.app.model.CloudAuthz).get(authz_id) - if authz is None: - raise ObjectNotFound("An authorization configuration with given ID not found.") - if trans.user.id != authz.user_id: - msg = "The request authorization configuration (with ID:`{}`) is not accessible for user with " \ - "ID:`{}`.".format(authz.id, trans.user.id) - log.warn(msg) - raise ItemAccessibilityException(msg) - return authz + return trans.app.authnz_manager.try_get_authz_config(trans, authz_id) def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_args=None): """ From e1d014939fe752cfcf99debc60e5756efe211fbf Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 16 Aug 2018 21:40:47 -0700 Subject: [PATCH 40/76] Change a deepcopy of cloudauthz config, instead of the reference. --- lib/galaxy/authnz/managers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 7e945e79b6a0..3ab88137f858 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -4,6 +4,7 @@ from galaxy import exceptions from galaxy import model +import copy import importlib import logging import xml.etree.ElementTree as ET @@ -189,7 +190,7 @@ def get_cloud_access_credentials(self, trans, cloudauthz): :param cloudauthz: :return: """ - config = cloudauthz.config + config = copy.deepcopy(cloudauthz.config) success, message, backend = self._get_authnz_backend(cloudauthz.authn.provider) strategy = Strategy(trans, Storage, backend.config) on_the_fly_config(trans) From 764ba3cdcedde28c0d96bfdb15e6321e096ea3cd Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 11:07:56 -0700 Subject: [PATCH 41/76] Encapsulate getting cloud authz credentials in a try-catch block. --- lib/galaxy/managers/cloud.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index d52b78ff0534..bed7bcc6c854 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -210,8 +210,11 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a if input_args is None: input_args = {} - authz = self._get_authz_config(trans, authz_id) - credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz) + try: + authz = self._get_authz_config(trans, authz_id) + credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz) + except Exception as e: + raise e connection = self._configure_provider(provider, credentials) try: bucket_obj = connection.storage.buckets.get(bucket) From 2915a2989a79742beb7e9a8bb31fcb422c3759f1 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 18:04:12 -0700 Subject: [PATCH 42/76] Use _future_expose_api in cloudauthz APIs for better exception handling. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 73e4df5cacd3..81f89b7b7785 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -10,7 +10,6 @@ import logging -from galaxy import web from galaxy.exceptions import ( ActionInputError, InternalServerError, @@ -22,6 +21,9 @@ ) from galaxy.managers import cloudauthzs from galaxy.web.base.controller import BaseAPIController +from galaxy.web import ( + _future_expose_api as expose_api +) log = logging.getLogger(__name__) @@ -36,7 +38,7 @@ def __init__(self, app): self.cloudauthz_manager = cloudauthzs.CloudAuthzManager(app) self.cloudauthz_serializer = cloudauthzs.CloudAuthzsSerializer(app) - @web.expose_api + @expose_api def index(self, trans, **kwargs): """ * GET /api/cloud/authz @@ -56,7 +58,7 @@ def index(self, trans, **kwargs): cloudauthz, user=trans.user, trans=trans, **self._parse_serialization_params(kwargs, 'summary'))) return rtv - @web.expose_api + @expose_api def create(self, trans, payload, **kwargs): """ * POST /api/cloud/authz/create From c8e5427e5d48c98a2075648c7ff97040e0e8300f Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 18:07:13 -0700 Subject: [PATCH 43/76] Encapsulate can_user_assume_authn in a try-catch block. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 81f89b7b7785..880fea16dab7 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -112,7 +112,10 @@ def create(self, trans, payload, **kwargs): except Exception: raise MalformedId('Invalid `authn_id`!') - trans.app.authnz_manager.can_user_assume_authn(trans, authn_id) + try: + trans.app.authnz_manager.can_user_assume_authn(trans, authn_id) + except Exception as e: + raise e # No two authorization configuration with # exact same key/value should not exist. From d223baa37ce4d61dd54e3b0dc29bf279d637fde9 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 18:15:04 -0700 Subject: [PATCH 44/76] Move getting cloudauthz by id from cloud API to authnz manager. --- lib/galaxy/authnz/managers.py | 5 +++-- lib/galaxy/managers/cloud.py | 7 +------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 3ab88137f858..e7e49802e07c 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -183,13 +183,14 @@ def disconnect(self, provider, trans, disconnect_redirect_url=None): log.exception(msg) return False, msg, None - def get_cloud_access_credentials(self, trans, cloudauthz): + def get_cloud_access_credentials(self, trans, authz_id): """ :param trans: - :param cloudauthz: + :param authz_id: :return: """ + cloudauthz = self.try_get_authz_config(trans, authz_id) config = copy.deepcopy(cloudauthz.config) success, message, backend = self._get_authnz_backend(cloudauthz.authn.provider) strategy = Strategy(trans, Storage, backend.config) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index bed7bcc6c854..c34cec653b2c 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -167,10 +167,6 @@ def _get_inputs(obj, key, input_args): 'files_0|url_paste': key.generate_url(expires_in=SINGED_URL_TTL), } - @staticmethod - def _get_authz_config(trans, authz_id): - return trans.app.authnz_manager.try_get_authz_config(trans, authz_id) - def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_args=None): """ Implements the logic of uploading a file from a cloud-based storage (e.g., Amazon S3) @@ -211,8 +207,7 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a input_args = {} try: - authz = self._get_authz_config(trans, authz_id) - credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz) + credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz_id) except Exception as e: raise e connection = self._configure_provider(provider, credentials) From 656fe5512765a636375380ef78dedc53cd73ad0f Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 18:50:06 -0700 Subject: [PATCH 45/76] Update the get_cloud_access_credentials doc-string. --- lib/galaxy/authnz/managers.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index e7e49802e07c..3820af139af3 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -185,10 +185,29 @@ def disconnect(self, provider, trans, disconnect_redirect_url=None): def get_cloud_access_credentials(self, trans, authz_id): """ - - :param trans: - :param authz_id: - :return: + This method leverages CloudAuthz (https://github.com/galaxyproject/cloudauthz) + to request a cloud-based resource provider (e.g., Amazon AWS, Microsoft Azure) + for temporary access credentials to a given resource. + + It first checks if a cloudauthz config with the given ID (`authz_id`) is + available and can be assumed by the user, and raises an exception if either + is false. Otherwise, it then extends the cloudauthz configuration as required + by the CloudAuthz library for the provider specified in the configuration. + For instance, it adds on-the-fly values such as a valid OpenID Connect + identity token, as required by CloudAuthz for AWS. Then requests temporary + credentials from the CloudAuthz library using the updated configuration. + + :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :param trans: Galaxy web transaction + + :type authz_id: int + :param authz_id: The ID of a CloudAuthz configuration to be used for + getting temporary credentials. + + :rtype: dict + :return: a dictionary containing credentials to access a cloud-based + resource provider. See CloudAuthz (https://github.com/galaxyproject/cloudauthz) + for details on the content of this dictionary. """ cloudauthz = self.try_get_authz_config(trans, authz_id) config = copy.deepcopy(cloudauthz.config) From 49976cb900c669a43b51d485475642e6b067d058 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 19:01:07 -0700 Subject: [PATCH 46/76] Update the try_get_authz_config doc-string. --- lib/galaxy/authnz/managers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 3820af139af3..6a8be0245046 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -131,6 +131,22 @@ def can_user_assume_authn(trans, authn_id): @staticmethod def try_get_authz_config(trans, authz_id): + """ + It returns a cloudauthz config (see model.CloudAuthz) with the + given ID; and raise an exception if either a config with given + ID does not exist, or the configuration is defined for a another + user than trans.user. + + :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :param trans: Galaxy web transaction + + :type authz_id: int + :param authz_id: The ID of a CloudAuthz configuration to be used for + getting temporary credentials. + + :rtype : model.CloudAuthz + :return: a cloudauthz configuration. + """ qres = trans.sa_session.query(model.CloudAuthz).get(authz_id) if qres is None: raise exceptions.ObjectNotFound("An authorization configuration with given ID not found.") From c934165b1df40f0487cf0e7a58c5f812dfa13856 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 19:06:22 -0700 Subject: [PATCH 47/76] Extend cloudauthz as per provider requirements. --- lib/galaxy/authnz/managers.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 6a8be0245046..324e3a8067c3 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -115,6 +115,15 @@ def _get_authnz_backend(self, provider): log.debug(msg) return False, msg, None + def _extend_cloudauthz(self, trans, cloudauthz): + config = copy.deepcopy(cloudauthz.config) + if config.provider == "aws": + success, message, backend = self._get_authnz_backend(cloudauthz.authn.provider) + strategy = Strategy(trans, Storage, backend.config) + on_the_fly_config(trans) + config['id_token'] = cloudauthz.authn.get_id_token(strategy) + return config + @staticmethod def can_user_assume_authn(trans, authn_id): qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) @@ -226,10 +235,6 @@ def get_cloud_access_credentials(self, trans, authz_id): for details on the content of this dictionary. """ cloudauthz = self.try_get_authz_config(trans, authz_id) - config = copy.deepcopy(cloudauthz.config) - success, message, backend = self._get_authnz_backend(cloudauthz.authn.provider) - strategy = Strategy(trans, Storage, backend.config) - on_the_fly_config(trans) - config['id_token'] = cloudauthz.authn.get_id_token(strategy) + config = self._extend_cloudauthz(trans, cloudauthz) ca = CloudAuthz() return ca.authorize(cloudauthz.provider, config) From e448a4e167957004611cd0f770dc233c0352ee99 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 19:12:10 -0700 Subject: [PATCH 48/76] Refactor a method name, and fix a bug in authnz manager. --- lib/galaxy/authnz/managers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 324e3a8067c3..e0045956fa47 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -115,9 +115,9 @@ def _get_authnz_backend(self, provider): log.debug(msg) return False, msg, None - def _extend_cloudauthz(self, trans, cloudauthz): + def _extend_cloudauthz_config(self, trans, cloudauthz): config = copy.deepcopy(cloudauthz.config) - if config.provider == "aws": + if cloudauthz.provider == "aws": success, message, backend = self._get_authnz_backend(cloudauthz.authn.provider) strategy = Strategy(trans, Storage, backend.config) on_the_fly_config(trans) @@ -235,6 +235,6 @@ def get_cloud_access_credentials(self, trans, authz_id): for details on the content of this dictionary. """ cloudauthz = self.try_get_authz_config(trans, authz_id) - config = self._extend_cloudauthz(trans, cloudauthz) + config = self._extend_cloudauthz_config(trans, cloudauthz) ca = CloudAuthz() return ca.authorize(cloudauthz.provider, config) From 9c8e1c41ddc25a60d80b87cfe10ff6cb4ec9a595 Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 17 Aug 2018 19:28:55 -0700 Subject: [PATCH 49/76] Update doc string of upload manager. --- lib/galaxy/managers/cloud.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index c34cec653b2c..c9291c6172fc 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -172,6 +172,10 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a Implements the logic of uploading a file from a cloud-based storage (e.g., Amazon S3) and persisting it as a Galaxy dataset. + This manager does NOT require use credentials, instead, it uses a more secure method, + which leverages CloudAuthz (https://github.com/galaxyproject/cloudauthz) and automatically + requests temporary credentials to access the defined resources. + :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction @@ -188,10 +192,10 @@ def upload(self, trans, history_id, provider, bucket, objects, authz_id, input_a :type objects: list of string :param objects: the name of objects to be uploaded. - :type credentials: dict - :param credentials: a dictionary containing all the credentials required to authenticated to the - specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, - "access_key": YOUR_AWS_ACCESS_TOKEN}). + :type authz_id: int + :param authz_id: the ID of CloudAuthz to be used for authorizing access to the resource provider. You may + get a list of the defined authorizations via `/api/cloud/authz`. Also, you can + use `/api/cloud/authz/create` to define a new authorization. :type input_args: dict :param input_args: a [Optional] a dictionary of input parameters: From f399d2981565d7395a02a80574a2d9c0dc12d950 Mon Sep 17 00:00:00 2001 From: vjalili Date: Tue, 11 Sep 2018 10:36:18 -0700 Subject: [PATCH 50/76] Add cloudauthz to the requirements, and update-dependencies. --- .../dependencies/pipfiles/default/Pipfile | 1 + .../default/pinned-dev-requirements.txt | 4 ++-- .../pipfiles/default/pinned-requirements.txt | 20 +++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 6f299105a786..9c3fa49cb7b7 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -71,6 +71,7 @@ pyparsing = "*" paramiko = "*" python-genomespaceclient = "<2.0" social_auth_core = {version = "==1.5.0", extras = ['openidconnect']} +cloudauthz = "<=0.1.0" [requires] python_version = "2.7" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index c3e94c4959ad..bf8693fa0c96 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -9,11 +9,11 @@ certifi==2018.8.24 chardet==3.0.4 commonmark==0.5.4 docutils==0.14 -funcsigs==1.0.2; python_version < '3.3' +funcsigs==1.0.2; python_version == '2.6' idna==2.7 imagesize==1.1.0; python_version != '3.3.*' jinja2==2.10 -lxml==4.2.4 +lxml==4.2.5 markupsafe==1.0 mock==2.0.0 more-itertools==4.3.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 3b253ccf162b..03b906bdbbd5 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -36,8 +36,9 @@ cffi==1.11.5 chardet==3.0.4 cheetah3==3.1.0 cliff==2.13.0; python_version >= '2.6' +cloudauthz==0.1.0 cloudbridge==1.0.1 -cmd2==0.8.9 +cmd2==0.8.9; python_version < '3.0' contextlib2==0.5.5; python_version < '3.5' cryptography==2.3.1 debtcollector==1.20.0 @@ -47,12 +48,12 @@ dictobj==0.4 docopt==0.6.2 docutils==0.14 dogpile.cache==0.6.7 -enum34==1.1.6; python_version < '3.4' +enum34==1.1.6; python_version < '3' fabric3==1.14.post1 -funcsigs==1.0.2; python_version < '3.3' +funcsigs==1.0.2; python_version == '2.6' functools32==3.2.3.post2; python_version == '2.7' future==0.16.0 -futures==3.2.0; python_version == '2.6' or python_version == '2.7' +futures==3.2.0; python_version < '3.0' galaxy-sequence-utils==1.1.3 h5py==2.8.0 html5lib==1.0.1 @@ -61,8 +62,8 @@ ipaddress==1.0.22; python_version < '3.3' iso8601==0.1.12 isodate==0.6.0 jmespath==0.9.3 -jsonpatch==1.23; python_version != '3.3.*' -jsonpointer==2.0; python_version != '3.3.*' +jsonpatch==1.23 +jsonpointer==2.0 jsonschema==2.6.0 keystoneauth1==3.10.0 kombu==4.2.1 @@ -79,8 +80,8 @@ netifaces==0.10.7 nose==1.3.7 numpy==1.15.1 oauthlib==2.1.0 -openstacksdk==0.17.0 -os-client-config==1.31.2 +openstacksdk==0.17.0; python_version >= '2.6' +os-client-config==1.31.2; python_version >= '2.6' os-service-types==1.3.0 osc-lib==1.11.1 oslo.config==6.4.0 @@ -103,7 +104,6 @@ pyasn1==0.4.4 pycparser==2.18 pycryptodome==3.6.6 pycryptodomex==3.6.6 -pyinotify==0.9.6; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' pyjwkest==1.4.0 pyjwt==1.6.4 pykwalify==1.6.1 @@ -136,7 +136,7 @@ simplejson==3.16.0 six==1.11.0 social-auth-core==1.5.0 sqlalchemy-migrate==0.11.0 -sqlalchemy-utils==0.33.3 +sqlalchemy-utils==0.33.4 sqlalchemy==1.2.11 sqlparse==0.2.4 stevedore==1.29.0 From b679bc038d1f9faf88590e1fdd0789142efbaa00 Mon Sep 17 00:00:00 2001 From: vjalili Date: Tue, 11 Sep 2018 15:18:31 -0700 Subject: [PATCH 51/76] Update requirements with patch in #6694 and manually revert dropping pyinotify. --- .../pipfiles/default/pinned-dev-requirements.txt | 2 +- .../pipfiles/default/pinned-requirements.txt | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index bf8693fa0c96..3740269733e4 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -9,7 +9,7 @@ certifi==2018.8.24 chardet==3.0.4 commonmark==0.5.4 docutils==0.14 -funcsigs==1.0.2; python_version == '2.6' +funcsigs==1.0.2; python_version < '3.3' idna==2.7 imagesize==1.1.0; python_version != '3.3.*' jinja2==2.10 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 03b906bdbbd5..94a995ad3494 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -38,7 +38,7 @@ cheetah3==3.1.0 cliff==2.13.0; python_version >= '2.6' cloudauthz==0.1.0 cloudbridge==1.0.1 -cmd2==0.8.9; python_version < '3.0' +cmd2==0.8.9 contextlib2==0.5.5; python_version < '3.5' cryptography==2.3.1 debtcollector==1.20.0 @@ -48,12 +48,12 @@ dictobj==0.4 docopt==0.6.2 docutils==0.14 dogpile.cache==0.6.7 -enum34==1.1.6; python_version < '3' +enum34==1.1.6; python_version < '3.4' fabric3==1.14.post1 -funcsigs==1.0.2; python_version == '2.6' +funcsigs==1.0.2; python_version < '3.3' functools32==3.2.3.post2; python_version == '2.7' future==0.16.0 -futures==3.2.0; python_version < '3.0' +futures==3.2.0; python_version == '2.6' or python_version == '2.7' galaxy-sequence-utils==1.1.3 h5py==2.8.0 html5lib==1.0.1 @@ -104,6 +104,7 @@ pyasn1==0.4.4 pycparser==2.18 pycryptodome==3.6.6 pycryptodomex==3.6.6 +pyinotify==0.9.6; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' pyjwkest==1.4.0 pyjwt==1.6.4 pykwalify==1.6.1 From 13cf6615c6e1c5b4a105b45a2e1619dc4e95c6ec Mon Sep 17 00:00:00 2001 From: vjalili Date: Tue, 11 Sep 2018 15:26:52 -0700 Subject: [PATCH 52/76] Fix lint issues in cloudauthz api. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 880fea16dab7..cfdd6b30464a 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -13,17 +13,15 @@ from galaxy.exceptions import ( ActionInputError, InternalServerError, - ItemAccessibilityException, MalformedId, - ObjectNotFound, RequestParameterInvalidException, RequestParameterMissingException ) from galaxy.managers import cloudauthzs -from galaxy.web.base.controller import BaseAPIController from galaxy.web import ( _future_expose_api as expose_api ) +from galaxy.web.base.controller import BaseAPIController log = logging.getLogger(__name__) From afed829a73131e158e3c4c08e19f8aedc2867631 Mon Sep 17 00:00:00 2001 From: vjalili Date: Tue, 11 Sep 2018 16:11:53 -0700 Subject: [PATCH 53/76] Fix lint issues with authnz/managers.py --- lib/galaxy/authnz/managers.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index e0045956fa47..1df2c3eb0253 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -1,16 +1,21 @@ -from cloudauthz import CloudAuthz - -from galaxy import exceptions -from galaxy import model - import copy import importlib import logging import xml.etree.ElementTree as ET from xml.etree.ElementTree import ParseError -from .psa_authnz import * +from cloudauthz import CloudAuthz + +from galaxy import exceptions +from galaxy import model +from .psa_authnz import ( + BACKENDS_NAME, + on_the_fly_config, + PSAAuthnz, + Storage, + Strategy +) log = logging.getLogger(__name__) From ba566a30563855f05f2fb667c3966afb4cd70334 Mon Sep 17 00:00:00 2001 From: vjalili Date: Thu, 13 Sep 2018 08:48:50 -0700 Subject: [PATCH 54/76] Upgrade to Cloudauthz v0.2.0. --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../dependencies/pipfiles/default/pinned-requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 9c3fa49cb7b7..49f082960c23 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -71,7 +71,7 @@ pyparsing = "*" paramiko = "*" python-genomespaceclient = "<2.0" social_auth_core = {version = "==1.5.0", extras = ['openidconnect']} -cloudauthz = "<=0.1.0" +cloudauthz = "<=0.2.0" [requires] python_version = "2.7" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 94a995ad3494..fb629d248518 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -36,7 +36,7 @@ cffi==1.11.5 chardet==3.0.4 cheetah3==3.1.0 cliff==2.13.0; python_version >= '2.6' -cloudauthz==0.1.0 +cloudauthz==0.2.0 cloudbridge==1.0.1 cmd2==0.8.9 contextlib2==0.5.5; python_version < '3.5' @@ -65,7 +65,7 @@ jmespath==0.9.3 jsonpatch==1.23 jsonpointer==2.0 jsonschema==2.6.0 -keystoneauth1==3.10.0 +keystoneauth1==3.11.0 kombu==4.2.1 mako==1.0.7 markupsafe==1.0 From c0a36ca20d0f95b56898ba06a6dd705b3c1afaea Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 14 Sep 2018 21:39:38 -0700 Subject: [PATCH 55/76] Raise an exception if cannot get/refresh an ID token in cloudauthz. --- lib/galaxy/authnz/managers.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 1df2c3eb0253..1de0da46ef2a 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -2,6 +2,7 @@ import copy import importlib import logging +import requests import xml.etree.ElementTree as ET from xml.etree.ElementTree import ParseError @@ -126,7 +127,18 @@ def _extend_cloudauthz_config(self, trans, cloudauthz): success, message, backend = self._get_authnz_backend(cloudauthz.authn.provider) strategy = Strategy(trans, Storage, backend.config) on_the_fly_config(trans) - config['id_token'] = cloudauthz.authn.get_id_token(strategy) + try: + config['id_token'] = cloudauthz.authn.get_id_token(strategy) + except requests.exceptions.HTTPError as e: + msg = "Sign-out from Galaxy and remove its access from `{}`, then log back in using `{}` " \ + "account.".format(self._unify_provider_name(cloudauthz.authn.provider),cloudauthz.authn.uid) + log.debug("Failed to get/refresh ID token for user with ID `{}` for assuming authz_id `{}`. " + "User may not have a refresh token. If the problem persists, set the `prompt` key to " + "`consent` in `oidc_backends_config.xml`, then restart Galaxy and ask user to: {}" + "Error Message: `{}`".format(trans.user.id, cloudauthz.id, msg, e.response.text)) + raise exceptions.AuthenticationFailed( + err_msg="An error occurred getting your ID token. {}. If the problem persists, please " + "contact Galaxy admin.".format(msg)) return config @staticmethod From cc2fa14932ae1b3029865dcd8200bbb9fe72660c Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 14 Sep 2018 21:41:20 -0700 Subject: [PATCH 56/76] Do not surround getting cloud access credentials in a try-catch block. --- lib/galaxy/managers/cloud.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 6106d82bce88..82577c0cd392 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -210,10 +210,7 @@ def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, in if input_args is None: input_args = {} - try: - credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz_id) - except Exception as e: - raise e + credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz_id) connection = self._configure_provider(provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) From 915fc9ef024980917af1198f691ecdc4602e6baa Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 14 Sep 2018 21:42:18 -0700 Subject: [PATCH 57/76] Catch CloudAuthz exceptions. --- lib/galaxy/authnz/managers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 1de0da46ef2a..cc94435be0af 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -7,6 +7,9 @@ from xml.etree.ElementTree import ParseError from cloudauthz import CloudAuthz +from cloudauthz.exceptions import ( + CloudAuthzBaseException +) from galaxy import exceptions from galaxy import model @@ -253,5 +256,10 @@ def get_cloud_access_credentials(self, trans, authz_id): """ cloudauthz = self.try_get_authz_config(trans, authz_id) config = self._extend_cloudauthz_config(trans, cloudauthz) - ca = CloudAuthz() - return ca.authorize(cloudauthz.provider, config) + try: + ca = CloudAuthz() + return ca.authorize(cloudauthz.provider, config) + except CloudAuthzBaseException as e: + log.exception(e.message) + raise exceptions.AuthenticationFailed(e.message) + From d7c26f33527b9ce2adfe5ef79432608dfcd8d4cb Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 14 Sep 2018 21:43:37 -0700 Subject: [PATCH 58/76] Remove an extra `objects` in getting objects from a bucket. --- lib/galaxy/managers/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 82577c0cd392..9e32771b13db 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -222,7 +222,7 @@ def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, in datasets = [] for obj in objects: try: - key = bucket.objects.objects.get(obj) + key = bucket.objects.get(obj) except Exception as e: raise MessageException("The following error occurred while getting the object {}: {}".format(obj, str(e))) if key is None: From 4ceab06486d6eb3adeedc86da64c260ff4adcd7e Mon Sep 17 00:00:00 2001 From: vjalili Date: Fri, 14 Sep 2018 21:50:35 -0700 Subject: [PATCH 59/76] Fix lint issues. --- lib/galaxy/authnz/managers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index cc94435be0af..9775d3bb1eae 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -2,10 +2,10 @@ import copy import importlib import logging -import requests import xml.etree.ElementTree as ET from xml.etree.ElementTree import ParseError +import requests from cloudauthz import CloudAuthz from cloudauthz.exceptions import ( CloudAuthzBaseException @@ -134,7 +134,7 @@ def _extend_cloudauthz_config(self, trans, cloudauthz): config['id_token'] = cloudauthz.authn.get_id_token(strategy) except requests.exceptions.HTTPError as e: msg = "Sign-out from Galaxy and remove its access from `{}`, then log back in using `{}` " \ - "account.".format(self._unify_provider_name(cloudauthz.authn.provider),cloudauthz.authn.uid) + "account.".format(self._unify_provider_name(cloudauthz.authn.provider), cloudauthz.authn.uid) log.debug("Failed to get/refresh ID token for user with ID `{}` for assuming authz_id `{}`. " "User may not have a refresh token. If the problem persists, set the `prompt` key to " "`consent` in `oidc_backends_config.xml`, then restart Galaxy and ask user to: {}" @@ -262,4 +262,3 @@ def get_cloud_access_credentials(self, trans, authz_id): except CloudAuthzBaseException as e: log.exception(e.message) raise exceptions.AuthenticationFailed(e.message) - From 60f08316ec9b66a6ebbd8a224b09cb0fb89825ae Mon Sep 17 00:00:00 2001 From: vjalili Date: Sat, 15 Sep 2018 11:01:09 -0700 Subject: [PATCH 60/76] Do not assert credentials using cloudbridge. --- lib/galaxy/managers/cloud.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 9e32771b13db..14bf77c615f3 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -42,7 +42,7 @@ def __init__(self, app, *args, **kwargs): super(CloudManager, self).__init__(app, *args, **kwargs) @staticmethod - def _configure_provider(provider, credentials): + def _configure_provider(provider, user, credentials): """ Given a provider name and required credentials, it configures and returns a cloudbridge connection to the provider. @@ -133,11 +133,22 @@ def _configure_provider(provider, credentials): raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported " "providers: {}.".format(provider, SUPPORTED_PROVIDERS)) - try: - if connection.authenticate(): - return connection - except ProviderConnectionException as e: - raise AuthenticationFailed("Could not authenticate to the '{}' provider. {}".format(provider, e)) + # The authorization-assertion mechanism of Cloudbridge assumes a user has an elevated privileges, + # such as Admin-level access to all resources (see https://github.com/CloudVE/cloudbridge/issues/135). + # As a result, a user who wants to authorize Galaxy to read/write an Amazon S3 bucket, need to + # also authorize Galaxy with full permission to Amazon EC2 (because Cloudbridge leverages EC2-specific + # operation to assert credentials). While the EC2 authorization is not required by Galaxy to + # read/write a S3 bucket, it can cause this exception. + # + # Until Cloudbridge implements an authorization-specific credentials assertion, we are not asserting + # the authorization/validity of the credentials, in order to avoid asking users to grant Galaxy with an + # elevated, yet unnecessary, privileges. + # + # Note, if user's credentials are invalid/expired to perform the authorized action, that can cause + # exceptions which we capture separately in related read/write attempts. + log.debug("Attempting to access cloud-based resources for user with ID `{}` while they may not have " + "full/admin authorization for all `{}` resources.".format(user.id, provider)) + return connection @staticmethod def _get_inputs(obj, key, input_args): @@ -211,7 +222,7 @@ def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, in input_args = {} credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz_id) - connection = self._configure_provider(provider, credentials) + connection = self._configure_provider(provider, trans.user, credentials) try: bucket = connection.storage.buckets.get(bucket_name) if bucket is None: @@ -286,7 +297,7 @@ def download(self, trans, history_id, provider, bucket_name, credentials, datase """ if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) - connection = self._configure_provider(provider, credentials) + connection = self._configure_provider(provider, trans.user, credentials) bucket = connection.storage.buckets.get(bucket_name) if bucket is None: From af6ded604dfe5fa2898e616820edf7948eca1261 Mon Sep 17 00:00:00 2001 From: vjalili Date: Sat, 15 Sep 2018 11:26:16 -0700 Subject: [PATCH 61/76] Update exception message (and log it) in a cloud object is not found. --- lib/galaxy/managers/cloud.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 14bf77c615f3..1e6dc2ca0ece 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -237,7 +237,12 @@ def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, in except Exception as e: raise MessageException("The following error occurred while getting the object {}: {}".format(obj, str(e))) if key is None: - raise ObjectNotFound("Could not get the object `{}`.".format(obj)) + log.exception( + "Could not get object `{}` for user `{}`. Object may not exist, or the provided credentials are " + "invalid or not authorized to read the bucket/object.".format(obj, trans.user.id)) + raise ObjectNotFound( + "Could not get the object `{}`. Please check if the object exists, and credentials are valid and " + "authorized to read the bucket and object. ".format(obj)) params = Params(self._get_inputs(obj, key, input_args), sanitize=False) incoming = params.__dict__ From e792480c74faf0500b2d1328a99c788482baa7ec Mon Sep 17 00:00:00 2001 From: vjalili Date: Sat, 15 Sep 2018 11:52:12 -0700 Subject: [PATCH 62/76] Surround downloading datasets to cloud in a try-catch to capture issues such as invalid credentials. --- lib/galaxy/managers/cloud.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 1e6dc2ca0ece..3a904b193921 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -316,6 +316,10 @@ def download(self, trans, history_id, provider, bucket_name, credentials, datase if overwrite_existing is False and bucket.objects.get(object_label) is not None: object_label += "-" + datetime.datetime.now().strftime("%y-%m-%d-%H-%M-%S") created_obj = bucket.objects.create(object_label) - created_obj.upload_from_file(hda.dataset.get_file_name()) - downloaded.append(object_label) + try: + created_obj.upload_from_file(hda.dataset.get_file_name()) + downloaded.append(object_label) + except Exception as e: + log.exception("Failed to download dataset to cloud, maybe invalid or unauthorized credentials. " + "{}".format(e.message)) return downloaded From e4184188323996513fba84794e80bfb86f4d122d Mon Sep 17 00:00:00 2001 From: vjalili Date: Sat, 15 Sep 2018 11:59:33 -0700 Subject: [PATCH 63/76] Report both successfully and unsuccessfully downloaded datasets to cloud --- lib/galaxy/managers/cloud.py | 9 ++++++--- lib/galaxy/webapps/galaxy/api/cloud.py | 21 +++++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 3a904b193921..57848c7776b9 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -297,8 +297,9 @@ def download(self, trans, history_id, provider, bucket_name, credentials, datase "False", Galaxy appends datetime to the dataset name to prevent overwriting the existing object. - :rtype: list - :return: A list of labels for the objects that were uploaded. + :rtype: tuple + :return: A tuple of two lists of labels of the objects that were successfully and + unsuccessfully downloaded to cloud.. """ if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) @@ -310,6 +311,7 @@ def download(self, trans, history_id, provider, bucket_name, credentials, datase history = trans.sa_session.query(trans.app.model.History).get(history_id) downloaded = [] + failed = [] for hda in history.datasets: if dataset_ids is None or hda.dataset.id in dataset_ids: object_label = hda.name @@ -322,4 +324,5 @@ def download(self, trans, history_id, provider, bucket_name, credentials, datase except Exception as e: log.exception("Failed to download dataset to cloud, maybe invalid or unauthorized credentials. " "{}".format(e.message)) - return downloaded + failed.append(object_label) + return downloaded, failed diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index c7efbe2f16f0..2660b5f5cd3d 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -170,8 +170,8 @@ def download(self, trans, payload, **kwargs): :param kwargs: :rtype: dictionary - :return: Information about the downloaded datasets, including downloaded_dataset_labels - and destination bucket name. + :return: Information about the downloaded and failed datasets, including downloaded_dataset_labels + failed_dataset_labels, and destination bucket name. """ missing_arguments = [] encoded_history_id = payload.get("history_id", None) @@ -213,12 +213,13 @@ def download(self, trans, payload, **kwargs): raise ActionInputError("The following provided dataset IDs are invalid, please correct them and retry. " "{}".format(invalid_dataset_ids)) - uploaded = self.cloud_manager.download(trans=trans, - history_id=history_id, - provider=provider, - bucket_name=bucket, - credentials=credentials, - dataset_ids=dataset_ids, - overwrite_existing=payload.get("overwrite_existing", False)) - return {'downloaded_dataset_labels': uploaded, + downloaded, failed = self.cloud_manager.download(trans=trans, + history_id=history_id, + provider=provider, + bucket_name=bucket, + credentials=credentials, + dataset_ids=dataset_ids, + overwrite_existing=payload.get("overwrite_existing", False)) + return {'downloaded_dataset_labels': downloaded, + 'failed_dataset_labels': failed, 'bucket_name': bucket} From cfb77903293ded0e829064da2786058463a3bfce Mon Sep 17 00:00:00 2001 From: vjalili Date: Sat, 15 Sep 2018 12:00:33 -0700 Subject: [PATCH 64/76] log unsuccessful cloud downloads as `debug` not `exception`. --- lib/galaxy/managers/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 57848c7776b9..d71d623418c7 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -322,7 +322,7 @@ def download(self, trans, history_id, provider, bucket_name, credentials, datase created_obj.upload_from_file(hda.dataset.get_file_name()) downloaded.append(object_label) except Exception as e: - log.exception("Failed to download dataset to cloud, maybe invalid or unauthorized credentials. " - "{}".format(e.message)) + log.debug("Failed to download dataset to cloud, maybe invalid or unauthorized credentials. " + "{}".format(e.message)) failed.append(object_label) return downloaded, failed From aecf9929027df8d6d3d348815acfc698c9661e7c Mon Sep 17 00:00:00 2001 From: vjalili Date: Sun, 16 Sep 2018 21:51:05 -0700 Subject: [PATCH 65/76] Fix lint issues in cloud manager. --- lib/galaxy/managers/cloud.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index d71d623418c7..3d10c4f68f12 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -6,7 +6,6 @@ import logging from galaxy.exceptions import ( - AuthenticationFailed, ItemAccessibilityException, MessageException, ObjectNotFound, @@ -18,7 +17,6 @@ try: from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList - from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException except ImportError: CloudProviderFactory = None ProviderList = None From 386eee076268964fe89299b82d5551ab0d08f75d Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 14:32:01 -0700 Subject: [PATCH 66/76] Remove a confusing debug log message. --- lib/galaxy/managers/cloud.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 3d10c4f68f12..78d3d8d4a5e6 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -13,7 +13,6 @@ RequestParameterMissingException ) from galaxy.managers import sharable -from galaxy.util import Params try: from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList @@ -144,8 +143,6 @@ def _configure_provider(provider, user, credentials): # # Note, if user's credentials are invalid/expired to perform the authorized action, that can cause # exceptions which we capture separately in related read/write attempts. - log.debug("Attempting to access cloud-based resources for user with ID `{}` while they may not have " - "full/admin authorization for all `{}` resources.".format(user.id, provider)) return connection @staticmethod From 00d883f2913767298c7616c4357048f6a8ae5d22 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 14:34:17 -0700 Subject: [PATCH 67/76] Remove an unused `user` argument from cloud connection config method. --- lib/galaxy/managers/cloud.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 78d3d8d4a5e6..9318a2cc95b9 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -39,7 +39,7 @@ def __init__(self, app, *args, **kwargs): super(CloudManager, self).__init__(app, *args, **kwargs) @staticmethod - def _configure_provider(provider, user, credentials): + def _configure_provider(provider, credentials): """ Given a provider name and required credentials, it configures and returns a cloudbridge connection to the provider. @@ -217,7 +217,7 @@ def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, in input_args = {} credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz_id) - connection = self._configure_provider(provider, trans.user, credentials) + connection = self._configure_provider(provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) if bucket is None: @@ -298,7 +298,7 @@ def download(self, trans, history_id, provider, bucket_name, credentials, datase """ if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) - connection = self._configure_provider(provider, trans.user, credentials) + connection = self._configure_provider(provider, credentials) bucket = connection.storage.buckets.get(bucket_name) if bucket is None: From 9bba80a322b85f12a9950f583eab39a4aa830215 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 15:41:09 -0700 Subject: [PATCH 68/76] Add more logging to cloudauthz api. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index cfdd6b30464a..a5d7c36ac826 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -81,6 +81,7 @@ def create(self, trans, payload, **kwargs): * status: HTTP response code * message: A message complementary to the response code. """ + msg_template = "Rejected user `" + str(trans.user.id) + "`'s request to create cloudauthz config because of {}." if not isinstance(payload, dict): raise ActionInputError('Invalid payload data type. The payload is expected to be a dictionary, but ' 'received data of type `{}`.'.format(str(type(payload)))) @@ -99,15 +100,18 @@ def create(self, trans, payload, **kwargs): missing_arguments.append('authn_id') if len(missing_arguments) > 0: + log.debug(msg_template.format("missing required config {}".format(missing_arguments))) raise RequestParameterMissingException('The following required arguments are missing in the payload: ' '{}'.format(missing_arguments)) if not isinstance(config, dict): + log.debug(msg_template.format("invalid config type `{}`, expect `dict`".format(type(config)))) raise RequestParameterInvalidException('Invalid type for the required `config` variable; expect `dict` ' 'but received `{}`.'.format(type(config))) try: authn_id = self.decode_id(authn_id) except Exception: + log.debug(msg_template.format("cannot decode authn_id `" + str(authn_id) + "`")) raise MalformedId('Invalid `authn_id`!') try: @@ -119,6 +123,8 @@ def create(self, trans, payload, **kwargs): # exact same key/value should not exist. for ca in trans.user.cloudauthzs: if ca.equals(trans.user.id, provider, authn_id, config): + log.debug("Rejected user `{}`'s request to create cloud authorization because a similar config " + "already exists.".format(trans.user.id)) raise ActionInputError("A similar cloud authorization configuration is already defined.") try: @@ -134,6 +140,6 @@ def create(self, trans, payload, **kwargs): return view except Exception as e: - msg = 'An unexpected error has occurred while responding to the create request of the cloudauthz API.' + str(e) - log.exception(msg) - raise InternalServerError(msg) + log.exception(msg_template.format(e.message)) + raise InternalServerError('An unexpected error has occurred while responding to the create request of the ' + 'cloudauthz API.' + str(e)) From 64d7418aae31cfee91f22112ac17d4f76f0f1276 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 16:21:18 -0700 Subject: [PATCH 69/76] Import Params in cloud manager. --- lib/galaxy/managers/cloud.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 9318a2cc95b9..e8c786fb5c07 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -13,6 +13,7 @@ RequestParameterMissingException ) from galaxy.managers import sharable +from galaxy.util import Params try: from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList From 0eb7892130b4b2099c2800919ad9cfb5c09a2cf7 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 16:30:36 -0700 Subject: [PATCH 70/76] Add a description field to cloudauthz. --- lib/galaxy/model/__init__.py | 3 ++- lib/galaxy/model/mapping.py | 3 ++- .../model/migrate/versions/0143_add_cloudauthz_tables.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1f258db42984..2e490745639c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4951,7 +4951,7 @@ def create_social_auth(cls, user, uid, provider): class CloudAuthz(object): - def __init__(self, user_id, provider, config, authn_id): + def __init__(self, user_id, provider, config, authn_id, description=""): self.user_id = user_id self.provider = provider self.config = config @@ -4959,6 +4959,7 @@ def __init__(self, user_id, provider, config, authn_id): self.tokens = None self.last_update = datetime.now() self.last_activity = datetime.now() + self.description = description def __eq__(self, other): if not isinstance(other, CloudAuthz): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 5367d4ca97ff..8802ccd2e54a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -142,7 +142,8 @@ Column('authn_id', Integer, ForeignKey("oidc_user_authnz_tokens.id"), index=True), Column('tokens', JSONType), Column('last_update', DateTime), - Column('last_activity', DateTime)) + Column('last_activity', DateTime), + Column('description', TEXT)) model.PasswordResetToken.table = Table( "password_reset_token", metadata, diff --git a/lib/galaxy/model/migrate/versions/0143_add_cloudauthz_tables.py b/lib/galaxy/model/migrate/versions/0143_add_cloudauthz_tables.py index 7f47e083cb79..6767e52b54b1 100644 --- a/lib/galaxy/model/migrate/versions/0143_add_cloudauthz_tables.py +++ b/lib/galaxy/model/migrate/versions/0143_add_cloudauthz_tables.py @@ -5,7 +5,7 @@ import logging -from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT from galaxy.model.custom_types import JSONType @@ -21,7 +21,8 @@ Column('authn_id', Integer, ForeignKey("oidc_user_authnz_tokens.id"), index=True), Column('tokens', JSONType), Column('last_update', DateTime), - Column('last_activity', DateTime)) + Column('last_activity', DateTime), + Column('description', TEXT)) def upgrade(migrate_engine): From f97c2550ec10d99a91cfe28dd3be96d1214d4499 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 16:47:50 -0700 Subject: [PATCH 71/76] Read cloudauthz description from api payload. --- lib/galaxy/webapps/galaxy/api/cloudauthz.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index a5d7c36ac826..f02b6c413ab9 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -67,13 +67,14 @@ def create(self, trans, payload, **kwargs): :type payload: dict :param payload: A dictionary structure containing the following keys: - * provider: the cloud-based resource provider to which this configuration belongs to. - * config: a dictionary containing all the configuration required to request temporary credentials - from the provider. - * authn_id: the (encoded) ID of a third-party authentication of a user. To have this ID, user must - have logged-in to this Galaxy server using third-party identity (e.g., Google), or has - associated his/her Galaxy account with a third-party OIDC-based identity. See this page: - https://galaxyproject.org/admin/authentication/ + * provider: the cloud-based resource provider to which this configuration belongs to. + * config: a dictionary containing all the configuration required to request temporary credentials + from the provider. + * authn_id: the (encoded) ID of a third-party authentication of a user. To have this ID, user must + have logged-in to this Galaxy server using third-party identity (e.g., Google), or has + associated his/her Galaxy account with a third-party OIDC-based identity. See this page: + https://galaxyproject.org/admin/authentication/ + * description: [Optional] a brief description for this configuration. :param kwargs: empty dict :rtype: dict @@ -104,6 +105,8 @@ def create(self, trans, payload, **kwargs): raise RequestParameterMissingException('The following required arguments are missing in the payload: ' '{}'.format(missing_arguments)) + description = payload.get("description", "") + if not isinstance(config, dict): log.debug(msg_template.format("invalid config type `{}`, expect `dict`".format(type(config)))) raise RequestParameterInvalidException('Invalid type for the required `config` variable; expect `dict` ' @@ -132,7 +135,8 @@ def create(self, trans, payload, **kwargs): user_id=trans.user.id, provider=provider, config=config, - authn_id=authn_id + authn_id=authn_id, + description=description ) view = self.cloudauthz_serializer.serialize_to_view(new_cloudauthz, trans=trans, **self._parse_serialization_params(kwargs, 'summary')) log.debug('Created a new cloudauthz record for the user id `{}` '.format(str(trans.user.id))) From 46183c91ec3d43927b4c9c83e297c7b3f2482635 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 19:17:07 -0700 Subject: [PATCH 72/76] Remove provider from cloud upload api, leverage authnz.provider instead. --- lib/galaxy/managers/cloud.py | 9 +++------ lib/galaxy/webapps/galaxy/api/cloud.py | 6 ------ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index e8c786fb5c07..6305a5caf8a1 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -174,7 +174,7 @@ def _get_inputs(obj, key, input_args): 'files_0|url_paste': key.generate_url(expires_in=SINGED_URL_TTL), } - def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, input_args=None): + def upload(self, trans, history_id, bucket_name, objects, authz_id, input_args=None): """ Implements the logic of uploading a file from a cloud-based storage (e.g., Amazon S3) and persisting it as a Galaxy dataset. @@ -189,10 +189,6 @@ def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, in :type history_id: string :param history_id: the (decoded) id of history to which the object should be uploaded to. - :type provider: string - :param provider: the name of cloud-based resource provided. A list of supported providers is given in - `SUPPORTED_PROVIDERS` variable. - :type bucket_name: string :param bucket_name: the name of a bucket from which data should be uploaded (e.g., a bucket name on AWS S3). @@ -217,8 +213,9 @@ def upload(self, trans, history_id, provider, bucket_name, objects, authz_id, in if input_args is None: input_args = {} + authz = trans.app.authnz_manager.try_get_authz_config(trans, authz_id) credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz_id) - connection = self._configure_provider(provider, credentials) + connection = self._configure_provider(authz.provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) if bucket is None: diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 2660b5f5cd3d..bfcedeb9253c 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -50,7 +50,6 @@ def upload(self, trans, payload, **kwargs): :type payload: dict :param payload: A dictionary structure containing the following keys: * history_id: the (encoded) id of history to which the object should be uploaded to. - * provider: the name of a cloud-based resource provided (e.g., `aws`, `azure`, or `openstack`). * bucket: the name of a bucket from which data should be uploaded from (e.g., a bucket name on AWS S3). * objects: a list of the names of objects to be uploaded. * credentials: a dictionary containing all the credentials required to authenticated to the @@ -93,10 +92,6 @@ def upload(self, trans, payload, **kwargs): if encoded_history_id is None: missing_arguments.append("history_id") - provider = payload.get("provider", None) - if provider is None: - missing_arguments.append("provider") - bucket = payload.get("bucket", None) if bucket is None: missing_arguments.append("bucket") @@ -128,7 +123,6 @@ def upload(self, trans, payload, **kwargs): datasets = self.cloud_manager.upload(trans=trans, history_id=history_id, - provider=provider, bucket_name=bucket, objects=objects, authz_id=authz_id, From 5e8cf269163773acf5674e8565615b9bd2b66b8d Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 19:21:50 -0700 Subject: [PATCH 73/76] Reuse a retrieved cloudauthz. --- lib/galaxy/authnz/managers.py | 3 +-- lib/galaxy/managers/cloud.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 9775d3bb1eae..410e321e8c47 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -228,7 +228,7 @@ def disconnect(self, provider, trans, disconnect_redirect_url=None): log.exception(msg) return False, msg, None - def get_cloud_access_credentials(self, trans, authz_id): + def get_cloud_access_credentials(self, trans, cloudauthz): """ This method leverages CloudAuthz (https://github.com/galaxyproject/cloudauthz) to request a cloud-based resource provider (e.g., Amazon AWS, Microsoft Azure) @@ -254,7 +254,6 @@ def get_cloud_access_credentials(self, trans, authz_id): resource provider. See CloudAuthz (https://github.com/galaxyproject/cloudauthz) for details on the content of this dictionary. """ - cloudauthz = self.try_get_authz_config(trans, authz_id) config = self._extend_cloudauthz_config(trans, cloudauthz) try: ca = CloudAuthz() diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 6305a5caf8a1..6662b96a81fb 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -214,7 +214,7 @@ def upload(self, trans, history_id, bucket_name, objects, authz_id, input_args=N input_args = {} authz = trans.app.authnz_manager.try_get_authz_config(trans, authz_id) - credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz_id) + credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz) connection = self._configure_provider(authz.provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) From 8f9a2c7c073b8d265f33b851467a6a4e906d86a4 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 19:22:34 -0700 Subject: [PATCH 74/76] Refactor authz -> cloudauthz. --- lib/galaxy/managers/cloud.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 6662b96a81fb..22e16de0f3fa 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -213,9 +213,9 @@ def upload(self, trans, history_id, bucket_name, objects, authz_id, input_args=N if input_args is None: input_args = {} - authz = trans.app.authnz_manager.try_get_authz_config(trans, authz_id) - credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, authz) - connection = self._configure_provider(authz.provider, credentials) + cloudauthz = trans.app.authnz_manager.try_get_authz_config(trans, authz_id) + credentials = trans.app.authnz_manager.get_cloud_access_credentials(trans, cloudauthz) + connection = self._configure_provider(cloudauthz.provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) if bucket is None: From 6d4163ae966a5d27753f2cd22cd43e341713aec4 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 19:23:51 -0700 Subject: [PATCH 75/76] Fix a bug using undefined variable in cloud manager. --- lib/galaxy/managers/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 22e16de0f3fa..339862024028 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -247,7 +247,7 @@ def upload(self, trans, history_id, bucket_name, objects, authz_id, input_args=N job_errors = output.get('job_errors', []) if job_errors: raise ValueError('Following error occurred while uploading the given object(s) from {}: {}'.format( - provider, job_errors)) + cloudauthz.provider, job_errors)) else: for d in output['out_data']: datasets.append(d[1].dataset) From cd571144b3ef74230bb68a19debe419cf8622249 Mon Sep 17 00:00:00 2001 From: vjalili Date: Mon, 17 Sep 2018 19:25:48 -0700 Subject: [PATCH 76/76] Update a function doc-string in authnz manager. --- lib/galaxy/authnz/managers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 410e321e8c47..2707721d4d56 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -245,9 +245,9 @@ def get_cloud_access_credentials(self, trans, cloudauthz): :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction - :type authz_id: int - :param authz_id: The ID of a CloudAuthz configuration to be used for - getting temporary credentials. + :type cloudauthz: CloudAuthz + :param cloudauthz: an instance of CloudAuthz to be used for getting temporary + credentials. :rtype: dict :return: a dictionary containing credentials to access a cloud-based