From 4090c38b9b335a023df09f00fc70b5b84f294d14 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Mon, 5 Nov 2018 12:08:37 +0000 Subject: [PATCH 01/27] modifications that match new api specification --- elixir_rems_proxy/app.py | 18 ++---- elixir_rems_proxy/schemas/get.json | 81 ++++++++++++----------- elixir_rems_proxy/schemas/patch.json | 92 +++++++++++++++------------ elixir_rems_proxy/schemas/post.json | 92 +++++++++++++++------------ elixir_rems_proxy/utils/db_actions.py | 36 ++++++----- elixir_rems_proxy/utils/process.py | 46 +++++++------- elixir_rems_proxy/utils/validate.py | 2 +- 7 files changed, 194 insertions(+), 173 deletions(-) diff --git a/elixir_rems_proxy/app.py b/elixir_rems_proxy/app.py index 0de49bf..23ac32a 100644 --- a/elixir_rems_proxy/app.py +++ b/elixir_rems_proxy/app.py @@ -33,10 +33,11 @@ async def user_post(request): missing_datasets = await process_post_request(request, db_pool) - if len(missing_datasets) == 0: + if not missing_datasets: return web.HTTPOk(text='Successful operation') else: - return web.HTTPCreated(text=f'Following datasets are missing from REMS ({missing_datasets}), check "GET /user" endpoint for added permissions') + return web.HTTPCreated(text=f'User was created, but following datasets are missing from REMS ({missing_datasets}), ' + 'check "GET /user" endpoint for added permissions') @routes.get('/user/') @@ -47,16 +48,9 @@ async def user_get(request): List all datasets user has access to. """ LOG.debug('GET Request received.') - user_identifier = None # username in REMS + user_identifier = None # ELIXIR id == REMS userid db_pool = request.app['pool'] - # try: - # # Optional query parameter, retrieved from /user/{user}?user_affiliation={organisation} - # # user_affiliation = request.query['user_affiliation'] # NOT IN USE - # except KeyError as key_error: - # LOG.debug(f'KeyError at optional key {key_error}, ignore and pass.') - # pass - if 'user' in request.match_info: user_identifier = request.match_info['user'] processed_request = await process_get_request(user_identifier, db_pool) @@ -80,7 +74,7 @@ async def user_patch(request): if 'user' in request.match_info: user_identifier = request.match_info['user'] processed_request = await process_patch_request(user_identifier, request, db_pool) - if len(processed_request) == 0 or processed_request is True: + if len(processed_request) == 0: return web.HTTPOk(text='Successful operation') else: return web.HTTPCreated(text=f'Following datasets are missing from REMS ({processed_request}), check "GET /user" endpoint for permissions') @@ -96,7 +90,7 @@ async def user_delete(request): Delete user. """ LOG.debug('DELETE Request received.') - user_identifier = None # username in REMS + user_identifier = None # ELIXIR id == REMS userid db_pool = request.app['pool'] if 'user' in request.match_info: diff --git a/elixir_rems_proxy/schemas/get.json b/elixir_rems_proxy/schemas/get.json index 11020f5..058e9da 100644 --- a/elixir_rems_proxy/schemas/get.json +++ b/elixir_rems_proxy/schemas/get.json @@ -1,42 +1,49 @@ { - "definitions": {}, - "type": "object", - "required": [ - "permissions" - ], - "properties": { - "permissions": { - "type": "array", - "items": { - "type": "object", - "required": [ - "affiliation", - "source_signature", - "url_prefix", - "datasets" - ], - "properties": { - "affiliation": { - "type": "string", - "default": "" - }, - "source_signature": { - "type": "string", - "default": "" - }, - "url_prefix": { - "type": "string", - "default": "" - }, - "datasets": { - "type": "array", - "items": { - "type": "string", - "default": "" - } - } + "definitions": {}, + "type": "object", + "required": [ + "sourceSignature", + "datasetPermissions" + ], + "properties": { + "sourceSignature": { + "type": "string", + "default": "" + }, + "datasetPermissions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "datasetOwner", + "urlPrefix", + "datasetId", + "startDate", + "endDate" + ], + "properties": { + "datasetOwner": { + "type": "string", + "default": "" + }, + "urlPrefix": { + "type": "string", + "default": "" + }, + "datasetId": { + "type": "string", + "default": "" + }, + "startDate": { + "type": "string", + "default": "" + }, + "endDate": { + "type": "string", + "default": "" } } } } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/elixir_rems_proxy/schemas/patch.json b/elixir_rems_proxy/schemas/patch.json index a52ff0e..98c6a96 100644 --- a/elixir_rems_proxy/schemas/patch.json +++ b/elixir_rems_proxy/schemas/patch.json @@ -2,59 +2,69 @@ "definitions": {}, "type": "object", "required": [ - "user_identifier", - "affiliation", - "datasets" + "userDetails", + "datasetPermissions" ], "properties": { - "user_identifier": { + "sourceSignature": { "type": "string", "default": "" }, - "affiliation": { - "type": "string", - "default": "" + "userDetails": { + "type": "object", + "required": [ + "elixirId" + ], + "properties": { + "elixirId": { + "type": "string", + "default": "", + "examples": [ + "c85cc6f750611ce89c775f560f10199e08f479ce@elixir-europe.org" + ], + "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" + }, + "eduPersonPrincipalName": { + "type": "string", + "default": "" + }, + "userEmail": { + "type": "string", + "default": "" + }, + "realName": { + "type": "string", + "default": "" + } + } }, - "datasets": { + "datasetPermissions": { "type": "array", "items": { "type": "object", "required": [ - "permissions" + "datasetId" ], "properties": { - "permissions": { - "type": "array", - "items": { - "type": "object", - "required": [ - "affiliation", - "source_signature", - "url_prefix", - "datasets" - ], - "properties": { - "affiliation": { - "type": "string", - "default": "" - }, - "source_signature": { - "type": "string", - "default": "" - }, - "url_prefix": { - "type": "string", - "default": "" - }, - "datasets": { - "type": "array", - "items": { - "type": "string", - "default": "" - } - } - } - } + "datasetOwner": { + "type": "string", + "default": "" + }, + "urlPrefix": { + "type": "string", + "default": "" + }, + "datasetId": { + "type": "string", + "default": "" + }, + "startDate": { + "type": "string", + "default": "" + }, + "endDate": { + "type": "string", + "default": "" } } } diff --git a/elixir_rems_proxy/schemas/post.json b/elixir_rems_proxy/schemas/post.json index a52ff0e..98c6a96 100644 --- a/elixir_rems_proxy/schemas/post.json +++ b/elixir_rems_proxy/schemas/post.json @@ -2,59 +2,69 @@ "definitions": {}, "type": "object", "required": [ - "user_identifier", - "affiliation", - "datasets" + "userDetails", + "datasetPermissions" ], "properties": { - "user_identifier": { + "sourceSignature": { "type": "string", "default": "" }, - "affiliation": { - "type": "string", - "default": "" + "userDetails": { + "type": "object", + "required": [ + "elixirId" + ], + "properties": { + "elixirId": { + "type": "string", + "default": "", + "examples": [ + "c85cc6f750611ce89c775f560f10199e08f479ce@elixir-europe.org" + ], + "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" + }, + "eduPersonPrincipalName": { + "type": "string", + "default": "" + }, + "userEmail": { + "type": "string", + "default": "" + }, + "realName": { + "type": "string", + "default": "" + } + } }, - "datasets": { + "datasetPermissions": { "type": "array", "items": { "type": "object", "required": [ - "permissions" + "datasetId" ], "properties": { - "permissions": { - "type": "array", - "items": { - "type": "object", - "required": [ - "affiliation", - "source_signature", - "url_prefix", - "datasets" - ], - "properties": { - "affiliation": { - "type": "string", - "default": "" - }, - "source_signature": { - "type": "string", - "default": "" - }, - "url_prefix": { - "type": "string", - "default": "" - }, - "datasets": { - "type": "array", - "items": { - "type": "string", - "default": "" - } - } - } - } + "datasetOwner": { + "type": "string", + "default": "" + }, + "urlPrefix": { + "type": "string", + "default": "" + }, + "datasetId": { + "type": "string", + "default": "" + }, + "startDate": { + "type": "string", + "default": "" + }, + "endDate": { + "type": "string", + "default": "" } } } diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 49282ce..88ea91d 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -1,5 +1,7 @@ """Database Queries.""" +from datetime import datetime + from aiohttp import web from ..utils.logging import LOG @@ -23,7 +25,7 @@ async def get_dataset_permissions(user, connection): """Return dataset permissions for given user.""" LOG.debug(f'Query database for {user}\'s dataset permissions') try: - query = f"""SELECT organization, a.resid, a.start, a.endt + query = f"""SELECT organization, a.resid, b.start, b.endt FROM resource a, entitlement b WHERE a.id=b.resid AND b.userid='{user}';""" @@ -49,14 +51,14 @@ async def create_user(user, connection): return False -async def get_dataset_index(ds, connection): +async def get_dataset_index(dataset, connection): """Check if given dataset exists. Returns dataset resource id for later use. """ - LOG.debug(f'Check if dataset {ds} exists') + LOG.debug(f'Check if dataset {dataset} exists') try: - query = f"""SELECT id FROM resource WHERE organization='{ds[0]}' AND resid='{ds[1]}';""" + query = f"""SELECT id FROM resource WHERE resid='{dataset}';""" statement = await connection.prepare(query) db_response = await statement.fetch() if db_response: @@ -70,25 +72,25 @@ async def get_dataset_index(ds, connection): return None -async def create_dataset_permissions(user, dataset_group, connection): +async def create_dataset_permissions(user, request_body, connection): """Create dataset permissions.""" LOG.debug('Create dataset permissions.') errors = [] try: # Make inserts inside of a transaction and commit all at once async with connection.transaction(): - for affiliation in dataset_group: - # aff..[0] = affiliation, aff..[1] = [datasets] - for dataset in affiliation[1]: - dataset_index = await get_dataset_index([affiliation[0], dataset], connection) - if dataset_index: - # Dataset belonging to given organisation exists, create permissions for user - await connection.execute(f"""INSERT INTO entitlement (resid, userid) VALUES ($1, $2)""", - dataset_index, user) - else: - # Dataset for this organisation doesn't exist - errors.append([affiliation[0], dataset]) - return errors + for dataset in request_body['datasetPermissions']: + dataset_index = await get_dataset_index(dataset['datasetId'], connection) + if dataset_index: + # Dataset exists, create permissions for user + await connection.execute(f"""INSERT INTO entitlement (resid, userid, start, endt) VALUES ($1, $2, $3, $4)""", + dataset_index, user, + datetime.strptime(dataset['startDate'], '%Y-%m-%d %H:%M:%S.%f%z') if dataset['startDate'] else datetime.now(), + datetime.strptime(dataset['endDate'], '%Y-%m-%d %H:%M:%S.%f%z') if dataset['endDate'] else None) + else: + # Dataset doesn't exist + errors.append(dataset) + return errors except Exception as e: LOG.debug(f'An error occurred while attempting to create permissions -> {e}') return None diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index ebff775..6dcce24 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -10,13 +10,16 @@ async def create_response_body(db_response): """Construct a dictionary for JSON response body.""" response_body = { - "permissions": [ + "sourceSignature": "", + "datasetPermissions": [ { - "affiliation": "", - "source_signature": "", - "url_prefix": "", - "datasets": [dict(item)['resid'] for item in db_response] + "datasetOwner": dict(record)['organization'], + "urlPrefix": "", + "datasetId": dict(record)['resid'], + "startDate": str(dict(record)['start']), + "endDate": str(dict(record)['endt']) } + for record in db_response ] } return response_body @@ -27,21 +30,19 @@ async def process_post_request(request, db_pool): LOG.debug('Process POST request.') # Put the JSON payload body into a dictionary object request_body = await request.json() + user_id = request_body['userDetails']['elixirId'] # Take one connection from the active database connection pool async with db_pool.acquire() as connection: # Create new user and dataset permissions - user_created = await create_user(request_body['user_identifier'], connection) + user_created = await create_user(user_id, connection) # Wait for user to be created, then add dataset permissions if user_created: # Parse datasets out of request_body - # ELIXIR API Specification has a typo in it, but this iterator satisfies it. - # Come back and fix this once the specification has been corrected - # dataset_group = [[p['affiliation'], p['datasets']] for p in body['permissions']] # Correct form - dataset_group = [[p['affiliation'], p['datasets']] for p in request_body['datasets'][0]['permissions']] - missing_datasets = await create_dataset_permissions(request_body['user_identifier'], dataset_group, connection) + # datasets = [item['datasetId'] for item in request_body['datasetPermissions']] + missing_datasets = await create_dataset_permissions(user_id, request_body, connection) LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') - return missing_datasets # If this list is empty, it means the request was processed fully + return missing_datasets # If this list is empty, it means the request was fully processed async def process_get_request(user, db_pool): @@ -74,18 +75,15 @@ async def process_patch_request(user, request, db_pool): # Try to remove permissions before adding new permissions await remove_dataset_permissions(user, connection) # Parse datasets out of request_body - # ELIXIR API Specification has a typo in it, but this iterator satisfies it. - # Come back and fix this once the specification has been corrected - # dataset_group = [[p['affiliation'], p['datasets']] for p in body['permissions']] # Correct form - dataset_group = [[p['affiliation'], p['datasets']] for p in request_body['datasets'][0]['permissions']] - if dataset_group: - # If any datasets were listed, add permissions for them - missing_datasets = await create_dataset_permissions(user, dataset_group, connection) - LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') - return missing_datasets # If this list is empty, it means the request was processed fully - else: - # Body contained no datasets, end operations here (permissions removed, none added) - return True + # datasets = [item['datasetId'] for item in request_body['datasetPermissions']] + # if datasets: + # If any datasets were listed, add permissions for them + missing_datasets = await create_dataset_permissions(user, request_body, connection) + LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') + return missing_datasets # If this list is empty, it means the request was fully processed + # else: + # # Body contained no datasets, end operations here (permissions removed, none added) + # return True async def process_delete_request(user, db_pool): diff --git a/elixir_rems_proxy/utils/validate.py b/elixir_rems_proxy/utils/validate.py index 2aab92c..7e822b4 100644 --- a/elixir_rems_proxy/utils/validate.py +++ b/elixir_rems_proxy/utils/validate.py @@ -107,7 +107,7 @@ async def parse_username(request): if request.method == 'POST': # Get user from payload request_body = await request.json() - return request_body['user_identifier'] + return request_body['userDetails']['elixirId'] elif request.method in ['GET', 'PATCH', 'DELETE']: # Get user from path return request.match_info['user'] From 3e21326e64877e138d3f26ff46a5335a6b4f24fc Mon Sep 17 00:00:00 2001 From: teemukataja Date: Mon, 5 Nov 2018 12:47:08 +0000 Subject: [PATCH 02/27] fix post and patch date handling --- elixir_rems_proxy/utils/db_actions.py | 6 +++--- elixir_rems_proxy/utils/validate.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 88ea91d..0e2ee3e 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -85,11 +85,11 @@ async def create_dataset_permissions(user, request_body, connection): # Dataset exists, create permissions for user await connection.execute(f"""INSERT INTO entitlement (resid, userid, start, endt) VALUES ($1, $2, $3, $4)""", dataset_index, user, - datetime.strptime(dataset['startDate'], '%Y-%m-%d %H:%M:%S.%f%z') if dataset['startDate'] else datetime.now(), - datetime.strptime(dataset['endDate'], '%Y-%m-%d %H:%M:%S.%f%z') if dataset['endDate'] else None) + datetime.strptime(dataset['startDate'], '%Y-%m-%d %H:%M:%S.%f%z') if dataset.get('startDate') else datetime.now(), + datetime.strptime(dataset['endDate'], '%Y-%m-%d %H:%M:%S.%f%z') if dataset.get('endDate') else None) else: # Dataset doesn't exist - errors.append(dataset) + errors.append(dataset['datasetId']) return errors except Exception as e: LOG.debug(f'An error occurred while attempting to create permissions -> {e}') diff --git a/elixir_rems_proxy/utils/validate.py b/elixir_rems_proxy/utils/validate.py index 7e822b4..1bf3293 100644 --- a/elixir_rems_proxy/utils/validate.py +++ b/elixir_rems_proxy/utils/validate.py @@ -122,13 +122,11 @@ async def check_user_middleware(request, handler): LOG.debug(f'Start user check: {request}') assert isinstance(request, web.Request) username = None - LOG.debug(request.path) if request.path.startswith('/user'): username = await parse_username(request) else: LOG.debug('hello') return await handler(request) - LOG.debug(username) if username: LOG.debug('Try to find user') userid_exists = False From 2432ef58b6a8ae3c9d8b6156728a4288d82884c7 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Mon, 5 Nov 2018 13:13:45 +0000 Subject: [PATCH 03/27] removed datasetowner key from schemas --- elixir_rems_proxy/schemas/patch.json | 10 +--------- elixir_rems_proxy/schemas/post.json | 6 +----- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/elixir_rems_proxy/schemas/patch.json b/elixir_rems_proxy/schemas/patch.json index 98c6a96..a8e4251 100644 --- a/elixir_rems_proxy/schemas/patch.json +++ b/elixir_rems_proxy/schemas/patch.json @@ -1,10 +1,6 @@ { "definitions": {}, "type": "object", - "required": [ - "userDetails", - "datasetPermissions" - ], "properties": { "sourceSignature": { "type": "string", @@ -46,7 +42,7 @@ "datasetId" ], "properties": { - "datasetOwner": { + "datasetId": { "type": "string", "default": "" }, @@ -54,10 +50,6 @@ "type": "string", "default": "" }, - "datasetId": { - "type": "string", - "default": "" - }, "startDate": { "type": "string", "default": "" diff --git a/elixir_rems_proxy/schemas/post.json b/elixir_rems_proxy/schemas/post.json index 98c6a96..8c5d42d 100644 --- a/elixir_rems_proxy/schemas/post.json +++ b/elixir_rems_proxy/schemas/post.json @@ -46,7 +46,7 @@ "datasetId" ], "properties": { - "datasetOwner": { + "datasetId": { "type": "string", "default": "" }, @@ -54,10 +54,6 @@ "type": "string", "default": "" }, - "datasetId": { - "type": "string", - "default": "" - }, "startDate": { "type": "string", "default": "" From 791b4401cfbedc681716afa8956f5ea462c2a357 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 6 Nov 2018 11:28:31 +0000 Subject: [PATCH 04/27] allow PATCHing of user details and other fixes --- elixir_rems_proxy/app.py | 9 ++-- elixir_rems_proxy/schemas/patch.json | 28 ++++-------- elixir_rems_proxy/schemas/post.json | 25 ++++------ elixir_rems_proxy/utils/db_actions.py | 66 +++++++++++++++++++++++++-- elixir_rems_proxy/utils/process.py | 62 ++++++++++++++++--------- 5 files changed, 123 insertions(+), 67 deletions(-) diff --git a/elixir_rems_proxy/app.py b/elixir_rems_proxy/app.py index 23ac32a..9aa6073 100644 --- a/elixir_rems_proxy/app.py +++ b/elixir_rems_proxy/app.py @@ -66,18 +66,19 @@ async def user_get(request): async def user_patch(request): """PATCH request to the /user endpoint. - Update dataset permissions for given user. + Update user details or dataset permissions for given user. """ LOG.debug('PATCH Request received.') db_pool = request.app['pool'] if 'user' in request.match_info: user_identifier = request.match_info['user'] - processed_request = await process_patch_request(user_identifier, request, db_pool) - if len(processed_request) == 0: + missing_datasets = await process_patch_request(user_identifier, request, db_pool) + if not missing_datasets: return web.HTTPOk(text='Successful operation') else: - return web.HTTPCreated(text=f'Following datasets are missing from REMS ({processed_request}), check "GET /user" endpoint for permissions') + return web.HTTPCreated(text=f'User was created, but following datasets are missing from REMS ({missing_datasets}), ' + 'check "GET /user" endpoint for added permissions') else: raise web.HTTPBadRequest(text='Username not provided') diff --git a/elixir_rems_proxy/schemas/patch.json b/elixir_rems_proxy/schemas/patch.json index a8e4251..549df0a 100644 --- a/elixir_rems_proxy/schemas/patch.json +++ b/elixir_rems_proxy/schemas/patch.json @@ -3,34 +3,26 @@ "type": "object", "properties": { "sourceSignature": { - "type": "string", - "default": "" + "type": "string" }, "userDetails": { "type": "object", - "required": [ - "elixirId" - ], "properties": { "elixirId": { "type": "string", - "default": "", "examples": [ "c85cc6f750611ce89c775f560f10199e08f479ce@elixir-europe.org" ], "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" }, "eduPersonPrincipalName": { - "type": "string", - "default": "" + "type": "string" }, "userEmail": { - "type": "string", - "default": "" + "type": "string" }, "realName": { - "type": "string", - "default": "" + "type": "string" } } }, @@ -43,20 +35,16 @@ ], "properties": { "datasetId": { - "type": "string", - "default": "" + "type": "string" }, "urlPrefix": { - "type": "string", - "default": "" + "type": "string" }, "startDate": { - "type": "string", - "default": "" + "type": "string" }, "endDate": { - "type": "string", - "default": "" + "type": "string" } } } diff --git a/elixir_rems_proxy/schemas/post.json b/elixir_rems_proxy/schemas/post.json index 8c5d42d..de9e4c3 100644 --- a/elixir_rems_proxy/schemas/post.json +++ b/elixir_rems_proxy/schemas/post.json @@ -7,8 +7,7 @@ ], "properties": { "sourceSignature": { - "type": "string", - "default": "" + "type": "string" }, "userDetails": { "type": "object", @@ -18,23 +17,19 @@ "properties": { "elixirId": { "type": "string", - "default": "", "examples": [ "c85cc6f750611ce89c775f560f10199e08f479ce@elixir-europe.org" ], "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" }, "eduPersonPrincipalName": { - "type": "string", - "default": "" + "type": "string" }, "userEmail": { - "type": "string", - "default": "" + "type": "string" }, "realName": { - "type": "string", - "default": "" + "type": "string" } } }, @@ -47,20 +42,16 @@ ], "properties": { "datasetId": { - "type": "string", - "default": "" + "type": "string" }, "urlPrefix": { - "type": "string", - "default": "" + "type": "string" }, "startDate": { - "type": "string", - "default": "" + "type": "string" }, "endDate": { - "type": "string", - "default": "" + "type": "string" } } } diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 0e2ee3e..d3b3808 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -1,5 +1,7 @@ """Database Queries.""" +import json + from datetime import datetime from aiohttp import web @@ -38,13 +40,26 @@ async def get_dataset_permissions(user, connection): return None -async def create_user(user, connection): +async def make_jsonb(user_details): + """Construct PostgreSQL jsonb from JSON payload.""" + LOG.debug('Construct jsonb') + user_attributes = { + "eppn": user_details.get("eduPersonPrincipalName", None), + "mail": user_details.get("userEmail", None), + "commonName": user_details.get("realName", None) + } + return json.dumps(user_attributes) + + +async def create_user(user_details, connection): """Create new user.""" - LOG.debug(f'Create new user {user}') + LOG.debug(f'Create new user {user_details}') + user_attributes = await make_jsonb(user_details) try: # Make inserts inside of a transaction async with connection.transaction(): - await connection.execute(f"""INSERT INTO users VALUES ($1)""", user) + await connection.execute(f"""INSERT INTO users (userid, userattrs) VALUES ($1, $2)""", + user_details['elixirId'], user_attributes) return True # User was created except Exception as e: LOG.debug(f'An error occurred while attempting to create user -> {e}') @@ -72,14 +87,14 @@ async def get_dataset_index(dataset, connection): return None -async def create_dataset_permissions(user, request_body, connection): +async def create_dataset_permissions(user, dataset_permissions, connection): """Create dataset permissions.""" LOG.debug('Create dataset permissions.') errors = [] try: # Make inserts inside of a transaction and commit all at once async with connection.transaction(): - for dataset in request_body['datasetPermissions']: + for dataset in dataset_permissions: dataset_index = await get_dataset_index(dataset['datasetId'], connection) if dataset_index: # Dataset exists, create permissions for user @@ -116,3 +131,44 @@ async def delete_user(user, connection): except Exception as e: LOG.debug(f'An error occurred while attempting to delete user -> {e}') raise web.HTTPInternalServerError(text='Database error when attempting to remove user') + + +async def update_user_attributes(new_user_details, old_user_details): + """Construct PostgreSQL jsonb from JSON payload.""" + LOG.debug('Construct jsonb') + user_attributes = { + "eppn": new_user_details.get("eduPersonPrincipalName", old_user_details.get("eppn", None)), + "mail": new_user_details.get("userEmail", old_user_details.get("mail", None)), + "commonName": new_user_details.get("realName", old_user_details.get("commonName", None)) + } + return json.dumps(user_attributes) + + +async def update_user(old_user, new_user_details, connection): + """Update user details.""" + LOG.debug(f'Update user={old_user} with={new_user_details}') + # Place new elixir id into variable, defaults to old user if it remains unchanged + new_user = new_user_details.get('elixirId', old_user) + + try: + # Get old user details + query = f"""SELECT * FROM users WHERE userid='{old_user}';""" + statement = await connection.prepare(query) + db_response = await statement.fetch() + # User existence is already checked at middleware, so no need to validate it again + LOG.debug(f'Response: {db_response}') + old_user_details = json.loads(dict(db_response[0])['userattrs']) + LOG.debug(type(old_user_details)) + # Create new user attribute jsonb + new_user_attributes = await update_user_attributes(new_user_details, old_user_details) + # Do updates inside of a transaction + async with connection.transaction(): + # Update users table + await connection.execute(f"""UPDATE users SET userid='{new_user}', userattrs='{new_user_attributes}' + WHERE userid='{old_user}';""") + # Update permissions table + await connection.execute(f"""UPDATE entitlement SET userid='{new_user}' WHERE userid='{old_user}';""") + return True + except Exception as e: + LOG.debug(f'An error occurred while attempting to update user -> {e}') + raise web.HTTPInternalServerError(text='Database error when attempting to update user') diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index 6dcce24..f5f4008 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -3,7 +3,7 @@ from aiohttp import web from ..utils.logging import LOG -from ..utils.db_actions import create_user, delete_user +from ..utils.db_actions import create_user, delete_user, update_user from ..utils.db_actions import get_dataset_permissions, create_dataset_permissions, remove_dataset_permissions @@ -30,17 +30,17 @@ async def process_post_request(request, db_pool): LOG.debug('Process POST request.') # Put the JSON payload body into a dictionary object request_body = await request.json() - user_id = request_body['userDetails']['elixirId'] + missing_datasets = [] # used to notify user of missing datasets in REMS # Take one connection from the active database connection pool async with db_pool.acquire() as connection: # Create new user and dataset permissions - user_created = await create_user(user_id, connection) + user_created = await create_user(request_body['userDetails'], connection) # Wait for user to be created, then add dataset permissions if user_created: - # Parse datasets out of request_body - # datasets = [item['datasetId'] for item in request_body['datasetPermissions']] - missing_datasets = await create_dataset_permissions(user_id, request_body, connection) + missing_datasets = await create_dataset_permissions(request_body['userDetails']['elixirId'], + request_body['datasetPermissions'], + connection) LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') return missing_datasets # If this list is empty, it means the request was fully processed @@ -61,29 +61,49 @@ async def process_get_request(user, db_pool): raise web.HTTPNotFound(text='Permissions for this user not found') +# async def process_patch_request(user, request, db_pool): +# """Update user's dataset permissions.""" +# LOG.debug('Process PATCH request.') +# # Put the JSON payload body into a dictionary object +# request_body = await request.json() + +# # Take one connection from the active database connection pool +# async with db_pool.acquire() as connection: +# # Check if user has permissions to be removed +# permissions = await get_dataset_permissions(user, connection) +# if permissions: +# # Try to remove permissions before adding new permissions +# await remove_dataset_permissions(user, connection) +# missing_datasets = await create_dataset_permissions(user, request_body, connection) +# LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') +# return missing_datasets # If this list is empty, it means the request was fully processed +# # else: +# # # Body contained no datasets, end operations here (permissions removed, none added) +# # return True + async def process_patch_request(user, request, db_pool): - """Update user's dataset permissions.""" + """Update user details and dataset permissions.""" LOG.debug('Process PATCH request.') # Put the JSON payload body into a dictionary object request_body = await request.json() + missing_datasets = [] # used to notify user of missing datasets in REMS # Take one connection from the active database connection pool async with db_pool.acquire() as connection: - # Check if user has permissions to be removed - permissions = await get_dataset_permissions(user, connection) - if permissions: - # Try to remove permissions before adding new permissions + # Check if payload contains dataset permissions + if request_body.get('datasetPermissions'): + LOG.debug('PATCHing dataset permissions') + # Remove old permissions await remove_dataset_permissions(user, connection) - # Parse datasets out of request_body - # datasets = [item['datasetId'] for item in request_body['datasetPermissions']] - # if datasets: - # If any datasets were listed, add permissions for them - missing_datasets = await create_dataset_permissions(user, request_body, connection) - LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') - return missing_datasets # If this list is empty, it means the request was fully processed - # else: - # # Body contained no datasets, end operations here (permissions removed, none added) - # return True + # Add new permissions + missing_datasets = await create_dataset_permissions(user, request_body['datasetPermissions'], connection) + LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') + # Check if payload contains user details + if request_body.get('userDetails'): + LOG.debug('PATCHing user details') + await update_user(user, request_body['userDetails'], connection) + + return missing_datasets # If this list is empty, it means the request was fully processed async def process_delete_request(user, db_pool): From 54ec91a2e3ecb9d6133981509b35a97028b5620d Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 6 Nov 2018 11:31:14 +0000 Subject: [PATCH 05/27] change PATCH OK from 200 to 204, remove comments --- elixir_rems_proxy/app.py | 2 +- elixir_rems_proxy/utils/db_actions.py | 1 - elixir_rems_proxy/utils/process.py | 20 -------------------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/elixir_rems_proxy/app.py b/elixir_rems_proxy/app.py index 9aa6073..802890a 100644 --- a/elixir_rems_proxy/app.py +++ b/elixir_rems_proxy/app.py @@ -75,7 +75,7 @@ async def user_patch(request): user_identifier = request.match_info['user'] missing_datasets = await process_patch_request(user_identifier, request, db_pool) if not missing_datasets: - return web.HTTPOk(text='Successful operation') + return web.HTTPNoContent() else: return web.HTTPCreated(text=f'User was created, but following datasets are missing from REMS ({missing_datasets}), ' 'check "GET /user" endpoint for added permissions') diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index d3b3808..bb74a59 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -158,7 +158,6 @@ async def update_user(old_user, new_user_details, connection): # User existence is already checked at middleware, so no need to validate it again LOG.debug(f'Response: {db_response}') old_user_details = json.loads(dict(db_response[0])['userattrs']) - LOG.debug(type(old_user_details)) # Create new user attribute jsonb new_user_attributes = await update_user_attributes(new_user_details, old_user_details) # Do updates inside of a transaction diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index f5f4008..0c937db 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -61,26 +61,6 @@ async def process_get_request(user, db_pool): raise web.HTTPNotFound(text='Permissions for this user not found') -# async def process_patch_request(user, request, db_pool): -# """Update user's dataset permissions.""" -# LOG.debug('Process PATCH request.') -# # Put the JSON payload body into a dictionary object -# request_body = await request.json() - -# # Take one connection from the active database connection pool -# async with db_pool.acquire() as connection: -# # Check if user has permissions to be removed -# permissions = await get_dataset_permissions(user, connection) -# if permissions: -# # Try to remove permissions before adding new permissions -# await remove_dataset_permissions(user, connection) -# missing_datasets = await create_dataset_permissions(user, request_body, connection) -# LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') -# return missing_datasets # If this list is empty, it means the request was fully processed -# # else: -# # # Body contained no datasets, end operations here (permissions removed, none added) -# # return True - async def process_patch_request(user, request, db_pool): """Update user details and dataset permissions.""" LOG.debug('Process PATCH request.') From b527644631c90c61497ec53c174104da94282712 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 6 Nov 2018 12:09:15 +0000 Subject: [PATCH 06/27] fix 400 at /user get, patch, delete --- elixir_rems_proxy/utils/validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elixir_rems_proxy/utils/validate.py b/elixir_rems_proxy/utils/validate.py index 1bf3293..c5acbd7 100644 --- a/elixir_rems_proxy/utils/validate.py +++ b/elixir_rems_proxy/utils/validate.py @@ -110,7 +110,7 @@ async def parse_username(request): return request_body['userDetails']['elixirId'] elif request.method in ['GET', 'PATCH', 'DELETE']: # Get user from path - return request.match_info['user'] + return request.match_info.get('user', None) def check_user(): @@ -125,7 +125,7 @@ async def check_user_middleware(request, handler): if request.path.startswith('/user'): username = await parse_username(request) else: - LOG.debug('hello') + LOG.debug('At info endpoint') return await handler(request) if username: LOG.debug('Try to find user') From 7c1491f91353281ab1b4f6946c5d998b9c29081d Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 6 Nov 2018 12:57:36 +0000 Subject: [PATCH 07/27] update docs and readme, remove suggestions and add suggestion for api spec 1.3 --- README.md | 60 +++--- docs/example.rst | 79 ++++---- elixir_rems_proxy/utils/db_actions.py | 8 + elixirapi.yaml | 272 ++++++++++++++++++++++++++ suggestions.md | 138 ------------- 5 files changed, 342 insertions(+), 215 deletions(-) create mode 100644 elixirapi.yaml delete mode 100644 suggestions.md diff --git a/README.md b/README.md index 2a27903..3cc4204 100644 --- a/README.md +++ b/README.md @@ -61,22 +61,24 @@ For more technical details, consult the [API Specification](https://app.swaggerh curl -X POST \ http://localhost:8080/user \ -H 'Content-Type: application/json' \ + -H 'elixir-api-key: secret' \ -d '{ - "user_identifier": "test_user", - "affiliation": "", - "datasets": [ + "userDetails": { + "elixirId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org", + "eduPersonPrincipalName": "user@org.org", + "userEmail": "firstname.lastname@organisation.org", + "realName": "Firstname Lastname" + }, + "datasetPermissions": [ { - "permissions": [ - { - "affiliation": "example-org", - "source_signature": "", - "url_prefix": "", - "datasets": [ - "urn:example-dataset-1", - "urn:example-dataset-2" - ] - } - ] + "datasetId": "urn:example-dataset-1", + "startDate": "2018-01-01 12:00:00.000000+0000", + "endDate": "2019-01-01 12:00:00.000000+0000" + }, + { + "datasetId": "urn:example-dataset-2", + "startDate": "", + "endDate": "" } ] }' @@ -85,30 +87,21 @@ curl -X POST \ #### GET /user/username `GET` method at `/user` endpoint is used to fetch dataset permissions for user. ``` -curl -X GET http://localhost:8080/user/test_user +curl -X GET -H 'elixir-api-key: secret' http://localhost:8080/user/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org ``` #### PATCH /user/username -`PATCH` method at `/user` endpoint is used to update dataset permissions for user. +`PATCH` method at `/user` endpoint is used to update user details and dataset permissions for user. `PATCH` endpoint consumes the same schema as `POST` endpoint, but all fields are optional instead of mandatory. ``` curl -X PATCH \ - http://localhost:8080/user/test_user \ + http://localhost:8080/user/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org \ -H 'Content-Type: application/json' \ + -H 'elixir-api-key: secret' \ -d '{ - "user_identifier": "", - "affiliation": "", - "datasets": [ + "datasetPermissions": [ { - "permissions": [ - { - "affiliation": "example-org", - "source_signature": "", - "url_prefix": "", - "datasets": [ - "urn:example-dataset-3" - ] - } - ] + "datasetId": "urn:example-dataset-1", + "endDate": "2020-01-01 12:00:00.000000+0000" } ] }' @@ -117,10 +110,5 @@ curl -X PATCH \ #### DELETE /user/username `DELETE` method at `/user` endpoint is used to delete user along with dataset permissions. ``` -curl -X DELETE http://localhost:8080/user/test_user +curl -X DELETE -H 'elixir-api-key: secret' http://localhost:8080/user/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org ``` - - - -### Other Business -The [Permissions API Specification](https://app.swaggerhub.com/apis-docs/ELIXIR-Finland/Permissions/1.2) contains some typos. A [suggestions](suggestions.md) document has been drafted to correct those issues. Expect changes to be made to the specification in the near future, along with changes to the API app. diff --git a/docs/example.rst b/docs/example.rst index 97247d8..7a87799 100644 --- a/docs/example.rst +++ b/docs/example.rst @@ -31,26 +31,25 @@ An example ``POST`` request and response to the ``user`` endpoint: .. code-block:: console - $ curl -X POST \ + curl -X POST \ http://localhost:8080/user \ -H 'Content-Type: application/json' \ -H 'elixir-api-key: secret' \ -d '{ - "user_identifier": "test_user", - "affiliation": "", - "datasets": [ + "userDetails": { + "elixirId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org", + "eduPersonPrincipalName": "user@org.org", + "userEmail": "firstname.lastname@organisation.org", + "realName": "Firstname Lastname" + }, + "datasetPermissions": [ { - "permissions": [ - { - "affiliation": "example-org", - "source_signature": "", - "url_prefix": "", - "datasets": [ - "urn:example-dataset-1", - "urn:example-dataset-2" - ] - } - ] + "datasetId": "urn:example-dataset-1", + "startDate": "2018-01-01 12:00:00.000000+0000", + "endDate": "2019-01-01 12:00:00.000000+0000" + }, + { + "datasetId": "urn:example-dataset-2", } ] }' @@ -65,23 +64,29 @@ An example ``GET`` request and response to the ``user`` endpoint: .. code-block:: console - $ curl -X GET -H 'elixir-api-key: secret' http://localhost:8080/user/test_user + $ curl -X GET -H 'elixir-api-key: secret' http://localhost:8080/user/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org Example Response: .. code-block:: javascript { - "permissions": [ - { - "affiliation": "", - "source_signature": "", - "url_prefix": "", - "datasets": [ - "urn:example-dataset-1", - "urn:example-dataset-2" - ] - } + "sourceSignature": "string", + "datasetPermissions": [ + { + "datasetOwner": "example-org", + "urlPrefix": "", + "datasetId": "urn:example-dataset-1", + "startDate": "2018-01-01 12:00:00.000000+0000", + "endDate": "2019-01-01 12:00:00.000000+0000" + }, + { + "datasetOwner": "example-org", + "urlPrefix": "", + "datasetId": "urn:example-dataset-2", + "startDate": "2018-11-06 12:00:00.000000+0000", + "endDate": "None" + } ] } @@ -89,29 +94,21 @@ An example ``PATCH`` request and response to the ``user`` endpoint: .. code-block:: console - $ curl -X PATCH \ - http://localhost:8080/user/test_user \ + curl -X PATCH \ + http://localhost:8080/user/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org \ -H 'Content-Type: application/json' \ -H 'elixir-api-key: secret' \ -d '{ - "user_identifier": "", - "affiliation": "", - "datasets": [ + "datasetPermissions": [ { - "permissions": [ - { - "affiliation": "example-org", - "source_signature": "", - "url_prefix": "", - "datasets": [ - "urn:example-dataset-3" - ] - } - ] + "datasetId": "urn:example-dataset-1", + "endDate": "2020-01-01 12:00:00.000000+0000" } ] }' +.. note:: ``PATCH`` endpoint consumes the same schema as ``POST`` endpoint, but all fields are optional instead of mandatory. + Example Response: .. code-block:: text diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index bb74a59..1fe7bec 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -150,7 +150,15 @@ async def update_user(old_user, new_user_details, connection): # Place new elixir id into variable, defaults to old user if it remains unchanged new_user = new_user_details.get('elixirId', old_user) + # Check if new username is taken + userid_exists = await user_exists(new_user, connection) + if userid_exists: + LOG.debug('PATCH Conflict: Username is taken') + # Username is taken, stop PATCH process here + raise web.HTTPConflict(text='Username is taken') + try: + LOG.debug('PATCHing begins..') # Get old user details query = f"""SELECT * FROM users WHERE userid='{old_user}';""" statement = await connection.prepare(query) diff --git a/elixirapi.yaml b/elixirapi.yaml new file mode 100644 index 0000000..3088d9e --- /dev/null +++ b/elixirapi.yaml @@ -0,0 +1,272 @@ +swagger: '2.0' +info: + description: | + This is a definition of Permissions API which is a part of an architecture for delivering dataset permissions on ELIXIR AAI. The whole architecture is explained at: https://docs.google.com/document/d/1rqCD75HRA99HKwq0s-OkWWBKiaEo2JO-_MYjlsyiSQ4 + + version: "1.3" + title: Permissions API + contact: + email: juha.tornroos@csc.fi + license: + name: License Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html +tags: +- name: user + description: User details and dataset permissions management +paths: + /user: + post: + tags: + - user + summary: Create a new user with access to given datasets + operationId: createUser + produces: + - application/json + parameters: + - in: body + name: body + description: Contains user details and dataset permissions + required: true + schema: + $ref: '#/definitions/PostRequest' + - name: elixir-api-key + in: header + description: Secret key to access the ELIXIR Permissions API + required: true + type: string + responses: + 200: + description: Successful operation + 201: + description: User was created, but some datasets are missing from REMS + 401: + description: Unauthorized api key + 400: + description: | + Could not validate request body (broken JSON) + \ + Missing headers "elixir-api-key" + 409: + description: Username is taken + + /user/{elixirId}: + get: + tags: + - user + summary: List all datasets user has access to + operationId: loginUser + produces: + - application/json + parameters: + - name: elixirId + in: path + description: The user's ELIXIR AAI issued identifier + required: true + type: string + - name: elixir-api-key + in: header + description: Secret key to access the ELIXIR Permissions API + required: true + type: string + responses: + 200: + description: Successful operation + schema: + $ref: '#/definitions/Response' + 400: + description: | + Username not provided + \ + Missing headers "elixir-api-key" + 404: + description: User not found + patch: + tags: + - user + summary: Update user details or dataset permissions for given user + description: This method overwrites old data, so it can be used to remove dataset permissions by sending an empty datasetPermissions object. PATCH uses the same payload model as POST, with the difference, that PATCH has no required fields in the request body. + operationId: logoutUser + produces: + - application/json + parameters: + - name: elixirId + in: path + description: The user's ELIXIR AAI issued identifier + required: true + type: string + - name: elixir-api-key + in: header + description: Secret key to access the ELIXIR Permissions API + required: true + type: string + - in: body + name: body + description: Contains user details and dataset permissions + required: true + schema: + $ref: '#/definitions/PatchRequest' + responses: + 201: + description: Some datasets are missing from REMS + 204: + description: Successful operation + 400: + description: | + Could not validate request body (broken JSON) + \ + Missing headers "elixir-api-key" + 404: + description: User not found + 409: + description: Username is taken + delete: + tags: + - user + summary: Delete user along with all dataset permissions + description: This can only be done by the logged in user. + operationId: deleteUser + produces: + - application/json + parameters: + - name: elixirId + in: path + description: The user's ELIXIR AAI issued identifier + required: true + type: string + - name: elixir-api-key + in: header + description: Secret key to access the ELIXIR Permissions API + required: true + type: string + responses: + 200: + description: User was deleted + 400: + description: | + Username not provided + \ + Missing headers "elixir-api-key" + 404: + description: User not found + +definitions: + Response: + type: object + properties: + sourceSignature: + type: string + datasetPermissions: + type: array + items: + $ref: '#/definitions/Permissions' + Permissions: + type: object + properties: + datasetOwner: + type: string + description: Organisation that owns the dataset + urlPrefix: + type: string + description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} + datasetId: + type: string + required: True + description: Globally unique dataset identifier, e.g. URN + startDate: + type: string + description: Start date of permission + endDate: + type: string + description: End date of permission + PostDatasets: + type: object + required: true + properties: + datasetId: + type: string + required: true + description: Globally unique dataset identifier, e.g. URN + urlPrefix: + type: string + description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} + startDate: + type: string + description: Start date of permission + endDate: + type: string + description: End date of permission + PostRequest: + type: object + properties: + sourceSignature: + type: string + userDetails: + type: object + required: true + properties: + elixirId: + type: string + required: true + description: Unique ELIXIR AAI issued user identifier + eduPersonPrincipalName: + type: string + format: email + description: User identifier at home organisation in email format, e.g. user@org.fi + userEmail: + type: string + format: email + description: User email address + realName: + type: string + description: Full legal name of the user + datasetPermissions: + type: array + required: true + items: + $ref: '#/definitions/PostDatasets' + PatchDatasets: + type: object + properties: + datasetId: + type: string + description: Globally unique dataset identifier, e.g. URN + urlPrefix: + type: string + description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} + startDate: + type: string + description: Start date of permission + endDate: + type: string + description: End date of permission + PatchRequest: + type: object + properties: + sourceSignature: + type: string + userDetails: + type: object + properties: + elixirId: + type: string + description: Unique ELIXIR AAI issued user identifier + eduPersonPrincipalName: + type: string + format: email + description: User identifier at home organisation in email format, e.g. user@org.fi + userEmail: + type: string + format: email + description: User email address + realName: + type: string + description: Full legal name of the user + datasetPermissions: + type: array + items: + $ref: '#/definitions/PatchDatasets' + +host: virtserver.swaggerhub.com +basePath: /ELIXIR-Finland/Permissions/1.0.0 +schemes: + - https \ No newline at end of file diff --git a/suggestions.md b/suggestions.md deleted file mode 100644 index ac48acd..0000000 --- a/suggestions.md +++ /dev/null @@ -1,138 +0,0 @@ -#### Suggestions for the ELIXIR Permissions API Specification -The [Permissions API Specification](https://app.swaggerhub.com/apis-docs/ELIXIR-Finland/Permissions/1.2) has some minor flaws, which this document aims to correct. - -### POST /user - -#### Payload -The `POST` method at `/user` endpoint has an extra `datasets` branch in the JSON payload. - -Current format: -``` -{ - "user_identifier": "string", - "affiliation": "string", - "datasets": [ - { - "permissions": [ - { - "affiliation": "user@example.com", - "source_signature": "string", - "url_prefix": "string", - "datasets": [ - "string" - ] - } - ] - } - ] -} -``` - -Suggested format: -``` -{ - "user_identifier": "string", - "affiliation": "string", - "permissions": [ - { - "affiliation": "user@example.com", - "source_signature": "string", - "url_prefix": "string", - "datasets": [ - "string" - ] - } - ] -} -``` - -#### Responses -The `POST` method at `/user` endpoint has the following response specified: - -| Code | Description | -| --- | --- | -| default | Successful operation | - -Suggested responses: - -| Code | Description | -| --- | --- | -| 200 | Successful operation | -| 201 | Partially successful operation (something was done, something was not done, usually happens when user is attempting to add permissions to datasets which don't exist in REMS. Response body can return the missing datasets which were in the request, see [app.py return http response](/api/app.py#L37).) | -| 409 | Username is taken | - -### GET /user/username - -#### Query Parameter -The `GET` method at `/user/username` endpoint has an optional query parameter `user_affiliation` which can't be tied to any item in the REMS database. As such, this parameter can't be leveraged with the current REMS database schema. The only thing that might indicate a user's affiliation is in the `user` -table's `userattrs` column, which is an email-address embedded within a jsonb: `{"eppn": "user_id", "mail": "user_id@org.org", "commonName": "User Name"}`. Attempting to parse this from all users could prove to be an expensive operation. - -#### Responses -The `GET` method at `/user/username` endpoint already has response codes `200` and `404` for `Successful operation` and `User not found` respectively. - -Suggested response addition: - -| Code | Description | -| --- | --- | -| 400 | Username not provided | - -### PATCH /user/username -#### Payload -The `PATCH` method at `/user/username` is decribed as a `PUT` method. - -The JSON payload for `PATCH` is identical to that of the `POST` endpoint, currently: -``` -{ - "user_identifier": "string", - "affiliation": "string", - "datasets": [ - { - "permissions": [ - { - "affiliation": "user@example.com", - "source_signature": "string", - "url_prefix": "string", - "datasets": [ - "string" - ] - } - ] - } - ] -} -``` -But this means the endpoint should in fact be a `PUT` method, because the endpoint is used to do a full replacement of data. To conform the endpoint to `PATCH` method standards, only the item that will be changed should be specified. - -Suggested payload format: -``` -{ - "permissions": [ - { - "affiliation": "user@example.com", - "source_signature": "string", - "url_prefix": "string", - "datasets": [ - "string" - ] - } - ] -} -``` - -#### Responses -The `PATCH` method at `/user/username` endpoint has the following response specified: - -| Code | Description | -| --- | --- | -| default | Successful operation | - -Suggested responses: - -| Code | Description | -| --- | --- | -| 200 | Successful operation | -| 201 | Partially successful operation (something was done, something was not done, usually happens when user is attempting to add permissions to datasets which don't exist in REMS. Response body can return the missing datasets which were in the request, see [app.py return http response](/api/app.py#L81).) | -| 400 | Username not provided | -| 404 | User not found | - -### DELETE /user/username -No suggestions for this endpoint. From e6d460ecf47de91d7542ee68d717d8db9fa3dc3a Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 6 Nov 2018 15:43:01 +0000 Subject: [PATCH 08/27] remove urlprefix from patch and post --- elixirapi.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/elixirapi.yaml b/elixirapi.yaml index 3088d9e..14db7be 100644 --- a/elixirapi.yaml +++ b/elixirapi.yaml @@ -186,9 +186,6 @@ definitions: type: string required: true description: Globally unique dataset identifier, e.g. URN - urlPrefix: - type: string - description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} startDate: type: string description: Start date of permission @@ -230,9 +227,6 @@ definitions: datasetId: type: string description: Globally unique dataset identifier, e.g. URN - urlPrefix: - type: string - description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} startDate: type: string description: Start date of permission From 981ddf0bd04ded827df1087392333863a52cdd03 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Wed, 7 Nov 2018 08:10:06 +0000 Subject: [PATCH 09/27] validate api key from database instead of environment, docs updates, schema fix, disable tests (to be rewritten) --- docs/example.rst | 36 ++++++++++++++++++++++++---- docs/instructions.rst | 2 -- elixir_rems_proxy/schemas/patch.json | 3 --- elixir_rems_proxy/utils/validate.py | 17 +++++++------ tests/test_app.py | 12 +++++----- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/docs/example.rst b/docs/example.rst index 7a87799..dc31b11 100644 --- a/docs/example.rst +++ b/docs/example.rst @@ -90,7 +90,9 @@ Example Response: ] } -An example ``PATCH`` request and response to the ``user`` endpoint: +Few examples of ``PATCH`` requests to the ``user`` endpoint: + +CASE 1: Update dataset permissions .. code-block:: console @@ -107,13 +109,37 @@ An example ``PATCH`` request and response to the ``user`` endpoint: ] }' -.. note:: ``PATCH`` endpoint consumes the same schema as ``POST`` endpoint, but all fields are optional instead of mandatory. +CASE 2: Clear dataset permissions by sending an empty ``datasetPermissions`` object -Example Response: +.. code-block:: console -.. code-block:: text + curl -X PATCH \ + http://localhost:8080/user/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org \ + -H 'Content-Type: application/json' \ + -H 'elixir-api-key: secret' \ + -d '{ + "datasetPermissions": [{}] + }' - Successful operation +CASE 3: Update user details + +.. code-block:: console + + curl -X PATCH \ + http://localhost:8080/user/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org \ + -H 'Content-Type: application/json' \ + -H 'elixir-api-key: secret' \ + -d '{ + "userDetails": { + "eduPersonPrincipalName": "user@org.fi", + "userEmail": "user.name@organisation.fi", + "realName": "User Name" + } + }' + +.. note:: ``PATCH`` endpoint consumes the same schema as ``POST`` endpoint, but all fields are optional instead of mandatory. + +.. note:: Successful ``PATCH`` requests respond with ``HTTP 204 No Content``. An example ``DELETE`` request and response to the ``user`` endpoint: diff --git a/docs/instructions.rst b/docs/instructions.rst index 577a172..0f8b2da 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -24,8 +24,6 @@ the table below. +------------- +-------------------------------+-------------------------------------+ | `APP_PORT` | `8080` | Default port for the Web Server. | +------------- +-------------------------------+-------------------------------------+ -| `PUBLIC_KEY` | `None` | Mandatory api key. | -+------------- +-------------------------------+-------------------------------------+ | `DEBUG` | `True` | If set to `True`, logs all actions. | +------------- +-------------------------------+-------------------------------------+ diff --git a/elixir_rems_proxy/schemas/patch.json b/elixir_rems_proxy/schemas/patch.json index 549df0a..4bc5040 100644 --- a/elixir_rems_proxy/schemas/patch.json +++ b/elixir_rems_proxy/schemas/patch.json @@ -30,9 +30,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "datasetId" - ], "properties": { "datasetId": { "type": "string" diff --git a/elixir_rems_proxy/utils/validate.py b/elixir_rems_proxy/utils/validate.py index c5acbd7..782f7f5 100644 --- a/elixir_rems_proxy/utils/validate.py +++ b/elixir_rems_proxy/utils/validate.py @@ -1,7 +1,5 @@ """JSON Schema Validation.""" -import os - from functools import wraps from aiohttp import web @@ -84,12 +82,17 @@ async def api_key_middleware(request, handler): raise web.HTTPBadRequest(text=f'Error with api key header: {e}') if elixir_api_key is not None: - try: - assert os.environ.get('PUBLIC_KEY', None) == elixir_api_key + # Take one connection from the active database connection pool + async with request.app['pool'].acquire() as connection: + # Check if api key exists in database + query = f"""SELECT comment FROM api_key WHERE apikey='{elixir_api_key}'""" + statement = await connection.prepare(query) + db_response = await statement.fetch() + LOG.debug(f'Response from API KEY CHECK: {db_response}') + if not db_response: + LOG.debug(f'ERROR: Bad api key: {db_response}') + raise web.HTTPUnauthorized(text='Unauthorized api key') LOG.debug('Provided api key is authorized') - except Exception as e: - LOG.debug(f'ERROR: Bad api key: {e}') - raise web.HTTPUnauthorized(text='Unauthorized api key') # Carry on with user request return await handler(request) elif '/user' not in request.path: diff --git a/tests/test_app.py b/tests/test_app.py index 0a96210..db7ec38 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -69,7 +69,7 @@ async def test_info(self): # assert 200 == resp.status # Test POST ... - + ''' @unittest_run_loop async def test_get_400(self): """Test the get endpoint.""" @@ -83,7 +83,7 @@ async def test_get_401(self): with asynctest.mock.patch('elixir_rems_proxy.app.user_get'): resp = await self.client.request("GET", "/user/", headers={"elixir-api-key": "invalid_key"}) assert 401 == resp.status - + ''' # 405's disabled: write tests that takes mandatory api key into account # @unittest_run_loop @@ -94,7 +94,7 @@ async def test_get_401(self): # assert 405 == resp.status # Test PATCH 200 - + ''' @unittest_run_loop async def test_patch_400(self): """Test the get endpoint.""" @@ -108,7 +108,7 @@ async def test_patch_401(self): with asynctest.mock.patch('elixir_rems_proxy.app.user_patch'): resp = await self.client.request("PATCH", "/user/", headers={"elixir-api-key": "invalid_key"}) assert 401 == resp.status - + ''' # @unittest_run_loop # async def test_patch_405(self): # """Test the get endpoint.""" @@ -117,7 +117,7 @@ async def test_patch_401(self): # assert 405 == resp.status # Test DELETE 200 - + ''' @unittest_run_loop async def test_delete_400(self): """Test the get endpoint.""" @@ -131,7 +131,7 @@ async def test_delete_401(self): with asynctest.mock.patch('elixir_rems_proxy.app.user_delete', side_effect={"smth": "value"}): resp = await self.client.request("DELETE", "/user/", headers={"elixir-api-key": "invalid_key"}) assert 401 == resp.status - + ''' # @unittest_run_loop # async def test_delete_405(self): # """Test the get endpoint.""" From e0d4180349dc681f0d7bbadd3ac82fdd9c508570 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Mon, 12 Nov 2018 11:08:59 +0200 Subject: [PATCH 10/27] move to openapi 3.0.0 --- elixirapi.yaml | 318 +++++++++++++++++++++++++------------------------ 1 file changed, 162 insertions(+), 156 deletions(-) diff --git a/elixirapi.yaml b/elixirapi.yaml index 14db7be..ec50eb9 100644 --- a/elixirapi.yaml +++ b/elixirapi.yaml @@ -1,8 +1,8 @@ -swagger: '2.0' +openapi: 3.0.0 info: - description: | + description: >- This is a definition of Permissions API which is a part of an architecture for delivering dataset permissions on ELIXIR AAI. The whole architecture is explained at: https://docs.google.com/document/d/1rqCD75HRA99HKwq0s-OkWWBKiaEo2JO-_MYjlsyiSQ4 - + version: "1.3" title: Permissions API contact: @@ -11,29 +11,29 @@ info: name: License Apache 2.0 url: http://www.apache.org/licenses/LICENSE-2.0.html tags: -- name: user - description: User details and dataset permissions management + - name: user + description: User details and dataset permissions management paths: /user: post: tags: - - user + - user summary: Create a new user with access to given datasets operationId: createUser - produces: - - application/json - parameters: - - in: body - name: body + requestBody: description: Contains user details and dataset permissions required: true - schema: - $ref: '#/definitions/PostRequest' - - name: elixir-api-key - in: header - description: Secret key to access the ELIXIR Permissions API - required: true - type: string + content: + application/json: + schema: + $ref: '#/components/schemas/PostRequest' + parameters: + - name: elixir-api-key + in: header + description: Secret key to access the ELIXIR Permissions API + required: true + schema: + type: string responses: 200: description: Successful operation @@ -52,27 +52,29 @@ paths: /user/{elixirId}: get: tags: - - user + - user summary: List all datasets user has access to operationId: loginUser - produces: - - application/json parameters: - name: elixirId in: path description: The user's ELIXIR AAI issued identifier required: true - type: string + schema: + type: string - name: elixir-api-key in: header description: Secret key to access the ELIXIR Permissions API required: true - type: string + schema: + type: string responses: 200: description: Successful operation - schema: - $ref: '#/definitions/Response' + content: + application/json: + schema: + $ref: '#/components/schemas/Response' 400: description: | Username not provided @@ -82,29 +84,30 @@ paths: description: User not found patch: tags: - - user + - user summary: Update user details or dataset permissions for given user description: This method overwrites old data, so it can be used to remove dataset permissions by sending an empty datasetPermissions object. PATCH uses the same payload model as POST, with the difference, that PATCH has no required fields in the request body. operationId: logoutUser - produces: - - application/json parameters: - name: elixirId in: path - description: The user's ELIXIR AAI issued identifier + description: The user's ELIXIR AAI issued identifier required: true - type: string + schema: + type: string - name: elixir-api-key in: header description: Secret key to access the ELIXIR Permissions API required: true - type: string - - in: body - name: body - description: Contains user details and dataset permissions - required: true schema: - $ref: '#/definitions/PatchRequest' + type: string + requestBody: + description: Contains user details and dataset permissions + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PatchRequest' responses: 201: description: Some datasets are missing from REMS @@ -121,23 +124,23 @@ paths: description: Username is taken delete: tags: - - user + - user summary: Delete user along with all dataset permissions description: This can only be done by the logged in user. operationId: deleteUser - produces: - - application/json parameters: - name: elixirId in: path - description: The user's ELIXIR AAI issued identifier + description: The user's ELIXIR AAI issued identifier required: true - type: string + schema: + type: string - name: elixir-api-key in: header description: Secret key to access the ELIXIR Permissions API required: true - type: string + schema: + type: string responses: 200: description: User was deleted @@ -149,118 +152,121 @@ paths: 404: description: User not found -definitions: - Response: - type: object - properties: - sourceSignature: - type: string - datasetPermissions: - type: array - items: - $ref: '#/definitions/Permissions' - Permissions: - type: object - properties: - datasetOwner: - type: string - description: Organisation that owns the dataset - urlPrefix: - type: string - description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} - datasetId: - type: string - required: True - description: Globally unique dataset identifier, e.g. URN - startDate: - type: string - description: Start date of permission - endDate: - type: string - description: End date of permission - PostDatasets: - type: object - required: true - properties: - datasetId: - type: string - required: true - description: Globally unique dataset identifier, e.g. URN - startDate: - type: string - description: Start date of permission - endDate: - type: string - description: End date of permission - PostRequest: - type: object - properties: - sourceSignature: - type: string - userDetails: - type: object - required: true - properties: - elixirId: - type: string - required: true - description: Unique ELIXIR AAI issued user identifier - eduPersonPrincipalName: - type: string - format: email - description: User identifier at home organisation in email format, e.g. user@org.fi - userEmail: - type: string - format: email - description: User email address - realName: - type: string - description: Full legal name of the user - datasetPermissions: - type: array - required: true - items: - $ref: '#/definitions/PostDatasets' - PatchDatasets: - type: object - properties: - datasetId: - type: string - description: Globally unique dataset identifier, e.g. URN - startDate: - type: string - description: Start date of permission - endDate: - type: string - description: End date of permission - PatchRequest: - type: object - properties: - sourceSignature: - type: string - userDetails: - type: object - properties: - elixirId: - type: string - description: Unique ELIXIR AAI issued user identifier - eduPersonPrincipalName: - type: string - format: email - description: User identifier at home organisation in email format, e.g. user@org.fi - userEmail: - type: string - format: email - description: User email address - realName: - type: string - description: Full legal name of the user - datasetPermissions: - type: array - items: - $ref: '#/definitions/PatchDatasets' +components: + schemas: + Response: + type: object + properties: + sourceSignature: + type: string + datasetPermissions: + type: array + items: + $ref: '#/components/schemas/Permissions' + Permissions: + type: object + required: + - datasetId + properties: + datasetOwner: + type: string + description: Organisation that owns the dataset + urlPrefix: + type: string + description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} + datasetId: + type: string + description: Globally unique dataset identifier, e.g. URN + startDate: + type: string + description: Start date of permission + endDate: + type: string + description: End date of permission + PostDatasets: + type: object + required: + - datasetId + properties: + datasetId: + type: string + description: Globally unique dataset identifier, e.g. URN + startDate: + type: string + description: Start date of permission + endDate: + type: string + description: End date of permission + PostRequest: + type: object + required: + - userDetails + - datasetPermissions + properties: + sourceSignature: + type: string + userDetails: + type: object + required: + - elixirId + properties: + elixirId: + type: string + description: Unique ELIXIR AAI issued user identifier + eduPersonPrincipalName: + type: string + format: email + description: User identifier at home organisation in email format, e.g. user@org.fi + userEmail: + type: string + format: email + description: User email address + realName: + type: string + description: Full legal name of the user + datasetPermissions: + type: array + items: + $ref: '#/components/schemas/PostDatasets' + PatchDatasets: + type: object + properties: + datasetId: + type: string + description: Globally unique dataset identifier, e.g. URN + startDate: + type: string + description: Start date of permission + endDate: + type: string + description: End date of permission + PatchRequest: + type: object + properties: + sourceSignature: + type: string + userDetails: + type: object + properties: + elixirId: + type: string + description: Unique ELIXIR AAI issued user identifier + eduPersonPrincipalName: + type: string + format: email + description: User identifier at home organisation in email format, e.g. user@org.fi + userEmail: + type: string + format: email + description: User email address + realName: + type: string + description: Full legal name of the user + datasetPermissions: + type: array + items: + $ref: '#/components/schemas/PatchDatasets' -host: virtserver.swaggerhub.com -basePath: /ELIXIR-Finland/Permissions/1.0.0 -schemes: - - https \ No newline at end of file +externalDocs: + url: https://virtserver.swaggerhub.com/ELIXIR-Finland/Permissions/1.0.0 + description: External Documentation From d62f7a0ad4db424622c95e2928629f4983ed7f0d Mon Sep 17 00:00:00 2001 From: teemukataja Date: Mon, 12 Nov 2018 12:46:52 +0000 Subject: [PATCH 11/27] add missing 401s --- elixirapi.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elixirapi.yaml b/elixirapi.yaml index ec50eb9..122b57d 100644 --- a/elixirapi.yaml +++ b/elixirapi.yaml @@ -80,6 +80,8 @@ paths: Username not provided \ Missing headers "elixir-api-key" + 401: + description: Unauthorized api key 404: description: User not found patch: @@ -118,6 +120,8 @@ paths: Could not validate request body (broken JSON) \ Missing headers "elixir-api-key" + 401: + description: Unauthorized api key 404: description: User not found 409: @@ -149,6 +153,8 @@ paths: Username not provided \ Missing headers "elixir-api-key" + 401: + description: Unauthorized api key 404: description: User not found From 5301710bc76bbdfb16a807424fb3927270693900 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 20 Nov 2018 08:50:35 +0000 Subject: [PATCH 12/27] #16 remove lingering user details on delete request --- elixir_rems_proxy/utils/db_actions.py | 13 +++++++++++++ elixir_rems_proxy/utils/process.py | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 1fe7bec..78fddaa 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -179,3 +179,16 @@ async def update_user(old_user, new_user_details, connection): except Exception as e: LOG.debug(f'An error occurred while attempting to update user -> {e}') raise web.HTTPInternalServerError(text='Database error when attempting to update user') + + +async def remove_lingering_userdetails(user, connection): + """Remove lingering userdetails from roles and application_event tables.""" + LOG.debug(f'Remove lingering userdetails for {user}') + try: + # Remove from application_event + await connection.execute(f"""DELETE FROM application_event WHERE userid='{user}'""") + # Remove from roles + await connection.execute(f"""DELETE FROM roles WHERE userid='{user}'""") + except Exception as e: + LOG.debug(f'An error occurred while attempting to delete lingering userdetails -> {e}') + raise web.HTTPInternalServerError(text='Database error when attempting to remove lingering userdetails') diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index 0c937db..63f543e 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -3,7 +3,7 @@ from aiohttp import web from ..utils.logging import LOG -from ..utils.db_actions import create_user, delete_user, update_user +from ..utils.db_actions import create_user, delete_user, update_user, remove_lingering_userdetails from ..utils.db_actions import get_dataset_permissions, create_dataset_permissions, remove_dataset_permissions @@ -97,5 +97,7 @@ async def process_delete_request(user, db_pool): if permissions: # Try to remove permissions before removing user await remove_dataset_permissions(user, connection) + # Remove userid from roles and application_event tables + await remove_lingering_userdetails(user, connection) # Finally remove the user await delete_user(user, connection) From 2f60a7379a3e15922dc67eeab324a89112e7b89f Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 20 Nov 2018 09:06:34 +0000 Subject: [PATCH 13/27] #16 restructure --- elixir_rems_proxy/utils/process.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index 63f543e..9f5bb57 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -92,12 +92,9 @@ async def process_delete_request(user, db_pool): # Take one connection from the active database connection pool async with db_pool.acquire() as connection: - # Check if user has permissions to be removed - permissions = await get_dataset_permissions(user, connection) - if permissions: - # Try to remove permissions before removing user - await remove_dataset_permissions(user, connection) - # Remove userid from roles and application_event tables - await remove_lingering_userdetails(user, connection) + # Try to remove permissions before removing user + await remove_dataset_permissions(user, connection) + # Remove userid from roles and application_event tables + await remove_lingering_userdetails(user, connection) # Finally remove the user await delete_user(user, connection) From d92050da24e612c67ba90db2a00e4952f2943c31 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 20 Nov 2018 09:50:49 +0000 Subject: [PATCH 14/27] #17 verify existance of datasets before POST/PATCH requests and change 201 to 404 --- elixir_rems_proxy/app.py | 14 ++++---------- elixir_rems_proxy/utils/db_actions.py | 12 ++++++++++++ elixir_rems_proxy/utils/process.py | 23 +++++++++++------------ 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/elixir_rems_proxy/app.py b/elixir_rems_proxy/app.py index 802890a..49f0ec9 100644 --- a/elixir_rems_proxy/app.py +++ b/elixir_rems_proxy/app.py @@ -31,13 +31,10 @@ async def user_post(request): LOG.debug('POST Request received.') db_pool = request.app['pool'] - missing_datasets = await process_post_request(request, db_pool) + processed_request = await process_post_request(request, db_pool) - if not missing_datasets: + if processed_request: return web.HTTPOk(text='Successful operation') - else: - return web.HTTPCreated(text=f'User was created, but following datasets are missing from REMS ({missing_datasets}), ' - 'check "GET /user" endpoint for added permissions') @routes.get('/user/') @@ -73,12 +70,9 @@ async def user_patch(request): if 'user' in request.match_info: user_identifier = request.match_info['user'] - missing_datasets = await process_patch_request(user_identifier, request, db_pool) - if not missing_datasets: + processed_request = await process_patch_request(user_identifier, request, db_pool) + if processed_request: return web.HTTPNoContent() - else: - return web.HTTPCreated(text=f'User was created, but following datasets are missing from REMS ({missing_datasets}), ' - 'check "GET /user" endpoint for added permissions') else: raise web.HTTPBadRequest(text='Username not provided') diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 78fddaa..90782a6 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -192,3 +192,15 @@ async def remove_lingering_userdetails(user, connection): except Exception as e: LOG.debug(f'An error occurred while attempting to delete lingering userdetails -> {e}') raise web.HTTPInternalServerError(text='Database error when attempting to remove lingering userdetails') + + +async def verify_datasets_exist(dataset_permissions, connection): + """Verify that requested datasets exist, raise exception on missing datasets.""" + LOG.debug('Verify that datasets exist') + missing_datasets = [] + for dataset in dataset_permissions: + dataset_index = await get_dataset_index(dataset['datasetId'], connection) + if not dataset_index: + missing_datasets.append(dataset['datasetId']) + if len(missing_datasets) > 0: + raise web.HTTPNotFound(text=f'Following datasets are missing from REMS: {missing_datasets}') diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index 9f5bb57..d11d19a 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -4,7 +4,7 @@ from ..utils.logging import LOG from ..utils.db_actions import create_user, delete_user, update_user, remove_lingering_userdetails -from ..utils.db_actions import get_dataset_permissions, create_dataset_permissions, remove_dataset_permissions +from ..utils.db_actions import get_dataset_permissions, create_dataset_permissions, remove_dataset_permissions, verify_datasets_exist async def create_response_body(db_response): @@ -30,19 +30,19 @@ async def process_post_request(request, db_pool): LOG.debug('Process POST request.') # Put the JSON payload body into a dictionary object request_body = await request.json() - missing_datasets = [] # used to notify user of missing datasets in REMS # Take one connection from the active database connection pool async with db_pool.acquire() as connection: + # Before doing any inserts, first check if requested datasets exist + await verify_datasets_exist(request_body['datasetPermissions'], connection) # raises http 404 if something is missing # Create new user and dataset permissions user_created = await create_user(request_body['userDetails'], connection) # Wait for user to be created, then add dataset permissions if user_created: - missing_datasets = await create_dataset_permissions(request_body['userDetails']['elixirId'], - request_body['datasetPermissions'], - connection) - LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') - return missing_datasets # If this list is empty, it means the request was fully processed + await create_dataset_permissions(request_body['userDetails']['elixirId'], + request_body['datasetPermissions'], + connection) + return True async def process_get_request(user, db_pool): @@ -66,24 +66,23 @@ async def process_patch_request(user, request, db_pool): LOG.debug('Process PATCH request.') # Put the JSON payload body into a dictionary object request_body = await request.json() - missing_datasets = [] # used to notify user of missing datasets in REMS # Take one connection from the active database connection pool async with db_pool.acquire() as connection: # Check if payload contains dataset permissions if request_body.get('datasetPermissions'): LOG.debug('PATCHing dataset permissions') + # Before doing any inserts or deletes, first check if requested datasets exist + await verify_datasets_exist(request_body['datasetPermissions'], connection) # raises http 404 if something is missing # Remove old permissions await remove_dataset_permissions(user, connection) # Add new permissions - missing_datasets = await create_dataset_permissions(user, request_body['datasetPermissions'], connection) - LOG.debug(f'List of datasets that are missing from REMS: {missing_datasets}. If empty, all were found.') + await create_dataset_permissions(user, request_body['datasetPermissions'], connection) # Check if payload contains user details if request_body.get('userDetails'): LOG.debug('PATCHing user details') await update_user(user, request_body['userDetails'], connection) - - return missing_datasets # If this list is empty, it means the request was fully processed + return True async def process_delete_request(user, db_pool): From f4c59ab4d8a39a37ce74505b930566a8b85f2062 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 20 Nov 2018 09:55:29 +0000 Subject: [PATCH 15/27] #17 add userid to roles table --- elixir_rems_proxy/utils/db_actions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 90782a6..b15d10a 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -60,6 +60,8 @@ async def create_user(user_details, connection): async with connection.transaction(): await connection.execute(f"""INSERT INTO users (userid, userattrs) VALUES ($1, $2)""", user_details['elixirId'], user_attributes) + await connection.execute(f"""INSERT INTO roles (userid, role) VALUES ($1, 'applicant')""", + user_details['elixirId']) return True # User was created except Exception as e: LOG.debug(f'An error occurred while attempting to create user -> {e}') From 96caa050a585506e751a5c48e4793cd9c815e622 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 20 Nov 2018 10:04:49 +0000 Subject: [PATCH 16/27] #18 remove eppn and use elixir id instead --- elixir_rems_proxy/schemas/patch.json | 3 --- elixir_rems_proxy/schemas/post.json | 3 --- elixir_rems_proxy/utils/db_actions.py | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/elixir_rems_proxy/schemas/patch.json b/elixir_rems_proxy/schemas/patch.json index 4bc5040..42e1036 100644 --- a/elixir_rems_proxy/schemas/patch.json +++ b/elixir_rems_proxy/schemas/patch.json @@ -15,9 +15,6 @@ ], "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" }, - "eduPersonPrincipalName": { - "type": "string" - }, "userEmail": { "type": "string" }, diff --git a/elixir_rems_proxy/schemas/post.json b/elixir_rems_proxy/schemas/post.json index de9e4c3..4460d17 100644 --- a/elixir_rems_proxy/schemas/post.json +++ b/elixir_rems_proxy/schemas/post.json @@ -22,9 +22,6 @@ ], "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" }, - "eduPersonPrincipalName": { - "type": "string" - }, "userEmail": { "type": "string" }, diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index b15d10a..9920308 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -44,7 +44,7 @@ async def make_jsonb(user_details): """Construct PostgreSQL jsonb from JSON payload.""" LOG.debug('Construct jsonb') user_attributes = { - "eppn": user_details.get("eduPersonPrincipalName", None), + "eppn": user_details.get("elixirId", None), "mail": user_details.get("userEmail", None), "commonName": user_details.get("realName", None) } @@ -139,7 +139,7 @@ async def update_user_attributes(new_user_details, old_user_details): """Construct PostgreSQL jsonb from JSON payload.""" LOG.debug('Construct jsonb') user_attributes = { - "eppn": new_user_details.get("eduPersonPrincipalName", old_user_details.get("eppn", None)), + "eppn": new_user_details.get("elixirId", old_user_details.get("eppn", None)), "mail": new_user_details.get("userEmail", old_user_details.get("mail", None)), "commonName": new_user_details.get("realName", old_user_details.get("commonName", None)) } From 257c83c4c0044dcec491f92f799f93bce5427bb7 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 20 Nov 2018 10:09:42 +0000 Subject: [PATCH 17/27] remove eppn --- elixirapi.yaml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/elixirapi.yaml b/elixirapi.yaml index 122b57d..cf1e872 100644 --- a/elixirapi.yaml +++ b/elixirapi.yaml @@ -37,8 +37,6 @@ paths: responses: 200: description: Successful operation - 201: - description: User was created, but some datasets are missing from REMS 401: description: Unauthorized api key 400: @@ -46,6 +44,8 @@ paths: Could not validate request body (broken JSON) \ Missing headers "elixir-api-key" + 404: + description: Following datasets are missing from REMS 409: description: Username is taken @@ -111,8 +111,6 @@ paths: schema: $ref: '#/components/schemas/PatchRequest' responses: - 201: - description: Some datasets are missing from REMS 204: description: Successful operation 400: @@ -123,7 +121,10 @@ paths: 401: description: Unauthorized api key 404: - description: User not found + description: | + User not found + \ + Following datasets are missing from REMS 409: description: Username is taken delete: @@ -219,10 +220,6 @@ components: elixirId: type: string description: Unique ELIXIR AAI issued user identifier - eduPersonPrincipalName: - type: string - format: email - description: User identifier at home organisation in email format, e.g. user@org.fi userEmail: type: string format: email @@ -257,10 +254,6 @@ components: elixirId: type: string description: Unique ELIXIR AAI issued user identifier - eduPersonPrincipalName: - type: string - format: email - description: User identifier at home organisation in email format, e.g. user@org.fi userEmail: type: string format: email From eccdf8ad4b432822555e07224ee5b919ab180083 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 20 Nov 2018 12:05:59 +0000 Subject: [PATCH 18/27] update docs --- docs/example.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/example.rst b/docs/example.rst index dc31b11..ca75c54 100644 --- a/docs/example.rst +++ b/docs/example.rst @@ -38,7 +38,6 @@ An example ``POST`` request and response to the ``user`` endpoint: -d '{ "userDetails": { "elixirId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org", - "eduPersonPrincipalName": "user@org.org", "userEmail": "firstname.lastname@organisation.org", "realName": "Firstname Lastname" }, @@ -131,7 +130,6 @@ CASE 3: Update user details -H 'elixir-api-key: secret' \ -d '{ "userDetails": { - "eduPersonPrincipalName": "user@org.fi", "userEmail": "user.name@organisation.fi", "realName": "User Name" } From ec226bddbe2748a2c43a317e8c4088ff5f51547d Mon Sep 17 00:00:00 2001 From: teemukataja Date: Wed, 28 Nov 2018 10:06:04 +0000 Subject: [PATCH 19/27] add affiliation key to userdetails and datasetpermissions --- elixir_rems_proxy/schemas/get.json | 4 ++-- elixir_rems_proxy/schemas/patch.json | 3 +++ elixir_rems_proxy/schemas/post.json | 3 +++ elixir_rems_proxy/utils/db_actions.py | 29 ++++++++++++++++++++++++--- elixir_rems_proxy/utils/process.py | 21 +++++++++++-------- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/elixir_rems_proxy/schemas/get.json b/elixir_rems_proxy/schemas/get.json index 058e9da..2eb6b11 100644 --- a/elixir_rems_proxy/schemas/get.json +++ b/elixir_rems_proxy/schemas/get.json @@ -15,14 +15,14 @@ "items": { "type": "object", "required": [ - "datasetOwner", + "userAffiliation", "urlPrefix", "datasetId", "startDate", "endDate" ], "properties": { - "datasetOwner": { + "userAffiliation": { "type": "string", "default": "" }, diff --git a/elixir_rems_proxy/schemas/patch.json b/elixir_rems_proxy/schemas/patch.json index 42e1036..d342f9f 100644 --- a/elixir_rems_proxy/schemas/patch.json +++ b/elixir_rems_proxy/schemas/patch.json @@ -15,6 +15,9 @@ ], "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" }, + "eduPersonPrincipalName": { + "type": "string" + }, "userEmail": { "type": "string" }, diff --git a/elixir_rems_proxy/schemas/post.json b/elixir_rems_proxy/schemas/post.json index 4460d17..7085c85 100644 --- a/elixir_rems_proxy/schemas/post.json +++ b/elixir_rems_proxy/schemas/post.json @@ -22,6 +22,9 @@ ], "pattern": "^([a-f0-9]{40}(@elixir-europe.org))$" }, + "eduPersonPrincipalName": { + "type": "string" + }, "userEmail": { "type": "string" }, diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 9920308..dd2876c 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -23,11 +23,32 @@ async def user_exists(user, connection): return None +async def get_user_affiliation(user, connection): + """Retrieve user attributes.""" + LOG.debug(f'Query database for {user}\'s user attributes') + try: + query = f"""SELECT userattrs + FROM users + WHERE userid='{user}';""" + statement = await connection.prepare(query) + db_response = await statement.fetch() + LOG.debug(f'Response: {db_response}') + try: + user_affiliation = json.loads(dict(db_response[0])['userattrs'])['affiliation'] + except KeyError as e: + user_affiliation = "" + LOG.debug(f'Parsed user affiliation: {user_affiliation}') + return user_affiliation + except Exception as e: + LOG.debug(f'An error occurred while attempting to fetch user attributes -> {e}') + return None + + async def get_dataset_permissions(user, connection): """Return dataset permissions for given user.""" LOG.debug(f'Query database for {user}\'s dataset permissions') try: - query = f"""SELECT organization, a.resid, b.start, b.endt + query = f"""SELECT a.resid, b.start, b.endt FROM resource a, entitlement b WHERE a.id=b.resid AND b.userid='{user}';""" @@ -46,7 +67,8 @@ async def make_jsonb(user_details): user_attributes = { "eppn": user_details.get("elixirId", None), "mail": user_details.get("userEmail", None), - "commonName": user_details.get("realName", None) + "commonName": user_details.get("realName", None), + "affiliation": user_details.get("eduPersonPrincipalName", None) } return json.dumps(user_attributes) @@ -141,7 +163,8 @@ async def update_user_attributes(new_user_details, old_user_details): user_attributes = { "eppn": new_user_details.get("elixirId", old_user_details.get("eppn", None)), "mail": new_user_details.get("userEmail", old_user_details.get("mail", None)), - "commonName": new_user_details.get("realName", old_user_details.get("commonName", None)) + "commonName": new_user_details.get("realName", old_user_details.get("commonName", None)), + "affiliation": new_user_details.get("eduPersonPrincipalName", old_user_details.get("affiliation", None)) } return json.dumps(user_attributes) diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index d11d19a..10cd871 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -1,25 +1,27 @@ """Process Requests.""" +import json + from aiohttp import web from ..utils.logging import LOG -from ..utils.db_actions import create_user, delete_user, update_user, remove_lingering_userdetails +from ..utils.db_actions import create_user, delete_user, update_user, remove_lingering_userdetails, get_user_affiliation from ..utils.db_actions import get_dataset_permissions, create_dataset_permissions, remove_dataset_permissions, verify_datasets_exist -async def create_response_body(db_response): +async def create_response_body(dataset_permissions, user_affiliation): """Construct a dictionary for JSON response body.""" response_body = { "sourceSignature": "", "datasetPermissions": [ { - "datasetOwner": dict(record)['organization'], + "userAffiliation": user_affiliation, "urlPrefix": "", - "datasetId": dict(record)['resid'], - "startDate": str(dict(record)['start']), - "endDate": str(dict(record)['endt']) + "datasetId": dict(ds)['resid'], + "startDate": str(dict(ds)['start']), + "endDate": str(dict(ds)['endt']) } - for record in db_response + for ds in dataset_permissions ] } return response_body @@ -51,10 +53,13 @@ async def process_get_request(user, db_pool): # Take one connection from the active database connection pool async with db_pool.acquire() as connection: + # Get user affiliation + user_affiliation = await get_user_affiliation(user, connection) + # Get permissions permissions = await get_dataset_permissions(user, connection) if permissions: # Return permitted datasets - permissions_response = await create_response_body(permissions) + permissions_response = await create_response_body(permissions, user_affiliation) return permissions_response else: # User has no dataset permissions From 14f78cbb7a399aa87142241d5de649586467508d Mon Sep 17 00:00:00 2001 From: teemukataja Date: Wed, 28 Nov 2018 10:06:42 +0000 Subject: [PATCH 20/27] change default value of logging debug env --- elixir_rems_proxy/utils/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elixir_rems_proxy/utils/logging.py b/elixir_rems_proxy/utils/logging.py index 048d4c1..44bf4c7 100644 --- a/elixir_rems_proxy/utils/logging.py +++ b/elixir_rems_proxy/utils/logging.py @@ -4,5 +4,5 @@ import logging formatting = '[%(asctime)s][%(name)s][%(process)d %(processName)s][%(levelname)-8s] (L:%(lineno)s) %(module)s | %(funcName)s: %(message)s' -logging.basicConfig(level=logging.DEBUG if os.environ.get('DEBUG', True) else logging.INFO, format=formatting) +logging.basicConfig(level=logging.DEBUG if os.environ.get('DEBUG', False) else logging.INFO, format=formatting) LOG = logging.getLogger("elixir") From fe3529094a11bafcece6c0fecd4ee169498d7c0a Mon Sep 17 00:00:00 2001 From: teemukataja Date: Wed, 28 Nov 2018 10:17:56 +0000 Subject: [PATCH 21/27] fix patch endpoint user check --- elixir_rems_proxy/utils/db_actions.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index dd2876c..5176b13 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -175,12 +175,14 @@ async def update_user(old_user, new_user_details, connection): # Place new elixir id into variable, defaults to old user if it remains unchanged new_user = new_user_details.get('elixirId', old_user) - # Check if new username is taken - userid_exists = await user_exists(new_user, connection) - if userid_exists: - LOG.debug('PATCH Conflict: Username is taken') - # Username is taken, stop PATCH process here - raise web.HTTPConflict(text='Username is taken') + # Check if user is going to change userid + if old_user is not new_user: + # Check if new username is taken + userid_exists = await user_exists(new_user, connection) + if userid_exists: + LOG.debug('PATCH Conflict: Username is taken') + # Username is taken, stop PATCH process here + raise web.HTTPConflict(text='Username is taken') try: LOG.debug('PATCHing begins..') @@ -198,8 +200,10 @@ async def update_user(old_user, new_user_details, connection): # Update users table await connection.execute(f"""UPDATE users SET userid='{new_user}', userattrs='{new_user_attributes}' WHERE userid='{old_user}';""") - # Update permissions table + # Update permissions, roles and application events await connection.execute(f"""UPDATE entitlement SET userid='{new_user}' WHERE userid='{old_user}';""") + await connection.execute(f"""UPDATE roles SET userid='{new_user}' WHERE userid='{old_user}';""") + await connection.execute(f"""UPDATE application_event SET userid='{new_user}' WHERE userid='{old_user}';""") return True except Exception as e: LOG.debug(f'An error occurred while attempting to update user -> {e}') From 6f5441442ac33c25db17446bf0fd8eb51e230e74 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Wed, 28 Nov 2018 10:21:06 +0000 Subject: [PATCH 22/27] update docs --- docs/example.rst | 5 +++-- docs/instructions.rst | 34 +++++++++++++++++----------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/example.rst b/docs/example.rst index ca75c54..ec552cf 100644 --- a/docs/example.rst +++ b/docs/example.rst @@ -38,6 +38,7 @@ An example ``POST`` request and response to the ``user`` endpoint: -d '{ "userDetails": { "elixirId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@elixir-europe.org", + "eduPersonPrincipalName": "username@organisation.org", "userEmail": "firstname.lastname@organisation.org", "realName": "Firstname Lastname" }, @@ -73,7 +74,7 @@ Example Response: "sourceSignature": "string", "datasetPermissions": [ { - "datasetOwner": "example-org", + "userAffiliation": "username@organisation.org", "urlPrefix": "", "datasetId": "urn:example-dataset-1", "startDate": "2018-01-01 12:00:00.000000+0000", @@ -130,7 +131,7 @@ CASE 3: Update user details -H 'elixir-api-key: secret' \ -d '{ "userDetails": { - "userEmail": "user.name@organisation.fi", + "userEmail": "user.name@organisation.org", "realName": "User Name" } }' diff --git a/docs/instructions.rst b/docs/instructions.rst index 0f8b2da..2c77890 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -9,23 +9,23 @@ Environment Setup The application requires some environmental arguments in order to run properly, these are illustrated in the table below. -+------------- +-------------------------------+-------------------------------------+ -| ENV | Default | Description | -+------------- +-------------------------------+-------------------------------------+ -| `DB_HOST` | `postgresql://localhost:5432` | The URL for the PostgreSQL server. | -+------------- +-------------------------------+-------------------------------------+ -| `DB_NAME` | `rems` | Name of the database. | -+------------- +-------------------------------+-------------------------------------+ -| `DB_USER` | `rems` | Database username. | -+------------- +-------------------------------+-------------------------------------+ -| `DB_PASS` | `rems` | Database password. | -+------------- +-------------------------------+-------------------------------------+ -| `APP_HOST` | `0.0.0.0` | Default Host for the Web Server. | -+------------- +-------------------------------+-------------------------------------+ -| `APP_PORT` | `8080` | Default port for the Web Server. | -+------------- +-------------------------------+-------------------------------------+ -| `DEBUG` | `True` | If set to `True`, logs all actions. | -+------------- +-------------------------------+-------------------------------------+ ++-------------+-------------------------------+-------------------------------------+ +| ENV | Default | Description | ++-------------+-------------------------------+-------------------------------------+ +| `DB_HOST` | `postgresql://localhost:5432` | The URL for the PostgreSQL server. | ++-------------+-------------------------------+-------------------------------------+ +| `DB_NAME` | `rems` | Name of the database. | ++-------------+-------------------------------+-------------------------------------+ +| `DB_USER` | `rems` | Database username. | ++-------------+-------------------------------+-------------------------------------+ +| `DB_PASS` | `rems` | Database password. | ++-------------+-------------------------------+-------------------------------------+ +| `APP_HOST` | `0.0.0.0` | Default Host for the Web Server. | ++-------------+-------------------------------+-------------------------------------+ +| `APP_PORT` | `8080` | Default port for the Web Server. | ++-------------+-------------------------------+-------------------------------------+ +| `DEBUG` | `True` | If set to `True`, logs all actions. | ++-------------+-------------------------------+-------------------------------------+ Setting the necessary environment variables can be done e.g. via the command line: From 466a861f1713174077c35672cfdbfb044eef1210 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Wed, 28 Nov 2018 12:53:23 +0000 Subject: [PATCH 23/27] changes --- elixir_rems_proxy/schemas/post.json | 5 ++++- elixirapi.yaml | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/elixir_rems_proxy/schemas/post.json b/elixir_rems_proxy/schemas/post.json index 7085c85..4d954a9 100644 --- a/elixir_rems_proxy/schemas/post.json +++ b/elixir_rems_proxy/schemas/post.json @@ -12,7 +12,10 @@ "userDetails": { "type": "object", "required": [ - "elixirId" + "elixirId", + "eduPersonPrincipalName", + "userEmail", + "realName" ], "properties": { "elixirId": { diff --git a/elixirapi.yaml b/elixirapi.yaml index cf1e872..4bcab47 100644 --- a/elixirapi.yaml +++ b/elixirapi.yaml @@ -175,9 +175,10 @@ components: required: - datasetId properties: - datasetOwner: + userAffiliation: type: string - description: Organisation that owns the dataset + format: email + description: Organisational affiliation of user that holds following dataset permission urlPrefix: type: string description: Prefix that can be used to construct URI, e.g. https://ebi.org/dataset/{datasetId} @@ -216,10 +217,17 @@ components: type: object required: - elixirId + - eduPersonPrincipalName + - userEmail + - realName properties: elixirId: type: string description: Unique ELIXIR AAI issued user identifier + eduPersonPrincipalName: + type: string + format: email + description: Organisational affiliation of user userEmail: type: string format: email @@ -254,6 +262,10 @@ components: elixirId: type: string description: Unique ELIXIR AAI issued user identifier + eduPersonPrincipalName: + type: string + format: email + description: Organisational affiliation of user userEmail: type: string format: email From 16c7f8bea16cf97835859ba564ec5e7bc9f03aa5 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Mon, 3 Dec 2018 07:44:39 +0000 Subject: [PATCH 24/27] flake8 --- elixir_rems_proxy/utils/process.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/elixir_rems_proxy/utils/process.py b/elixir_rems_proxy/utils/process.py index 10cd871..0f5d32a 100644 --- a/elixir_rems_proxy/utils/process.py +++ b/elixir_rems_proxy/utils/process.py @@ -1,7 +1,5 @@ """Process Requests.""" -import json - from aiohttp import web from ..utils.logging import LOG From f8192bb30dbb427da70f8e86cba85611db905a28 Mon Sep 17 00:00:00 2001 From: teemukataja Date: Mon, 3 Dec 2018 08:16:37 +0000 Subject: [PATCH 25/27] forgot to add file --- elixir_rems_proxy/utils/db_actions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elixir_rems_proxy/utils/db_actions.py b/elixir_rems_proxy/utils/db_actions.py index 5176b13..3b9772f 100644 --- a/elixir_rems_proxy/utils/db_actions.py +++ b/elixir_rems_proxy/utils/db_actions.py @@ -36,6 +36,7 @@ async def get_user_affiliation(user, connection): try: user_affiliation = json.loads(dict(db_response[0])['userattrs'])['affiliation'] except KeyError as e: + LOG.debug(f'user_affiliation key was not found in userattrs object :: {e}') user_affiliation = "" LOG.debug(f'Parsed user affiliation: {user_affiliation}') return user_affiliation From 02858bc54d2e776ca9ee1b0f8feb0e75f6742c7b Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 11 Dec 2018 08:05:23 +0000 Subject: [PATCH 26/27] update docs --- docs/instructions.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index 2c77890..3ebfbeb 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -9,23 +9,23 @@ Environment Setup The application requires some environmental arguments in order to run properly, these are illustrated in the table below. -+-------------+-------------------------------+-------------------------------------+ -| ENV | Default | Description | -+-------------+-------------------------------+-------------------------------------+ -| `DB_HOST` | `postgresql://localhost:5432` | The URL for the PostgreSQL server. | -+-------------+-------------------------------+-------------------------------------+ -| `DB_NAME` | `rems` | Name of the database. | -+-------------+-------------------------------+-------------------------------------+ -| `DB_USER` | `rems` | Database username. | -+-------------+-------------------------------+-------------------------------------+ -| `DB_PASS` | `rems` | Database password. | -+-------------+-------------------------------+-------------------------------------+ -| `APP_HOST` | `0.0.0.0` | Default Host for the Web Server. | -+-------------+-------------------------------+-------------------------------------+ -| `APP_PORT` | `8080` | Default port for the Web Server. | -+-------------+-------------------------------+-------------------------------------+ -| `DEBUG` | `True` | If set to `True`, logs all actions. | -+-------------+-------------------------------+-------------------------------------+ ++-------------+-------------------------------+-----------------------------------------------+ +| ENV | Default | Description | ++-------------+-------------------------------+-----------------------------------------------+ +| `DB_HOST` | `postgresql://localhost:5432` | The URL for the PostgreSQL server. | ++-------------+-------------------------------+-----------------------------------------------+ +| `DB_NAME` | `rems` | Name of the database. | ++-------------+-------------------------------+-----------------------------------------------+ +| `DB_USER` | `rems` | Database username. | ++-------------+-------------------------------+-----------------------------------------------+ +| `DB_PASS` | `rems` | Database password. | ++-------------+-------------------------------+-----------------------------------------------+ +| `APP_HOST` | `0.0.0.0` | Default Host for the Web Server. | ++-------------+-------------------------------+-----------------------------------------------+ +| `APP_PORT` | `8080` | Default port for the Web Server. | ++-------------+-------------------------------+-----------------------------------------------+ +| `DEBUG` | `False` | If set to any string value, logs all actions. | ++-------------+-------------------------------+-----------------------------------------------+ Setting the necessary environment variables can be done e.g. via the command line: From 3ba2584801038559496973adf2548e6bc3fcbebd Mon Sep 17 00:00:00 2001 From: teemukataja Date: Tue, 11 Dec 2018 08:15:17 +0000 Subject: [PATCH 27/27] update docs, remove old envvar --- docs/instructions.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index 3ebfbeb..bb7b45e 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -37,7 +37,6 @@ Setting the necessary environment variables can be done e.g. via the command li $ export DB_PASS=rems $ export HOST=0.0.0.0 $ export PORT=8080 - $ export PUBLIC_KEY=secret_string $ export DEBUG=True .. _app-setup: