From 45c1b33e0cf780bab8d2e76a8f472b55b9cc9248 Mon Sep 17 00:00:00 2001 From: dbalintx <132444646+dbalintx@users.noreply.github.com> Date: Wed, 17 May 2023 14:20:53 +0200 Subject: [PATCH] Worksheets modularization (#449) Modularization of Worksheets - Moved Worksheet related code to its own new module - Merged AthenaQueryResult object into Worksheets - Worksheet related permissions moved to new module - Removed worksheet sharing related (unused) code from the entire repo By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --- .../api/Objects/AthenaQueryResult/__init__.py | 7 - .../api/Objects/AthenaQueryResult/helpers.py | 83 ------- .../api/Objects/AthenaQueryResult/schema.py | 45 ---- .../api/Objects/AthenaQueryResult/wrapper.py | 79 ------- backend/dataall/api/Objects/Feed/registry.py | 1 - .../dataall/api/Objects/Worksheet/__init__.py | 9 - .../api/Objects/Worksheet/mutations.py | 59 ----- .../api/Objects/Worksheet/resolvers.py | 169 --------------- backend/dataall/api/Objects/__init__.py | 2 - backend/dataall/api/constants.py | 8 - .../group/services/group_resource_manager.py | 6 +- backend/dataall/db/api/__init__.py | 1 - backend/dataall/db/api/environment.py | 8 +- backend/dataall/db/api/worksheet.py | 204 ------------------ backend/dataall/db/models/Enums.py | 8 - backend/dataall/db/models/__init__.py | 1 - backend/dataall/db/permissions.py | 24 --- .../modules/datasets/db/dataset_repository.py | 4 +- .../dataall/modules/worksheets/__init__.py | 28 +++ .../modules/worksheets/api/__init__.py | 9 + .../worksheets/api}/input_types.py | 11 +- .../modules/worksheets/api/mutations.py | 29 +++ .../worksheets/api}/queries.py | 4 +- .../modules/worksheets/api/resolvers.py | 90 ++++++++ .../worksheets/api/types.py} | 79 ++++--- .../modules/worksheets/aws/__init__.py | 1 + .../modules/worksheets/aws/athena_client.py | 47 ++++ .../dataall/modules/worksheets/db/__init__.py | 1 + .../worksheets/db/models.py} | 18 +- .../modules/worksheets/db/repositories.py | 56 +++++ .../modules/worksheets/services/__init__.py | 7 + .../services/worksheet_permissions.py | 40 ++++ .../worksheets/services/worksheet_services.py | 120 +++++++++++ ...fc49baecea4_add_enviromental_parameters.py | 17 ++ config.json | 3 + frontend/src/api/Worksheet/index.js | 8 - .../src/api/Worksheet/listWorksheetShares.js | 28 --- .../src/api/Worksheet/removeWorksheetShare.js | 14 -- frontend/src/api/Worksheet/shareWorksheet.js | 20 -- .../src/api/Worksheet/updateWorksheetShare.js | 24 --- tests/api/test_feed.py | 107 --------- .../modules/feed/__init__.py | 0 tests/modules/feed/testhelper.py | 98 +++++++++ tests/modules/worksheets/__init__.py | 0 .../worksheets}/test_worksheet.py | 135 +----------- .../modules/worksheets/test_worksheet_feed.py | 36 ++++ 46 files changed, 644 insertions(+), 1104 deletions(-) delete mode 100644 backend/dataall/api/Objects/AthenaQueryResult/__init__.py delete mode 100644 backend/dataall/api/Objects/AthenaQueryResult/helpers.py delete mode 100644 backend/dataall/api/Objects/AthenaQueryResult/schema.py delete mode 100644 backend/dataall/api/Objects/AthenaQueryResult/wrapper.py delete mode 100644 backend/dataall/api/Objects/Worksheet/__init__.py delete mode 100644 backend/dataall/api/Objects/Worksheet/mutations.py delete mode 100644 backend/dataall/api/Objects/Worksheet/resolvers.py delete mode 100644 backend/dataall/db/api/worksheet.py create mode 100644 backend/dataall/modules/worksheets/__init__.py create mode 100644 backend/dataall/modules/worksheets/api/__init__.py rename backend/dataall/{api/Objects/Worksheet => modules/worksheets/api}/input_types.py (86%) create mode 100644 backend/dataall/modules/worksheets/api/mutations.py rename backend/dataall/{api/Objects/Worksheet => modules/worksheets/api}/queries.py (90%) create mode 100644 backend/dataall/modules/worksheets/api/resolvers.py rename backend/dataall/{api/Objects/Worksheet/schema.py => modules/worksheets/api/types.py} (67%) create mode 100644 backend/dataall/modules/worksheets/aws/__init__.py create mode 100644 backend/dataall/modules/worksheets/aws/athena_client.py create mode 100644 backend/dataall/modules/worksheets/db/__init__.py rename backend/dataall/{db/models/Worksheet.py => modules/worksheets/db/models.py} (68%) create mode 100644 backend/dataall/modules/worksheets/db/repositories.py create mode 100644 backend/dataall/modules/worksheets/services/__init__.py create mode 100644 backend/dataall/modules/worksheets/services/worksheet_permissions.py create mode 100644 backend/dataall/modules/worksheets/services/worksheet_services.py delete mode 100644 frontend/src/api/Worksheet/listWorksheetShares.js delete mode 100644 frontend/src/api/Worksheet/removeWorksheetShare.js delete mode 100644 frontend/src/api/Worksheet/shareWorksheet.js delete mode 100644 frontend/src/api/Worksheet/updateWorksheetShare.js delete mode 100644 tests/api/test_feed.py rename frontend/src/api/Worksheet/deleteWorksheetShare.js => tests/modules/feed/__init__.py (100%) create mode 100644 tests/modules/feed/testhelper.py create mode 100644 tests/modules/worksheets/__init__.py rename tests/{api => modules/worksheets}/test_worksheet.py (50%) create mode 100644 tests/modules/worksheets/test_worksheet_feed.py diff --git a/backend/dataall/api/Objects/AthenaQueryResult/__init__.py b/backend/dataall/api/Objects/AthenaQueryResult/__init__.py deleted file mode 100644 index d130be5ef..000000000 --- a/backend/dataall/api/Objects/AthenaQueryResult/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from . import schema, helpers -from .wrapper import ( - AthenaQueryResult, - AthenaQueryResultStatus, -) - -__all__ = ['schema', 'helpers', 'AthenaQueryResult', 'AthenaQueryResultStatus'] diff --git a/backend/dataall/api/Objects/AthenaQueryResult/helpers.py b/backend/dataall/api/Objects/AthenaQueryResult/helpers.py deleted file mode 100644 index e90f05e71..000000000 --- a/backend/dataall/api/Objects/AthenaQueryResult/helpers.py +++ /dev/null @@ -1,83 +0,0 @@ -import nanoid -from pyathena import connect - -from ....db import models -from ....aws.handlers.sts import SessionHelper - - -def random_key(): - return nanoid.generate() - - -def run_query(environment: models.Environment, sql=None): - - boto3_session = SessionHelper.remote_session(accountid=environment.AwsAccountId) - creds = boto3_session.get_credentials() - connection = connect( - aws_access_key_id=creds.access_key, - aws_secret_access_key=creds.secret_key, - aws_session_token=creds.token, - work_group='primary', - s3_staging_dir=f's3://{environment.EnvironmentDefaultBucketName}/preview/', - region_name=environment.region, - ) - cursor = connection.cursor() - cursor.execute(sql) - columns = [] - for f in cursor.description: - columns.append({'columnName': f[0], 'typeName': 'String'}) - - rows = [] - for row in cursor: - record = {'cells': []} - for col_position, column in enumerate(columns): - cell = {} - cell['columnName'] = column['columnName'] - cell['typeName'] = column['typeName'] - cell['value'] = str(row[col_position]) - record['cells'].append(cell) - rows.append(record) - return { - 'error': None, - 'AthenaQueryId': cursor.query_id, - 'ElapsedTime': cursor.total_execution_time_in_millis, - 'rows': rows, - 'columns': columns, - } - - -def run_query_with_role(environment: models.Environment, environment_group: models.EnvironmentGroup, sql=None): - base_session = SessionHelper.remote_session(accountid=environment.AwsAccountId) - boto3_session = SessionHelper.get_session(base_session=base_session, role_arn=environment_group.environmentIAMRoleArn) - creds = boto3_session.get_credentials() - connection = connect( - aws_access_key_id=creds.access_key, - aws_secret_access_key=creds.secret_key, - aws_session_token=creds.token, - work_group=environment_group.environmentAthenaWorkGroup, - s3_staging_dir=f's3://{environment.EnvironmentDefaultBucketName}/athenaqueries/{environment_group.environmentAthenaWorkGroup}/', - region_name=environment.region, - ) - cursor = connection.cursor() - cursor.execute(sql) - columns = [] - for f in cursor.description: - columns.append({'columnName': f[0], 'typeName': 'String'}) - - rows = [] - for row in cursor: - record = {'cells': []} - for col_position, column in enumerate(columns): - cell = {} - cell['columnName'] = column['columnName'] - cell['typeName'] = column['typeName'] - cell['value'] = str(row[col_position]) - record['cells'].append(cell) - rows.append(record) - return { - 'error': None, - 'AthenaQueryId': cursor.query_id, - 'ElapsedTime': cursor.total_execution_time_in_millis, - 'rows': rows, - 'columns': columns, - } diff --git a/backend/dataall/api/Objects/AthenaQueryResult/schema.py b/backend/dataall/api/Objects/AthenaQueryResult/schema.py deleted file mode 100644 index 34ff2fb3e..000000000 --- a/backend/dataall/api/Objects/AthenaQueryResult/schema.py +++ /dev/null @@ -1,45 +0,0 @@ -from ... import gql - -AthenaResultColumnDescriptor = gql.ObjectType( - name='AthenaResultColumnDescriptor', - fields=[ - gql.Field(name='columnName', type=gql.NonNullableType(gql.String)), - gql.Field(name='typeName', type=gql.NonNullableType(gql.String)), - ], -) - - -AthenaResultRecordCell = gql.ObjectType( - name='AthenaResultRecordCell', - fields=[ - gql.Field(name='value', type=gql.String), - gql.Field(name='typeName', type=gql.NonNullableType(gql.String)), - gql.Field(name='columnName', type=gql.NonNullableType(gql.String)), - ], -) - -AthenaResultRecord = gql.ObjectType( - name='AthenaResultRecord', - fields=[ - gql.Field(name='cells', type=gql.ArrayType(gql.Ref('AthenaResultRecordCell'))) - ], -) - - -AthenaQueryResult = gql.ObjectType( - name='AthenaQueryResult', - fields=[ - gql.Field(name='Error', type=gql.String), - gql.Field(name='OutputLocation', type=gql.String), - gql.Field(name='AthenaQueryId', type=gql.String), - gql.Field(name='AwsAccountId', type=gql.String), - gql.Field(name='region', type=gql.String), - gql.Field(name='ElapsedTimeInMs', type=gql.Integer), - gql.Field(name='DataScannedInBytes', type=gql.Integer), - gql.Field(name='Status', type=gql.String), - gql.Field( - name='columns', type=gql.ArrayType(gql.Ref('AthenaResultColumnDescriptor')) - ), - gql.Field(name='rows', type=gql.ArrayType(gql.Ref('AthenaResultRecord'))), - ], -) diff --git a/backend/dataall/api/Objects/AthenaQueryResult/wrapper.py b/backend/dataall/api/Objects/AthenaQueryResult/wrapper.py deleted file mode 100644 index 394ec44ae..000000000 --- a/backend/dataall/api/Objects/AthenaQueryResult/wrapper.py +++ /dev/null @@ -1,79 +0,0 @@ -from enum import Enum -from typing import List - - -class AthenaQueryResultStatus(Enum): - CANCELLED = 'CANCELLED' - FAILED = 'FAILED' - QUEUED = 'QUEUED' - RUNNING = 'RUNNING' - SUCCEEDED = 'SUCCEEDED' - - -class AthenaQueryResult: - props = [ - 'Status', - 'Error', - 'AthenaQueryId', - 'ElapsedTimeInMs', - 'DataScannedInBytes', - 'OutputLocation', - 'rows', - 'columns', - ] - - def __init__( - self, - Error: str = None, - Status: str = None, - AthenaQueryId: str = None, - ElapsedTimeInMs: int = None, - DataScannedInBytes: int = None, - OutputLocation: str = None, - rows: List = None, - columns: List = None, - **kwargs - ): - self._error = Error - self._status = Status - self._query_id = AthenaQueryId - self._elapsed_time = ElapsedTimeInMs - self._data_scanned = DataScannedInBytes - self._loc = OutputLocation - self._rows = rows - self._columns = columns - - def to_dict(self): - return {k: getattr(self, k) for k in AthenaQueryResult.props} - - @property - def Status(self) -> AthenaQueryResultStatus: - return self._status - - @property - def Error(self) -> str: - return self._error - - @property - def AthenaQueryId(self): - return self._query_id - - @property - def ElapsedTimeInMs(self): - return self._elapsed_time - - @property - def DataScannedInBytes(self): - return self._data_scanned - - @property - def OutputLocation(self): - return self._loc - - @property - def rows(self): - return self._rows - - @property - def columns(self): - return self._columns diff --git a/backend/dataall/api/Objects/Feed/registry.py b/backend/dataall/api/Objects/Feed/registry.py index 4fedd252a..ad5ce108a 100644 --- a/backend/dataall/api/Objects/Feed/registry.py +++ b/backend/dataall/api/Objects/Feed/registry.py @@ -36,6 +36,5 @@ def types(cls): return [gql.Ref(target_type) for target_type in cls._DEFINITIONS.keys()] -FeedRegistry.register(FeedDefinition("Worksheet", models.Worksheet)) FeedRegistry.register(FeedDefinition("DataPipeline", models.DataPipeline)) FeedRegistry.register(FeedDefinition("Dashboard", models.Dashboard)) diff --git a/backend/dataall/api/Objects/Worksheet/__init__.py b/backend/dataall/api/Objects/Worksheet/__init__.py deleted file mode 100644 index dfa46b264..000000000 --- a/backend/dataall/api/Objects/Worksheet/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from . import ( - input_types, - mutations, - queries, - resolvers, - schema, -) - -__all__ = ['resolvers', 'schema', 'input_types', 'queries', 'mutations'] diff --git a/backend/dataall/api/Objects/Worksheet/mutations.py b/backend/dataall/api/Objects/Worksheet/mutations.py deleted file mode 100644 index 62f6d2010..000000000 --- a/backend/dataall/api/Objects/Worksheet/mutations.py +++ /dev/null @@ -1,59 +0,0 @@ -from ... import gql -from .resolvers import * - - -createWorksheet = gql.MutationField( - name='createWorksheet', - args=[gql.Argument(name='input', type=gql.Ref('NewWorksheetInput'))], - type=gql.Ref('Worksheet'), - resolver=create_worksheet, -) - -updateWorksheet = gql.MutationField( - name='updateWorksheet', - resolver=update_worksheet, - args=[ - gql.Argument(name='worksheetUri', type=gql.NonNullableType(gql.String)), - gql.Argument(name='input', type=gql.Ref('UpdateWorksheetInput')), - ], - type=gql.Ref('Worksheet'), -) - -shareWorksheet = gql.MutationField( - name='shareWorksheet', - resolver=share_worksheet, - args=[ - gql.Argument(name='worksheetUri', type=gql.NonNullableType(gql.String)), - gql.Argument(name='input', type=gql.Ref('WorksheetShareInput')), - ], - type=gql.Ref('WorksheetShare'), -) - -updateShareWorksheet = gql.MutationField( - name='updateShareWorksheet', - resolver=update_worksheet_share, - args=[ - gql.Argument(name='worksheetShareUri', type=gql.NonNullableType(gql.String)), - gql.Argument(name='canEdit', type=gql.NonNullableType(gql.Boolean)), - ], - type=gql.Ref('WorksheetShare'), -) - -deleteShareWorksheet = gql.MutationField( - name='deleteShareWorksheet', - resolver=remove_worksheet_share, - args=[ - gql.Argument(name='worksheetShareUri', type=gql.NonNullableType(gql.String)), - ], - type=gql.Boolean, -) - - -deleteWorksheet = gql.MutationField( - name='deleteWorksheet', - resolver=delete_worksheet, - args=[ - gql.Argument(name='worksheetUri', type=gql.NonNullableType(gql.String)), - ], - type=gql.Boolean, -) diff --git a/backend/dataall/api/Objects/Worksheet/resolvers.py b/backend/dataall/api/Objects/Worksheet/resolvers.py deleted file mode 100644 index d84adda1a..000000000 --- a/backend/dataall/api/Objects/Worksheet/resolvers.py +++ /dev/null @@ -1,169 +0,0 @@ -from sqlalchemy import and_ - -from .... import db -from ..AthenaQueryResult import helpers as athena_helpers -from ....api.constants import WorksheetRole -from ....api.context import Context -from ....db import paginate, exceptions, permissions, models -from ....db.api import ResourcePolicy - - -def create_worksheet(context: Context, source, input: dict = None): - with context.engine.scoped_session() as session: - return db.api.Worksheet.create_worksheet( - session=session, - username=context.username, - groups=context.groups, - uri=None, - data=input, - check_perm=True, - ) - - -def update_worksheet( - context: Context, source, worksheetUri: str = None, input: dict = None -): - with context.engine.scoped_session() as session: - return db.api.Worksheet.update_worksheet( - session=session, - username=context.username, - groups=context.groups, - uri=worksheetUri, - data=input, - check_perm=True, - ) - - -def get_worksheet(context: Context, source, worksheetUri: str = None): - with context.engine.scoped_session() as session: - return db.api.Worksheet.get_worksheet( - session=session, - username=context.username, - groups=context.groups, - uri=worksheetUri, - data=None, - check_perm=True, - ) - - -def resolve_user_role(context: Context, source: models.Worksheet): - if context.username and source.owner == context.username: - return WorksheetRole.Creator.value - elif context.groups and source.SamlAdminGroupName in context.groups: - return WorksheetRole.Admin.value - return WorksheetRole.NoPermission.value - - -def list_worksheets(context, source, filter: dict = None): - if not filter: - filter = {} - with context.engine.scoped_session() as session: - return db.api.Worksheet.paginated_user_worksheets( - session=session, - username=context.username, - groups=context.groups, - uri=None, - data=filter, - check_perm=True, - ) - - -def share_worksheet( - context: Context, source, worksheetUri: str = None, input: dict = None -): - with context.engine.scoped_session() as session: - return db.api.Worksheet.share_worksheet( - session=session, - username=context.username, - groups=context.groups, - uri=worksheetUri, - data=input, - check_perm=True, - ) - - -def update_worksheet_share( - context, source, worksheetShareUri: str = None, canEdit: bool = None -): - with context.engine.scoped_session() as session: - share: models.WorksheetShare = session.query(models.WorksheetShare).get( - worksheetShareUri - ) - if not share: - raise exceptions.ObjectNotFound('WorksheetShare', worksheetShareUri) - - return db.api.Worksheet.update_share_worksheet( - session=session, - username=context.username, - groups=context.groups, - uri=share.worksheetUri, - data={'canEdit': canEdit, 'share': share}, - check_perm=True, - ) - - return share - - -def remove_worksheet_share(context, source, worksheetShareUri): - with context.engine.scoped_session() as session: - share: models.WorksheetShare = session.query(models.WorksheetShare).get( - worksheetShareUri - ) - if not share: - raise exceptions.ObjectNotFound('WorksheetShare', worksheetShareUri) - - return db.api.Worksheet.delete_share_worksheet( - session=session, - username=context.username, - groups=context.groups, - uri=share.worksheetUri, - data={'share': share}, - check_perm=True, - ) - - -def resolve_shares(context: Context, source: models.Worksheet, filter: dict = None): - if not filter: - filter = {} - with context.engine.scoped_session() as session: - q = session.query(models.WorksheetShare).filter( - models.WorksheetShare.worksheetUri == source.worksheetUri - ) - return paginate( - q, page_size=filter.get('pageSize', 15), page=filter.get('page', 1) - ).to_dict() - - -def run_sql_query( - context: Context, source, environmentUri: str = None, worksheetUri: str = None, sqlQuery: str = None -): - with context.engine.scoped_session() as session: - ResourcePolicy.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=environmentUri, - permission_name=permissions.RUN_ATHENA_QUERY, - ) - environment = db.api.Environment.get_environment_by_uri(session, environmentUri) - worksheet = db.api.Worksheet.get_worksheet_by_uri(session, worksheetUri) - - env_group = db.api.Environment.get_environment_group( - session, worksheet.SamlAdminGroupName, environment.environmentUri - ) - - return athena_helpers.run_query_with_role( - environment=environment, environment_group=env_group, sql=sqlQuery - ) - - -def delete_worksheet(context, source, worksheetUri: str = None): - with context.engine.scoped_session() as session: - return db.api.Worksheet.delete_worksheet( - session=session, - username=context.username, - groups=context.groups, - uri=worksheetUri, - data=None, - check_perm=True, - ) diff --git a/backend/dataall/api/Objects/__init__.py b/backend/dataall/api/Objects/__init__.py index 94a2ed2ba..9ee40a5a1 100644 --- a/backend/dataall/api/Objects/__init__.py +++ b/backend/dataall/api/Objects/__init__.py @@ -27,8 +27,6 @@ SagemakerStudio, RedshiftCluster, Glossary, - AthenaQueryResult, - Worksheet, Feed, Notification, Vpc, diff --git a/backend/dataall/api/constants.py b/backend/dataall/api/constants.py index d3c2a74e8..db07990a0 100644 --- a/backend/dataall/api/constants.py +++ b/backend/dataall/api/constants.py @@ -214,12 +214,4 @@ class Topic(GraphQLEnumMapper): Misc = 'Misc' -class WorksheetRole(GraphQLEnumMapper): - Creator = '950' - Admin = '900' - SharedWithWritePermission = '500' - SharedWithReadPermission = '400' - NoPermission = '000' - - GLUEBUSINESSPROPERTIES = ['EXAMPLE_GLUE_PROPERTY_TO_BE_ADDED_ON_ES'] diff --git a/backend/dataall/core/group/services/group_resource_manager.py b/backend/dataall/core/group/services/group_resource_manager.py index 01d858495..a5aeaba4a 100644 --- a/backend/dataall/core/group/services/group_resource_manager.py +++ b/backend/dataall/core/group/services/group_resource_manager.py @@ -3,7 +3,7 @@ class GroupResource(ABC): - def count_resources(self, session, environment_uri, group_uri) -> int: + def count_resources(self, session, environment, group_uri) -> int: raise NotImplementedError() @@ -18,8 +18,8 @@ def register(resource: GroupResource): GroupResourceManager._resources.append(resource) @staticmethod - def count_group_resources(session, environment_uri, group_uri) -> int: + def count_group_resources(session, environment, group_uri) -> int: counter = 0 for resource in GroupResourceManager._resources: - counter += resource.count_resources(session, environment_uri, group_uri) + counter += resource.count_resources(session, environment, group_uri) return counter diff --git a/backend/dataall/db/api/__init__.py b/backend/dataall/db/api/__init__.py index ed19787aa..b32221fca 100644 --- a/backend/dataall/db/api/__init__.py +++ b/backend/dataall/db/api/__init__.py @@ -17,4 +17,3 @@ from .sgm_studio_notebook import SgmStudioNotebook from .dashboard import Dashboard from .pipeline import Pipeline -from .worksheet import Worksheet diff --git a/backend/dataall/db/api/environment.py b/backend/dataall/db/api/environment.py index 734e3dcbb..890e26b05 100644 --- a/backend/dataall/db/api/environment.py +++ b/backend/dataall/db/api/environment.py @@ -305,7 +305,6 @@ def validate_permissions(session, uri, g_permissions, group): if permissions.CREATE_NETWORK in g_permissions: g_permissions.append(permissions.LIST_ENVIRONMENT_NETWORKS) - g_permissions.append(permissions.RUN_ATHENA_QUERY) g_permissions.append(permissions.GET_ENVIRONMENT) g_permissions.append(permissions.LIST_ENVIRONMENT_GROUPS) g_permissions.append(permissions.LIST_ENVIRONMENT_GROUP_PERMISSIONS) @@ -370,11 +369,6 @@ def remove_group(session, username, groups, uri, data=None, check_perm=None): models.Dashboard, models.Dashboard.environmentUri == models.Environment.environmentUri, ) - .outerjoin( - models.WorksheetQueryResult, - models.WorksheetQueryResult.AwsAccountId - == models.Environment.AwsAccountId, - ) .filter( and_( models.Environment.environmentUri == environment.environmentUri, @@ -391,7 +385,7 @@ def remove_group(session, username, groups, uri, data=None, check_perm=None): group_env_objects_count += GroupResourceManager.count_group_resources( session=session, - environment_uri=environment.environmentUri, + environment=environment, group_uri=group ) diff --git a/backend/dataall/db/api/worksheet.py b/backend/dataall/db/api/worksheet.py deleted file mode 100644 index 7f975b536..000000000 --- a/backend/dataall/db/api/worksheet.py +++ /dev/null @@ -1,204 +0,0 @@ -import logging - -from sqlalchemy import and_, or_ -from sqlalchemy.orm import Query - -from .. import exceptions, permissions, paginate -from .. import models -from . import has_tenant_perm, ResourcePolicy, has_resource_perm - -logger = logging.getLogger(__name__) - - -class Worksheet: - @staticmethod - def get_worksheet_by_uri(session, uri: str) -> models.Worksheet: - if not uri: - raise exceptions.RequiredParameter(param_name='worksheetUri') - worksheet = Worksheet.find_worksheet_by_uri(session, uri) - if not worksheet: - raise exceptions.ObjectNotFound('Worksheet', uri) - return worksheet - - @staticmethod - def find_worksheet_by_uri(session, uri) -> models.Worksheet: - return session.query(models.Worksheet).get(uri) - - @staticmethod - @has_tenant_perm(permissions.MANAGE_WORKSHEETS) - def create_worksheet( - session, username, groups, uri, data=None, check_perm=None - ) -> models.Worksheet: - if not data: - raise exceptions.RequiredParameter(data) - if not data.get('SamlAdminGroupName'): - raise exceptions.RequiredParameter('groupUri') - if not data.get('label'): - raise exceptions.RequiredParameter('label') - - worksheet = models.Worksheet( - owner=username, - label=data.get('label'), - description=data.get('description', 'No description provided'), - tags=data.get('tags'), - chartConfig={'dimensions': [], 'measures': [], 'chartType': 'bar'}, - SamlAdminGroupName=data['SamlAdminGroupName'], - ) - session.add(worksheet) - session.commit() - - activity = models.Activity( - action='WORKSHEET:CREATE', - label='WORKSHEET:CREATE', - owner=username, - summary=f'{username} created worksheet {worksheet.name} ', - targetUri=worksheet.worksheetUri, - targetType='worksheet', - ) - session.add(activity) - - ResourcePolicy.attach_resource_policy( - session=session, - group=data['SamlAdminGroupName'], - permissions=permissions.WORKSHEET_ALL, - resource_uri=worksheet.worksheetUri, - resource_type=models.Worksheet.__name__, - ) - return worksheet - - @staticmethod - @has_resource_perm(permissions.UPDATE_WORKSHEET) - def update_worksheet(session, username, groups, uri, data=None, check_perm=None): - worksheet = Worksheet.get_worksheet_by_uri(session, uri) - for field in data.keys(): - setattr(worksheet, field, data.get(field)) - session.commit() - - activity = models.Activity( - action='WORKSHEET:UPDATE', - label='WORKSHEET:UPDATE', - owner=username, - summary=f'{username} updated worksheet {worksheet.name} ', - targetUri=worksheet.worksheetUri, - targetType='worksheet', - ) - session.add(activity) - return worksheet - - @staticmethod - @has_resource_perm(permissions.GET_WORKSHEET) - def get_worksheet(session, username, groups, uri, data=None, check_perm=None): - worksheet = Worksheet.get_worksheet_by_uri(session, uri) - return worksheet - - @staticmethod - def query_user_worksheets(session, username, groups, filter) -> Query: - query = session.query(models.Worksheet).filter( - or_( - models.Worksheet.owner == username, - models.Worksheet.SamlAdminGroupName.in_(groups), - ) - ) - if filter and filter.get('term'): - query = query.filter( - or_( - models.Worksheet.label.ilike('%' + filter.get('term') + '%'), - models.Worksheet.description.ilike('%' + filter.get('term') + '%'), - models.Worksheet.tags.contains(f"{{{filter.get('term')}}}"), - ) - ) - return query - - @staticmethod - def paginated_user_worksheets( - session, username, groups, uri, data=None, check_perm=None - ) -> dict: - return paginate( - query=Worksheet.query_user_worksheets(session, username, groups, data), - page=data.get('page', 1), - page_size=data.get('pageSize', 10), - ).to_dict() - - @staticmethod - @has_resource_perm(permissions.SHARE_WORKSHEET) - def share_worksheet( - session, username, groups, uri, data=None, check_perm=None - ) -> models.WorksheetShare: - share = ( - session.query(models.WorksheetShare) - .filter( - and_( - models.WorksheetShare.worksheetUri == uri, - models.WorksheetShare.principalId == data.get('principalId'), - models.WorksheetShare.principalType == data.get('principalType'), - ) - ) - .first() - ) - - if not share: - share = models.WorksheetShare( - worksheetUri=uri, - principalType=data['principalType'], - principalId=data['principalId'], - canEdit=data.get('canEdit', True), - owner=username, - ) - session.add(share) - - ResourcePolicy.attach_resource_policy( - session=session, - group=data['principalId'], - permissions=permissions.WORKSHEET_SHARED, - resource_uri=uri, - resource_type=models.Worksheet.__name__, - ) - return share - - @staticmethod - @has_resource_perm(permissions.SHARE_WORKSHEET) - def update_share_worksheet( - session, username, groups, uri, data=None, check_perm=None - ) -> models.WorksheetShare: - share: models.WorksheetShare = data['share'] - share.canEdit = data['canEdit'] - worksheet = Worksheet.get_worksheet_by_uri(session, uri) - ResourcePolicy.attach_resource_policy( - session=session, - group=share.principalId, - permissions=permissions.WORKSHEET_SHARED, - resource_uri=uri, - resource_type=models.Worksheet.__name__, - ) - return share - - @staticmethod - @has_resource_perm(permissions.SHARE_WORKSHEET) - def delete_share_worksheet( - session, username, groups, uri, data=None, check_perm=None - ) -> bool: - share: models.WorksheetShare = data['share'] - ResourcePolicy.delete_resource_policy( - session=session, - group=share.principalId, - resource_uri=uri, - resource_type=models.Worksheet.__name__, - ) - session.delete(share) - session.commit() - return True - - @staticmethod - @has_resource_perm(permissions.DELETE_WORKSHEET) - def delete_worksheet( - session, username, groups, uri, data=None, check_perm=None - ) -> bool: - worksheet = Worksheet.get_worksheet_by_uri(session, uri) - session.delete(worksheet) - ResourcePolicy.delete_resource_policy( - session=session, - group=worksheet.SamlAdminGroupName, - resource_uri=uri, - resource_type=models.Worksheet.__name__, - ) - return True diff --git a/backend/dataall/db/models/Enums.py b/backend/dataall/db/models/Enums.py index e31718fbb..4602fed13 100644 --- a/backend/dataall/db/models/Enums.py +++ b/backend/dataall/db/models/Enums.py @@ -175,11 +175,3 @@ class Topic(Enum): Energy = 'Energy' Customers = 'Customers' Misc = 'Misc' - - -class WorksheetRole(Enum): - Creator = '950' - Admin = '900' - SharedWithWritePermission = '500' - SharedWithReadPermission = '400' - NoPermission = '000' diff --git a/backend/dataall/db/models/__init__.py b/backend/dataall/db/models/__init__.py index fff02245e..94084267a 100644 --- a/backend/dataall/db/models/__init__.py +++ b/backend/dataall/db/models/__init__.py @@ -34,5 +34,4 @@ from .TenantAdministrator import TenantAdministrator from .User import User from .Vpc import Vpc -from .Worksheet import Worksheet, WorksheetQueryResult, WorksheetShare from .Vote import Vote diff --git a/backend/dataall/db/permissions.py b/backend/dataall/db/permissions.py index bec9fd4cd..f5c80ddff 100644 --- a/backend/dataall/db/permissions.py +++ b/backend/dataall/db/permissions.py @@ -27,7 +27,6 @@ MANAGE_PIPELINES = 'MANAGE_PIPELINES' MANAGE_GROUPS = 'MANAGE_GROUPS' MANAGE_ENVIRONMENT = 'MANAGE_ENVIRONMENT' -MANAGE_WORKSHEETS = 'MANAGE_WORKSHEETS' MANAGE_GLOSSARIES = 'MANAGE_GLOSSARIES' MANAGE_ENVIRONMENTS = 'MANAGE_ENVIRONMENTS' MANAGE_ORGANIZATIONS = 'MANAGE_ORGANIZATIONS' @@ -49,7 +48,6 @@ CREDENTIALS_ENVIRONMENT = 'CREDENTIALS_ENVIRONMENT' ENABLE_ENVIRONMENT_SUBSCRIPTIONS = 'ENABLE_ENVIRONMENT_SUBSCRIPTIONS' DISABLE_ENVIRONMENT_SUBSCRIPTIONS = 'DISABLE_ENVIRONMENT_SUBSCRIPTIONS' -RUN_ATHENA_QUERY = 'RUN_ATHENA_QUERY' CREATE_SHARE_OBJECT = 'CREATE_SHARE_OBJECT' LIST_ENVIRONMENT_SHARED_WITH_OBJECTS = 'LIST_ENVIRONMENT_SHARED_WITH_OBJECTS' CREATE_REDSHIFT_CLUSTER = 'CREATE_REDSHIFT_CLUSTER' @@ -71,7 +69,6 @@ LIST_ENVIRONMENT_CONSUMPTION_ROLES, CREATE_SHARE_OBJECT, LIST_ENVIRONMENT_SHARED_WITH_OBJECTS, - RUN_ATHENA_QUERY, CREATE_REDSHIFT_CLUSTER, LIST_ENVIRONMENT_REDSHIFT_CLUSTERS, CREATE_SGMSTUDIO_NOTEBOOK, @@ -109,7 +106,6 @@ CREDENTIALS_ENVIRONMENT, ENABLE_ENVIRONMENT_SUBSCRIPTIONS, DISABLE_ENVIRONMENT_SUBSCRIPTIONS, - RUN_ATHENA_QUERY, CREATE_SHARE_OBJECT, CREATE_REDSHIFT_CLUSTER, LIST_ENVIRONMENT_REDSHIFT_CLUSTERS, @@ -198,7 +194,6 @@ MANAGE_REDSHIFT_CLUSTERS, MANAGE_DASHBOARDS, MANAGE_PIPELINES, - MANAGE_WORKSHEETS, MANAGE_GLOSSARIES, MANAGE_GROUPS, MANAGE_ENVIRONMENTS, @@ -210,7 +205,6 @@ TENANT_ALL_WITH_DESC[MANAGE_DASHBOARDS] = 'Manage dashboards' TENANT_ALL_WITH_DESC[MANAGE_REDSHIFT_CLUSTERS] = 'Manage Redshift clusters' TENANT_ALL_WITH_DESC[MANAGE_GLOSSARIES] = 'Manage glossaries' -TENANT_ALL_WITH_DESC[MANAGE_WORKSHEETS] = 'Manage worksheets' TENANT_ALL_WITH_DESC[MANAGE_ENVIRONMENTS] = 'Manage environments' TENANT_ALL_WITH_DESC[MANAGE_GROUPS] = 'Manage teams' TENANT_ALL_WITH_DESC[MANAGE_PIPELINES] = 'Manage pipelines' @@ -295,23 +289,6 @@ LIST_PIPELINES, ] -""" -WORKSHEETS -""" -GET_WORKSHEET = 'GET_WORKSHEET' -UPDATE_WORKSHEET = 'UPDATE_WORKSHEET' -DELETE_WORKSHEET = 'DELETE_WORKSHEET' -SHARE_WORKSHEET = 'SHARE_WORKSHEET' -RUN_WORKSHEET_QUERY = 'RUN_WORKSHEET_QUERY' -WORKSHEET_ALL = [ - GET_WORKSHEET, - UPDATE_WORKSHEET, - DELETE_WORKSHEET, - SHARE_WORKSHEET, - RUN_WORKSHEET_QUERY, -] -WORKSHEET_SHARED = [GET_WORKSHEET, UPDATE_WORKSHEET, RUN_WORKSHEET_QUERY] - """ NETWORKS """ @@ -332,7 +309,6 @@ + GLOSSARY_ALL + SGMSTUDIO_NOTEBOOK_ALL + DASHBOARD_ALL - + WORKSHEET_ALL + PIPELINE_ALL + NETWORK_ALL ) diff --git a/backend/dataall/modules/datasets/db/dataset_repository.py b/backend/dataall/modules/datasets/db/dataset_repository.py index 8159147cc..3a08adc1e 100644 --- a/backend/dataall/modules/datasets/db/dataset_repository.py +++ b/backend/dataall/modules/datasets/db/dataset_repository.py @@ -15,12 +15,12 @@ def get_dataset_by_uri(session, dataset_uri) -> Dataset: raise exceptions.ObjectNotFound('Dataset', dataset_uri) return dataset - def count_resources(self, session, environment_uri, group_uri) -> int: + def count_resources(self, session, environment, group_uri) -> int: return ( session.query(Dataset) .filter( and_( - Dataset.environmentUri == environment_uri, + Dataset.environmentUri == environment.environmentUri, Dataset.SamlAdminGroupName == group_uri )) .count() diff --git a/backend/dataall/modules/worksheets/__init__.py b/backend/dataall/modules/worksheets/__init__.py new file mode 100644 index 000000000..900c55b2a --- /dev/null +++ b/backend/dataall/modules/worksheets/__init__.py @@ -0,0 +1,28 @@ +"""Contains the code related to worksheets""" +import logging + +from dataall.core.group.services.group_resource_manager import GroupResourceManager +from dataall.modules.loader import ImportMode, ModuleInterface +from dataall.modules.worksheets.db.models import Worksheet +from dataall.modules.worksheets.db.repositories import WorksheetRepository + +log = logging.getLogger(__name__) + + +class WorksheetApiModuleInterface(ModuleInterface): + """Implements ModuleInterface for worksheet GraphQl lambda""" + + @classmethod + def is_supported(cls, modes): + return ImportMode.API in modes + + def __init__(self): + from dataall.api.Objects.Feed.registry import FeedRegistry, FeedDefinition + + import dataall.modules.worksheets.api + + FeedRegistry.register(FeedDefinition("Worksheet", Worksheet)) + + GroupResourceManager.register(WorksheetRepository()) + + log.info("API of worksheets has been imported") \ No newline at end of file diff --git a/backend/dataall/modules/worksheets/api/__init__.py b/backend/dataall/modules/worksheets/api/__init__.py new file mode 100644 index 000000000..8aaff7db0 --- /dev/null +++ b/backend/dataall/modules/worksheets/api/__init__.py @@ -0,0 +1,9 @@ +from dataall.modules.worksheets.api import ( + input_types, + mutations, + queries, + resolvers, + types, +) + +__all__ = ['resolvers', 'types', 'input_types', 'queries', 'mutations'] diff --git a/backend/dataall/api/Objects/Worksheet/input_types.py b/backend/dataall/modules/worksheets/api/input_types.py similarity index 86% rename from backend/dataall/api/Objects/Worksheet/input_types.py rename to backend/dataall/modules/worksheets/api/input_types.py index 50724028f..b292a99df 100644 --- a/backend/dataall/api/Objects/Worksheet/input_types.py +++ b/backend/dataall/modules/worksheets/api/input_types.py @@ -1,4 +1,4 @@ -from ... import gql +from dataall.api import gql NewWorksheetInput = gql.InputType( name='NewWorksheetInput', @@ -50,15 +50,6 @@ ], ) -WorksheetShareInput = gql.InputType( - name='WorksheetShareInput', - arguments=[ - gql.Argument(name='principalId', type=gql.NonNullableType(gql.String)), - gql.Argument(name='principalType', type=gql.NonNullableType(gql.String)), - gql.Argument(name='canEdit', type=gql.NonNullableType(gql.Boolean)), - ], -) - WorksheetDimensionInput = gql.InputType( name='WorksheetDimensionInput', diff --git a/backend/dataall/modules/worksheets/api/mutations.py b/backend/dataall/modules/worksheets/api/mutations.py new file mode 100644 index 000000000..46c5c0a8d --- /dev/null +++ b/backend/dataall/modules/worksheets/api/mutations.py @@ -0,0 +1,29 @@ +from dataall.api import gql +from dataall.modules.worksheets.api.resolvers import * + + +createWorksheet = gql.MutationField( + name='createWorksheet', + args=[gql.Argument(name='input', type=gql.Ref('NewWorksheetInput'))], + type=gql.Ref('Worksheet'), + resolver=create_worksheet, +) + +updateWorksheet = gql.MutationField( + name='updateWorksheet', + resolver=update_worksheet, + args=[ + gql.Argument(name='worksheetUri', type=gql.NonNullableType(gql.String)), + gql.Argument(name='input', type=gql.Ref('UpdateWorksheetInput')), + ], + type=gql.Ref('Worksheet'), +) + +deleteWorksheet = gql.MutationField( + name='deleteWorksheet', + resolver=delete_worksheet, + args=[ + gql.Argument(name='worksheetUri', type=gql.NonNullableType(gql.String)), + ], + type=gql.Boolean, +) diff --git a/backend/dataall/api/Objects/Worksheet/queries.py b/backend/dataall/modules/worksheets/api/queries.py similarity index 90% rename from backend/dataall/api/Objects/Worksheet/queries.py rename to backend/dataall/modules/worksheets/api/queries.py index 488f5d3cb..64b19afbf 100644 --- a/backend/dataall/api/Objects/Worksheet/queries.py +++ b/backend/dataall/modules/worksheets/api/queries.py @@ -1,5 +1,5 @@ -from ... import gql -from .resolvers import * +from dataall.api import gql +from dataall.modules.worksheets.api.resolvers import * getWorksheet = gql.QueryField( diff --git a/backend/dataall/modules/worksheets/api/resolvers.py b/backend/dataall/modules/worksheets/api/resolvers.py new file mode 100644 index 000000000..154189fd4 --- /dev/null +++ b/backend/dataall/modules/worksheets/api/resolvers.py @@ -0,0 +1,90 @@ +from dataall.api.constants import GraphQLEnumMapper +from dataall.db import exceptions +from dataall.modules.worksheets.db.models import Worksheet +from dataall.modules.worksheets.db.repositories import WorksheetRepository +from dataall.modules.worksheets.services.worksheet_services import WorksheetService +from dataall.api.context import Context + + +class WorksheetRole(GraphQLEnumMapper): + Creator = '950' + Admin = '900' + NoPermission = '000' + +def create_worksheet(context: Context, source, input: dict = None): + if not input: + raise exceptions.RequiredParameter(input) + if not input.get('SamlAdminGroupName'): + raise exceptions.RequiredParameter('groupUri') + if not input.get('label'): + raise exceptions.RequiredParameter('label') + + with context.engine.scoped_session() as session: + return WorksheetService.create_worksheet( + session=session, + username=context.username, + uri=None, + data=input, + ) + + +def update_worksheet( + context: Context, source, worksheetUri: str = None, input: dict = None +): + with context.engine.scoped_session() as session: + return WorksheetService.update_worksheet( + session=session, + username=context.username, + uri=worksheetUri, + data=input + ) + + +def get_worksheet(context: Context, source, worksheetUri: str = None): + with context.engine.scoped_session() as session: + return WorksheetService.get_worksheet( + session=session, + uri=worksheetUri, + ) + + +def resolve_user_role(context: Context, source: Worksheet): + if context.username and source.owner == context.username: + return WorksheetRole.Creator.value + elif context.groups and source.SamlAdminGroupName in context.groups: + return WorksheetRole.Admin.value + return WorksheetRole.NoPermission.value + + +def list_worksheets(context, source, filter: dict = None): + if not filter: + filter = {} + with context.engine.scoped_session() as session: + return WorksheetRepository.paginated_user_worksheets( + session=session, + username=context.username, + groups=context.groups, + uri=None, + data=filter, + check_perm=True, + ) + + +def run_sql_query( + context: Context, source, environmentUri: str = None, worksheetUri: str = None, sqlQuery: str = None +): + with context.engine.scoped_session() as session: + return WorksheetService.run_sql_query( + session=session, + uri=environmentUri, + worksheetUri=worksheetUri, + sqlQuery=sqlQuery + ) + + +def delete_worksheet(context, source, worksheetUri: str = None): + with context.engine.scoped_session() as session: + return WorksheetService.delete_worksheet( + session=session, + uri=worksheetUri + ) diff --git a/backend/dataall/api/Objects/Worksheet/schema.py b/backend/dataall/modules/worksheets/api/types.py similarity index 67% rename from backend/dataall/api/Objects/Worksheet/schema.py rename to backend/dataall/modules/worksheets/api/types.py index af51aeae9..490636418 100644 --- a/backend/dataall/api/Objects/Worksheet/schema.py +++ b/backend/dataall/modules/worksheets/api/types.py @@ -1,5 +1,50 @@ -from ... import gql -from ..Worksheet.resolvers import * +from dataall.api import gql +from dataall.modules.worksheets.api.resolvers import resolve_user_role, WorksheetRole + + +AthenaResultColumnDescriptor = gql.ObjectType( + name='AthenaResultColumnDescriptor', + fields=[ + gql.Field(name='columnName', type=gql.NonNullableType(gql.String)), + gql.Field(name='typeName', type=gql.NonNullableType(gql.String)), + ], +) + + +AthenaResultRecordCell = gql.ObjectType( + name='AthenaResultRecordCell', + fields=[ + gql.Field(name='value', type=gql.String), + gql.Field(name='typeName', type=gql.NonNullableType(gql.String)), + gql.Field(name='columnName', type=gql.NonNullableType(gql.String)), + ], +) + +AthenaResultRecord = gql.ObjectType( + name='AthenaResultRecord', + fields=[ + gql.Field(name='cells', type=gql.ArrayType(gql.Ref('AthenaResultRecordCell'))) + ], +) + + +AthenaQueryResult = gql.ObjectType( + name='AthenaQueryResult', + fields=[ + gql.Field(name='Error', type=gql.String), + gql.Field(name='OutputLocation', type=gql.String), + gql.Field(name='AthenaQueryId', type=gql.String), + gql.Field(name='AwsAccountId', type=gql.String), + gql.Field(name='region', type=gql.String), + gql.Field(name='ElapsedTimeInMs', type=gql.Integer), + gql.Field(name='DataScannedInBytes', type=gql.Integer), + gql.Field(name='Status', type=gql.String), + gql.Field( + name='columns', type=gql.ArrayType(gql.Ref('AthenaResultColumnDescriptor')) + ), + gql.Field(name='rows', type=gql.ArrayType(gql.Ref('AthenaResultRecord'))), + ], +) Worksheet = gql.ObjectType( @@ -20,12 +65,6 @@ name='lastSavedQueryResult', type=gql.Ref('AthenaQueryResult'), ), - gql.Field( - args=[gql.Argument(name='filter', type=gql.Ref('WorksheetFilter'))], - name='shares', - resolver=resolve_shares, - type=gql.Ref('WorksheetShares'), - ), gql.Field( name='userRoleForWorksheet', type=gql.Ref('WorksheetRole'), @@ -48,30 +87,6 @@ ) -WorksheetShare = gql.ObjectType( - name='WorksheetShare', - fields=[ - gql.Field(name='worksheetShareUri', type=gql.ID), - gql.Field(name='principalId', type=gql.NonNullableType(gql.String)), - gql.Field(name='principalType', type=gql.NonNullableType(gql.String)), - gql.Field(name='canEdit', type=gql.Boolean), - ], -) - - -WorksheetShares = gql.ObjectType( - name='WorksheetShares', - fields=[ - gql.Field(name='count', type=gql.Integer), - gql.Field(name='page', type=gql.Integer), - gql.Field(name='pages', type=gql.Integer), - gql.Field(name='hasNext', type=gql.Boolean), - gql.Field(name='hasPrevious', type=gql.Boolean), - gql.Field(name='nodes', type=gql.ArrayType(gql.Ref('WorksheetShare'))), - ], -) - - WorksheetQueryResult = gql.ObjectType( name='WorksheetQueryResult', fields=[ diff --git a/backend/dataall/modules/worksheets/aws/__init__.py b/backend/dataall/modules/worksheets/aws/__init__.py new file mode 100644 index 000000000..1315c5b1f --- /dev/null +++ b/backend/dataall/modules/worksheets/aws/__init__.py @@ -0,0 +1 @@ +"""Contains code that send requests to AWS Athena """ diff --git a/backend/dataall/modules/worksheets/aws/athena_client.py b/backend/dataall/modules/worksheets/aws/athena_client.py new file mode 100644 index 000000000..88ebddfdc --- /dev/null +++ b/backend/dataall/modules/worksheets/aws/athena_client.py @@ -0,0 +1,47 @@ +from pyathena import connect +from dataall.aws.handlers.sts import SessionHelper + +class AthenaClient: + """ Makes requests to AWS Athena """ + + @staticmethod + def run_athena_query(aws_account_id, env_group, s3_staging_dir, region, sql=None): + base_session = SessionHelper.remote_session(accountid=aws_account_id) + boto3_session = SessionHelper.get_session(base_session=base_session, role_arn=env_group.environmentIAMRoleArn) + creds = boto3_session.get_credentials() + connection = connect( + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + aws_session_token=creds.token, + work_group=env_group.environmentAthenaWorkGroup, + s3_staging_dir=s3_staging_dir, + region_name=region, + ) + cursor = connection.cursor() + cursor.execute(sql) + + return cursor + + @staticmethod + def convert_query_output(cursor): + columns = [] + for f in cursor.description: + columns.append({'columnName': f[0], 'typeName': 'String'}) + + rows = [] + for row in cursor: + record = {'cells': []} + for col_position, column in enumerate(columns): + cell = {} + cell['columnName'] = column['columnName'] + cell['typeName'] = column['typeName'] + cell['value'] = str(row[col_position]) + record['cells'].append(cell) + rows.append(record) + return { + 'error': None, + 'AthenaQueryId': cursor.query_id, + 'ElapsedTime': cursor.total_execution_time_in_millis, + 'rows': rows, + 'columns': columns, + } \ No newline at end of file diff --git a/backend/dataall/modules/worksheets/db/__init__.py b/backend/dataall/modules/worksheets/db/__init__.py new file mode 100644 index 000000000..dbd4edb44 --- /dev/null +++ b/backend/dataall/modules/worksheets/db/__init__.py @@ -0,0 +1 @@ +"""Contains code to interact with the database""" diff --git a/backend/dataall/db/models/Worksheet.py b/backend/dataall/modules/worksheets/db/models.py similarity index 68% rename from backend/dataall/db/models/Worksheet.py rename to backend/dataall/modules/worksheets/db/models.py index c78b12056..e8d6801a9 100644 --- a/backend/dataall/db/models/Worksheet.py +++ b/backend/dataall/modules/worksheets/db/models.py @@ -1,12 +1,12 @@ import datetime import enum -from sqlalchemy import Column, Boolean, DateTime, Integer, Enum, String +from sqlalchemy import Column, DateTime, Integer, Enum, String from sqlalchemy.dialects import postgresql from sqlalchemy.orm import query_expression -from .. import Base -from .. import Resource, utils +from dataall.db import Base +from dataall.db import Resource, utils class QueryType(enum.Enum): @@ -39,15 +39,3 @@ class WorksheetQueryResult(Base): ElapsedTimeInMs = Column(Integer, nullable=True) DataScannedInBytes = Column(Integer, nullable=True) created = Column(DateTime, default=datetime.datetime.now) - - -class WorksheetShare(Base): - __tablename__ = 'worksheet_share' - worksheetShareUri = Column(String, primary_key=True, default=utils.uuid('_')) - worksheetUri = Column(String, nullable=False) - principalId = Column(String, nullable=False) - principalType = Column(String, nullable=False) - canEdit = Column(Boolean, default=False) - owner = Column(String, nullable=False) - created = Column(DateTime, default=datetime.datetime.now) - updated = Column(DateTime, onupdate=datetime.datetime.now) diff --git a/backend/dataall/modules/worksheets/db/repositories.py b/backend/dataall/modules/worksheets/db/repositories.py new file mode 100644 index 000000000..420c06909 --- /dev/null +++ b/backend/dataall/modules/worksheets/db/repositories.py @@ -0,0 +1,56 @@ +""" +DAO layer that encapsulates the logic and interaction with the database for worksheets +""" +from sqlalchemy import or_ +from sqlalchemy.orm import Query + +from dataall.core.group.services.group_resource_manager import GroupResource +from dataall.db import paginate +from dataall.modules.worksheets.db.models import Worksheet, WorksheetQueryResult + + +class WorksheetRepository(GroupResource): + """DAO layer for worksheets""" + _DEFAULT_PAGE = 1 + _DEFAULT_PAGE_SIZE = 10 + + def count_resources(self, session, environment, group_uri) -> int: + return ( + session.query(WorksheetQueryResult) + .filter( + WorksheetQueryResult.AwsAccountId == environment.AwsAccountId + ) + .count() + ) + + @staticmethod + def find_worksheet_by_uri(session, uri) -> Worksheet: + return session.query(Worksheet).get(uri) + + @staticmethod + def query_user_worksheets(session, username, groups, filter) -> Query: + query = session.query(Worksheet).filter( + or_( + Worksheet.owner == username, + Worksheet.SamlAdminGroupName.in_(groups), + ) + ) + if filter and filter.get('term'): + query = query.filter( + or_( + Worksheet.label.ilike('%' + filter.get('term') + '%'), + Worksheet.description.ilike('%' + filter.get('term') + '%'), + Worksheet.tags.contains(f"{{{filter.get('term')}}}"), + ) + ) + return query + + @staticmethod + def paginated_user_worksheets( + session, username, groups, uri, data=None, check_perm=None + ) -> dict: + return paginate( + query=WorksheetRepository.query_user_worksheets(session, username, groups, data), + page=data.get('page', WorksheetRepository._DEFAULT_PAGE), + page_size=data.get('pageSize', WorksheetRepository._DEFAULT_PAGE_SIZE), + ).to_dict() diff --git a/backend/dataall/modules/worksheets/services/__init__.py b/backend/dataall/modules/worksheets/services/__init__.py new file mode 100644 index 000000000..6b724007d --- /dev/null +++ b/backend/dataall/modules/worksheets/services/__init__.py @@ -0,0 +1,7 @@ +""" +Contains the code needed for service layer. +The service layer is a layer where all business logic is aggregated +""" +from dataall.modules.worksheets.services import worksheet_services, worksheet_permissions + +__all__ = ["worksheet_services", "worksheet_permissions"] diff --git a/backend/dataall/modules/worksheets/services/worksheet_permissions.py b/backend/dataall/modules/worksheets/services/worksheet_permissions.py new file mode 100644 index 000000000..c255c1238 --- /dev/null +++ b/backend/dataall/modules/worksheets/services/worksheet_permissions.py @@ -0,0 +1,40 @@ +from dataall.db.permissions import TENANT_ALL, TENANT_ALL_WITH_DESC, RESOURCES_ALL, \ + RESOURCES_ALL_WITH_DESC, ENVIRONMENT_INVITED, ENVIRONMENT_INVITATION_REQUEST, ENVIRONMENT_ALL + +MANAGE_WORKSHEETS = 'MANAGE_WORKSHEETS' + +TENANT_ALL.append(MANAGE_WORKSHEETS) +TENANT_ALL_WITH_DESC[MANAGE_WORKSHEETS] = 'Manage worksheets' + +""" +WORKSHEETS +""" +GET_WORKSHEET = 'GET_WORKSHEET' +UPDATE_WORKSHEET = 'UPDATE_WORKSHEET' +DELETE_WORKSHEET = 'DELETE_WORKSHEET' +RUN_WORKSHEET_QUERY = 'RUN_WORKSHEET_QUERY' +WORKSHEET_ALL = [ + GET_WORKSHEET, + UPDATE_WORKSHEET, + DELETE_WORKSHEET, + RUN_WORKSHEET_QUERY, +] + +RESOURCES_ALL.extend(WORKSHEET_ALL) + +for perm in WORKSHEET_ALL: + RESOURCES_ALL_WITH_DESC[perm] = perm + +""" +RUN ATHENA QUERY +""" +RUN_ATHENA_QUERY = 'RUN_ATHENA_QUERY' + +ENVIRONMENT_INVITED.append(RUN_ATHENA_QUERY) + +ENVIRONMENT_INVITATION_REQUEST.append(RUN_ATHENA_QUERY) + +ENVIRONMENT_ALL.append(RUN_ATHENA_QUERY) + +RESOURCES_ALL.append(RUN_ATHENA_QUERY) +RESOURCES_ALL_WITH_DESC[RUN_ATHENA_QUERY] = RUN_ATHENA_QUERY \ No newline at end of file diff --git a/backend/dataall/modules/worksheets/services/worksheet_services.py b/backend/dataall/modules/worksheets/services/worksheet_services.py new file mode 100644 index 000000000..22abbf27a --- /dev/null +++ b/backend/dataall/modules/worksheets/services/worksheet_services.py @@ -0,0 +1,120 @@ +import logging + +from dataall import db +from dataall.core.permission_checker import has_tenant_permission, has_resource_permission +from dataall.db import exceptions +from dataall.db import models +from dataall.db.api import ResourcePolicy +from dataall.modules.worksheets.aws.athena_client import AthenaClient +from dataall.modules.worksheets.db.models import Worksheet +from dataall.modules.worksheets.db.repositories import WorksheetRepository +from dataall.modules.worksheets.services.worksheet_permissions import MANAGE_WORKSHEETS, UPDATE_WORKSHEET, \ + WORKSHEET_ALL, GET_WORKSHEET, DELETE_WORKSHEET, RUN_ATHENA_QUERY + + +logger = logging.getLogger(__name__) + + +class WorksheetService: + @staticmethod + def get_worksheet_by_uri(session, uri: str) -> Worksheet: + if not uri: + raise exceptions.RequiredParameter(param_name='worksheetUri') + worksheet = WorksheetRepository.find_worksheet_by_uri(session, uri) + if not worksheet: + raise exceptions.ObjectNotFound('Worksheet', uri) + return worksheet + + + @staticmethod + @has_tenant_permission(MANAGE_WORKSHEETS) + def create_worksheet( + session, username, uri, data=None) -> Worksheet: + worksheet = Worksheet( + owner=username, + label=data.get('label'), + description=data.get('description', 'No description provided'), + tags=data.get('tags'), + chartConfig={'dimensions': [], 'measures': [], 'chartType': 'bar'}, + SamlAdminGroupName=data['SamlAdminGroupName'], + ) + + session.add(worksheet) + session.commit() + + activity = models.Activity( + action='WORKSHEET:CREATE', + label='WORKSHEET:CREATE', + owner=username, + summary=f'{username} created worksheet {worksheet.name} ', + targetUri=worksheet.worksheetUri, + targetType='worksheet', + ) + session.add(activity) + + ResourcePolicy.attach_resource_policy( + session=session, + group=data['SamlAdminGroupName'], + permissions=WORKSHEET_ALL, + resource_uri=worksheet.worksheetUri, + resource_type=Worksheet.__name__, + ) + return worksheet + + @staticmethod + @has_resource_permission(UPDATE_WORKSHEET) + def update_worksheet(session, username, uri, data=None): + worksheet = WorksheetService.get_worksheet_by_uri(session, uri) + for field in data.keys(): + setattr(worksheet, field, data.get(field)) + session.commit() + + activity = models.Activity( + action='WORKSHEET:UPDATE', + label='WORKSHEET:UPDATE', + owner=username, + summary=f'{username} updated worksheet {worksheet.name} ', + targetUri=worksheet.worksheetUri, + targetType='worksheet', + ) + session.add(activity) + return worksheet + + @staticmethod + @has_resource_permission(GET_WORKSHEET) + def get_worksheet(session, uri): + worksheet = WorksheetService.get_worksheet_by_uri(session, uri) + return worksheet + + @staticmethod + @has_resource_permission(DELETE_WORKSHEET) + def delete_worksheet(session, uri) -> bool: + worksheet = WorksheetService.get_worksheet_by_uri(session, uri) + session.delete(worksheet) + ResourcePolicy.delete_resource_policy( + session=session, + group=worksheet.SamlAdminGroupName, + resource_uri=uri, + resource_type=Worksheet.__name__, + ) + return True + + @staticmethod + @has_resource_permission(RUN_ATHENA_QUERY) + def run_sql_query(session, uri, worksheetUri, sqlQuery): + environment = db.api.Environment.get_environment_by_uri(session, uri) + worksheet = WorksheetService.get_worksheet_by_uri(session, worksheetUri) + + env_group = db.api.Environment.get_environment_group( + session, worksheet.SamlAdminGroupName, environment.environmentUri + ) + + cursor = AthenaClient.run_athena_query( + aws_account_id=environment.AwsAccountId, + env_group=env_group, + s3_staging_dir=f's3://{environment.EnvironmentDefaultBucketName}/athenaqueries/{env_group.environmentAthenaWorkGroup}/', + region=environment.region, + sql=sqlQuery + ) + + return AthenaClient.convert_query_output(cursor) diff --git a/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py b/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py index 628ae38ab..7c1d13d1f 100644 --- a/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py +++ b/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py @@ -5,6 +5,8 @@ Create Date: 2023-02-20 14:28:13.331670 """ +import sqlalchemy as sa + from typing import List from alembic import op @@ -109,6 +111,9 @@ def upgrade(): sid=permission.sid, permissionUri=manage_mlstudio.permissionUri, )) + + op.drop_table('worksheet_share') + session.commit() migrate_groups_permissions(session) @@ -135,6 +140,18 @@ def downgrade(): session.add_all(envs) op.drop_table("environment_parameters") + op.create_table( + 'worksheet_share', + sa.Column('worksheetShareUri', sa.String(), nullable=False), + sa.Column('worksheetUri', sa.String(), nullable=False), + sa.Column('principalId', sa.String(), nullable=False), + sa.Column('principalType', sa.String(), nullable=False), + sa.Column('canEdit', sa.Boolean(), nullable=True), + sa.Column('owner', sa.String(), nullable=False), + sa.Column('created', sa.DateTime(), nullable=True), + sa.Column('updated', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('worksheetShareUri'), + ) except Exception as ex: print(f"Failed to execute the rollback script due to: {ex}") diff --git a/config.json b/config.json index 4aed5ef87..6f60f494f 100644 --- a/config.json +++ b/config.json @@ -5,6 +5,9 @@ }, "datasets": { "active": true + }, + "worksheets": { + "active": true } } } \ No newline at end of file diff --git a/frontend/src/api/Worksheet/index.js b/frontend/src/api/Worksheet/index.js index 7d6c7fc01..c02efe1b0 100644 --- a/frontend/src/api/Worksheet/index.js +++ b/frontend/src/api/Worksheet/index.js @@ -2,10 +2,6 @@ import listWorksheets from './listWorksheets'; import createWorksheet from './createWorksheet'; import updateWorksheet from './updateWorksheet'; import getWorksheet from './getWorksheet'; -import listWorksheetShares from './listWorksheetShares'; -import shareWorksheet from './shareWorksheet'; -import removeWorksheetShare from './removeWorksheetShare'; -import updateWorksheetShare from './updateWorksheetShare'; import deleteWorksheet from './deleteWorksheet'; import runAthenaSqlQuery from './runAthenaSqlQuery'; @@ -15,9 +11,5 @@ export { runAthenaSqlQuery, updateWorksheet, getWorksheet, - listWorksheetShares, - shareWorksheet, - removeWorksheetShare, - updateWorksheetShare, deleteWorksheet }; diff --git a/frontend/src/api/Worksheet/listWorksheetShares.js b/frontend/src/api/Worksheet/listWorksheetShares.js deleted file mode 100644 index e14da5702..000000000 --- a/frontend/src/api/Worksheet/listWorksheetShares.js +++ /dev/null @@ -1,28 +0,0 @@ -import { gql } from 'apollo-boost'; - -const listWorksheetShares = ({ worksheetUri, filter }) => ({ - variables: { - worksheetUri, - filter - }, - query: gql` - query GetWorksheet($worksheetUri: String!, $filter: WorksheetFilter) { - getWorksheet(worksheetUri: $worksheetUri) { - shares(filter: $filter) { - count - page - pages - hasNext - hasPrevious - nodes { - worksheetShareUri - principalId - principalType - } - } - } - } - ` -}); - -export default listWorksheetShares; diff --git a/frontend/src/api/Worksheet/removeWorksheetShare.js b/frontend/src/api/Worksheet/removeWorksheetShare.js deleted file mode 100644 index f4d82f036..000000000 --- a/frontend/src/api/Worksheet/removeWorksheetShare.js +++ /dev/null @@ -1,14 +0,0 @@ -import { gql } from 'apollo-boost'; - -const removeWorksheetShare = (worksheetShareUri) => ({ - variables: { - worksheetShareUri - }, - mutation: gql` - mutation RemoveWorksheetShare($worksheetShareUri: String!) { - removeWorksheetShare(worksheetShareUri: $worksheetShareUri) - } - ` -}); - -export default removeWorksheetShare; diff --git a/frontend/src/api/Worksheet/shareWorksheet.js b/frontend/src/api/Worksheet/shareWorksheet.js deleted file mode 100644 index 3d6278aea..000000000 --- a/frontend/src/api/Worksheet/shareWorksheet.js +++ /dev/null @@ -1,20 +0,0 @@ -import { gql } from 'apollo-boost'; - -const shareWorksheet = ({ worksheetUri, input }) => ({ - variables: { - worksheetUri, - input - }, - mutation: gql` - mutation ShareWorksheet( - $worksheetUri: String! - $input: WorksheetShareInput! - ) { - shareWorksheet(worksheetUri: $worksheetUri, input: $input) { - worksheetShareUri - } - } - ` -}); - -export default shareWorksheet; diff --git a/frontend/src/api/Worksheet/updateWorksheetShare.js b/frontend/src/api/Worksheet/updateWorksheetShare.js deleted file mode 100644 index 557bb6aae..000000000 --- a/frontend/src/api/Worksheet/updateWorksheetShare.js +++ /dev/null @@ -1,24 +0,0 @@ -import { gql } from 'apollo-boost'; - -const updateWorksheetShare = ({ worksheetShareUri, canEdit }) => ({ - variables: { - worksheetShareUri, - canEdit - }, - mutation: gql` - mutation RemoveWorksheetShare( - $worksheetShareUri: String! - $canEdit: Boolean - ) { - updateWorksheetShare( - worksheetShareUri: $worksheetShareUri - canEdit: $canEdit - ) { - worksheetShareUri - canEdit - } - } - ` -}); - -export default updateWorksheetShare; diff --git a/tests/api/test_feed.py b/tests/api/test_feed.py deleted file mode 100644 index f1d8aaf7f..000000000 --- a/tests/api/test_feed.py +++ /dev/null @@ -1,107 +0,0 @@ -import pytest - -from dataall.db import models - - -@pytest.fixture(scope='module', autouse=True) -def worksheet(db): - with db.scoped_session() as session: - w = models.Worksheet( - owner='me', - label='xxx', - SamlAdminGroupName='g', - ) - session.add(w) - return w - - -def test_post_message(client, worksheet): - response = client.query( - """ - mutation PostFeedMessage( - $targetUri : String!, - $targetType: String!, - $input:FeedMessageInput - ){ - postFeedMessage(targetUri:$targetUri, targetType:$targetType,input:$input){ - feedMessageUri - content - created - creator - } - } - """, - username='me', - targetUri=worksheet.worksheetUri, - targetType='Worksheet', - input={'content': 'hello'}, - ) - - assert response.data.postFeedMessage.content == 'hello' - assert response.data.postFeedMessage.creator == 'me' - - -def test_list_messages(client, worksheet): - response = client.query( - """ - query GetFeed( - $targetUri:String!, - $targetType:String!, - $filter:FeedMessageFilter! - ){ - getFeed( - targetUri:$targetUri, - targetType:$targetType, - - ){ - messages( filter:$filter){ - count - page - pages - hasNext - hasPrevious - nodes{ - content - created - } - } - } - } - """, - username='me', - targetUri=worksheet.worksheetUri, - targetType='Worksheet', - filter={}, - ) - - assert response.data.getFeed.messages.count == 1 - assert response.data.getFeed.messages.nodes[0].content == 'hello' - - -def test_get_target(client, worksheet): - response = client.query( - """ - query GetFeed( - $targetUri:String!, - $targetType:String!, - ){ - getFeed( - targetUri:$targetUri, - targetType:$targetType, - - ){ - - target{ - ... on Worksheet{ - worksheetUri - } - } - } - } - """, - targetUri=worksheet.worksheetUri, - targetType='Worksheet', - username='me', - ) - assert response.data.getFeed.target.worksheetUri == worksheet.worksheetUri - diff --git a/frontend/src/api/Worksheet/deleteWorksheetShare.js b/tests/modules/feed/__init__.py similarity index 100% rename from frontend/src/api/Worksheet/deleteWorksheetShare.js rename to tests/modules/feed/__init__.py diff --git a/tests/modules/feed/testhelper.py b/tests/modules/feed/testhelper.py new file mode 100644 index 000000000..512c894a6 --- /dev/null +++ b/tests/modules/feed/testhelper.py @@ -0,0 +1,98 @@ +from tests.api.conftest import * +from tests.api.client import * + +class FeedTestHelper: + + def __init__(self, test_object_fixture, client_fixture): + self._test_object_fixture = test_object_fixture + self._client_fixture = client_fixture + + def test_post_message(self, object_name, uri): + response = self._client_fixture.query( + """ + mutation PostFeedMessage( + $targetUri : String!, + $targetType: String!, + $input:FeedMessageInput + ){ + postFeedMessage(targetUri:$targetUri, targetType:$targetType,input:$input){ + feedMessageUri + content + created + creator + } + } + """, + username='me', + targetUri=getattr(self._test_object_fixture, uri), + targetType=object_name, + input={'content': 'hello'}, + ) + + assert response.data.postFeedMessage.content == 'hello' + assert response.data.postFeedMessage.creator == 'me' + + def test_list_messages(self, object_name, uri): + response = self._client_fixture.query( + """ + query GetFeed( + $targetUri:String!, + $targetType:String!, + $filter:FeedMessageFilter! + ){ + getFeed( + targetUri:$targetUri, + targetType:$targetType, + + ){ + messages( filter:$filter){ + count + page + pages + hasNext + hasPrevious + nodes{ + content + created + } + } + } + } + """, + username='me', + targetUri=getattr(self._test_object_fixture, uri), + targetType=object_name, + filter={}, + ) + + assert response.data.getFeed.messages.count == 1 + assert response.data.getFeed.messages.nodes[0].content == 'hello' + + def test_get_target(self, object_name, uri): + query_request = """ + query GetFeed( + $targetUri:String!, + $targetType:String!, + ){ + getFeed( + targetUri:$targetUri, + targetType:$targetType, + + ){ + + target{ + ... on """ + object_name + """{ + """ + uri + """ + } + } + } + } + """ + + response = self._client_fixture.query( + query_request, + targetUri=getattr(self._test_object_fixture, uri), + targetType=object_name, + username='me', + ) + assert getattr(response.data.getFeed.target, uri) == getattr(self._test_object_fixture, uri) \ No newline at end of file diff --git a/tests/modules/worksheets/__init__.py b/tests/modules/worksheets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/api/test_worksheet.py b/tests/modules/worksheets/test_worksheet.py similarity index 50% rename from tests/api/test_worksheet.py rename to tests/modules/worksheets/test_worksheet.py index 0cbae9da7..35a1fd89c 100644 --- a/tests/api/test_worksheet.py +++ b/tests/modules/worksheets/test_worksheet.py @@ -1,5 +1,7 @@ import pytest -from dataall.api.constants import WorksheetRole +from dataall.modules.worksheets.api.resolvers import WorksheetRole +from tests.api.client import client, app +from tests.api.conftest import * @pytest.fixture(scope='module', autouse=True) @@ -144,134 +146,3 @@ def test_update_worksheet(client, worksheet, group): ) assert response.data.updateWorksheet.label == 'change label' - - -def test_share_with_individual(client, worksheet, group2, group): - response = client.query( - """ - mutation ShareWorksheet( - $worksheetUri:String!, - $input: WorksheetShareInput! - ){ - shareWorksheet(worksheetUri:$worksheetUri,input:$input){ - worksheetShareUri - canEdit - } - } - """, - worksheetUri=worksheet.worksheetUri, - input={'principalId': group2.name, 'principalType': 'Group', 'canEdit': False}, - username='alice', - groups=[group.name], - ) - share_uri = response.data.shareWorksheet.worksheetShareUri - assert share_uri - assert not response.data.shareWorksheet.canEdit - - response = client.query( - """ - mutation UpdateShareWorksheet( - $worksheetShareUri:String!, - $canEdit: Boolean! - ){ - updateShareWorksheet(worksheetShareUri:$worksheetShareUri,canEdit:$canEdit){ - worksheetShareUri - canEdit - } - } - """, - worksheetShareUri=share_uri, - canEdit=True, - username='alice', - groups=[group.name], - ) - share_uri = response.data.updateShareWorksheet.worksheetShareUri - assert share_uri - assert response.data.updateShareWorksheet.canEdit - - response = client.query( - """ - query GetWorksheet($worksheetUri:String!){ - getWorksheet(worksheetUri:$worksheetUri){ - label - description - userRoleForWorksheet - } - } - """, - worksheetUri=worksheet.worksheetUri, - username='bob', - groups=[group2.name], - ) - - assert response.data.getWorksheet.label == 'change label' - - response = client.query( - """ - query GetWorksheet($worksheetUri:String!){ - getWorksheet(worksheetUri:$worksheetUri){ - label - description - userRoleForWorksheet - shares{ - count - } - lastSavedQueryResult - { - AthenaQueryId - } - - } - } - """, - worksheetUri=worksheet.worksheetUri, - username='bob', - groups=[group2.name], - ) - - assert response.data.getWorksheet.label == 'change label' - - response = client.query( - """ - mutation deleteShareWorksheet( - $worksheetShareUri:String! - ){ - deleteShareWorksheet(worksheetShareUri:$worksheetShareUri) - } - """, - worksheetShareUri=share_uri, - username='alice', - groups=[group.name], - ) - assert response.data.deleteShareWorksheet - - response = client.query( - """ - query GetWorksheet($worksheetUri:String!){ - getWorksheet(worksheetUri:$worksheetUri){ - label - description - userRoleForWorksheet - } - } - """, - worksheetUri=worksheet.worksheetUri, - username='bob', - groups=[group2.name], - ) - - assert 'UnauthorizedOperation' in response.errors[0].message - - response = client.query( - """ - mutation deleteWorksheet( - $worksheetUri:String! - ){ - deleteWorksheet(worksheetUri:$worksheetUri) - } - """, - worksheetUri=worksheet.worksheetUri, - username='alice', - groups=[group.name], - ) - assert response.data.deleteWorksheet diff --git a/tests/modules/worksheets/test_worksheet_feed.py b/tests/modules/worksheets/test_worksheet_feed.py new file mode 100644 index 000000000..08efced6a --- /dev/null +++ b/tests/modules/worksheets/test_worksheet_feed.py @@ -0,0 +1,36 @@ +from tests.modules.feed.testhelper import FeedTestHelper +from tests.api.conftest import * +from dataall.modules.worksheets.db.models import Worksheet + + +@pytest.fixture(scope='module', autouse=True) +def worksheet(db): + with db.scoped_session() as session: + w = Worksheet( + owner='me', + label='xxx', + SamlAdminGroupName='g', + ) + session.add(w) + return w + + +def test_post_message_worksheet(worksheet, client): + FeedTestHelper( + test_object_fixture=worksheet, + client_fixture=client + ).test_post_message(object_name='Worksheet', uri='worksheetUri') + + +def test_list_messages_worksheet(worksheet, client): + FeedTestHelper( + test_object_fixture=worksheet, + client_fixture=client + ).test_list_messages(object_name='Worksheet', uri='worksheetUri') + + +def test_get_target_worksheet(worksheet, client): + FeedTestHelper( + test_object_fixture=worksheet, + client_fixture=client + ).test_get_target(object_name='Worksheet', uri='worksheetUri') \ No newline at end of file