Datasets modularization pt.5 (data-dot-all#442)

### Feature or Bugfix
- Refactoring

### Detail
Refactoring of the Dataset entity and related to it code.
Refactoring for Votes
Introduced DataPolicy (the same way as ServicePolicy was used used)
Extracted dataset related permissions.
Used new `has_tenant_permission` instead of `has_tenant_perm` that
allows not to pass unused parameters

### Relates
data-dot-all#412 and data-dot-all#295

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

.gitignore

@@ -70,5 +70,3 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 .idea
-/backend/dataall/cdkproxy/assets/gluedataqualityjob/datahubdq.zip
-/backend/dataall/cdkproxy/assets/glueprofilingjob/datahubdq.zip

backend/dataall/api/Objects/Environment/queries.py

@@ -48,18 +48,6 @@
 )
 
 
-listDatasetsCreatedInEnvironment = gql.QueryField(
-    name='listDatasetsCreatedInEnvironment',
-    type=gql.Ref('DatasetSearchResult'),
-    args=[
-        gql.Argument(name='environmentUri', type=gql.NonNullableType(gql.String)),
-        gql.Argument(name='filter', type=gql.Ref('DatasetFilter')),
-    ],
-    resolver=list_datasets_created_in_environment,
-    test_scope='Dataset',
-)
-
-
 searchEnvironmentDataItems = gql.QueryField(
     name='searchEnvironmentDataItems',
     args=[

backend/dataall/api/Objects/Environment/resolvers.py

@@ -402,23 +402,6 @@ def list_environment_group_permissions(
             check_perm=True,
         )
 
-
-def list_datasets_created_in_environment(
-    context: Context, source, environmentUri: str = None, filter: dict = None
-):
-    if not filter:
-        filter = {}
-    with context.engine.scoped_session() as session:
-        return db.api.Environment.paginated_environment_datasets(
-            session=session,
-            username=context.username,
-            groups=context.groups,
-            uri=environmentUri,
-            data=filter,
-            check_perm=True,
-        )
-
-
 def list_shared_with_environment_data_items(
     context: Context, source, environmentUri: str = None, filter: dict = None
 ):

backend/dataall/api/Objects/Group/queries.py

@@ -1,5 +1,5 @@
 from ... import gql
-from .resolvers import get_group, list_datasets_owned_by_env_group, list_data_items_shared_with_env_group, list_cognito_groups
+from .resolvers import get_group, list_data_items_shared_with_env_group, list_cognito_groups
 
 getGroup = gql.QueryField(
     name='getGroup',
@@ -8,20 +8,6 @@
     resolver=get_group,
 )
 
-
-listDatasetsOwnedByEnvGroup = gql.QueryField(
-    name='listDatasetsOwnedByEnvGroup',
-    type=gql.Ref('DatasetSearchResult'),
-    args=[
-        gql.Argument(name='environmentUri', type=gql.NonNullableType(gql.String)),
-        gql.Argument(name='groupUri', type=gql.NonNullableType(gql.String)),
-        gql.Argument(name='filter', type=gql.Ref('DatasetFilter')),
-    ],
-    resolver=list_datasets_owned_by_env_group,
-    test_scope='Dataset',
-)
-
-
 listDataItemsSharedWithEnvGroup = gql.QueryField(
     name='listDataItemsSharedWithEnvGroup',
     args=[

backend/dataall/api/Objects/Group/resolvers.py

@@ -43,23 +43,6 @@ def get_group(context, source, groupUri):
     return Group(groupUri=groupUri, name=groupUri, label=groupUri)
 
 
-def list_datasets_owned_by_env_group(
-    context, source, environmentUri: str = None, groupUri: str = None, filter: dict = None
-):
-    if not filter:
-        filter = {}
-    with context.engine.scoped_session() as session:
-        return db.api.Environment.paginated_environment_group_datasets(
-            session=session,
-            username=context.username,
-            groups=context.groups,
-            envUri=environmentUri,
-            groupUri=groupUri,
-            data=filter,
-            check_perm=True,
-        )
-
-
 def list_data_items_shared_with_env_group(
     context, source, environmentUri: str = None, groupUri: str = None, filter: dict = None
 ):

backend/dataall/api/Objects/ShareObject/resolvers.py

@@ -7,7 +7,8 @@
 from ....api.context import Context
 from ....aws.handlers.service_handlers import Worker
 from ....db import models
-from dataall.modules.datasets.db.models import DatasetStorageLocation, DatasetTable
+from dataall.modules.datasets.db.models import DatasetStorageLocation, DatasetTable, Dataset
+from dataall.modules.datasets.services.dataset_service import DatasetService
 
 log = logging.getLogger(__name__)
 
@@ -19,7 +20,7 @@ def get_share_object_dataset(context, source, **kwargs):
         share: models.ShareObject = session.query(models.ShareObject).get(
             source.shareUri
         )
-        return session.query(models.Dataset).get(share.datasetUri)
+        return session.query(Dataset).get(share.datasetUri)
 
 
 def create_share_object(
@@ -32,7 +33,7 @@ def create_share_object(
 ):
 
     with context.engine.scoped_session() as session:
-        dataset: models.Dataset = db.api.Dataset.get_dataset_by_uri(session, datasetUri)
+        dataset: Dataset = DatasetService.get_dataset_by_uri(session, datasetUri)
         environment: models.Environment = db.api.Environment.get_environment_by_uri(
             session, input['environmentUri']
         )
@@ -222,7 +223,7 @@ def resolve_user_role(context: Context, source: models.ShareObject, **kwargs):
     if not source:
         return None
     with context.engine.scoped_session() as session:
-        dataset: models.Dataset = db.api.Dataset.get_dataset_by_uri(session, source.datasetUri)
+        dataset: Dataset = DatasetService.get_dataset_by_uri(session, source.datasetUri)
         if dataset and dataset.stewards in context.groups:
             return ShareObjectPermission.Approvers.value
         if (
@@ -250,7 +251,7 @@ def resolve_dataset(context: Context, source: models.ShareObject, **kwargs):
     if not source:
         return None
     with context.engine.scoped_session() as session:
-        ds: models.Dataset = db.api.Dataset.get_dataset_by_uri(session, source.datasetUri)
+        ds: Dataset = DatasetService.get_dataset_by_uri(session, source.datasetUri)
         if ds:
             env: models.Environment = db.api.Environment.get_environment_by_uri(session, ds.environmentUri)
             return {
@@ -292,7 +293,7 @@ def resolve_consumption_data(context: Context, source: models.ShareObject, **kwa
     if not source:
         return None
     with context.engine.scoped_session() as session:
-        ds: models.Dataset = db.api.Dataset.get_dataset_by_uri(session, source.datasetUri)
+        ds: Dataset = DatasetService.get_dataset_by_uri(session, source.datasetUri)
         if ds:
             S3AccessPointName = utils.slugify(
                 source.datasetUri + '-' + source.principalId,

backend/dataall/api/Objects/Stack/stack_helper.py

@@ -3,7 +3,6 @@
 import requests
 
 from .... import db
-from ....api.context import Context
 from ....aws.handlers.service_handlers import Worker
 from ....aws.handlers.ecs import Ecs
 from ....db import models
@@ -84,15 +83,6 @@ def deploy_stack(targetUri):
         return stack
 
 
-def deploy_dataset_stack(dataset: models.Dataset):
-    """
-    Each dataset stack deployment triggers environment stack update
-    to rebuild teams IAM roles data access policies
-    """
-    deploy_stack(dataset.datasetUri)
-    deploy_stack(dataset.environmentUri)
-
-
 def delete_stack(
     target_uri, accountid, cdk_role_arn, region
 ):

backend/dataall/api/Objects/Vote/resolvers.py

@@ -1,7 +1,15 @@
-from .... import db
-from ....api.context import Context
+from typing import Dict, Type
+
+from dataall import db
+from dataall.api.context import Context
 from dataall.searchproxy.indexers import DashboardIndexer
-from dataall.modules.datasets.indexers.dataset_indexer import DatasetIndexer
+from dataall.searchproxy.base_indexer import BaseIndexer
+
+_VOTE_TYPES: Dict[str, Type[BaseIndexer]] = {}
+
+
+def add_vote_type(target_type: str, indexer: Type[BaseIndexer]):
+    _VOTE_TYPES[target_type] = indexer
 
 
 def count_upvotes(
@@ -28,15 +36,9 @@ def upvote(context: Context, source, input=None):
             data=input,
             check_perm=True,
         )
-        reindex(session, vote)
-        return vote
 
-
-def reindex(session, vote):
-    if vote.targetType == 'dataset':
-        DatasetIndexer.upsert(session=session, dataset_uri=vote.targetUri)
-    elif vote.targetType == 'dashboard':
-        DashboardIndexer.upsert(session=session, dashboard_uri=vote.targetUri)
+        _VOTE_TYPES[vote.targetType].upsert(session, vote.targetUri)
+        return vote
 
 
 def get_vote(context: Context, source, targetUri: str = None, targetType: str = None):
@@ -49,3 +51,7 @@ def get_vote(context: Context, source, targetUri: str = None, targetType: str =
             data={'targetType': targetType},
             check_perm=True,
         )
+
+
+# TODO should migrate after into the Dashboard module
+add_vote_type("dashboard", DashboardIndexer)

backend/dataall/api/Objects/__init__.py

@@ -17,7 +17,6 @@
     DataPipeline,
     Environment,
     Activity,
-    Dataset,
     Group,
     Principal,
     Dashboard,
@@ -83,7 +82,6 @@ def adapted(obj, info, **kwargs):
         response = resolver(
             context=Namespace(
                 engine=info.context['engine'],
-                es=info.context['es'],
                 username=info.context['username'],
                 groups=info.context['groups'],
                 schema=info.context['schema'],

backend/dataall/api/constants.py

@@ -80,16 +80,6 @@ class DataPipelineRole(GraphQLEnumMapper):
     NoPermission = '000'
 
 
-class DatasetRole(GraphQLEnumMapper):
-    # Permissions on a dataset
-    BusinessOwner = '999'
-    DataSteward = '998'
-    Creator = '950'
-    Admin = '900'
-    Shared = '300'
-    NoPermission = '000'
-
-
 class GlossaryRole(GraphQLEnumMapper):
     # Permissions on a glossary
     Admin = '900'