From 84b42d28b953edcb64ff0d1bc39500ce45515d3c Mon Sep 17 00:00:00 2001 From: Grace Guo Date: Wed, 19 Feb 2020 09:06:52 -0800 Subject: [PATCH] [dashboard] use filter_scopes metadata when import old dashboard (#9145) * [dashboard] update filter_scopes metadata when import old dashboard * fix review comments --- superset/models/dashboard.py | 52 +++++++++++++++++++-------------- superset/models/helpers.py | 5 ++++ superset/views/dashboard/api.py | 3 +- tests/import_export_tests.py | 13 +++++++-- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/superset/models/dashboard.py b/superset/models/dashboard.py index 14d24b4916f0..60af209da21c 100644 --- a/superset/models/dashboard.py +++ b/superset/models/dashboard.py @@ -44,6 +44,7 @@ from superset.models.tags import DashboardUpdater from superset.models.user_attributes import UserAttribute from superset.utils import core as utils +from superset.utils.dashboard_filter_scopes_converter import convert_filter_scopes if TYPE_CHECKING: # pylint: disable=unused-import @@ -286,11 +287,11 @@ def alter_positions(dashboard, old_to_new_slc_id_dict): # copy slices object as Slice.import_slice will mutate the slice # and will remove the existing dashboard - slice association slices = copy(dashboard_to_import.slices) + old_json_metadata = json.loads(dashboard_to_import.json_metadata or "{}") old_to_new_slc_id_dict = {} - new_filter_immune_slices = [] - new_filter_immune_slice_fields = {} new_timed_refresh_immune_slices = [] new_expanded_slices = {} + new_filter_scopes = {} i_params_dict = dashboard_to_import.params_dict remote_id_slice_map = { slc.params_dict["remote_id"]: slc @@ -309,18 +310,6 @@ def alter_positions(dashboard, old_to_new_slc_id_dict): # update json metadata that deals with slice ids new_slc_id_str = "{}".format(new_slc_id) old_slc_id_str = "{}".format(slc.id) - if ( - "filter_immune_slices" in i_params_dict - and old_slc_id_str in i_params_dict["filter_immune_slices"] - ): - new_filter_immune_slices.append(new_slc_id_str) - if ( - "filter_immune_slice_fields" in i_params_dict - and old_slc_id_str in i_params_dict["filter_immune_slice_fields"] - ): - new_filter_immune_slice_fields[new_slc_id_str] = i_params_dict[ - "filter_immune_slice_fields" - ][old_slc_id_str] if ( "timed_refresh_immune_slices" in i_params_dict and old_slc_id_str in i_params_dict["timed_refresh_immune_slices"] @@ -334,6 +323,29 @@ def alter_positions(dashboard, old_to_new_slc_id_dict): old_slc_id_str ] + # since PR #9109, filter_immune_slices and filter_immune_slice_fields + # are converted to filter_scopes + # but dashboard create from import may still have old dashboard filter metadata + # here we convert them to new filter_scopes metadata first + filter_scopes: Dict = {} + if ( + "filter_immune_slices" in i_params_dict + or "filter_immune_slice_fields" in i_params_dict + ): + filter_scopes = convert_filter_scopes(old_json_metadata, slices) + + if "filter_scopes" in i_params_dict: + filter_scopes = old_json_metadata.get("filter_scopes") + + # then replace old slice id to new slice id: + for (slice_id, scopes) in filter_scopes.items(): + new_filter_key = old_to_new_slc_id_dict[int(slice_id)] + new_filter_scopes[str(new_filter_key)] = scopes + for scope in scopes.values(): + scope["immune"] = [ + old_to_new_slc_id_dict[slice_id] for slice_id in scope.get("immune") + ] + # override the dashboard existing_dashboard = None for dash in session.query(Dashboard).all(): @@ -351,16 +363,12 @@ def alter_positions(dashboard, old_to_new_slc_id_dict): if dashboard_to_import.position_json: alter_positions(dashboard_to_import, old_to_new_slc_id_dict) dashboard_to_import.alter_params(import_time=import_time) + dashboard_to_import.remove_params(param_to_remove="filter_immune_slices") + dashboard_to_import.remove_params(param_to_remove="filter_immune_slice_fields") + if new_filter_scopes: + dashboard_to_import.alter_params(filter_scopes=new_filter_scopes) if new_expanded_slices: dashboard_to_import.alter_params(expanded_slices=new_expanded_slices) - if new_filter_immune_slices: - dashboard_to_import.alter_params( - filter_immune_slices=new_filter_immune_slices - ) - if new_filter_immune_slice_fields: - dashboard_to_import.alter_params( - filter_immune_slice_fields=new_filter_immune_slice_fields - ) if new_timed_refresh_immune_slices: dashboard_to_import.alter_params( timed_refresh_immune_slices=new_timed_refresh_immune_slices diff --git a/superset/models/helpers.py b/superset/models/helpers.py index b4d5edcc5faa..7a1433453571 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -278,6 +278,11 @@ def alter_params(self, **kwargs): d.update(kwargs) self.params = json.dumps(d) + def remove_params(self, param_to_remove: str) -> None: + d = self.params_dict + d.pop(param_to_remove, None) + self.params = json.dumps(d) + def reset_ownership(self): """ object will belong to the user the current user """ # make sure the object doesn't have relations to a user diff --git a/superset/views/dashboard/api.py b/superset/views/dashboard/api.py index d11def5baf7c..5e09019ec644 100644 --- a/superset/views/dashboard/api.py +++ b/superset/views/dashboard/api.py @@ -47,9 +47,10 @@ class DashboardJSONMetadataSchema(Schema): expanded_slices = fields.Dict() refresh_frequency = fields.Integer() default_filters = fields.Str() - filter_immune_slice_fields = fields.Dict() stagger_refresh = fields.Boolean() stagger_time = fields.Integer() + color_scheme = fields.Str() + label_colors = fields.Dict() def validate_json(value): diff --git a/tests/import_export_tests.py b/tests/import_export_tests.py index bf5f6dd55016..50a1d04335e5 100644 --- a/tests/import_export_tests.py +++ b/tests/import_export_tests.py @@ -400,11 +400,16 @@ def test_import_dashboard_2_slices(self): dash_with_2_slices.json_metadata = json.dumps( { "remote_id": 10003, - "filter_immune_slices": ["{}".format(e_slc.id)], "expanded_slices": { "{}".format(e_slc.id): True, "{}".format(b_slc.id): False, }, + # mocked filter_scope metadata + "filter_scopes": { + str(e_slc.id): { + "region": {"scope": ["ROOT_ID"], "immune": [b_slc.id]} + } + }, } ) @@ -421,7 +426,11 @@ def test_import_dashboard_2_slices(self): expected_json_metadata = { "remote_id": 10003, "import_time": 1991, - "filter_immune_slices": ["{}".format(i_e_slc.id)], + "filter_scopes": { + str(i_e_slc.id): { + "region": {"scope": ["ROOT_ID"], "immune": [i_b_slc.id]} + } + }, "expanded_slices": { "{}".format(i_e_slc.id): True, "{}".format(i_b_slc.id): False,