Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improving speed of dashboard import #3958

Merged
merged 2 commits into from Dec 1, 2017
Merged

Improving speed of dashboard import #3958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
99feaf9
Improve dashboard import
michellethomas Nov 30, 2017
4c112b3
Updating tests for Slice.import_obj
michellethomas Nov 30, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 13 additions & 9 deletions superset/models/core.py
View file
Open in desktop
Expand Up @@ -248,26 +248,24 @@ def get_viz(self):
)

@classmethod
def import_obj(cls, slc_to_import, import_time=None):
def import_obj(cls, slc_to_import, slc_to_override, import_time=None):
"""Inserts or overrides slc in the database.

remote_id and import_time fields in params_dict are set to track the
slice origin and ensure correct overrides for multiple imports.
Slice.perm is used to find the datasources and connect them.

:param Slice slc_to_import: Slice object to import
:param Slice slc_to_override: Slice to replace, id matches remote_id
:returns: The resulting id for the imported slice
:rtype: int
"""
session = db.session
make_transient(slc_to_import)
slc_to_import.dashboards = []
slc_to_import.alter_params(
remote_id=slc_to_import.id, import_time=import_time)

# find if the slice was already imported
slc_to_override = None
for slc in session.query(Slice).all():
if ('remote_id' in slc.params_dict and
slc.params_dict['remote_id'] == slc_to_import.id):
slc_to_override = slc

slc_to_import = slc_to_import.copy()
params = slc_to_import.params_dict
slc_to_import.datasource_id = ConnectorRegistry.get_datasource_by_name(
Expand Down Expand Up @@ -432,10 +430,16 @@ def alter_positions(dashboard, old_to_new_slc_id_dict):
new_timed_refresh_immune_slices = []
new_expanded_slices = {}
i_params_dict = dashboard_to_import.params_dict
remote_id_slice_map = {
slc.params_dict['remote_id']: slc
for slc in session.query(Slice).all()
if 'remote_id' in slc.params_dict
}
for slc in slices:
logging.info('Importing slice {} from the dashboard: {}'.format(
slc.to_json(), dashboard_to_import.dashboard_title))
new_slc_id = Slice.import_obj(slc, import_time=import_time)
remote_slc = remote_id_slice_map.get(slc.id)
new_slc_id = Slice.import_obj(slc, remote_slc, import_time=import_time)
old_to_new_slc_id_dict[slc.id] = new_slc_id
# update json metadata that deals with slice ids
new_slc_id_str = '{}'.format(new_slc_id)
Expand Down
18 changes: 10 additions & 8 deletions tests/import_export_tests.py
View file
Open in desktop
Expand Up @@ -248,7 +248,7 @@ def test_export_2_dashboards(self):

def test_import_1_slice(self):
expected_slice = self.create_slice('Import Me', id=10001)
slc_id = models.Slice.import_obj(expected_slice, import_time=1989)
slc_id = models.Slice.import_obj(expected_slice, None, import_time=1989)
slc = self.get_slice(slc_id)
self.assertEquals(slc.datasource.perm, slc.perm)
self.assert_slice_equals(expected_slice, slc)
Expand All @@ -260,9 +260,9 @@ def test_import_2_slices_for_same_table(self):
table_id = self.get_table_by_name('wb_health_population').id
# table_id != 666, import func will have to find the table
slc_1 = self.create_slice('Import Me 1', ds_id=666, id=10002)
slc_id_1 = models.Slice.import_obj(slc_1)
slc_id_1 = models.Slice.import_obj(slc_1, None)
slc_2 = self.create_slice('Import Me 2', ds_id=666, id=10003)
slc_id_2 = models.Slice.import_obj(slc_2)
slc_id_2 = models.Slice.import_obj(slc_2, None)

imported_slc_1 = self.get_slice(slc_id_1)
imported_slc_2 = self.get_slice(slc_id_2)
Expand All @@ -277,17 +277,19 @@ def test_import_2_slices_for_same_table(self):
def test_import_slices_for_non_existent_table(self):
with self.assertRaises(IndexError):
models.Slice.import_obj(self.create_slice(
'Import Me 3', id=10004, table_name='non_existent'))
'Import Me 3', id=10004, table_name='non_existent'), None)

def test_import_slices_override(self):
slc = self.create_slice('Import Me New', id=10005)
slc_1_id = models.Slice.import_obj(slc, import_time=1990)
slc_1_id = models.Slice.import_obj(slc, None, import_time=1990)
slc.slice_name = 'Import Me New'
imported_slc_1 = self.get_slice(slc_1_id)
slc_2 = self.create_slice('Import Me New', id=10005)
slc_2_id = models.Slice.import_obj(
self.create_slice('Import Me New', id=10005), import_time=1990)
slc_2, imported_slc_1, import_time=1990)
self.assertEquals(slc_1_id, slc_2_id)
imported_slc = self.get_slice(slc_2_id)
self.assert_slice_equals(slc, imported_slc)
imported_slc_2 = self.get_slice(slc_2_id)
self.assert_slice_equals(slc, imported_slc_2)

def test_import_empty_dashboard(self):
empty_dash = self.create_dashboard('empty_dashboard', id=10001)
Expand Down