Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: fix redash dashboard exporter #422

Merged
merged 2 commits into from
Dec 9, 2020
Merged

fix: fix redash dashboard exporter #422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
40f6dd4
fix redash dashboard exporter
gjxdxh Dec 8, 2020
6fd743a
fix mypy issue
gjxdxh Dec 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 16 additions & 6 deletions databuilder/extractor/dashboard/redash/redash_dashboard_extractor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from databuilder.models.dashboard.dashboard_owner import DashboardOwner
from databuilder.models.dashboard.dashboard_query import DashboardQuery
from databuilder.models.dashboard.dashboard_table import DashboardTable
from databuilder.models.dashboard.dashboard_chart import DashboardChart
from databuilder.models.table_metadata import TableMetadata
from databuilder.extractor.base_extractor import Extractor
from databuilder.rest_api.rest_api_query import RestApiQuery
Expand Down Expand Up @@ -113,8 +114,8 @@ def _get_extract_iter(self) -> Iterator[Any]:
identity_data = {
'cluster': self._cluster,
'product': RedashDashboardExtractor.PRODUCT,
'dashboard_group_id': RedashDashboardExtractor.DASHBOARD_GROUP_ID,
'dashboard_id': record['dashboard_id']
'dashboard_group_id': str(RedashDashboardExtractor.DASHBOARD_GROUP_ID),
'dashboard_id': str(record['dashboard_id'])
}

dash_data = {
Expand All @@ -125,8 +126,8 @@ def _get_extract_iter(self) -> Iterator[Any]:
'dashboard_name':
record['dashboard_name'],
'dashboard_url':
'{redash}/dashboard/{slug}'
.format(redash=self._redash_base_url, slug=record['slug']),
'{redash}/dashboards/{id}'
.format(redash=self._redash_base_url, id=record['dashboard_id']),
'created_timestamp':
record['created_timestamp']
}
Expand Down Expand Up @@ -155,7 +156,7 @@ def _get_extract_iter(self) -> Iterator[Any]:

for viz in viz_widgets:
query_data = {
'query_id': viz.query_id,
'query_id': str(viz.query_id),
'query_name': viz.query_name,
'url': self._redash_base_url + viz.query_relative_url,
'query_text': viz.raw_query
Expand All @@ -164,6 +165,15 @@ def _get_extract_iter(self) -> Iterator[Any]:
query_data.update(identity_data)
yield DashboardQuery(**query_data)

chart_data = {
'query_id': str(viz.query_id),
'chart_id': str(viz.visualization_id),
'chart_name': viz.visualization_name,
'chart_type': viz.visualization_type,
}
chart_data.update(identity_data)
yield DashboardChart(**chart_data)

# if a table parser is provided, retrieve tables from this viz
if self._parse_tables:
for tbl in self._parse_tables(viz):
Expand Down Expand Up @@ -197,7 +207,7 @@ def _build_restapi_query(self) -> RestApiQuery:

return RestApiQuery(
query_to_join=dashes_query,
url='{redash_api}/dashboards/{{slug}}'.format(redash_api=self._api_base_url),
url='{redash_api}/dashboards/{{dashboard_id}}'.format(redash_api=self._api_base_url),
params=self._get_default_api_query_params(),
json_path='widgets',
field_names=['widgets'],
Expand Down
14 changes: 13 additions & 1 deletion databuilder/extractor/dashboard/redash/redash_dashboard_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,18 @@ def query_relative_url(self) -> str:
def query_name(self) -> str:
return self._data['visualization']['query']['name']

@property
def visualization_id(self) -> int:
return self._data['visualization']['id']

@property
def visualization_name(self) -> str:
return self._data['visualization']['name']

@property
def visualization_type(self) -> str:
return self._data['visualization']['type']


class RedashTextWidget:
"""
Expand Down Expand Up @@ -135,7 +147,7 @@ def generate_dashboard_description(text_widgets: List[RedashTextWidget],
if len(text_widgets) > 0:
return '\n\n'.join([w.text for w in text_widgets])
elif len(viz_widgets) > 0:
query_list = '\n'.join(['- {}'.format(v.query_name) for v in set(viz_widgets)])
query_list = '\n'.join(set(['- {}'.format(v.query_name) for v in set(viz_widgets)]))
return 'A dashboard containing the following queries:\n\n' + query_list

return 'This dashboard appears to be empty!'
31 changes: 23 additions & 8 deletions tests/unit/extractor/dashboard/redash/test_redash_dashboard_extractor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from databuilder.models.dashboard.dashboard_owner import DashboardOwner
from databuilder.models.dashboard.dashboard_query import DashboardQuery
from databuilder.models.dashboard.dashboard_table import DashboardTable
from databuilder.models.dashboard.dashboard_chart import DashboardChart


logging.basicConfig(level=logging.INFO)
Expand Down Expand Up @@ -43,18 +44,21 @@ def test_table_relation_data(self) -> None:

def test_with_one_dashboard(self) -> None:
def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:
if 'test-dash' in url:
if '1000' in url:
return MockApiResponse({
'id': 123,
'id': 1000,
'widgets': [
{
'visualization': {
'query': {
'data_source_id': 1,
'id': '1234',
'id': 1234,
'name': 'Test Query',
'query': 'SELECT id FROM users'
}
},
'id': 12345,
'name': 'test_widget',
'type': 'CHART',
},
'options': {}
}
Expand All @@ -67,7 +71,7 @@ def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:
'page_size': 50,
'results': [
{
'id': 123,
'id': 1000,
'name': 'Test Dash',
'slug': 'test-dash',
'created_at': '2020-01-01T00:00:00.000Z',
Expand Down Expand Up @@ -96,20 +100,20 @@ def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:

# DashboardMetadata
record = extractor.extract()
self.assertEqual(record.dashboard_id, 123)
self.assertEqual(record.dashboard_id, '1000')
self.assertEqual(record.dashboard_name, 'Test Dash')
self.assertEqual(record.dashboard_group_id, RedashDashboardExtractor.DASHBOARD_GROUP_ID)
self.assertEqual(record.dashboard_group, RedashDashboardExtractor.DASHBOARD_GROUP_NAME)
self.assertEqual(record.product, RedashDashboardExtractor.PRODUCT)
self.assertEqual(record.cluster, RedashDashboardExtractor.DEFAULT_CLUSTER)
self.assertEqual(record.created_timestamp, 1577836800)
self.assertTrue(redash_base_url in record.dashboard_url)
self.assertTrue('test-dash' in record.dashboard_url)
self.assertTrue('1000' in record.dashboard_url)

# DashboardLastModified
record = extractor.extract()
identity: Dict[str, Any] = {
'dashboard_id': 123,
'dashboard_id': '1000',
'dashboard_group_id': RedashDashboardExtractor.DASHBOARD_GROUP_ID,
'product': RedashDashboardExtractor.PRODUCT,
'cluster': u'prod'
Expand All @@ -136,6 +140,17 @@ def mock_api_get(url: str, *args: Any, **kwargs: Any) -> MockApiResponse:
)
self.assertEqual(record.__repr__(), expected_query.__repr__())

# DashboardChart
record = extractor.extract()
expected_chart = DashboardChart(
query_id='1234',
chart_id='12345',
chart_name='test_widget',
chart_type='CHART',
**identity
)
self.assertEqual(record.__repr__(), expected_chart.__repr__())

# DashboardTable
record = extractor.extract()
expected_table = DashboardTable(
Expand Down
34 changes: 33 additions & 1 deletion tests/unit/extractor/dashboard/redash/test_redash_dashboard_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ def test_visualization_widget_props(self) -> None:
'data_source_id': 1,
'query': 'SELECT 2+2 FROM DUAL',
'name': 'Test'
}
},
'id': 12345,
'name': 'test_widget',
'type': 'CHART'
}
}
widget = get_visualization_widgets([widget_data])[0]
Expand All @@ -75,6 +78,9 @@ def test_visualization_widget_props(self) -> None:
self.assertEqual(widget.data_source_id, 1)
self.assertEqual(widget.raw_query, 'SELECT 2+2 FROM DUAL')
self.assertEqual(widget.query_name, 'Test')
self.assertEqual(widget.visualization_id, 12345)
self.assertEqual(widget.visualization_name, 'test_widget')
self.assertEqual(widget.visualization_type, 'CHART')

def test_descriptions_from_text(self) -> None:
text_widgets = get_text_widgets([
Expand Down Expand Up @@ -123,6 +129,32 @@ def test_descriptions_from_text(self) -> None:
desc4 = generate_dashboard_description([], [])
self.assertTrue('empty' in desc4)

def test_descriptions_remove_duplicate(self) -> None:
viz_widgets = get_visualization_widgets([
{
'visualization': {
'query': {
'id': 1,
'data_source_id': 1,
'name': 'same_query_name',
'query': 'n/a'
}
}
},
{
'visualization': {
'query': {
'id': 2,
'data_source_id': 1,
'name': 'same_query_name',
'query': 'n/a'
}
}
}
])
desc1 = generate_dashboard_description([], viz_widgets)
self.assertEqual('A dashboard containing the following queries:\n\n- same_query_name', desc1)

def test_auth_headers(self) -> None:
headers = get_auth_headers('testkey')
self.assertTrue('testkey' in headers['Authorization'])
Expand Down