From 1baec33899df8394b42b6a667ff2a5d1224b8528 Mon Sep 17 00:00:00 2001 From: Alex Levene <37639249+alevene@users.noreply.github.com> Date: Thu, 27 Aug 2020 17:46:48 -0600 Subject: [PATCH] fix: Set Tableau URLs (base + API) via config (#349) Signed-off-by: Tao Feng --- .../tableau/tableau_dashboard_constants.py | 3 +- .../tableau/tableau_dashboard_extractor.py | 13 ++++---- ...bleau_dashboard_last_modified_extractor.py | 2 +- .../tableau_dashboard_query_extractor.py | 2 +- .../tableau_dashboard_table_extractor.py | 2 +- .../tableau/tableau_dashboard_utils.py | 32 ++++++++++--------- .../tableau_external_table_extractor.py | 2 +- example/scripts/sample_tableau_data_loader.py | 16 ++++++---- .../test_tableau_dashboard_extractor.py | 3 +- ...bleau_dashboard_last_modified_extractor.py | 2 +- .../test_tableau_dashboard_query_extractor.py | 2 +- .../test_tableau_dashboard_table_extractor.py | 2 +- 12 files changed, 44 insertions(+), 37 deletions(-) diff --git a/databuilder/extractor/dashboard/tableau/tableau_dashboard_constants.py b/databuilder/extractor/dashboard/tableau/tableau_dashboard_constants.py index 036318ca3..8df066667 100644 --- a/databuilder/extractor/dashboard/tableau/tableau_dashboard_constants.py +++ b/databuilder/extractor/dashboard/tableau/tableau_dashboard_constants.py @@ -1,5 +1,6 @@ API_VERSION = 'api_version' -TABLEAU_HOST = 'tableau_host' +API_BASE_URL = 'api_base_url' +TABLEAU_BASE_URL = 'tableau_base_url' SITE_NAME = 'site_name' TABLEAU_ACCESS_TOKEN_NAME = 'tableau_personal_access_token_name' TABLEAU_ACCESS_TOKEN_SECRET = 'tableau_personal_access_token_secret' diff --git a/databuilder/extractor/dashboard/tableau/tableau_dashboard_extractor.py b/databuilder/extractor/dashboard/tableau/tableau_dashboard_extractor.py index 06c6e2ccb..85725f45e 100644 --- a/databuilder/extractor/dashboard/tableau/tableau_dashboard_extractor.py +++ b/databuilder/extractor/dashboard/tableau/tableau_dashboard_extractor.py @@ -26,7 +26,7 @@ class TableauGraphQLApiMetadataExtractor(TableauGraphQLApiExtractor): CLUSTER = const.CLUSTER EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS - TABLEAU_HOST = const.TABLEAU_HOST + TABLEAU_BASE_URL = const.TABLEAU_BASE_URL def execute(self) -> Iterator[Dict[str, Any]]: response = self.execute_query() @@ -41,12 +41,12 @@ def execute(self) -> Iterator[Dict[str, Any]]: 'dashboard_name': TableauDashboardUtils.sanitize_workbook_name(workbook['name']), 'description': workbook.get('description', ''), 'created_timestamp': workbook['createdAt'], - 'dashboard_group_url': 'https://{}/#/projects/{}'.format( - self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_HOST), + 'dashboard_group_url': '{}/#/projects/{}'.format( + self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_BASE_URL), workbook['projectVizportalUrlId'] ), - 'dashboard_url': 'https://{}/#/workbooks/{}/views'.format( - self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_HOST), + 'dashboard_url': '{}/#/workbooks/{}/views'.format( + self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_BASE_URL), workbook['vizportalUrlId'] ), 'cluster': self._conf.get_string(TableauGraphQLApiMetadataExtractor.CLUSTER) @@ -66,11 +66,12 @@ class TableauDashboardExtractor(Extractor): Uses the Metadata API: https://help.tableau.com/current/api/metadata_api/en-us/index.html """ + API_BASE_URL = const.API_BASE_URL API_VERSION = const.API_VERSION CLUSTER = const.CLUSTER EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS SITE_NAME = const.SITE_NAME - TABLEAU_HOST = const.TABLEAU_HOST + TABLEAU_BASE_URL = const.TABLEAU_BASE_URL TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET VERIFY_REQUEST = const.VERIFY_REQUEST diff --git a/databuilder/extractor/dashboard/tableau/tableau_dashboard_last_modified_extractor.py b/databuilder/extractor/dashboard/tableau/tableau_dashboard_last_modified_extractor.py index b82cf460e..10dab164a 100644 --- a/databuilder/extractor/dashboard/tableau/tableau_dashboard_last_modified_extractor.py +++ b/databuilder/extractor/dashboard/tableau/tableau_dashboard_last_modified_extractor.py @@ -52,11 +52,11 @@ class TableauDashboardLastModifiedExtractor(Extractor): Dashboard last modified timestamp (Workbook last modified timestamp) """ + API_BASE_URL = const.API_BASE_URL API_VERSION = const.API_VERSION CLUSTER = const.CLUSTER EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS SITE_NAME = const.SITE_NAME - TABLEAU_HOST = const.TABLEAU_HOST TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET VERIFY_REQUEST = const.VERIFY_REQUEST diff --git a/databuilder/extractor/dashboard/tableau/tableau_dashboard_query_extractor.py b/databuilder/extractor/dashboard/tableau/tableau_dashboard_query_extractor.py index 8e87be3fb..4d8b057c1 100644 --- a/databuilder/extractor/dashboard/tableau/tableau_dashboard_query_extractor.py +++ b/databuilder/extractor/dashboard/tableau/tableau_dashboard_query_extractor.py @@ -52,11 +52,11 @@ class TableauDashboardQueryExtractor(Extractor): workbook that uses the query. """ + API_BASE_URL = const.API_BASE_URL API_VERSION = const.API_VERSION CLUSTER = const.CLUSTER EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS SITE_NAME = const.SITE_NAME - TABLEAU_HOST = const.TABLEAU_HOST TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET VERIFY_REQUEST = const.VERIFY_REQUEST diff --git a/databuilder/extractor/dashboard/tableau/tableau_dashboard_table_extractor.py b/databuilder/extractor/dashboard/tableau/tableau_dashboard_table_extractor.py index a184964fa..1e2e4671b 100644 --- a/databuilder/extractor/dashboard/tableau/tableau_dashboard_table_extractor.py +++ b/databuilder/extractor/dashboard/tableau/tableau_dashboard_table_extractor.py @@ -89,13 +89,13 @@ class TableauDashboardTableExtractor(Extractor): Assumes that all the nodes for both the dashboards and the tables have already been created. """ + API_BASE_URL = const.API_BASE_URL API_VERSION = const.API_VERSION CLUSTER = const.CLUSTER DATABASE = const.DATABASE EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS EXTERNAL_CLUSTER_NAME = const.EXTERNAL_CLUSTER_NAME SITE_NAME = const.SITE_NAME - TABLEAU_HOST = const.TABLEAU_HOST TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET VERIFY_REQUEST = const.VERIFY_REQUEST diff --git a/databuilder/extractor/dashboard/tableau/tableau_dashboard_utils.py b/databuilder/extractor/dashboard/tableau/tableau_dashboard_utils.py index 3ddce8428..901140274 100644 --- a/databuilder/extractor/dashboard/tableau/tableau_dashboard_utils.py +++ b/databuilder/extractor/dashboard/tableau/tableau_dashboard_utils.py @@ -65,9 +65,9 @@ class TableauGraphQLApiExtractor(Extractor): Base class for querying the Tableau Metdata API, which uses a GraphQL schema. """ + API_BASE_URL = const.API_BASE_URL QUERY = 'query' QUERY_VARIABLES = 'query_variables' - TABLEAU_HOST = const.TABLEAU_HOST VERIFY_REQUEST = 'verify_request' def init(self, conf: ConfigTree) -> None: @@ -76,11 +76,11 @@ def init(self, conf: ConfigTree) -> None: self._query = self._conf.get(TableauGraphQLApiExtractor.QUERY) self._iterator: Optional[Iterator[Dict[str, Any]]] = None self._static_dict = conf.get(STATIC_RECORD_DICT, dict()) - self._metadata_url = 'https://{TABLEAU_HOST}/api/metadata/graphql'.format( - TABLEAU_HOST=self._conf.get_string(TableauGraphQLApiExtractor.TABLEAU_HOST) + self._metadata_url = '{api_base_url}/api/metadata/graphql'.format( + api_base_url=self._conf.get_string(TableauGraphQLApiExtractor.API_BASE_URL) ) self._query_variables = self._conf.get(TableauGraphQLApiExtractor.QUERY_VARIABLES, {}) - self._verify_request = self._conf.get(TableauGraphQLApiExtractor.VERIFY_REQUEST, True) + self._verify_request = self._conf.get(TableauGraphQLApiExtractor.VERIFY_REQUEST, None) def execute_query(self) -> Dict[str, Any]: """ @@ -95,12 +95,12 @@ def execute_query(self) -> Dict[str, Any]: 'X-Tableau-Auth': self._auth_token } params = { - 'data': query_payload, - 'headers': headers, - 'verify': self._verify_request + 'headers': headers } + if self._verify_request is not None: + params['verify'] = self._verify_request - response = requests.post(url=self._metadata_url, **params) + response = requests.post(url=self._metadata_url, data=query_payload, **params) return response.json()['data'] def execute(self) -> Iterator[Dict[str, Any]]: @@ -136,9 +136,9 @@ class TableauDashboardAuth: https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_auth.htm """ + API_BASE_URL = const.API_BASE_URL API_VERSION = const.API_VERSION SITE_NAME = const.SITE_NAME - TABLEAU_HOST = const.TABLEAU_HOST TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET VERIFY_REQUEST = const.VERIFY_REQUEST @@ -150,8 +150,8 @@ def __init__(self, conf: ConfigTree) -> None: self._access_token_secret = self._conf.get_string(TableauDashboardAuth.TABLEAU_ACCESS_TOKEN_SECRET) self._api_version = self._conf.get_string(TableauDashboardAuth.API_VERSION) self._site_name = self._conf.get_string(TableauDashboardAuth.SITE_NAME) - self._tableau_host = self._conf.get_string(TableauDashboardAuth.TABLEAU_HOST) - self._verify_request = self._conf.get(TableauDashboardAuth.VERIFY_REQUEST, True) + self._api_base_url = self._conf.get_string(TableauDashboardAuth.API_BASE_URL) + self._verify_request = self._conf.get(TableauDashboardAuth.VERIFY_REQUEST, None) @property def token(self) -> Optional[str]: @@ -166,10 +166,11 @@ def _authenticate(self) -> str: See https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_versions.htm for details or ask your Tableau server administrator. """ - self._auth_url = "https://{tableau_host}/api/{api_version}/auth/signin".format( - tableau_host=self._tableau_host, + self._auth_url = "{api_base_url}/api/{api_version}/auth/signin".format( + api_base_url=self._api_base_url, api_version=self._api_version ) + payload = json.dumps({ 'credentials': { 'personalAccessTokenName': self._access_token_name, @@ -185,9 +186,10 @@ def _authenticate(self) -> str: } # verify = False is needed bypass occasional (valid) self-signed cert errors. TODO: actually fix it!! params = { - 'headers': headers, - 'verify': self._verify_request + 'headers': headers } + if self._verify_request is not None: + params['verify'] = self._verify_request response_json = requests.post(url=self._auth_url, data=payload, **params).json() return response_json['credentials']['token'] diff --git a/databuilder/extractor/dashboard/tableau/tableau_external_table_extractor.py b/databuilder/extractor/dashboard/tableau/tableau_external_table_extractor.py index ad969a195..0c6e7baf5 100644 --- a/databuilder/extractor/dashboard/tableau/tableau_external_table_extractor.py +++ b/databuilder/extractor/dashboard/tableau/tableau_external_table_extractor.py @@ -76,6 +76,7 @@ class TableauDashboardExternalTableExtractor(Extractor): googlesheets://external.growth_by_region_county/FY_20_Report """ + API_BASE_URL = const.API_BASE_URL API_VERSION = const.API_VERSION CLUSTER = const.CLUSTER EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS @@ -83,7 +84,6 @@ class TableauDashboardExternalTableExtractor(Extractor): EXTERNAL_SCHEMA_NAME = const.EXTERNAL_SCHEMA_NAME EXTERNAL_TABLE_TYPES = const.EXTERNAL_TABLE_TYPES SITE_NAME = const.SITE_NAME - TABLEAU_HOST = const.TABLEAU_HOST TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET VERIFY_REQUEST = const.VERIFY_REQUEST diff --git a/example/scripts/sample_tableau_data_loader.py b/example/scripts/sample_tableau_data_loader.py index 33162971c..07a093eab 100644 --- a/example/scripts/sample_tableau_data_loader.py +++ b/example/scripts/sample_tableau_data_loader.py @@ -68,7 +68,8 @@ LOGGER = logging.getLogger(__name__) -tableau_host = "" +tableau_base_url = "" +tableau_api_base_url = "" tableau_api_version = 0 tableau_site_name = "" tableau_personal_access_token_name = "" @@ -79,7 +80,7 @@ tableau_external_table_cluster = "" tableau_external_table_schema = "" tableau_external_table_types = [] -tableau_verify_request = True +tableau_verify_request = None common_tableau_config = { 'publisher.neo4j.neo4j_endpoint': neo4j_endpoint, @@ -156,7 +157,8 @@ def run_tableau_metadata_job(): dict_config = common_tableau_config dict_config.update({ - 'extractor.tableau_dashboard_metadata.tableau_host': tableau_host, + 'extractor.tableau_dashboard_metadata.api_base_url': tableau_api_base_url, + 'extractor.tableau_dashboard_metadata.tableau_base_url': tableau_base_url, 'extractor.tableau_dashboard_metadata.api_version': tableau_api_version, 'extractor.tableau_dashboard_metadata.site_name': tableau_site_name, 'extractor.tableau_dashboard_metadata.tableau_personal_access_token_name': @@ -195,7 +197,7 @@ def run_tableau_last_modified_job(): dict_config = common_tableau_config dict_config.update({ - 'extractor.tableau_dashboard_last_modified.tableau_host': tableau_host, + 'extractor.tableau_dashboard_last_modified.api_base_url': tableau_api_base_url, 'extractor.tableau_dashboard_last_modified.api_version': tableau_api_version, 'extractor.tableau_dashboard_last_modified.site_name': tableau_site_name, 'extractor.tableau_dashboard_last_modified.tableau_personal_access_token_name': @@ -234,7 +236,7 @@ def run_tableau_query_job(): dict_config = common_tableau_config dict_config.update({ - 'extractor.tableau_dashboard_query.tableau_host': tableau_host, + 'extractor.tableau_dashboard_query.api_base_url': tableau_api_base_url, 'extractor.tableau_dashboard_query.api_version': tableau_api_version, 'extractor.tableau_dashboard_query.site_name': tableau_site_name, 'extractor.tableau_dashboard_query.tableau_personal_access_token_name': @@ -273,7 +275,7 @@ def run_tableau_table_job(): dict_config = common_tableau_config dict_config.update({ - 'extractor.tableau_dashboard_table.tableau_host': tableau_host, + 'extractor.tableau_dashboard_table.api_base_url': tableau_api_base_url, 'extractor.tableau_dashboard_table.api_version': tableau_api_version, 'extractor.tableau_dashboard_table.site_name': tableau_site_name, 'extractor.tableau_dashboard_table.tableau_personal_access_token_name': @@ -313,7 +315,7 @@ def run_tableau_external_table_job(): dict_config = common_tableau_config dict_config.update({ - 'extractor.tableau_external_table.tableau_host': tableau_host, + 'extractor.tableau_external_table.api_base_url': tableau_api_base_url, 'extractor.tableau_external_table.api_version': tableau_api_version, 'extractor.tableau_external_table.site_name': tableau_site_name, 'extractor.tableau_external_table.tableau_personal_access_token_name': diff --git a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_extractor.py b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_extractor.py index e2a6f54ef..4ba14fd15 100644 --- a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_extractor.py +++ b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_extractor.py @@ -44,7 +44,8 @@ class TestTableauDashboardExtractor(unittest.TestCase): def test_dashboard_metadata_extractor(self) -> None: config = ConfigFactory.from_dict({ - 'extractor.tableau_dashboard_metadata.tableau_host': 'tableau_host', + 'extractor.tableau_dashboard_metadata.api_base_url': 'api_base_url', + 'extractor.tableau_dashboard_metadata.tableau_base_url': 'tableau_base_url', 'extractor.tableau_dashboard_metadata.api_version': 'tableau_api_version', 'extractor.tableau_dashboard_metadata.site_name': 'tableau_site_name', 'extractor.tableau_dashboard_metadata.tableau_personal_access_token_name': diff --git a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_last_modified_extractor.py b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_last_modified_extractor.py index f0a315037..77e66a001 100644 --- a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_last_modified_extractor.py +++ b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_last_modified_extractor.py @@ -42,7 +42,7 @@ class TestTableauDashboardLastModified(unittest.TestCase): def test_dashboard_last_modified_extractor(self) -> None: config = ConfigFactory.from_dict({ - 'extractor.tableau_dashboard_last_modified.tableau_host': 'tableau_host', + 'extractor.tableau_dashboard_last_modified.api_base_url': 'api_base_url', 'extractor.tableau_dashboard_last_modified.api_version': 'tableau_api_version', 'extractor.tableau_dashboard_last_modified.site_name': 'tableau_site_name', 'extractor.tableau_dashboard_last_modified.tableau_personal_access_token_name': diff --git a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_query_extractor.py b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_query_extractor.py index 96dcd2a4e..18e282fa8 100644 --- a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_query_extractor.py +++ b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_query_extractor.py @@ -46,7 +46,7 @@ class TestTableauDashboardQuery(unittest.TestCase): def test_dashboard_query_extractor(self) -> None: config = ConfigFactory.from_dict({ - 'extractor.tableau_dashboard_query.tableau_host': 'tableau_host', + 'extractor.tableau_dashboard_query.api_base_url': 'api_base_url', 'extractor.tableau_dashboard_query.api_version': 'tableau_api_version', 'extractor.tableau_dashboard_query.site_name': 'tableau_site_name', 'extractor.tableau_dashboard_query.tableau_personal_access_token_name': diff --git a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_table_extractor.py b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_table_extractor.py index ca327f74d..47a84b9e0 100644 --- a/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_table_extractor.py +++ b/tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_table_extractor.py @@ -57,7 +57,7 @@ class TestTableauDashboardTable(unittest.TestCase): def test_dashboard_table_extractor(self) -> None: config = ConfigFactory.from_dict({ - 'extractor.tableau_dashboard_table.tableau_host': 'tableau_host', + 'extractor.tableau_dashboard_table.api_base_url': 'api_base_url', 'extractor.tableau_dashboard_table.api_version': 'tableau_api_version', 'extractor.tableau_dashboard_table.site_name': 'tableau_site_name', 'extractor.tableau_dashboard_table.tableau_personal_access_token_name':