Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(tableau): make Tableau ingestor resilient to timeout exceptions #7333

Merged
merged 9 commits into from
Feb 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions metadata-ingestion/src/datahub/ingestion/source/tableau.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import tableauserverclient as TSC
from pydantic import validator
from pydantic.fields import Field
from requests.adapters import ConnectionError
from tableauserverclient import (
PersonalAccessTokenAuth,
Server,
Expand Down Expand Up @@ -362,8 +363,16 @@ def __init__(
self._authenticate()

def close(self) -> None:
if self.server is not None:
self.server.auth.sign_out()
try:
if self.server is not None:
self.server.auth.sign_out()
except ConnectionError as err:
logger.warning(
"During graceful closing of Tableau source a sign-out call was tried but ended up with"
" a ConnectionError (%s). Continuing closing of the source",
err,
)
self.server = None
super().close()

def _populate_usage_stat_registry(self):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import pytest
from freezegun import freeze_time
from requests.adapters import ConnectionError
from tableauserverclient.models import ViewItem

from datahub.configuration.source_common import DEFAULT_ENV
Expand Down Expand Up @@ -92,6 +93,7 @@ def tableau_ingest_common(
output_file_name,
mock_datahub_graph,
pipeline_config=config_source_default,
sign_out_side_effect=lambda: None,
):
test_resources_dir = pathlib.Path(
pytestconfig.rootpath / "tests/integration/tableau"
Expand All @@ -111,7 +113,7 @@ def tableau_ingest_common(
mock_client.views = mock.Mock()
mock_client.views.get.side_effect = side_effect_usage_stat
mock_client.auth.sign_in.return_value = None
mock_client.auth.sign_out.return_value = None
mock_client.auth.sign_out.side_effect = sign_out_side_effect
mock_sdk.return_value = mock_client
mock_sdk._auth_token = "ABC"

Expand Down Expand Up @@ -414,3 +416,26 @@ def test_tableau_no_verify():
report = source.get_report().as_string()
assert "SSL" not in report
assert "Unable to login" in report


@freeze_time(FROZEN_TIME)
@pytest.mark.slow_unit
def test_tableau_signout_timeout(pytestconfig, tmp_path, mock_datahub_graph):
output_file_name: str = "tableau_mces.json"
golden_file_name: str = "tableau_mces_golden.json"
tableau_ingest_common(
pytestconfig,
tmp_path,
[
read_response(pytestconfig, "workbooksConnection_all.json"),
read_response(pytestconfig, "sheetsConnection_all.json"),
read_response(pytestconfig, "dashboardsConnection_all.json"),
read_response(pytestconfig, "embeddedDatasourcesConnection_all.json"),
read_response(pytestconfig, "publishedDatasourcesConnection_all.json"),
read_response(pytestconfig, "customSQLTablesConnection_all.json"),
],
golden_file_name,
output_file_name,
mock_datahub_graph,
sign_out_side_effect=ConnectionError,
)