Skip to content

Commit

Permalink
vdk-impala: uncomment tests that were not passing due to core change (#…
Browse files Browse the repository at this point in the history 
…2845)

Uncomment tests after
#2840

Signed-off-by: Dilyan Marinov <mdilyan@vmware.com>
Co-authored-by: Dilyan Marinov <mdilyan@vmware.com>
  • Loading branch information
@DeltaMichael
DeltaMichael and Dilyan Marinov committed Oct 27, 2023
1 parent 6743881 commit 895ee29
Show file tree
Hide file tree
Showing 2 changed files with 154 additions and 160 deletions.
53 changes: 26 additions & 27 deletions projects/vdk-plugins/vdk-csv/tests/functional/test_csv_plugin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,33 +151,32 @@ def test_export_csv_with_already_existing_file(tmpdir):
cli_assert_equal(1, result)


# TODO: Uncomment after https://github.com/vmware/versatile-data-kit/pull/2840 is merged
# def test_csv_export_with_nonexistent_table(tmpdir):
# db_dir = str(tmpdir) + "vdk-sqlite.db"
# with mock.patch.dict(
# os.environ,
# {
# "VDK_DB_DEFAULT_TYPE": "SQLITE",
# "VDK_SQLITE_FILE": db_dir,
# },
# ):
# runner = CliEntryBasedTestRunner(sqlite_plugin, csv_plugin)
# drop_table(runner, "test_table")
# result = runner.invoke(
# [
# "export-csv",
# "--query",
# "SELECT * FROM test_table",
# "--file",
# "result3.csv",
# ]
# )
# assert isinstance(result.exception, OperationalError)
# assert hasattr(result.exception, "_vdk_resolvable_actual")
# assert (
# getattr(result.exception, "_vdk_resolvable_actual")
# == ResolvableByActual.USER
# )
def test_csv_export_with_nonexistent_table(tmpdir):
db_dir = str(tmpdir) + "vdk-sqlite.db"
with mock.patch.dict(
os.environ,
{
"VDK_DB_DEFAULT_TYPE": "SQLITE",
"VDK_SQLITE_FILE": db_dir,
},
):
runner = CliEntryBasedTestRunner(sqlite_plugin, csv_plugin)
drop_table(runner, "test_table")
result = runner.invoke(
[
"export-csv",
"--query",
"SELECT * FROM test_table",
"--file",
"result3.csv",
]
)
assert isinstance(result.exception, OperationalError)
assert hasattr(result.exception, "_vdk_resolvable_actual")
assert (
getattr(result.exception, "_vdk_resolvable_actual")
== ResolvableByActual.USER
)


def test_csv_export_with_no_data(tmpdir):
Expand Down
261 changes: 128 additions & 133 deletions projects/vdk-plugins/vdk-impala/tests/functional/template_regression_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,26 +43,25 @@ def setUp(self) -> None:
self._run_query("CREATE DATABASE IF NOT EXISTS vdkprototypes")
self._run_query("CREATE DATABASE IF NOT EXISTS staging_vdkprototypes")

# TODO: Uncomment after https://github.com/vmware/versatile-data-kit/pull/2840 is merged
# def test_load_dimension_scd1(self) -> None:
# test_schema = "vdkprototypes"
# source_view = "vw_dim_org"
# target_table = "dw_dim_org"

# res = self._run_job(
# "load_dimension_scd1_template_job",
# {
# "source_schema": test_schema,
# "source_view": source_view,
# "target_schema": test_schema,
# "target_table": target_table,
# },
# )
# cli_assert(not res.exception, res)
# actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
# expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{source_view}")
# assert actual_rs.output and expected_rs.output
# assert actual_rs.output == expected_rs.output
def test_load_dimension_scd1(self) -> None:
test_schema = "vdkprototypes"
source_view = "vw_dim_org"
target_table = "dw_dim_org"

res = self._run_job(
"load_dimension_scd1_template_job",
{
"source_schema": test_schema,
"source_view": source_view,
"target_schema": test_schema,
"target_table": target_table,
},
)
cli_assert(not res.exception, res)
actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{source_view}")
assert actual_rs.output and expected_rs.output
assert actual_rs.output == expected_rs.output

def test_load_dimension_scd1_partitioned(self) -> None:
test_schema = "vdkprototypes"
Expand Down Expand Up @@ -91,20 +90,19 @@ def test_load_dimension_scd1_partitioned(self) -> None:
expected, actual, f"Elements in {source_view} and {target_table} differ."
)

# TODO: Uncomment after https://github.com/vmware/versatile-data-kit/pull/2840 is merged
# def test_load_dimension_scd1_parameter_validation(self) -> None:
# self._run_template_with_bad_arguments(
# template_name="load_dimension_scd1_template_only",
# template_args={},
# num_exp_errors=4,
# params_class_name="SlowlyChangingDimensionTypeOverwriteParams",
# )
# self._run_template_with_bad_arguments(
# template_name="load_dimension_scd1_template_only",
# template_args={"source_view": "foo", "extra_parameter": "bar"},
# num_exp_errors=3,
# params_class_name="SlowlyChangingDimensionTypeOverwriteParams",
# )
def test_load_dimension_scd1_parameter_validation(self) -> None:
self._run_template_with_bad_arguments(
template_name="load_dimension_scd1_template_only",
template_args={},
num_exp_errors=4,
params_class_name="SlowlyChangingDimensionTypeOverwriteParams",
)
self._run_template_with_bad_arguments(
template_name="load_dimension_scd1_template_only",
template_args={"source_view": "foo", "extra_parameter": "bar"},
num_exp_errors=3,
params_class_name="SlowlyChangingDimensionTypeOverwriteParams",
)

def test_load_dimension_scd1_bad_target_schema(self) -> None:
template_args = {
Expand Down Expand Up @@ -202,20 +200,19 @@ def test_load_dimension_scd2(self) -> None:
expected, actual, f"Elements in {expect_table} and {target_table} differ."
)

# TODO: Uncomment after https://github.com/vmware/versatile-data-kit/pull/2840 is merged
# def test_load_dimension_scd2_parameter_validation(self) -> None:
# self._run_template_with_bad_arguments(
# template_name="load_dimension_scd2_template_only",
# template_args={},
# num_exp_errors=9,
# params_class_name="SlowlyChangingDimensionType2Params",
# )
# self._run_template_with_bad_arguments(
# template_name="load_dimension_scd2_template_only",
# template_args={"source_view": "foo", "extra_parameter": "bar"},
# num_exp_errors=8,
# params_class_name="SlowlyChangingDimensionType2Params",
# )
def test_load_dimension_scd2_parameter_validation(self) -> None:
self._run_template_with_bad_arguments(
template_name="load_dimension_scd2_template_only",
template_args={},
num_exp_errors=9,
params_class_name="SlowlyChangingDimensionType2Params",
)
self._run_template_with_bad_arguments(
template_name="load_dimension_scd2_template_only",
template_args={"source_view": "foo", "extra_parameter": "bar"},
num_exp_errors=8,
params_class_name="SlowlyChangingDimensionType2Params",
)

def test_load_dimension_scd2_bad_target_schema(self) -> None:
template_args = {
Expand Down Expand Up @@ -334,77 +331,76 @@ def test_load_versioned_partitioned(self) -> None:
actual, expected, f"Elements in {expect_table} and {target_table} differ."
)

# TODO: Uncomment after https://github.com/vmware/versatile-data-kit/pull/2840 is merged
# def test_load_versioned_parameter_validation(self) -> None:
# self._run_template_with_bad_arguments(
# template_name="load_versioned_template_only",
# template_args={},
# num_exp_errors=7,
# params_class_name="LoadVersionedParams",
# )

# good_template_args = {
# "source_schema": "vdkprototypes",
# "source_view": "vw_sddc_h_updates",
# "target_schema": "vdkprototypes",
# "target_table": "dim_sddc_h_as_textfile",
# "id_column": "sddc_id",
# "sk_column": "sddc_sk",
# "value_columns": [
# "updated_by_user_id",
# "state",
# "is_nsxt",
# "cloud_vendor",
# "version",
# ],
# "tracked_columns": ["updated_by_user_id", "state", "is_nsxt", "version"],
# "active_from_column": "active_from",
# "active_to_column": "active_to",
# "active_to_max_value": "9999-12-31",
# "updated_at_column": "updated_at",
# "extra_parameter": "bar",
# }

# self._run_template_with_bad_arguments(
# template_name="load_versioned_template_only",
# template_args={
# **good_template_args,
# **{
# "value_columns": [
# "updated_by_user_id",
# "state",
# "is_nsxt",
# "cloud_vendor",
# ],
# "tracked_columns": [
# "updated_by_user_id",
# "state",
# "is_nsxt",
# "version",
# ],
# },
# },
# num_exp_errors=1,
# params_class_name="LoadVersionedParams",
# )

# self._run_template_with_bad_arguments(
# template_name="load_versioned_template_only",
# template_args={
# **good_template_args,
# **{
# "value_columns": [
# "updated_by_user_id",
# "state",
# "is_nsxt",
# "cloud_vendor",
# ],
# "tracked_columns": [],
# },
# },
# num_exp_errors=1,
# params_class_name="LoadVersionedParams",
# )
def test_load_versioned_parameter_validation(self) -> None:
self._run_template_with_bad_arguments(
template_name="load_versioned_template_only",
template_args={},
num_exp_errors=7,
params_class_name="LoadVersionedParams",
)

good_template_args = {
"source_schema": "vdkprototypes",
"source_view": "vw_sddc_h_updates",
"target_schema": "vdkprototypes",
"target_table": "dim_sddc_h_as_textfile",
"id_column": "sddc_id",
"sk_column": "sddc_sk",
"value_columns": [
"updated_by_user_id",
"state",
"is_nsxt",
"cloud_vendor",
"version",
],
"tracked_columns": ["updated_by_user_id", "state", "is_nsxt", "version"],
"active_from_column": "active_from",
"active_to_column": "active_to",
"active_to_max_value": "9999-12-31",
"updated_at_column": "updated_at",
"extra_parameter": "bar",
}

self._run_template_with_bad_arguments(
template_name="load_versioned_template_only",
template_args={
**good_template_args,
**{
"value_columns": [
"updated_by_user_id",
"state",
"is_nsxt",
"cloud_vendor",
],
"tracked_columns": [
"updated_by_user_id",
"state",
"is_nsxt",
"version",
],
},
},
num_exp_errors=1,
params_class_name="LoadVersionedParams",
)

self._run_template_with_bad_arguments(
template_name="load_versioned_template_only",
template_args={
**good_template_args,
**{
"value_columns": [
"updated_by_user_id",
"state",
"is_nsxt",
"cloud_vendor",
],
"tracked_columns": [],
},
},
num_exp_errors=1,
params_class_name="LoadVersionedParams",
)

def test_load_versioned_bad_target_schema(self) -> None:
template_args = {
Expand Down Expand Up @@ -528,20 +524,19 @@ def test_load_fact_snapshot_partition(self) -> None:
actual, expected, f"Elements in {expect_table} and {target_table} differ."
)

# TODO: Uncomment after https://github.com/vmware/versatile-data-kit/pull/2840 is merged
# def test_load_fact_snapshot_parameter_validation(self) -> None:
# self._run_template_with_bad_arguments(
# template_name="load_fact_snapshot_template_only",
# template_args={},
# num_exp_errors=5,
# params_class_name="FactDailySnapshotParams",
# )
# self._run_template_with_bad_arguments(
# template_name="load_fact_snapshot_template_only",
# template_args={"source_view": "foo", "target_table": None},
# num_exp_errors=4,
# params_class_name="FactDailySnapshotParams",
# )
def test_load_fact_snapshot_parameter_validation(self) -> None:
self._run_template_with_bad_arguments(
template_name="load_fact_snapshot_template_only",
template_args={},
num_exp_errors=5,
params_class_name="FactDailySnapshotParams",
)
self._run_template_with_bad_arguments(
template_name="load_fact_snapshot_template_only",
template_args={"source_view": "foo", "target_table": None},
num_exp_errors=4,
params_class_name="FactDailySnapshotParams",
)

def test_load_fact_snapshot_bad_target_schema(self) -> None:
template_args = {
Expand Down

0 comments on commit 895ee29

Please sign in to comment.