Skip to content

Commit

Permalink
Implement evrs
Browse files Browse the repository at this point in the history
  • Loading branch information
abegong committed Jun 13, 2019
1 parent 6156aa2 commit e4f5f37
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 8 deletions.
2 changes: 1 addition & 1 deletion great_expectations/profile/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def profile(cls, dataset):

batch_kwargs = dataset.get_batch_kwargs()
expectations_config = cls.add_meta(expectations_config, batch_kwargs)
validation_results = None # dataset.validate(expectations_config)
validation_results = dataset.validate(expectations_config)
return expectations_config, validation_results

@classmethod
Expand Down
31 changes: 24 additions & 7 deletions tests/test_profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,10 @@ def test_BasicDatasetProfiler_with_context(empty_data_context, filesystem_csv_2)
not_so_empty_data_context = empty_data_context

batch = not_so_empty_data_context.get_batch("my_datasource", "f1")
expectations_config, evr_config = BasicDatasetProfiler.profile(batch)
expectations_config, validation_results = BasicDatasetProfiler.profile(
batch)

print(batch.get_batch_kwargs())
# print(batch.get_batch_kwargs())
# print(json.dumps(expectations_config, indent=2))

assert expectations_config["data_asset_name"] == "f1"
Expand All @@ -125,6 +126,13 @@ def test_BasicDatasetProfiler_with_context(empty_data_context, filesystem_csv_2)
"confidence": "very low"
}

print(json.dumps(validation_results, indent=2))

assert validation_results["meta"]["data_asset_name"] == "f1"
assert set(validation_results["meta"].keys()) == {
"great_expectations.__version__", "data_asset_name", "run_id", "batch_kwargs"
}


@pytest.fixture()
def filesystem_csv_2(tmp_path_factory):
Expand All @@ -140,14 +148,23 @@ def filesystem_csv_2(tmp_path_factory):
def test_context_profiler(empty_data_context, filesystem_csv_2):
empty_data_context.add_datasource(
"my_datasource", "pandas", base_directory=str(filesystem_csv_2))
not_so_empty_data_context = empty_data_context

assert empty_data_context.list_expectations_configs() == []
empty_data_context.profile_datasource("my_datasource")
assert not_so_empty_data_context.list_expectations_configs() == []
not_so_empty_data_context.profile_datasource("my_datasource")

print(empty_data_context.list_expectations_configs())
assert empty_data_context.list_expectations_configs() != []
print(not_so_empty_data_context.list_expectations_configs())
assert not_so_empty_data_context.list_expectations_configs() != []

profiled_expectations = empty_data_context.get_expectations('f1')
profiled_expectations = not_so_empty_data_context.get_expectations('f1')
print(json.dumps(profiled_expectations, indent=2))

assert len(profiled_expectations["expectations"]) > 0

# print(json.dumps(validation_results, indent=2))

# # Note: deliberately not testing context file storage in this test.
# context_expectations_config = not_so_empty_data_context.get_expectations(
# "my_datasource", "f1")

# assert context_expectations_config == profiled_expectations

0 comments on commit e4f5f37

Please sign in to comment.