diff --git a/great_expectations/datasource/fluent/config.py b/great_expectations/datasource/fluent/config.py index 2c6b2abf8dee..c60bce4701ec 100644 --- a/great_expectations/datasource/fluent/config.py +++ b/great_expectations/datasource/fluent/config.py @@ -74,6 +74,14 @@ def _load_datasource_subtype(cls, v: Dict[str, dict]): f"'{ds_name}' has unsupported 'type' - {type_lookup_err}" ) from type_lookup_err + config["name"] = ds_name + + if "assets" not in config: + config["assets"] = {} + + for asset_name, asset_config in config["assets"].items(): + asset_config["name"] = asset_name + datasource = ds_type(**config) # the ephemeral asset should never be serialized @@ -96,6 +104,7 @@ def _load_datasource_subtype(cls, v: Dict[str, dict]): if v and not loaded_datasources: logger.info(f"Of {len(v)} entries, no 'datasources' could be loaded") + return loaded_datasources @classmethod @@ -127,4 +136,6 @@ def parse_yaml( "`_allow_empty` does not prevent unrelated validation errors" ) raise + + # noinspection PyTypeChecker return super().parse_yaml(f) diff --git a/tests/datasource/fluent/great_expectations.yml b/tests/datasource/fluent/great_expectations.yml index 4a900509815a..c9a9dd1f0561 100644 --- a/tests/datasource/fluent/great_expectations.yml +++ b/tests/datasource/fluent/great_expectations.yml @@ -1,23 +1,19 @@ fluent_datasources: my_pg_ds: type: postgres - name: my_pg_ds connection_string: "postgresql://userName:@hostname/dbName" assets: my_table_asset_wo_splitters: type: table - name: my_table_asset_wo_splitters table_name: my_table with_splitter: type: table - name: with_splitter table_name: another_table splitter: - method_name: split_on_year_and_month - column_name: my_column + method_name: split_on_year_and_month + column_name: my_column with_sorters: type: table - name: with_sorters table_name: another_table order_by: - key: year @@ -25,7 +21,6 @@ fluent_datasources: reverse: true with_dsl_ish_sorters: type: table - name: with_dsl_ish_sorters table_name: another_table order_by: - year diff --git a/tests/datasource/fluent/test_config.py b/tests/datasource/fluent/test_config.py index a7e78480d0be..5a7aad97049c 100644 --- a/tests/datasource/fluent/test_config.py +++ b/tests/datasource/fluent/test_config.py @@ -38,11 +38,9 @@ "fluent_datasources": { "my_pg_ds": { "connection_string": "postgresql://userName:@hostname/dbName", - "name": "my_pg_ds", "type": "postgres", "assets": { "my_table_asset_wo_splitters": { - "name": "my_table_asset_wo_splitters", "table_name": "my_table", "type": "table", }, @@ -51,7 +49,6 @@ "column_name": "my_column", "method_name": "split_on_year_and_month", }, - "name": "with_splitter", "table_name": "another_table", "type": "table", }, @@ -60,13 +57,11 @@ {"key": "year"}, {"key": "month", "reverse": True}, ], - "name": "with_sorters", "table_name": "yet_another_table", "type": "table", }, "with_dslish_sorters": { "order_by": ["year", "-month"], - "name": "with_sorters", "table_name": "yet_another_table", "type": "table", }, @@ -74,18 +69,15 @@ }, "my_pandas_filesystem_ds": { "type": "pandas_filesystem", - "name": "my_pandas_filesystem_ds", "base_directory": __file__, "assets": { "my_csv_asset": { - "name": "my_csv_asset", "type": "csv", "batching_regex": r"yellow_tripdata_sample_(?P\d{4})-(?P\d{2}).csv", "sep": "|", "names": ["col1", "col2"], }, "my_json_asset": { - "name": "my_json_asset", "type": "json", "batching_regex": r"yellow_tripdata_sample_(?P\d{4})-(?P\d{2}).json", "orient": "records", @@ -99,7 +91,6 @@ SIMPLE_DS_DICT = { "fluent_datasources": { "my_ds": { - "name": "my_ds", "type": "sql", "connection_string": "sqlite://", } @@ -109,7 +100,6 @@ COMBINED_FLUENT_AND_OLD_STYLE_CFG_DICT = { "fluent_datasources": { "my_ds": { - "name": "my_ds", "type": "sql", "connection_string": "sqlite://", }