Skip to content

Commit

Permalink
Fix missing info when loading some datasets from Parquet export (#6635)
Browse files Browse the repository at this point in the history
* set default config when single config parquet export

* fix tests

* fix test

* Update src/datasets/utils/metadata.py

Co-authored-by: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com>

---------

Co-authored-by: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com>
  • Loading branch information
lhoestq and albertvillanova committed Feb 7, 2024
1 parent ba3cfad commit 14d9afb
Show file tree
Hide file tree
Showing 3 changed files with 4 additions and 4 deletions.
2 changes: 1 addition & 1 deletion src/datasets/utils/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None:
def get_default_config_name(self) -> Optional[str]:
default_config_name = None
for config_name, metadata_config in self.items():
if config_name == "default" or metadata_config.get("default"):
if len(self) == 1 or config_name == "default" or metadata_config.get("default"):
if default_config_name is None:
default_config_name = config_name
else:
Expand Down
4 changes: 2 additions & 2 deletions tests/test_load.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@ def test_LocalDatasetModuleFactoryWithoutScript_with_single_config_in_metadata(s
assert module_builder_configs[0].drop_labels is True # parameter is passed from metadata

# config named "default" is automatically considered to be a default config
assert module_factory_result.builder_configs_parameters.default_config_name is None
assert module_factory_result.builder_configs_parameters.default_config_name == "custom"

# we don't pass config params to builder in builder_kwargs, they are stored in builder_configs directly
assert "drop_labels" not in module_factory_result.builder_kwargs
Expand Down Expand Up @@ -695,7 +695,7 @@ def test_HubDatasetModuleFactoryWithoutScript_with_one_default_config_in_metadat
assert module_builder_configs[0].drop_labels is True # parameter is passed from metadata

# config named "default" is automatically considered to be a default config
assert module_factory_result.builder_configs_parameters.default_config_name is None
assert module_factory_result.builder_configs_parameters.default_config_name == "custom"

# we don't pass config params to builder in builder_kwargs, they are stored in builder_configs directly
assert "drop_labels" not in module_factory_result.builder_kwargs
Expand Down
2 changes: 1 addition & 1 deletion tests/test_metadata_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ def test_from_yaml_string(self):
@pytest.mark.parametrize(
"readme_content, expected_metadata_configs_dict, expected_default_config_name",
[
(README_METADATA_SINGLE_CONFIG, EXPECTED_METADATA_SINGLE_CONFIG, None),
(README_METADATA_SINGLE_CONFIG, EXPECTED_METADATA_SINGLE_CONFIG, "custom"),
(README_METADATA_TWO_CONFIGS_WITH_DEFAULT_FLAG, EXPECTED_METADATA_TWO_CONFIGS_DEFAULT_FLAG, "v2"),
(README_METADATA_TWO_CONFIGS_WITH_DEFAULT_NAME, EXPECTED_METADATA_TWO_CONFIGS_DEFAULT_NAME, "default"),
],
Expand Down

0 comments on commit 14d9afb

Please sign in to comment.