Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix dataset features and dataset preview for HuggingFaceDatasetSaver #5135

Merged
merged 6 commits into from Aug 9, 2023
Merged

Fix dataset features and dataset preview for HuggingFaceDatasetSaver #5135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2f313ba
Add tests
freddyaboulton Aug 8, 2023
f1f0393
Use hf token
freddyaboulton Aug 8, 2023
ccefddd
lint
freddyaboulton Aug 8, 2023
d8b3b94
add changeset
gradio-pr-bot Aug 8, 2023
9185f71
Add empty string for None
freddyaboulton Aug 9, 2023
141f0bf
add changeset
gradio-pr-bot Aug 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/new-icons-refuse.md
View file
Open in desktop
@@ -0,0 +1,5 @@
---
"gradio": minor
---

fix:Fix dataset features and dataset preview for HuggingFaceDatasetSaver
48 changes: 34 additions & 14 deletions gradio/flagging.py
View file
Open in desktop
Expand Up @@ -248,6 +248,21 @@ def setup(self, components: list[IOComponent], flagging_dir: str):
repo_type="dataset",
exist_ok=True,
).repo_id
path_glob = "**/*.jsonl" if self.separate_dirs else "data.csv"
abidlabs marked this conversation as resolved.
Show resolved Hide resolved
huggingface_hub.metadata_update(
repo_id=self.dataset_id,
repo_type="dataset",
metadata={
"configs": [
{
"config_name": "default",
"data_files": [{"split": "train", "path": path_glob}],
}
]
},
overwrite=True,
token=self.hf_token,
)

# Setup flagging dir
self.components = components
Expand Down Expand Up @@ -284,7 +299,7 @@ def flag(
if self.separate_dirs:
# JSONL files to support dataset preview on the Hub
unique_id = str(uuid.uuid4())
components_dir = self.dataset_dir / str(uuid.uuid4())
components_dir = self.dataset_dir / unique_id
abidlabs marked this conversation as resolved.
Show resolved Hide resolved
data_file = components_dir / "metadata.jsonl"
path_in_repo = unique_id # upload in sub folder (safer for concurrency)
else:
Expand Down Expand Up @@ -416,28 +431,33 @@ def _deserialize_components(
features[label] = {"dtype": "string", "_type": "Value"}
try:
assert Path(deserialized).exists()
row.append(Path(deserialized).name)
row.append(str(Path(deserialized).relative_to(self.dataset_dir)))
except (AssertionError, TypeError, ValueError):
row.append(str(deserialized))
deserialized = "" if deserialized is None else str(deserialized)
abidlabs marked this conversation as resolved.
Show resolved Hide resolved
row.append(deserialized)

# If component is eligible for a preview, add the URL of the file
# Be mindful that images and audio can be None
if isinstance(component, tuple(file_preview_types)): # type: ignore
for _component, _type in file_preview_types.items():
if isinstance(component, _component):
features[label + " file"] = {"_type": _type}
break
path_in_repo = str( # returned filepath is absolute, we want it relative to compute URL
Path(deserialized).relative_to(self.dataset_dir)
).replace(
"\\", "/"
)
row.append(
huggingface_hub.hf_hub_url(
repo_id=self.dataset_id,
filename=path_in_repo,
repo_type="dataset",
if deserialized:
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was causing errors if the image was None

path_in_repo = str( # returned filepath is absolute, we want it relative to compute URL
Path(deserialized).relative_to(self.dataset_dir)
).replace(
"\\", "/"
)
)
row.append(
huggingface_hub.hf_hub_url(
repo_id=self.dataset_id,
filename=path_in_repo,
repo_type="dataset",
)
)
else:
row.append("")
features["flag"] = {"dtype": "string", "_type": "Value"}
features["username"] = {"dtype": "string", "_type": "Value"}
row.append(flag_option)
Expand Down
9 changes: 6 additions & 3 deletions test/test_flagging.py
View file
Open in desktop
Expand Up @@ -46,7 +46,8 @@ class TestHuggingFaceDatasetSaver:
return_value=MagicMock(repo_id="gradio-tests/test"),
)
@patch("huggingface_hub.hf_hub_download")
def test_saver_setup(self, mock_download, mock_create):
@patch("huggingface_hub.metadata_update")
def test_saver_setup(self, metadata_update, mock_download, mock_create):
flagger = flagging.HuggingFaceDatasetSaver("test_token", "test")
with tempfile.TemporaryDirectory() as tmpdirname:
flagger.setup([gr.Audio, gr.Textbox], tmpdirname)
Expand All @@ -60,8 +61,9 @@ def test_saver_setup(self, mock_download, mock_create):
@patch("huggingface_hub.hf_hub_download")
@patch("huggingface_hub.upload_folder")
@patch("huggingface_hub.upload_file")
@patch("huggingface_hub.metadata_update")
def test_saver_flag_same_dir(
self, mock_upload_file, mock_upload, mock_download, mock_create
self, metadata_update, mock_upload_file, mock_upload, mock_download, mock_create
):
with tempfile.TemporaryDirectory() as tmpdirname:
io = gr.Interface(
Expand Down Expand Up @@ -89,8 +91,9 @@ def test_saver_flag_same_dir(
@patch("huggingface_hub.hf_hub_download")
@patch("huggingface_hub.upload_folder")
@patch("huggingface_hub.upload_file")
@patch("huggingface_hub.metadata_update")
def test_saver_flag_separate_dirs(
self, mock_upload_file, mock_upload, mock_download, mock_create
self, metadata_update, mock_upload_file, mock_upload, mock_download, mock_create
):
with tempfile.TemporaryDirectory() as tmpdirname:
io = gr.Interface(
Expand Down