Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix HfApi.create_repo when repo_type is 'space' #394

Merged
merged 13 commits into from
Oct 29, 2021
Merged

Fix HfApi.create_repo when repo_type is 'space' #394

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
8aaa68c
:bug: fix create spaces
nateraw Oct 6, 2021
dba8272
:bug: fix create spaces bug
nateraw Oct 6, 2021
b6a0549
:sparkles: add spaces sdk types as constant
nateraw Oct 6, 2021
e1e9555
:test_tube: add a commit that unskips spaces test
nateraw Oct 6, 2021
976f966
:white_check_mark: skip spaces tests while in beta
nateraw Oct 6, 2021
5c00196
:construction: update cli
nateraw Oct 28, 2021
d6c573b
:bug: fix missed merge confict
nateraw Oct 28, 2021
1f36461
:lipstick: style
nateraw Oct 28, 2021
ea50acd
:bug: add missing import
nateraw Oct 28, 2021
8f5412e
:art: only hardcode gated param for spaces repos
nateraw Oct 28, 2021
4532084
:white_check_mark: Update spaces tests to use prod
nateraw Oct 28, 2021
652880f
:white_check_mark: use staging endpoint for spaces tests
nateraw Oct 28, 2021
4784508
:fire: remove hack
nateraw Oct 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 12 additions & 1 deletion src/huggingface_hub/commands/user.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@
from typing import List, Union

from huggingface_hub.commands import BaseHuggingfaceCLICommand
from huggingface_hub.constants import REPO_TYPES, REPO_TYPES_URL_PREFIXES
from huggingface_hub.constants import (
REPO_TYPES,
REPO_TYPES_URL_PREFIXES,
SPACES_SDK_TYPES,
)
from huggingface_hub.hf_api import HfApi, HfFolder
from requests.exceptions import HTTPError

Expand Down Expand Up @@ -68,6 +72,12 @@ def register_subcommand(parser: ArgumentParser):
repo_create_parser.add_argument(
"--organization", type=str, help="Optional: organization namespace."
)
repo_create_parser.add_argument(
"--space_sdk",
type=str,
help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".',
choices=SPACES_SDK_TYPES,
)
repo_create_parser.add_argument(
"-y",
"--yes",
Expand Down Expand Up @@ -263,6 +273,7 @@ def run(self):
token=token,
organization=self.args.organization,
repo_type=self.args.type,
space_sdk=self.args.space_sdk,
)
except HTTPError as e:
print(e)
Expand Down
1 change: 1 addition & 0 deletions src/huggingface_hub/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
REPO_TYPE_DATASET = "dataset"
REPO_TYPE_SPACE = "space"
REPO_TYPES = [None, REPO_TYPE_DATASET, REPO_TYPE_SPACE]
SPACES_SDK_TYPES = ["gradio", "streamlit", "static"]

REPO_TYPES_URL_PREFIXES = {
REPO_TYPE_DATASET: "datasets/",
Expand Down
30 changes: 28 additions & 2 deletions src/huggingface_hub/hf_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,13 @@
import requests
from requests.exceptions import HTTPError

from .constants import ENDPOINT, REPO_TYPES, REPO_TYPES_MAPPING, REPO_TYPES_URL_PREFIXES
from .constants import (
ENDPOINT,
REPO_TYPES,
REPO_TYPES_MAPPING,
REPO_TYPES_URL_PREFIXES,
SPACES_SDK_TYPES,
)


if sys.version_info >= (3, 8):
Expand Down Expand Up @@ -664,6 +670,7 @@ def create_repo(
repo_type: Optional[str] = None,
exist_ok=False,
lfsmultipartthresh: Optional[int] = None,
space_sdk: Optional[str] = None,
) -> str:
"""
HuggingFace git-based system, used for models, datasets, and spaces.
Expand All @@ -679,6 +686,8 @@ def create_repo(

lfsmultipartthresh: Optional: internal param for testing purposes.

space_sdk: Choice of SDK to use if repo_type is "space". Can be "streamlit", "gradio", or "static".

Returns:
URL to the newly created repo.
"""
Expand Down Expand Up @@ -707,6 +716,22 @@ def create_repo(
json = {"name": name, "organization": organization, "private": private}
if repo_type is not None:
json["type"] = repo_type
if repo_type == "space":
if space_sdk is None:
raise ValueError(
"No space_sdk provided. `create_repo` expects space_sdk to be one of "
f"{SPACES_SDK_TYPES} when repo_type is 'space'`"
)
if space_sdk not in SPACES_SDK_TYPES:
raise ValueError(
f"Invalid space_sdk. Please choose one of {SPACES_SDK_TYPES}."
)
json["sdk"] = space_sdk
if space_sdk is not None and repo_type != "space":
warnings.warn(
"Ignoring provided space_sdk because repo_type is not 'space'."
)

if lfsmultipartthresh is not None:
json["lfsmultipartthresh"] = lfsmultipartthresh
r = requests.post(
Expand Down Expand Up @@ -821,7 +846,8 @@ def update_repo_visibility(
path_prefix += REPO_TYPES_URL_PREFIXES[repo_type]

path = "{}{}/{}/settings".format(path_prefix, namespace, name)
json = {"private": private}
# HACK - hard coded recently added 'gated' param for now. Decide how to deal with this in the future.
json = {"private": private, "gated": False}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@SBrandeis Repo updates fail if gated is not included as a parameter. Saw this was added recently, so just hardcoded False for now. Is that ok?

Referring to moon-landing PR: https://github.com/huggingface/moon-landing/pull/1316

CC @julien-c

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

API endpoint should probably default to false server-side, @SBrandeis

Does this currently break all repo creation from hf_api, @nateraw? not just spaces, right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh actually its only failing for spaces. Can update accordingly here for now

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

very weird, but good to hear that =)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

all done 😄

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Super weird that's it's failing only for spaces 🤔

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the meantime, you can also test in production by adding the @with_production_testing decorator to your tests. These will run on the production server instead of the staging, so can be helpful while spaces are unavailable on staging.

Copy link
Member

@julien-c julien-c Oct 28, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not a great idea in my opinion, but we've just updated the staging env so should might work OTB now

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Production tests are better than no tests :) if the staging is available, then that's definitely the way to go.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@LysandreJik , after this convo earlier, we got the staging endpoint working 😀. All good!


r = requests.put(
path,
Expand Down
67 changes: 46 additions & 21 deletions tests/test_hf_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,14 @@
import unittest
from io import BytesIO

import pytest

import requests
from huggingface_hub.constants import REPO_TYPE_DATASET, REPO_TYPE_SPACE
from huggingface_hub.constants import (
REPO_TYPE_DATASET,
REPO_TYPE_SPACE,
SPACES_SDK_TYPES,
)
from huggingface_hub.file_download import cached_download, hf_hub_download
from huggingface_hub.hf_api import (
DatasetInfo,
Expand Down Expand Up @@ -155,26 +161,45 @@ def test_create_update_and_delete_dataset_repo(self):

@unittest.skip("skipped while spaces in beta")
def test_create_update_and_delete_space_repo(self):
self._api.create_repo(
name=SPACE_REPO_NAME, token=self._token, repo_type=REPO_TYPE_SPACE
)
res = self._api.update_repo_visibility(
name=SPACE_REPO_NAME,
token=self._token,
private=True,
repo_type=REPO_TYPE_SPACE,
)
self.assertTrue(res["private"])
res = self._api.update_repo_visibility(
name=SPACE_REPO_NAME,
token=self._token,
private=False,
repo_type=REPO_TYPE_SPACE,
)
self.assertFalse(res["private"])
self._api.delete_repo(
name=SPACE_REPO_NAME, token=self._token, repo_type=REPO_TYPE_SPACE
)
with pytest.raises(ValueError, match=r"No space_sdk provided.*"):
self._api.create_repo(
token=self._token,
name=SPACE_REPO_NAME,
repo_type=REPO_TYPE_SPACE,
space_sdk=None,
)
with pytest.raises(ValueError, match=r"Invalid space_sdk.*"):
self._api.create_repo(
token=self._token,
name=SPACE_REPO_NAME,
repo_type=REPO_TYPE_SPACE,
space_sdk="asdfasdf",
)

for sdk in SPACES_SDK_TYPES:
self._api.create_repo(
name=SPACE_REPO_NAME,
token=self._token,
repo_type=REPO_TYPE_SPACE,
space_sdk=sdk,
)
res = self._api.update_repo_visibility(
name=SPACE_REPO_NAME,
token=self._token,
private=True,
repo_type=REPO_TYPE_SPACE,
)
self.assertTrue(res["private"])
res = self._api.update_repo_visibility(
name=SPACE_REPO_NAME,
token=self._token,
private=False,
repo_type=REPO_TYPE_SPACE,
)
self.assertFalse(res["private"])
self._api.delete_repo(
name=SPACE_REPO_NAME, token=self._token, repo_type=REPO_TYPE_SPACE
)


class HfApiUploadFileTest(HfApiCommonTestWithLogin):
Expand Down