fix(core): change project-id to be based on project slug instead of n…

…ame (#2345)

renku/core/models/git.py

@@ -26,7 +26,7 @@
 import attr
 
 from renku.core import errors
-from renku.core.utils.scm import is_ascii
+from renku.core.utils.scm import is_ascii, normalize_to_ascii
 
 _RE_PROTOCOL = r"(?P<protocol>(git\+)?(https?|git|ssh|rsync))\://"
 
@@ -95,6 +95,7 @@ class GitURL(object):
     port = attr.ib(default=None)
     owner = attr.ib(default=None)
     name = attr.ib(default=None, converter=filter_repo_name)
+    slug = attr.ib(default=None)
     _regex = attr.ib(default=None, eq=False, order=False)
 
     def __attrs_post_init__(self):
@@ -107,6 +108,8 @@ def __attrs_post_init__(self):
         if not self.name and self.pathname:
             self.name = filter_repo_name(Path(self.pathname).name)
 
+        self.slug = normalize_to_ascii(self.name)
+
     @classmethod
     def parse(cls, href):
         """Derive URI components."""

renku/core/models/project.py

@@ -29,6 +29,7 @@
 from renku.core.models.provenance.agent import Person, PersonSchema
 from renku.core.models.provenance.annotation import Annotation, AnnotationSchema
 from renku.core.utils.datetime8601 import fix_timezone, local_now, parse_date
+from renku.core.utils.scm import normalize_to_ascii
 
 
 class Project(persistent.Persistent):
@@ -80,7 +81,12 @@ def __init__(
 
     @classmethod
     def from_client(
-        cls, client, name: str = None, description: str = None, custom_metadata: Dict = None, creator: Person = None
+        cls,
+        client,
+        name: str = None,
+        description: str = None,
+        custom_metadata: Dict = None,
+        creator: Person = None,
     ) -> "Project":
         """Create an instance from a LocalClient."""
         namespace, name = cls.get_namespace_and_name(client=client, name=name, creator=creator)
@@ -121,9 +127,9 @@ def generate_id(namespace: str, name: str):
         assert name, "Cannot generate Project id with no name"
 
         namespace = quote(namespace.strip("/"), safe="/")
-        name = quote(name, safe="")
+        slug = normalize_to_ascii(name)
 
-        return f"/projects/{namespace}/{name}"
+        return f"/projects/{namespace}/{slug}"
 
     def update_metadata(self, custom_metadata=None, **kwargs):
         """Updates metadata."""

renku/service/cache/models/project.py

@@ -45,6 +45,7 @@ class Project(Model):
     git_url = TextField(index=True)
 
     name = TextField()
+    slug = TextField()
     fullname = TextField()
     description = TextField()
     email = TextField()
@@ -55,7 +56,7 @@ class Project(Model):
     @property
     def abs_path(self):
         """Full path of cached project."""
-        return CACHE_PROJECTS_PATH / self.user_id / self.project_id / self.owner / self.name
+        return CACHE_PROJECTS_PATH / self.user_id / self.project_id / self.owner / self.slug
 
     def read_lock(self):
         """Shared read lock on the project."""

renku/service/cache/serializers/project.py

@@ -36,6 +36,7 @@ class ProjectSchema(CreationSchema, MandatoryUserSchema):
     git_url = fields.String()
 
     name = fields.String(required=True)
+    slug = fields.String(required=True)
     description = fields.String(missing=None)
     fullname = fields.String(required=True)
     email = fields.String(required=True)

renku/service/controllers/templates_create_project.py

@@ -90,7 +90,8 @@ def setup_new_project(self):
         new_project_data = {
             "clone_depth": self.ctx["depth"],
             "git_url": self.ctx["new_project_url"],
-            "name": self.ctx["project_name_stripped"],
+            "name": self.ctx["project_name"],
+            "slug": self.ctx["project_name_stripped"],
             "description": self.ctx["project_description"],
             "fullname": self.ctx["fullname"],
             "email": self.ctx["email"],
@@ -167,7 +168,8 @@ def new_project(self):
         return {
             "url": self.ctx["new_project_url"],
             "namespace": self.ctx["project_namespace"],
-            "name": self.ctx["project_name_stripped"],
+            "name": self.ctx["project_name"],
+            "slug": self.ctx["project_name_stripped"],
             "project_id": new_project.project_id,
         }
 

renku/service/serializers/cache.py

@@ -25,6 +25,7 @@
 
 from renku.core.errors import ConfigurationError
 from renku.core.models.git import GitURL
+from renku.core.utils.scm import normalize_to_ascii
 from renku.service.config import PROJECT_CLONE_DEPTH_DEFAULT
 from renku.service.serializers.common import (
     ArchiveSchema,
@@ -103,6 +104,7 @@ class ProjectCloneContext(RepositoryCloneRequest):
 
     # user data
     name = fields.String()
+    slug = fields.String()
     fullname = fields.String()
     email = fields.String()
     owner = fields.String()
@@ -133,6 +135,7 @@ def set_owner_name(self, data, **kwargs):
         if git_url.name is None:
             raise ValidationError("Invalid `git_url`")
         data["name"] = git_url.name
+        data["slug"] = normalize_to_ascii(data["name"])
 
         return data
 

renku/service/serializers/templates.py

@@ -136,6 +136,7 @@ class ProjectTemplateResponse(Schema):
     url = fields.String(required=True)
     namespace = fields.String(required=True)
     name = fields.String(required=True)
+    slug = fields.String(required=True)
     project_id = fields.String(required=False, default=None)
 
 

renku/service/utils/__init__.py

@@ -28,7 +28,7 @@ def make_project_path(user, project):
     valid_project = project and "owner" in project and "name" in project and "project_id" in project
 
     if valid_user and valid_project:
-        return CACHE_PROJECTS_PATH / user["user_id"] / project["project_id"] / project["owner"] / project["name"]
+        return CACHE_PROJECTS_PATH / user["user_id"] / project["project_id"] / project["owner"] / project["slug"]
 
 
 def make_file_path(user, cached_file):

tests/cli/test_init.py

@@ -454,16 +454,18 @@ def test_default_init_parameters(isolated_runner, mocker, project_init, template
 def test_init_with_description(isolated_runner, template):
     """Test project initialization with description."""
     result = isolated_runner.invoke(
-        cli, ["init", "--description", "my project description", "new-project", "--template-id", template["id"]]
+        cli, ["init", "--description", "my project description", "new project", "--template-id", template["id"]]
     )
 
     assert 0 == result.exit_code, format_result_exception(result)
 
-    database = Database.from_path(Path("new-project") / ".renku" / "metadata")
+    database = Database.from_path(Path("new project") / ".renku" / "metadata")
     project = database.get("project")
 
+    assert "new project" == project.name
+    assert project.id.endswith("new-project")  # make sure id uses slug version of name without space
     assert "my project description" in project.template_metadata
     assert "my project description" == project.description
 
-    readme_content = (Path("new-project") / "README.md").read_text()
+    readme_content = (Path("new project") / "README.md").read_text()
     assert "my project description" in readme_content

tests/service/cache/test_cache.py

@@ -253,6 +253,7 @@ def test_service_cache_make_project(svc_client_cache):
     user = cache.ensure_user({"user_id": uuid.uuid4().hex})
     project_data = {
         "name": "renku-project-template",
+        "slug": "renku-project-template",
         "depth": 1,
         "git_url": "https://github.com/SwissDataScienceCenter/renku-project-template",
         "email": "contact@renkulab.io",

tests/service/controllers/test_templates_create_project.py

@@ -36,11 +36,11 @@ def test_template_create_project_ctrl(ctrl_init, svc_client_templates_creation,
 
     # Check response.
     assert {"result"} == response.json.keys()
-    assert {"project_id", "url", "namespace", "name"} == response.json["result"].keys()
+    assert {"project_id", "url", "namespace", "name", "slug"} == response.json["result"].keys()
 
     # Check ctrl_mock.
     assert ctrl_mock.call_count == 1
-    assert response.json["result"]["name"] == ctrl_mock.call_args[0][0].name
+    assert response.json["result"]["slug"] == ctrl_mock.call_args[0][0].name
 
     # Ctrl state.
     expected_context = {
@@ -55,6 +55,7 @@ def test_template_create_project_ctrl(ctrl_init, svc_client_templates_creation,
         "parameters",
         "project_name",
         "name",
+        "slug",
         "project_description",
         "new_project_url",
         "fullname",
@@ -144,8 +145,8 @@ def test_project_name_handler(project_name, expected_name, ctrl_init, svc_client
 
     # Check response.
     assert {"result"} == response.json.keys()
-    assert {"project_id", "url", "namespace", "name"} == response.json["result"].keys()
-    assert expected_name == response.json["result"]["name"]
+    assert {"project_id", "url", "namespace", "name", "slug"} == response.json["result"].keys()
+    assert expected_name == response.json["result"]["slug"]
 
 
 @pytest.mark.parametrize("project_name", ["здрасти", "---- --üäü ----", "-.-", "...", "----", "~.---", "`~~"])

tests/service/fixtures/service_integration.py

@@ -62,6 +62,7 @@ def integration_repo_path(headers, project_id, url_components):
         "project_id": project_id,
         "owner": url_components.owner,
         "name": url_components.name,
+        "slug": url_components.slug,
     }
 
     project_path = make_project_path(user, project)
@@ -219,6 +220,7 @@ def _mock_owner(self, data, **kwargs):
         data["owner"] = "dummy"
 
         data["name"] = "project"
+        data["slug"] = "project"
 
         return data
 
@@ -265,7 +267,7 @@ def _mock_owner(self, data, **kwargs):
             response = svc_client.post("/cache.project_clone", data=json.dumps(payload), headers=identity_headers)
 
             assert response
-            assert {"result"} == set(response.json.keys())
+            assert {"result"} == set(response.json.keys()), response.json
 
             project_id = response.json["result"]["project_id"]
             assert isinstance(uuid.UUID(project_id), uuid.UUID)

tests/service/fixtures/service_projects.py

@@ -25,13 +25,17 @@
 import git
 import pytest
 
+from renku.core.utils.scm import normalize_to_ascii
+
 
 @pytest.fixture
 def project_metadata(project):
     """Create project with metadata."""
+    name = Path(project).name
     metadata = {
         "project_id": uuid.uuid4().hex,
-        "name": Path(project).name,
+        "name": name,
+        "slug": normalize_to_ascii(name),
         "fullname": "full project name",
         "email": "my@email.com",
         "owner": "me",

tests/service/jobs/test_datasets.py

@@ -67,7 +67,13 @@ def test_dataset_url_import_job(url, svc_client_with_repo):
     assert {"job_id", "created_at"} == set(response.json["result"].keys())
 
     dest = make_project_path(
-        user_data, {"owner": url_components.owner, "name": url_components.name, "project_id": project_id}
+        user_data,
+        {
+            "owner": url_components.owner,
+            "name": url_components.name,
+            "slug": url_components.slug,
+            "project_id": project_id,
+        },
     )
 
     old_commit = Repo(dest).head.commit
@@ -109,7 +115,13 @@ def test_dataset_import_job(doi, svc_client_with_repo):
     assert {"job_id", "created_at"} == set(response.json["result"].keys())
 
     dest = make_project_path(
-        user, {"owner": url_components.owner, "name": url_components.name, "project_id": project_id}
+        user,
+        {
+            "owner": url_components.owner,
+            "name": url_components.name,
+            "slug": url_components.slug,
+            "project_id": project_id,
+        },
     )
 
     old_commit = Repo(dest).head.commit
@@ -158,7 +170,13 @@ def test_dataset_import_junk_job(doi, expected_err, svc_client_with_repo):
     assert {"job_id", "created_at"} == set(response.json["result"].keys())
 
     dest = make_project_path(
-        user, {"owner": url_components.owner, "name": url_components.name, "project_id": project_id}
+        user,
+        {
+            "owner": url_components.owner,
+            "name": url_components.name,
+            "slug": url_components.slug,
+            "project_id": project_id,
+        },
     )
 
     old_commit = Repo(dest).head.commit
@@ -201,7 +219,13 @@ def test_dataset_import_twice_job(doi, svc_client_with_repo):
     assert {"job_id", "created_at"} == set(response.json["result"].keys())
 
     dest = make_project_path(
-        user, {"owner": url_components.owner, "name": url_components.name, "project_id": project_id}
+        user,
+        {
+            "owner": url_components.owner,
+            "name": url_components.name,
+            "slug": url_components.slug,
+            "project_id": project_id,
+        },
     )
 
     old_commit = Repo(dest).head.commit
@@ -248,7 +272,13 @@ def test_dataset_add_remote_file(url, svc_client_with_repo):
     assert {"files", "name", "project_id", "remote_branch"} == set(response.json["result"].keys())
 
     dest = make_project_path(
-        user, {"owner": url_components.owner, "name": url_components.name, "project_id": project_id}
+        user,
+        {
+            "owner": url_components.owner,
+            "name": url_components.name,
+            "slug": url_components.slug,
+            "project_id": project_id,
+        },
     )
     old_commit = Repo(dest).head.commit
     job_id = response.json["result"]["files"][0]["job_id"]
@@ -363,7 +393,13 @@ def test_dataset_project_lock(doi, svc_client_with_repo):
     assert {"job_id", "created_at"} == set(response.json["result"].keys())
 
     dest = make_project_path(
-        user, {"owner": url_components.owner, "name": url_components.name, "project_id": project_id}
+        user,
+        {
+            "owner": url_components.owner,
+            "name": url_components.name,
+            "slug": url_components.slug,
+            "project_id": project_id,
+        },
     )
 
     old_commit = Repo(dest).head.commit

tests/service/jobs/test_jobs.py

@@ -136,6 +136,7 @@ def test_cleanup_project_old_keys(svc_client_with_user, service_job):
     project = {
         "project_id": uuid.uuid4().hex,
         "name": "my-project",
+        "slug": "my-project",
         "fullname": "full project name",
         "email": "my@email.com",
         "owner": "me",
@@ -172,6 +173,7 @@ def test_job_constructor_lock(svc_client_with_user, service_job):
     project = {
         "project_id": uuid.uuid4().hex,
         "name": "my-project",
+        "slug": "my-project",
         "fullname": "full project name",
         "email": "my@email.com",
         "owner": "me",

tests/service/views/test_dataset_views.py

@@ -27,6 +27,7 @@
 import pytest
 from werkzeug.utils import secure_filename
 
+from renku.core.utils.scm import normalize_to_ascii
 from renku.service.config import (
     GIT_ACCESS_DENIED_ERROR_CODE,
     INVALID_HEADERS_ERROR_CODE,
@@ -1126,9 +1127,12 @@ def test_cached_import_dataset_job(doi, svc_client_cache, project):
     user_id = encode_b64(secure_filename("9ab2fc80-3a5c-426d-ae78-56de01d214df"))
     user = cache.ensure_user({"user_id": user_id})
 
+    name = Path(project).name
+
     project_meta = {
         "project_id": uuid.uuid4().hex,
-        "name": Path(project).name,
+        "name": name,
+        "slug": normalize_to_ascii(name),
         "fullname": "full project name",
         "email": "my@email.com",
         "owner": "me",