fix: clean cache after trying to fetch projects from non-existing rep…

…os (#2789)

fix #2787

renku/service/cache/projects.py

@@ -64,7 +64,7 @@ def invalidate_project(user, project_id):
         project_obj = ProjectManagementCache.get_project(user, project_id)
 
         if project_obj:
-            project_obj.delete()
+            project_obj.purge()
 
         return project_obj
 

renku/service/controllers/utils/project_clone.py

@@ -17,6 +17,7 @@
 # limitations under the License.
 """Utilities for renku service controllers."""
 from renku.core.commands.clone import project_clone_command
+from renku.core.errors import GitCommandError
 from renku.service.logger import service_log
 from renku.service.views.decorators import requires_cache
 
@@ -31,24 +32,28 @@ def user_project_clone(cache, user_data, project_data):
     project = cache.make_project(user, project_data)
     project.abs_path.mkdir(parents=True, exist_ok=True)
 
-    with project.write_lock():
-        repo, project.initialized = (
-            project_clone_command()
-            .build()
-            .execute(
-                project_data["url_with_auth"],
-                path=project.abs_path,
-                depth=project_data["depth"],
-                raise_git_except=True,
-                config={
-                    "user.name": project_data["fullname"],
-                    "user.email": project_data["email"],
-                    "pull.rebase": False,
-                },
-                checkout_revision=project_data["ref"],
-            )
-        ).output
-        project.save()
+    try:
+        with project.write_lock():
+            repo, project.initialized = (
+                project_clone_command()
+                .build()
+                .execute(
+                    project_data["url_with_auth"],
+                    path=project.abs_path,
+                    depth=project_data["depth"],
+                    raise_git_except=True,
+                    config={
+                        "user.name": project_data["fullname"],
+                        "user.email": project_data["email"],
+                        "pull.rebase": False,
+                    },
+                    checkout_revision=project_data["ref"],
+                )
+            ).output
+            project.save()
+    except GitCommandError as e:
+        cache.invalidate_project(user, project.project_id)
+        raise e
 
     service_log.debug(f"project successfully cloned: {repo}")
     service_log.debug(f"project folder exists: {project.exists()}")

tests/service/controllers/utils/test_project_clone.py

@@ -16,15 +16,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Renku service project clone tests."""
+import json
 import time
 import uuid
 
 import pytest
 from marshmallow import EXCLUDE
+from werkzeug.utils import secure_filename
 
 from renku.service.controllers.utils.project_clone import user_project_clone
+from renku.service.serializers.headers import encode_b64
 from renku.service.serializers.templates import ProjectTemplateRequest
-from tests.utils import modified_environ, retry_failed
+from tests.utils import assert_rpc_response, modified_environ, retry_failed
 
 
 @pytest.mark.integration
@@ -72,3 +75,34 @@ def test_service_user_project_clone(svc_client_cache):
         projects = [project.project_id for project in cache.get_projects(user)]
         assert project_one.project_id in projects
         assert project_two.project_id in projects
+
+
+@pytest.mark.service
+@pytest.mark.integration
+@retry_failed
+def test_service_user_non_existing_project_clone(svc_client_cache, it_remote_repo_url):
+    """Check reading manifest template."""
+    svc_client, headers, cache = svc_client_cache
+    user_id = encode_b64(secure_filename("9ab2fc80-3a5c-426d-ae78-56de01d214df"))
+    user = cache.ensure_user({"user_id": user_id})
+
+    # NOTE: clone a valid repo and verify there is one project in the cache
+    payload = {"git_url": it_remote_repo_url, "depth": -1}
+    response = svc_client.post("/cache.project_clone", data=json.dumps(payload), headers=headers)
+
+    assert_rpc_response(response)
+    projects = list(cache.get_projects(user))
+    assert 1 == len(projects)
+
+    # NOTE: invalidate the project
+    cache.invalidate_project(user, projects[0].project_id)
+    projects = list(cache.get_projects(user))
+    assert 0 == len(projects)
+
+    # NOTE: try to clone a non-existing repo and verify no other projects are added to the cache
+    payload["git_url"] = f"{it_remote_repo_url}-non-existing-project-url"
+    response = svc_client.post("/cache.project_clone", data=json.dumps(payload), headers=headers)
+
+    assert_rpc_response(response, "error")
+    projects = list(cache.get_projects(user))
+    assert 0 == len(projects)