[API] Fix projects leader to sync enrichment to followers (#2009)

(cherry picked from commit 93f8bb1)

mlrun/api/utils/projects/leader.py

@@ -1,4 +1,5 @@
 import collections
+import traceback
 import typing
 
 import humanfriendly
@@ -258,6 +259,7 @@ def _ensure_project_synced(
                     project=project,
                     project_name=project_name,
                     exc=str(exc),
+                    traceback=traceback.format_exc(),
                 )
             else:
                 project_in_leader = True
@@ -270,38 +272,95 @@ def _ensure_project_synced(
             missing_followers = set(follower_names).symmetric_difference(
                 self._followers.keys()
             )
-            if missing_followers:
-                # projects name validation is enforced on creation, the only way for a project name to be invalid is
-                # if it was created prior to 0.6.0, and the version was upgraded
-                # we do not want to sync these projects since it will anyways fail (Nuclio doesn't allow these names
-                # as well)
-                if not mlrun.projects.ProjectMetadata.validate_project_name(
-                    project_name, raise_on_failure=False
-                ):
-                    return
-                for missing_follower in missing_followers:
-                    logger.debug(
-                        "Project is missing from follower. Creating",
-                        missing_follower_name=missing_follower,
-                        project_follower_name=project_follower_name,
-                        project_name=project_name,
-                        project=project,
+            if self._should_sync_project_to_followers(project_name):
+                if missing_followers:
+                    self._create_project_in_missing_followers(
+                        db_session,
+                        missing_followers,
+                        project_follower_name,
+                        project_name,
+                        project,
                     )
-                    try:
-                        self._enrich_and_validate_before_creation(project)
-                        self._followers[missing_follower].create_project(
-                            db_session,
-                            project,
-                        )
-                    except Exception as exc:
-                        logger.warning(
-                            "Failed creating missing project in follower",
-                            missing_follower_name=missing_follower,
-                            project_follower_name=project_follower_name,
-                            project_name=project_name,
-                            project=project,
-                            exc=str(exc),
-                        )
+
+                # we possibly enriched the project we found in the follower, so let's update the followers that had it
+                self._store_project_in_followers(
+                    db_session, follower_names, project_name, project
+                )
+
+    def _store_project_in_followers(
+        self,
+        db_session: sqlalchemy.orm.Session,
+        follower_names: typing.Set[str],
+        project_name: str,
+        project: mlrun.api.schemas.Project,
+    ):
+        for follower_name in follower_names:
+            logger.debug(
+                "Updating project in follower",
+                follower_name=follower_name,
+                project_name=project_name,
+                project=project,
+            )
+            try:
+                self._enrich_and_validate_before_creation(project)
+                self._followers[follower_name].store_project(
+                    db_session,
+                    project_name,
+                    project,
+                )
+            except Exception as exc:
+                logger.warning(
+                    "Failed updating project in follower",
+                    follower_name=follower_name,
+                    project_name=project_name,
+                    project=project,
+                    exc=str(exc),
+                    traceback=traceback.format_exc(),
+                )
+
+    def _create_project_in_missing_followers(
+        self,
+        db_session: sqlalchemy.orm.Session,
+        missing_followers: typing.Set[str],
+        # the name of the follower which we took the missing project from
+        project_follower_name: str,
+        project_name: str,
+        project: mlrun.api.schemas.Project,
+    ):
+        for missing_follower in missing_followers:
+            logger.debug(
+                "Project is missing from follower. Creating",
+                missing_follower_name=missing_follower,
+                project_follower_name=project_follower_name,
+                project_name=project_name,
+                project=project,
+            )
+            try:
+                self._enrich_and_validate_before_creation(project)
+                self._followers[missing_follower].create_project(
+                    db_session,
+                    project,
+                )
+            except Exception as exc:
+                logger.warning(
+                    "Failed creating missing project in follower",
+                    missing_follower_name=missing_follower,
+                    project_follower_name=project_follower_name,
+                    project_name=project_name,
+                    project=project,
+                    exc=str(exc),
+                    traceback=traceback.format_exc(),
+                )
+
+    def _should_sync_project_to_followers(self, project_name: str) -> bool:
+        """
+        projects name validation is enforced on creation, the only way for a project name to be invalid is if it was
+        created prior to 0.6.0, and the version was upgraded we do not want to sync these projects since it will
+        anyways fail (Nuclio doesn't allow these names as well)
+        """
+        return mlrun.projects.ProjectMetadata.validate_project_name(
+            project_name, raise_on_failure=False
+        )
 
     def _run_on_all_followers(
         self, leader_first: bool, method: str, *args, **kwargs

mlrun/api/utils/projects/remotes/nop_follower.py

@@ -18,15 +18,17 @@ def create_project(
     ):
         if project.metadata.name in self._projects:
             raise mlrun.errors.MLRunConflictError("Project already exists")
-        self._projects[project.metadata.name] = project
+        # deep copy so we won't accidentally get changes from tests
+        self._projects[project.metadata.name] = project.copy(deep=True)
 
     def store_project(
         self,
         session: sqlalchemy.orm.Session,
         name: str,
         project: mlrun.api.schemas.Project,
     ):
-        self._projects[name] = project
+        # deep copy so we won't accidentally get changes from tests
+        self._projects[name] = project.copy(deep=True)
 
     def patch_project(
         self,
@@ -52,7 +54,8 @@ def delete_project(
     def get_project(
         self, session: sqlalchemy.orm.Session, name: str
     ) -> mlrun.api.schemas.Project:
-        return self._projects[name]
+        # deep copy so we won't accidentally get changes from tests
+        return self._projects[name].copy(deep=True)
 
     def list_projects(
         self,
@@ -68,6 +71,8 @@ def list_projects(
                 "Filtering by owner, labels or state is not supported"
             )
         projects = list(self._projects.values())
+        # deep copy so we won't accidentally get changes from tests
+        projects = [project.copy(deep=True) for project in projects]
         if names:
             projects = [
                 project

tests/api/utils/projects/test_leader_member.py

@@ -131,7 +131,10 @@ def test_projects_sync_leader_project_syncing(
         metadata=mlrun.api.schemas.ProjectMetadata(name=project_name),
         spec=mlrun.api.schemas.ProjectSpec(description=project_description),
     )
-    leader_follower.create_project(None, project)
+    enriched_project = project.copy(deep=True)
+    # simulate project enrichment
+    enriched_project.status.state = enriched_project.spec.desired_state
+    leader_follower.create_project(None, enriched_project)
     invalid_project_name = "invalid_name"
     invalid_project = mlrun.api.schemas.Project(
         metadata=mlrun.api.schemas.ProjectMetadata(name=invalid_project_name),