[Projects] Add option to mark project as archived (#610)

mlrun/api/api/endpoints/projects.py

@@ -61,6 +61,7 @@ def list_projects(
     format_: schemas.Format = Query(schemas.Format.full, alias="format"),
     owner: str = None,
     labels: typing.List[str] = Query(None, alias="label"),
+    state: schemas.ProjectState = None,
     db_session: Session = Depends(deps.get_db_session),
 ):
-    return get_project_member().list_projects(db_session, owner, format_, labels)
+    return get_project_member().list_projects(db_session, owner, format_, labels, state)

mlrun/api/db/base.py

@@ -187,6 +187,7 @@ def list_projects(
         owner: str = None,
         format_: schemas.Format = schemas.Format.full,
         labels: List[str] = None,
+        state: schemas.ProjectState = None,
     ) -> schemas.ProjectsOutput:
         pass
 

mlrun/api/db/filedb/db.py

@@ -140,9 +140,10 @@ def list_projects(
         owner: str = None,
         format_: schemas.Format = schemas.Format.full,
         labels: List[str] = None,
+        state: schemas.ProjectState = None,
     ) -> schemas.ProjectsOutput:
         return self._transform_run_db_error(
-            self.db.list_projects, owner, format_, labels
+            self.db.list_projects, owner, format_, labels, state
         )
 
     def store_project(self, session, name: str, project: schemas.Project):

mlrun/api/db/sqldb/db.py

@@ -718,8 +718,9 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> schemas.ProjectsOutput:
-        query = self._query(session, Project, owner=owner)
+        query = self._query(session, Project, owner=owner, state=state)
         if labels:
             query = self._add_labels_filter(session, query, Project, labels)
         projects = []

mlrun/api/initial_data.py

@@ -1,7 +1,10 @@
 import os
+import sqlalchemy.orm
 import pathlib
 
 from mlrun.api.db.init_db import init_db
+import mlrun.api.db.sqldb.db
+import mlrun.api.schemas
 from mlrun.api.db.session import create_session, close_session
 from mlrun.utils import logger
 from .utils.alembic import AlembicUtil
@@ -10,7 +13,7 @@
 def init_data(from_scratch: bool = False) -> None:
     logger.info("Creating initial data")
 
-    # run migrations on existing DB or create it with alembic
+    # run schema migrations on existing DB or create it with alembic
     dir_path = pathlib.Path(os.path.dirname(os.path.realpath(__file__)))
     alembic_config_path = dir_path / "alembic.ini"
 
@@ -20,11 +23,39 @@ def init_data(from_scratch: bool = False) -> None:
     db_session = create_session()
     try:
         init_db(db_session)
+        _perform_data_migrations(db_session)
     finally:
         close_session(db_session)
     logger.info("Initial data created")
 
 
+def _perform_data_migrations(db_session: sqlalchemy.orm.Session):
+    # FileDB is not really a thing anymore, so using SQLDB directly
+    db = mlrun.api.db.sqldb.db.SQLDB("")
+    logger.info("Performing data migrations")
+    _fill_project_state(db, db_session)
+
+
+def _fill_project_state(
+    db: mlrun.api.db.sqldb.db.SQLDB, db_session: sqlalchemy.orm.Session
+):
+    projects = db.list_projects(db_session)
+    for project in projects.projects:
+        changed = False
+        if not project.spec.desired_state:
+            changed = True
+            project.spec.desired_state = mlrun.api.schemas.ProjectState.online
+        if not project.status.state:
+            changed = True
+            project.status.state = project.spec.desired_state
+        if changed:
+            logger.debug(
+                "Found project without state data. Enriching",
+                name=project.metadata.name,
+            )
+            db.store_project(db_session, project.metadata.name, project)
+
+
 def main() -> None:
     init_data()
 

mlrun/api/schemas/__init__.py

@@ -34,6 +34,8 @@
     ProjectMetadata,
     ProjectSpec,
     ProjectsOutput,
+    ProjectStatus,
+    ProjectState,
 )
 from .schedule import (
     SchedulesOutput,

mlrun/api/schemas/project.py

@@ -1,4 +1,5 @@
 import datetime
+import enum
 import typing
 
 import pydantic
@@ -18,6 +19,15 @@ class Config:
         extra = pydantic.Extra.allow
 
 
+class ProjectState(str, enum.Enum):
+    online = "online"
+    archived = "archived"
+
+
+class ProjectStatus(ObjectStatus):
+    state: typing.Optional[ProjectState]
+
+
 class ProjectSpec(pydantic.BaseModel):
     description: typing.Optional[str] = None
     goals: typing.Optional[str] = None
@@ -30,6 +40,7 @@ class ProjectSpec(pydantic.BaseModel):
     source: typing.Optional[str] = None
     subpath: typing.Optional[str] = None
     origin_url: typing.Optional[str] = None
+    desired_state: typing.Optional[ProjectState] = ProjectState.online
 
     class Config:
         extra = pydantic.Extra.allow

mlrun/api/utils/clients/nuclio.py

@@ -90,6 +90,7 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
         if owner:
             raise NotImplementedError(
@@ -99,6 +100,10 @@ def list_projects(
             raise NotImplementedError(
                 "Filtering nuclio projects by labels is currently not supported"
             )
+        if state:
+            raise NotImplementedError(
+                "Filtering nuclio projects by state is currently not supported"
+            )
         response = self._send_request_to_api("GET", "projects")
         response_body = response.json()
         projects = []

mlrun/api/utils/projects/leader.py

@@ -55,7 +55,7 @@ def ensure_project(self, session: sqlalchemy.orm.Session, name: str):
     def create_project(
         self, session: sqlalchemy.orm.Session, project: mlrun.api.schemas.Project
     ) -> mlrun.api.schemas.Project:
-        self._validate_project_name(project.metadata.name)
+        self._enrich_and_validate_before_creation(project)
         self._run_on_all_followers("create_project", session, project)
         return self.get_project(session, project.metadata.name)
 
@@ -65,6 +65,7 @@ def store_project(
         name: str,
         project: mlrun.api.schemas.Project,
     ):
+        self._enrich_project(project)
         self._validate_project_name(name)
         self._validate_body_and_path_names_matches(name, project)
         self._run_on_all_followers("store_project", session, name, project)
@@ -77,6 +78,7 @@ def patch_project(
         project: dict,
         patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
     ):
+        self._enrich_project_patch(project)
         self._validate_body_and_path_names_matches(name, project)
         self._run_on_all_followers("patch_project", session, name, project, patch_mode)
         return self.get_project(session, name)
@@ -95,8 +97,11 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
-        return self._leader_follower.list_projects(session, owner, format_, labels)
+        return self._leader_follower.list_projects(
+            session, owner, format_, labels, state
+        )
 
     def _start_periodic_sync(self):
         # if no followers no need for sync
@@ -192,9 +197,8 @@ def _ensure_project_synced(
                 # Heuristically pick the first follower
                 project_follower_name = list(follower_names)[0]
                 project = followers_projects_map[project_follower_name][project_name]
-                self._leader_follower.create_project(
-                    session, mlrun.api.schemas.Project(**project.dict())
-                )
+                self._enrich_and_validate_before_creation(project)
+                self._leader_follower.create_project(session, project)
             except Exception as exc:
                 logger.warning(
                     "Failed creating missing project in leader",
@@ -232,8 +236,9 @@ def _ensure_project_synced(
                         project=project,
                     )
                     try:
+                        self._enrich_and_validate_before_creation(project)
                         self._followers[missing_follower].create_project(
-                            session, mlrun.api.schemas.Project(**project.dict()),
+                            session, project,
                         )
                     except Exception as exc:
                         logger.warning(
@@ -290,6 +295,21 @@ def _initialize_follower(
             raise ValueError(f"Unknown follower name: {name}")
         return followers_classes_map[name]
 
+    def _enrich_and_validate_before_creation(self, project: mlrun.api.schemas.Project):
+        self._enrich_project(project)
+        self._validate_project_name(project.metadata.name)
+
+    @staticmethod
+    def _enrich_project(project: mlrun.api.schemas.Project):
+        project.status.state = project.spec.desired_state
+
+    @staticmethod
+    def _enrich_project_patch(project_patch: dict):
+        if project_patch.get("spec", {}).get("desired_state"):
+            project_patch.setdefault("status", {})["state"] = project_patch["spec"][
+                "desired_state"
+            ]
+
     @staticmethod
     def _validate_project_name(name: str, raise_on_failure: bool = True) -> bool:
         try:

mlrun/api/utils/projects/member.py

@@ -63,5 +63,6 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
         pass

mlrun/api/utils/projects/remotes/member.py

@@ -49,5 +49,6 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
         pass

mlrun/api/utils/projects/remotes/nop.py

@@ -55,9 +55,12 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
-        if owner or labels:
-            raise NotImplementedError("Filtering by owner or labels is not supported")
+        if owner or labels or state:
+            raise NotImplementedError(
+                "Filtering by owner, labels or state is not supported"
+            )
         if format_ == mlrun.api.schemas.Format.full:
             return mlrun.api.schemas.ProjectsOutput(
                 projects=list(self._projects.values())

mlrun/db/base.py

@@ -144,6 +144,7 @@ def list_projects(
         owner: str = None,
         format_: schemas.Format = schemas.Format.full,
         labels: List[str] = None,
+        state: schemas.ProjectState = None,
     ) -> schemas.ProjectsOutput:
         pass
 

mlrun/db/filedb.py

@@ -417,8 +417,9 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
-        if owner or format_ == mlrun.api.schemas.Format.full or labels:
+        if owner or format_ == mlrun.api.schemas.Format.full or labels or state:
             raise NotImplementedError()
         run_dir = path.join(self.dirpath, run_logs)
         if not path.isdir(run_dir):

mlrun/db/httpdb.py

@@ -971,9 +971,13 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> List[Union[mlrun.projects.MlrunProject, str]]:
+        if isinstance(state, mlrun.api.schemas.ProjectState):
+            state = state.value
         params = {
             "owner": owner,
+            "state": state,
             "format": format_,
             "label": labels or [],
         }

mlrun/db/sqldb.py

@@ -218,6 +218,7 @@ def list_projects(
         owner: str = None,
         format_: mlrun.api.schemas.Format = mlrun.api.schemas.Format.full,
         labels: List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
         raise NotImplementedError()
 

mlrun/projects/project.py

@@ -48,6 +48,7 @@
     RunNotifications,
 )
 from ..runtimes.utils import add_code_metadata
+import mlrun.api.schemas
 
 
 class ProjectError(Exception):
@@ -224,6 +225,7 @@ def __init__(
         subpath=None,
         origin_url=None,
         goals=None,
+        desired_state=mlrun.api.schemas.ProjectState.online.value,
     ):
         self.repo = None
 
@@ -235,6 +237,7 @@ def __init__(
         self.subpath = subpath or ""
         self.origin_url = origin_url or ""
         self.goals = goals
+        self.desired_state = desired_state
         self.branch = None
         self.tag = ""
         self.params = params or {}