[API] Add schedules and pipelines counters to project summary & creat…

…e project summary endpoints (#1342)

mlrun/api/api/endpoints/projects.py

@@ -215,6 +215,69 @@ def list_projects(
     )
 
 
+@router.get(
+    "/project-summaries", response_model=mlrun.api.schemas.ProjectSummariesOutput
+)
+def list_project_summaries(
+    owner: str = None,
+    labels: typing.List[str] = fastapi.Query(None, alias="label"),
+    state: mlrun.api.schemas.ProjectState = None,
+    auth_verifier: mlrun.api.api.deps.AuthVerifierDep = fastapi.Depends(
+        mlrun.api.api.deps.AuthVerifierDep
+    ),
+    db_session: sqlalchemy.orm.Session = fastapi.Depends(
+        mlrun.api.api.deps.get_db_session
+    ),
+):
+    projects_output = get_project_member().list_projects(
+        db_session,
+        owner,
+        mlrun.api.schemas.ProjectsFormat.name_only,
+        labels,
+        state,
+        auth_verifier.auth_info.projects_role,
+        auth_verifier.auth_info.session,
+    )
+    allowed_project_names = projects_output.projects
+    # skip permission check if it's the leader
+    if not _is_request_from_leader(auth_verifier.auth_info.projects_role):
+        allowed_project_names = mlrun.api.utils.clients.opa.Client().filter_projects_by_permissions(
+            projects_output.projects, auth_verifier.auth_info,
+        )
+    return get_project_member().list_project_summaries(
+        db_session,
+        owner,
+        labels,
+        state,
+        auth_verifier.auth_info.projects_role,
+        auth_verifier.auth_info.session,
+        allowed_project_names,
+    )
+
+
+@router.get(
+    "/project-summaries/{name}", response_model=mlrun.api.schemas.ProjectSummary
+)
+def get_project_summary(
+    name: str,
+    db_session: sqlalchemy.orm.Session = fastapi.Depends(
+        mlrun.api.api.deps.get_db_session
+    ),
+    auth_verifier: mlrun.api.api.deps.AuthVerifierDep = fastapi.Depends(
+        mlrun.api.api.deps.AuthVerifierDep
+    ),
+):
+    project_summary = get_project_member().get_project_summary(
+        db_session, name, auth_verifier.auth_info.session
+    )
+    # skip permission check if it's the leader
+    if not _is_request_from_leader(auth_verifier.auth_info.projects_role):
+        mlrun.api.utils.clients.opa.Client().query_project_permissions(
+            name, mlrun.api.schemas.AuthorizationAction.read, auth_verifier.auth_info,
+        )
+    return project_summary
+
+
 def _is_request_from_leader(
     projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole],
 ) -> bool:

mlrun/api/crud/projects.py

@@ -1,3 +1,4 @@
+import collections
 import typing
 
 import sqlalchemy.orm
@@ -119,3 +120,59 @@ def list_projects(
         return mlrun.api.utils.singletons.db.get_db().list_projects(
             session, owner, format_, labels, state, names
         )
+
+    def list_project_summaries(
+        self,
+        session: sqlalchemy.orm.Session,
+        owner: str = None,
+        labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
+        names: typing.Optional[typing.List[str]] = None,
+    ) -> mlrun.api.schemas.ProjectSummariesOutput:
+        project_summaries = mlrun.api.utils.singletons.db.get_db().list_project_summaries(
+            session, owner, labels, state, names
+        )
+        self._add_pipeline_running_count_to_project_summaries(
+            session, project_summaries.project_summaries
+        )
+        return project_summaries
+
+    def get_project_summary(
+        self, session: sqlalchemy.orm.Session, name: str
+    ) -> mlrun.api.schemas.ProjectSummary:
+        project_summary = mlrun.api.utils.singletons.db.get_db().get_project_summary(
+            session, name
+        )
+        self._add_pipeline_running_count_to_project_summaries(
+            session, [project_summary]
+        )
+        return project_summary
+
+    def generate_projects_summaries(
+        self, session: sqlalchemy.orm.Session, projects: typing.List[str]
+    ) -> typing.List[mlrun.api.schemas.ProjectSummary]:
+        project_summaries = mlrun.api.utils.singletons.db.get_db().generate_projects_summaries(
+            session, projects
+        )
+        self._add_pipeline_running_count_to_project_summaries(
+            session, project_summaries
+        )
+        return project_summaries
+
+    def _add_pipeline_running_count_to_project_summaries(
+        self,
+        session: sqlalchemy.orm.Session,
+        project_summaries: typing.List[mlrun.api.schemas.ProjectSummary],
+    ):
+        _, _, pipelines = mlrun.api.crud.Pipelines().list_pipelines(
+            session, "*", format_=mlrun.api.schemas.PipelinesFormat.metadata_only
+        )
+        project_to_running_pipelines_count = collections.defaultdict(int)
+        for pipeline in pipelines:
+            if pipeline["status"] not in mlrun.run.RunStatuses.stable_statuses():
+                project_to_running_pipelines_count[pipeline["project"]] += 1
+        for project_summary in project_summaries:
+            project_summary.pipelines_running_count = project_to_running_pipelines_count.get(
+                project_summary.name, 0
+            )
+        return project_summaries

mlrun/api/db/base.py

@@ -230,6 +230,21 @@ def get_project(
     ) -> schemas.Project:
         pass
 
+    @abstractmethod
+    def list_project_summaries(
+        self,
+        session,
+        owner: str = None,
+        labels: List[str] = None,
+        state: schemas.ProjectState = None,
+        names: Optional[List[str]] = None,
+    ) -> schemas.ProjectSummariesOutput:
+        pass
+
+    @abstractmethod
+    def get_project_summary(self, session, name: str) -> schemas.ProjectSummary:
+        pass
+
     @abstractmethod
     def create_project(self, session, project: schemas.Project):
         pass

mlrun/api/db/filedb/db.py

@@ -166,6 +166,19 @@ def list_projects(
             self.db.list_projects, owner, format_, labels, state
         )
 
+    def list_project_summaries(
+        self,
+        session,
+        owner: str = None,
+        labels: List[str] = None,
+        state: schemas.ProjectState = None,
+        names: Optional[List[str]] = None,
+    ) -> schemas.ProjectSummariesOutput:
+        raise NotImplementedError()
+
+    def get_project_summary(self, session, name: str) -> schemas.ProjectSummary:
+        raise NotImplementedError("Get project summary is not supported")
+
     def store_project(self, session, name: str, project: schemas.Project):
         raise NotImplementedError()
 

mlrun/api/db/sqldb/db.py

@@ -60,7 +60,7 @@
 unversioned_tagged_object_uid_prefix = "unversioned-"
 
 
-class SQLDB(mlrun.api.utils.projects.remotes.follower.Member, DBInterface):
+class SQLDB(DBInterface):
     def __init__(self, dsn):
         self.dsn = dsn
         self._cache = {
@@ -858,6 +858,30 @@ def list_projects(
                     )
         return schemas.ProjectsOutput(projects=projects)
 
+    def list_project_summaries(
+        self,
+        session: Session,
+        owner: str = None,
+        labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
+        names: typing.Optional[typing.List[str]] = None,
+    ) -> mlrun.api.schemas.ProjectSummariesOutput:
+        query = self._query(session, Project.name, owner=owner, state=state)
+        if labels:
+            query = self._add_labels_filter(session, query, Project, labels)
+        if names:
+            query = query.filter(Project.name.in_(names))
+        project_names = [name for name, in query]
+        project_summaries = self.generate_projects_summaries(session, project_names)
+        return schemas.ProjectSummariesOutput(project_summaries=project_summaries)
+
+    def get_project_summary(
+        self, session: Session, name: str
+    ) -> mlrun.api.schemas.ProjectSummary:
+        # Call get project so we'll explode if project doesn't exists
+        self.get_project(session, name)
+        return self.generate_projects_summaries(session, [name])[0]
+
     def _get_project_resources_counters(self, session: Session):
         now = datetime.now()
         if (
@@ -878,6 +902,14 @@ def _get_project_resources_counters(self, session: Session):
             project_to_function_count = {
                 result[0]: result[1] for result in functions_count_per_project
             }
+            schedules_count_per_project = (
+                session.query(Schedule.project, func.count(distinct(Schedule.name)))
+                .group_by(Schedule.project)
+                .all()
+            )
+            project_to_schedule_count = {
+                result[0]: result[1] for result in schedules_count_per_project
+            }
             feature_sets_count_per_project = (
                 session.query(FeatureSet.project, func.count(distinct(FeatureSet.name)))
                 .group_by(FeatureSet.project)
@@ -942,6 +974,7 @@ def _get_project_resources_counters(self, session: Session):
 
             self._cache["project_resources_counters"]["result"] = (
                 project_to_function_count,
+                project_to_schedule_count,
                 project_to_feature_set_count,
                 project_to_models_count,
                 project_to_recent_failed_runs_count,
@@ -961,6 +994,7 @@ def generate_projects_summaries(
     ) -> List[mlrun.api.schemas.ProjectSummary]:
         (
             project_to_function_count,
+            project_to_schedule_count,
             project_to_feature_set_count,
             project_to_models_count,
             project_to_recent_failed_runs_count,
@@ -972,12 +1006,16 @@ def generate_projects_summaries(
                 mlrun.api.schemas.ProjectSummary(
                     name=project,
                     functions_count=project_to_function_count.get(project, 0),
+                    schedules_count=project_to_schedule_count.get(project, 0),
                     feature_sets_count=project_to_feature_set_count.get(project, 0),
                     models_count=project_to_models_count.get(project, 0),
                     runs_failed_recent_count=project_to_recent_failed_runs_count.get(
                         project, 0
                     ),
                     runs_running_count=project_to_running_runs_count.get(project, 0),
+                    # This is a mandatory field - filling here with 0, it will be filled with the real number in the
+                    # crud layer
+                    pipelines_running_count=0,
                 )
             )
         return project_summaries

mlrun/api/schemas/__init__.py

@@ -88,6 +88,7 @@
     ProjectSpec,
     ProjectState,
     ProjectStatus,
+    ProjectSummariesOutput,
     ProjectSummary,
 )
 from .runtime_resource import (

mlrun/api/schemas/project.py

@@ -90,6 +90,8 @@ class ProjectSummary(pydantic.BaseModel):
     models_count: int
     runs_failed_recent_count: int
     runs_running_count: int
+    schedules_count: int
+    pipelines_running_count: int
 
 
 class IguazioProject(pydantic.BaseModel):
@@ -107,3 +109,7 @@ class ProjectsOutput(pydantic.BaseModel):
     # to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
     # https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
     projects: typing.List[typing.Union[Project, str, ProjectSummary, IguazioProject]]
+
+
+class ProjectSummariesOutput(pydantic.BaseModel):
+    project_summaries: typing.List[ProjectSummary]

mlrun/api/utils/clients/nuclio.py

@@ -121,6 +121,7 @@ def list_projects(
         format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
         labels: typing.List[str] = None,
         state: mlrun.api.schemas.ProjectState = None,
+        names: typing.Optional[typing.List[str]] = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
         if owner:
             raise NotImplementedError(
@@ -134,6 +135,10 @@ def list_projects(
             raise NotImplementedError(
                 "Filtering nuclio projects by state is currently not supported"
             )
+        if names:
+            raise NotImplementedError(
+                "Filtering nuclio projects by names is currently not supported"
+            )
         response = self._send_request_to_api("GET", "projects")
         response_body = response.json()
         projects = []
@@ -150,6 +155,21 @@ def list_projects(
                 f"Provided format is not supported. format={format_}"
             )
 
+    def list_project_summaries(
+        self,
+        session: sqlalchemy.orm.Session,
+        owner: str = None,
+        labels: typing.List[str] = None,
+        state: mlrun.api.schemas.ProjectState = None,
+        names: typing.Optional[typing.List[str]] = None,
+    ) -> mlrun.api.schemas.ProjectSummariesOutput:
+        raise NotImplementedError("Listing project summaries is not supported")
+
+    def get_project_summary(
+        self, session: sqlalchemy.orm.Session, name: str
+    ) -> mlrun.api.schemas.ProjectSummary:
+        raise NotImplementedError("Get project summary is not supported")
+
     def get_dashboard_version(self) -> str:
         response = self._send_request_to_api("GET", "versions")
         response_body = response.json()