[API] Align Old Formatted Objects To Use New ObjectFormat (#5661)

mlrun/api/schemas/__init__.py

@@ -25,6 +25,7 @@
 import sys
 import warnings
 
+import mlrun.common.formatters
 import mlrun.common.schemas
 import mlrun.common.schemas.artifact as old_artifact
 import mlrun.common.schemas.auth as old_auth
@@ -96,7 +97,7 @@ def __getattr__(self, attr):
 # and return the new schema. This is done for backwards compatibility with mlrun.api.schemas.
 ArtifactCategories = DeprecationHelper(mlrun.common.schemas.ArtifactCategories)
 ArtifactIdentifier = DeprecationHelper(mlrun.common.schemas.ArtifactIdentifier)
-ArtifactsFormat = DeprecationHelper(mlrun.common.schemas.ArtifactsFormat)
+ArtifactsFormat = DeprecationHelper(mlrun.common.formatters.ArtifactsFormat)
 AuthInfo = DeprecationHelper(mlrun.common.schemas.AuthInfo)
 AuthorizationAction = DeprecationHelper(mlrun.common.schemas.AuthorizationAction)
 AuthorizationResourceTypes = DeprecationHelper(
@@ -221,15 +222,15 @@ def __getattr__(self, attr):
 ObjectMetadata = DeprecationHelper(mlrun.common.schemas.ObjectMetadata)
 ObjectSpec = DeprecationHelper(mlrun.common.schemas.ObjectSpec)
 ObjectStatus = DeprecationHelper(mlrun.common.schemas.ObjectStatus)
-PipelinesFormat = DeprecationHelper(mlrun.common.schemas.PipelinesFormat)
+PipelinesFormat = DeprecationHelper(mlrun.common.formatters.PipelinesFormat)
 PipelinesOutput = DeprecationHelper(mlrun.common.schemas.PipelinesOutput)
 PipelinesPagination = DeprecationHelper(mlrun.common.schemas.PipelinesPagination)
 IguazioProject = DeprecationHelper(mlrun.common.schemas.IguazioProject)
 Project = DeprecationHelper(mlrun.common.schemas.Project)
 ProjectDesiredState = DeprecationHelper(mlrun.common.schemas.ProjectDesiredState)
 ProjectMetadata = DeprecationHelper(mlrun.common.schemas.ProjectMetadata)
 ProjectOwner = DeprecationHelper(mlrun.common.schemas.ProjectOwner)
-ProjectsFormat = DeprecationHelper(mlrun.common.schemas.ProjectsFormat)
+ProjectsFormat = DeprecationHelper(mlrun.common.formatters.ProjectsFormat)
 ProjectsOutput = DeprecationHelper(mlrun.common.schemas.ProjectsOutput)
 ProjectSpec = DeprecationHelper(mlrun.common.schemas.ProjectSpec)
 ProjectState = DeprecationHelper(mlrun.common.schemas.ProjectState)

mlrun/common/formatters/__init__.py

@@ -13,4 +13,7 @@
 # limitations under the License.
 #
 
+from .artifact import ArtifactFormat  # noqa
 from .function import FunctionFormat  # noqa
+from .pipeline import PipelineFormat  # noqa
+from .project import ProjectFormat  # noqa

mlrun/common/formatters/artifact.py

@@ -0,0 +1,21 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import mlrun.common.types
+
+
+# TODO: add a format that returns a minimal response with ObjectFormat
+class ArtifactFormat(mlrun.common.types.StrEnum):
+    full = "full"

mlrun/common/formatters/base.py

@@ -15,6 +15,8 @@
 
 import typing
 
+import mlrun.errors
+
 
 class ObjectFormat:
     full = "full"
@@ -26,9 +28,26 @@ def format_method(_format: str) -> typing.Optional[typing.Callable]:
         }[_format]
 
     @classmethod
-    def format_obj(cls, obj: typing.Any, _format: str) -> typing.Any:
+    def format_obj(
+        cls,
+        obj: typing.Any,
+        _format: str,
+        exclude_formats: typing.Optional[list[str]] = None,
+    ) -> typing.Any:
+        exclude_formats = exclude_formats or []
         _format = _format or cls.full
-        format_method = cls.format_method(_format)
+        invalid_format_exc = mlrun.errors.MLRunBadRequestError(
+            f"Provided format is not supported. format={_format}"
+        )
+
+        if _format in exclude_formats:
+            raise invalid_format_exc
+
+        try:
+            format_method = cls.format_method(_format)
+        except KeyError:
+            raise invalid_format_exc
+
         if not format_method:
             return obj
 

mlrun/common/formatters/pipeline.py

@@ -0,0 +1,53 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun_pipelines.common.ops
+import mlrun_pipelines.models
+
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class PipelineFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    metadata_only = "metadata_only"
+    name_only = "name_only"
+    summary = "summary"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _full(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return run.to_dict()
+
+        def _metadata_only(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun.utils.helpers.format_run(run, with_project=True)
+
+        def _name_only(run: mlrun_pipelines.models.PipelineRun) -> str:
+            return run.get("name")
+
+        def _summary(run: mlrun_pipelines.models.PipelineRun) -> dict:
+            return mlrun_pipelines.common.ops.format_summary_from_kfp_run(
+                run, run["project"]
+            )
+
+        return {
+            PipelineFormat.full: _full,
+            PipelineFormat.metadata_only: _metadata_only,
+            PipelineFormat.name_only: _name_only,
+            PipelineFormat.summary: _summary,
+        }[_format]

mlrun/common/formatters/project.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+
+import mlrun.common.schemas
+import mlrun.common.types
+
+from .base import ObjectFormat
+
+
+class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
+    full = "full"
+    name_only = "name_only"
+    # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
+    # and is used for faster response times (in the UI)
+    minimal = "minimal"
+    # internal - allowed only in follower mode, only for the leader for upgrade purposes
+    leader = "leader"
+
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        def _name_only(project: mlrun.common.schemas.Project) -> str:
+            return project.metadata.name
+
+        def _minimal(
+            project: mlrun.common.schemas.Project,
+        ) -> mlrun.common.schemas.Project:
+            project.spec.functions = None
+            project.spec.workflows = None
+            project.spec.artifacts = None
+            return project
+
+        return {
+            ProjectFormat.full: None,
+            ProjectFormat.name_only: _name_only,
+            ProjectFormat.minimal: _minimal,
+            ProjectFormat.leader: None,
+        }[_format]

mlrun/common/schemas/__init__.py

@@ -37,7 +37,6 @@
     ArtifactCategories,
     ArtifactIdentifier,
     ArtifactMetadata,
-    ArtifactsFormat,
     ArtifactSpec,
 )
 from .auth import (
@@ -161,15 +160,14 @@
 )
 from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
 from .pagination import PaginationInfo
-from .pipeline import PipelinesFormat, PipelinesOutput, PipelinesPagination
+from .pipeline import PipelinesOutput, PipelinesPagination
 from .project import (
     IguazioProject,
     Project,
     ProjectDesiredState,
     ProjectMetadata,
     ProjectOutput,
     ProjectOwner,
-    ProjectsFormat,
     ProjectsOutput,
     ProjectSpec,
     ProjectState,

mlrun/common/schemas/artifact.py

@@ -58,11 +58,6 @@ class ArtifactIdentifier(pydantic.BaseModel):
     # hash: typing.Optional[str]
 
 
-class ArtifactsFormat(mlrun.common.types.StrEnum):
-    # TODO: add a format that returns a minimal response
-    full = "full"
-
-
 class ArtifactMetadata(pydantic.BaseModel):
     key: str
     project: str

mlrun/common/schemas/pipeline.py

@@ -16,15 +16,6 @@
 
 import pydantic
 
-import mlrun.common.types
-
-
-class PipelinesFormat(mlrun.common.types.StrEnum):
-    full = "full"
-    metadata_only = "metadata_only"
-    summary = "summary"
-    name_only = "name_only"
-
 
 class PipelinesPagination(str):
     default_page_size = 20

mlrun/common/schemas/project.py

@@ -23,16 +23,6 @@
 from .object import ObjectKind, ObjectStatus
 
 
-class ProjectsFormat(mlrun.common.types.StrEnum):
-    full = "full"
-    name_only = "name_only"
-    # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
-    # and is used for faster response times (in the UI)
-    minimal = "minimal"
-    # internal - allowed only in follower mode, only for the leader for upgrade purposes
-    leader = "leader"
-
-
 class ProjectMetadata(pydantic.BaseModel):
     name: str
     created: typing.Optional[datetime.datetime] = None

mlrun/db/base.py

@@ -17,6 +17,7 @@
 from typing import Optional, Union
 
 import mlrun.alerts
+import mlrun.common.formatters
 import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.model_monitoring
@@ -267,7 +268,7 @@ def create_project(
     def list_projects(
         self,
         owner: str = None,
-        format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.name_only,
+        format_: mlrun.common.formatters.ProjectFormat = mlrun.common.formatters.ProjectFormat.name_only,
         labels: list[str] = None,
         state: mlrun.common.schemas.ProjectState = None,
     ) -> mlrun.common.schemas.ProjectsOutput:
@@ -443,8 +444,8 @@ def get_pipeline(
         namespace: str = None,
         timeout: int = 30,
         format_: Union[
-            str, mlrun.common.schemas.PipelinesFormat
-        ] = mlrun.common.schemas.PipelinesFormat.summary,
+            str, mlrun.common.formatters.PipelineFormat
+        ] = mlrun.common.formatters.PipelineFormat.summary,
         project: str = None,
     ):
         pass
@@ -458,8 +459,8 @@ def list_pipelines(
         page_token: str = "",
         filter_: str = "",
         format_: Union[
-            str, mlrun.common.schemas.PipelinesFormat
-        ] = mlrun.common.schemas.PipelinesFormat.metadata_only,
+            str, mlrun.common.formatters.PipelineFormat
+        ] = mlrun.common.formatters.PipelineFormat.metadata_only,
         page_size: int = None,
     ) -> mlrun.common.schemas.PipelinesOutput:
         pass

mlrun/db/httpdb.py

@@ -30,6 +30,7 @@
 from mlrun_pipelines.utils import compile_pipeline
 
 import mlrun
+import mlrun.common.formatters
 import mlrun.common.runtimes
 import mlrun.common.schemas
 import mlrun.common.types
@@ -995,7 +996,7 @@ def read_artifact(
         error = f"read artifact {project}/{key}"
         # explicitly set artifacts format to 'full' since old servers may default to 'legacy'
         params = {
-            "format": mlrun.common.schemas.ArtifactsFormat.full.value,
+            "format": mlrun.common.formatters.ArtifactFormat.full.value,
             "tag": tag,
             "tree": tree,
             "uid": uid,
@@ -1111,7 +1112,7 @@ def list_artifacts(
             "kind": kind,
             "category": category,
             "tree": tree,
-            "format": mlrun.common.schemas.ArtifactsFormat.full.value,
+            "format": mlrun.common.formatters.ArtifactFormat.full.value,
             "producer_uri": producer_uri,
         }
         error = "list artifacts"
@@ -1933,8 +1934,8 @@ def list_pipelines(
         page_token: str = "",
         filter_: str = "",
         format_: Union[
-            str, mlrun.common.schemas.PipelinesFormat
-        ] = mlrun.common.schemas.PipelinesFormat.metadata_only,
+            str, mlrun.common.formatters.PipelineFormat
+        ] = mlrun.common.formatters.PipelineFormat.metadata_only,
         page_size: int = None,
     ) -> mlrun.common.schemas.PipelinesOutput:
         """Retrieve a list of KFP pipelines. This function can be invoked to get all pipelines from all projects,
@@ -1980,8 +1981,8 @@ def get_pipeline(
         namespace: str = None,
         timeout: int = 30,
         format_: Union[
-            str, mlrun.common.schemas.PipelinesFormat
-        ] = mlrun.common.schemas.PipelinesFormat.summary,
+            str, mlrun.common.formatters.PipelineFormat
+        ] = mlrun.common.formatters.PipelineFormat.summary,
         project: str = None,
     ):
         """Retrieve details of a specific pipeline using its run ID (as provided when the pipeline was executed)."""
@@ -2623,8 +2624,8 @@ def list_projects(
         self,
         owner: str = None,
         format_: Union[
-            str, mlrun.common.schemas.ProjectsFormat
-        ] = mlrun.common.schemas.ProjectsFormat.name_only,
+            str, mlrun.common.formatters.ProjectFormat
+        ] = mlrun.common.formatters.ProjectFormat.name_only,
         labels: list[str] = None,
         state: Union[str, mlrun.common.schemas.ProjectState] = None,
     ) -> list[Union[mlrun.projects.MlrunProject, str]]:
@@ -2650,7 +2651,7 @@ def list_projects(
 
         error_message = f"Failed listing projects, query: {params}"
         response = self.api_call("GET", "projects", error_message, params=params)
-        if format_ == mlrun.common.schemas.ProjectsFormat.name_only:
+        if format_ == mlrun.common.formatters.ProjectFormat.name_only:
             # projects is just a list of strings
             return response.json()["projects"]