[Model Monitoring] Update disable_model_monitoring sdk and api & Ad…

…ded new api and sdk for`delete_model_monitoring_function` (#5685)

mlrun/common/constants.py

@@ -42,6 +42,7 @@ class MLRunInternalLabels:
 
     ### nuclio
     nuclio_project_name = f"{NUCLIO_LABEL_PREFIX}project-name"
+    nuclio_function_name = f"{NUCLIO_LABEL_PREFIX}function-name"
     nuclio_class = f"{NUCLIO_LABEL_PREFIX}class"
 
     ### mlrun

mlrun/common/types.py

@@ -29,3 +29,4 @@ def __repr__(self):
 class HTTPMethod(StrEnum):
     GET = "GET"
     POST = "POST"
+    DELETE = "DELETE"

mlrun/db/base.py

@@ -841,6 +841,24 @@ def enable_model_monitoring(
     ) -> None:
         pass
 
+    @abstractmethod
+    def disable_model_monitoring(
+        self,
+        project: str,
+        delete_resources: bool = True,
+        delete_stream_function: bool = False,
+        delete_histogram_data_drift_app: bool = True,
+        delete_user_applications: bool = False,
+        user_application_list: list[str] = None,
+    ) -> bool:
+        pass
+
+    @abstractmethod
+    def delete_model_monitoring_function(
+        self, project: str, functions: list[str]
+    ) -> bool:
+        pass
+
     @abstractmethod
     def deploy_histogram_data_drift_app(
         self, project: str, image: str = "mlrun/mlrun"

mlrun/db/httpdb.py

@@ -3347,6 +3347,113 @@ def enable_model_monitoring(
             },
         )
 
+    def disable_model_monitoring(
+        self,
+        project: str,
+        delete_resources: bool = True,
+        delete_stream_function: bool = False,
+        delete_histogram_data_drift_app: bool = True,
+        delete_user_applications: bool = False,
+        user_application_list: list[str] = None,
+    ) -> bool:
+        """
+        Disable model monitoring application controller, writer, stream, histogram data drift application
+        and the user's applications functions, according to the given params.
+
+        :param project:                             Project name.
+        :param delete_resources:                    If True, it would delete the model monitoring controller & writer
+                                                    functions. Default True
+        :param delete_stream_function:              If True, it would delete model monitoring stream function,
+                                                    need to use wisely because if you're deleting this function
+                                                    this can cause data loss in case you will want to
+                                                    enable the model monitoring capability to the project.
+                                                    Default False.
+        :param delete_histogram_data_drift_app:     If True, it would delete the default histogram-based data drift
+                                                    application. Default False.
+        :param delete_user_applications:            If True, it would delete the user's model monitoring
+                                                    application according to user_application_list, Default False.
+        :param user_application_list:               List of the user's model monitoring application to disable.
+                                                    Default all the applications.
+                                                    Note: you have to set delete_user_applications to True
+                                                    in order to delete the desired application.
+
+        :returns:                                   True if the deletion was successful, False otherwise.
+        """
+        response = self.api_call(
+            method=mlrun.common.types.HTTPMethod.DELETE,
+            path=f"projects/{project}/model-monitoring/disable-model-monitoring",
+            params={
+                "delete_resources": delete_resources,
+                "delete_stream_function": delete_stream_function,
+                "delete_histogram_data_drift_app": delete_histogram_data_drift_app,
+                "delete_user_applications": delete_user_applications,
+                "user_application_list": user_application_list,
+            },
+        )
+        deletion_failed = False
+        if response.status_code == http.HTTPStatus.ACCEPTED:
+            if delete_resources:
+                logger.info(
+                    "Model Monitoring is being disable",
+                    project_name=project,
+                )
+            if delete_user_applications:
+                logger.info("User applications are being deleted", project_name=project)
+            background_tasks = mlrun.common.schemas.BackgroundTaskList(
+                **response.json()
+            ).background_tasks
+            for task in background_tasks:
+                task = self._wait_for_background_task_to_reach_terminal_state(
+                    task.metadata.name, project=project
+                )
+                if (
+                    task.status.state
+                    == mlrun.common.schemas.BackgroundTaskState.succeeded
+                ):
+                    continue
+                elif (
+                    task.status.state == mlrun.common.schemas.BackgroundTaskState.failed
+                ):
+                    deletion_failed = True
+        return not deletion_failed
+
+    def delete_model_monitoring_function(
+        self, project: str, functions: list[str]
+    ) -> bool:
+        """
+        Delete a model monitoring application.
+
+        :param functions:            List of the model monitoring function to delete.
+        :param project:              Project name.
+
+        :returns:                    True if the deletion was successful, False otherwise.
+        """
+        response = self.api_call(
+            method=mlrun.common.types.HTTPMethod.DELETE,
+            path=f"projects/{project}/model-monitoring/functions",
+            params={"functions": functions},
+        )
+        deletion_failed = False
+        if response.status_code == http.HTTPStatus.ACCEPTED:
+            logger.info("User applications are being deleted", project_name=project)
+            background_tasks = mlrun.common.schemas.BackgroundTaskList(
+                **response.json()
+            ).background_tasks
+            for task in background_tasks:
+                task = self._wait_for_background_task_to_reach_terminal_state(
+                    task.metadata.name, project=project
+                )
+                if (
+                    task.status.state
+                    == mlrun.common.schemas.BackgroundTaskState.succeeded
+                ):
+                    continue
+                elif (
+                    task.status.state == mlrun.common.schemas.BackgroundTaskState.failed
+                ):
+                    deletion_failed = True
+        return not deletion_failed
+
     def deploy_histogram_data_drift_app(
         self, project: str, image: str = "mlrun/mlrun"
     ) -> None:

mlrun/db/nopdb.py

@@ -678,6 +678,22 @@ def enable_model_monitoring(
     ) -> None:
         pass
 
+    def disable_model_monitoring(
+        self,
+        project: str,
+        delete_resources: bool = True,
+        delete_stream_function: bool = False,
+        delete_histogram_data_drift_app: bool = True,
+        delete_user_applications: bool = False,
+        user_application_list: list[str] = None,
+    ) -> bool:
+        pass
+
+    def delete_model_monitoring_function(
+        self, project: str, functions: list[str]
+    ) -> bool:
+        pass
+
     def deploy_histogram_data_drift_app(
         self, project: str, image: str = "mlrun/mlrun"
     ) -> None:

mlrun/projects/project.py

@@ -2215,23 +2215,67 @@ def update_model_monitoring_controller(
             )
 
     def disable_model_monitoring(
-        self, *, delete_histogram_data_drift_app: bool = True
+        self,
+        *,
+        delete_resources: bool = True,
+        delete_stream_function: bool = False,
+        delete_histogram_data_drift_app: bool = True,
+        delete_user_applications: bool = False,
+        user_application_list: list[str] = None,
     ) -> None:
         """
-        Note: This method is currently not advised for use. See ML-3432.
-        Disable model monitoring by deleting the underlying functions infrastructure from MLRun database.
-
-        :param delete_histogram_data_drift_app: Whether to delete the histogram data drift app.
+        Disable model monitoring application controller, writer, stream, histogram data drift application
+        and the user's applications functions, according to the given params.
+
+        :param delete_resources:                    If True, it would delete the model monitoring controller & writer
+                                                    functions. Default True
+        :param delete_stream_function:              If True, it would delete model monitoring stream function,
+                                                    need to use wisely because if you're deleting this function
+                                                    this can cause data loss in case you will want to
+                                                    enable the model monitoring capability to the project.
+                                                    Default False.
+        :param delete_histogram_data_drift_app:     If True, it would delete the default histogram-based data drift
+                                                    application. Default False.
+        :param delete_user_applications:            If True, it would delete the user's model monitoring
+                                                    application according to user_application_list, Default False.
+        :param user_application_list:               List of the user's model monitoring application to disable.
+                                                    Default all the applications.
+                                                    Note: you have to set delete_user_applications to True
+                                                    in order to delete the desired application.
         """
-        db = mlrun.db.get_run_db(secrets=self._secrets)
-        for fn_name in mm_constants.MonitoringFunctionNames.list():
-            db.delete_function(project=self.name, name=fn_name)
-        if delete_histogram_data_drift_app:
-            db.delete_function(
-                project=self.name,
-                name=mm_constants.HistogramDataDriftApplicationConstants.NAME,
+        if not delete_user_applications and user_application_list:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "user_application_list can be specified only if delete_user_applications is set to True"
             )
 
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        succeed = db.disable_model_monitoring(
+            project=self.name,
+            delete_resources=delete_resources,
+            delete_stream_function=delete_stream_function,
+            delete_histogram_data_drift_app=delete_histogram_data_drift_app,
+            delete_user_applications=delete_user_applications,
+            user_application_list=user_application_list,
+        )
+        if succeed and delete_resources:
+            if delete_resources:
+                logger.info("Model Monitoring disabled", project=self.name)
+            if delete_user_applications:
+                logger.info(
+                    "All the desired monitoring application were deleted",
+                    project=self.name,
+                )
+        else:
+            if delete_resources:
+                logger.info(
+                    "Model Monitoring was not disabled properly", project=self.name
+                )
+            if delete_user_applications:
+                logger.info(
+                    "Some of the desired monitoring application were not deleted",
+                    project=self.name,
+                )
+
     def set_function(
         self,
         func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
@@ -2426,22 +2470,47 @@ def remove_function(self, name):
         """
         self.spec.remove_function(name)
 
-    def remove_model_monitoring_function(self, name):
-        """remove the specified model-monitoring-app function from the project and from the db
+    def remove_model_monitoring_function(self, name: Union[str, list[str]]):
+        """remove the specified model-monitoring-app function/s from the project spec
 
-        :param name: name of the model-monitoring-app function (under the project)
+        :param name: name of the model-monitoring-function/s (under the project)
         """
-        function = self.get_function(key=name)
-        if (
-            function.metadata.labels.get(mm_constants.ModelMonitoringAppLabel.KEY)
-            == mm_constants.ModelMonitoringAppLabel.VAL
-        ):
-            self.remove_function(name=name)
-            mlrun.db.get_run_db().delete_function(name=name.lower())
-            logger.info(f"{name} function has been removed from {self.name} project")
+        names = name if isinstance(name, list) else [name]
+        for func_name in names:
+            function = self.get_function(key=func_name)
+            if (
+                function.metadata.labels.get(mm_constants.ModelMonitoringAppLabel.KEY)
+                == mm_constants.ModelMonitoringAppLabel.VAL
+            ):
+                self.remove_function(name=func_name)
+                logger.info(
+                    f"{func_name} function has been removed from {self.name} project"
+                )
+            else:
+                raise logger.warn(
+                    f"There is no model monitoring function with {func_name} name"
+                )
+
+    def delete_model_monitoring_function(self, name: Union[str, list[str]]):
+        """delete the specified model-monitoring-app function/s
+
+        :param name: name of the model-monitoring-function/s (under the project)
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        succeed = db.delete_model_monitoring_function(
+            project=self.name,
+            functions=name if isinstance(name, list) else [name],
+        )
+        if succeed:
+            logger.info(
+                "All the desired monitoring functions were deleted",
+                project=self.name,
+                functions=name,
+            )
         else:
-            raise logger.error(
-                f"There is no model monitoring function with {name} name"
+            logger.info(
+                "Some of the desired monitoring functions were not deleted",
+                project=self.name,
             )
 
     def get_function(