From 73dcf54e53641d307dc9a22349b7e2b47f072628 Mon Sep 17 00:00:00 2001 From: danielperezz Date: Mon, 10 Nov 2025 12:56:51 +0200 Subject: [PATCH 1/3] sphinx build docs bug fix --- cli/marketplace/conf.template | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/marketplace/conf.template b/cli/marketplace/conf.template index 93c83c9d3..e26f065aa 100644 --- a/cli/marketplace/conf.template +++ b/cli/marketplace/conf.template @@ -15,8 +15,14 @@ import re import sys import os -sys.path.insert(0, "{{sphinx_docs_target}}") -sys.path.insert(0, os.path.abspath(os.path.join("{{sphinx_docs_target}}", "../functions"))) +import pathlib + +DOCS_DIR = pathlib.Path(__file__).resolve().parent +REPO_ROOT = DOCS_DIR.parent + +# Add both source trees +sys.path.insert(0, str(REPO_ROOT / "functions")) +sys.path.insert(0, str(REPO_ROOT / "modules")) # -- Project information ----------------------------------------------------- From f429b94701429240ad676c6cd147a20024c82307 Mon Sep 17 00:00:00 2001 From: danielperezz Date: Mon, 10 Nov 2025 12:57:15 +0200 Subject: [PATCH 2/3] add evidently demo app module (empty example notebook) --- modules/src/evidently/evidently_iris.ipynb | 37 ++++++ modules/src/evidently/evidently_iris.py | 117 +++++++++++++++++++ modules/src/evidently/item.yaml | 21 ++++ modules/src/evidently/requirements.txt | 3 + modules/src/evidently/test_evidently_iris.py | 72 ++++++++++++ 5 files changed, 250 insertions(+) create mode 100644 modules/src/evidently/evidently_iris.ipynb create mode 100644 modules/src/evidently/evidently_iris.py create mode 100644 modules/src/evidently/item.yaml create mode 100644 modules/src/evidently/requirements.txt create mode 100644 modules/src/evidently/test_evidently_iris.py diff --git a/modules/src/evidently/evidently_iris.ipynb b/modules/src/evidently/evidently_iris.ipynb new file mode 100644 index 000000000..54f657bb0 --- /dev/null +++ b/modules/src/evidently/evidently_iris.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "initial_id", + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modules/src/evidently/evidently_iris.py b/modules/src/evidently/evidently_iris.py new file mode 100644 index 000000000..3d8896107 --- /dev/null +++ b/modules/src/evidently/evidently_iris.py @@ -0,0 +1,117 @@ +# Copyright 2025 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import pandas as pd +from sklearn.datasets import load_iris + +import mlrun.model_monitoring.applications.context as mm_context +from mlrun.common.schemas.model_monitoring.constants import ( + ResultKindApp, + ResultStatusApp, +) +from mlrun.feature_store.api import norm_column_name +from mlrun.model_monitoring.applications import ModelMonitoringApplicationResult +from mlrun.model_monitoring.applications.evidently import ( + _HAS_EVIDENTLY, + EvidentlyModelMonitoringApplicationBase, +) + +if _HAS_EVIDENTLY: + from evidently.core.report import Report, Snapshot + from evidently.metrics import DatasetMissingValueCount, ValueDrift + from evidently.presets import DataDriftPreset, DataSummaryPreset + from evidently.ui.workspace import ( + STR_UUID, + OrgID, + ) + + _PROJECT_NAME = "Iris Monitoring" + _PROJECT_DESCRIPTION = "Test project using iris dataset" + + +class EvidentlyIrisMonitoringApp(EvidentlyModelMonitoringApplicationBase): + NAME = "Evidently-App-Test" + + def __init__( + self, + evidently_project_id: Optional["STR_UUID"] = None, + evidently_workspace_path: Optional[str] = None, + cloud_workspace: bool = False, + evidently_organization_id: Optional["OrgID"] = None, + ) -> None: + self.org_id = evidently_organization_id + self._init_iris_data() + super().__init__( + evidently_project_id=evidently_project_id, + evidently_workspace_path=evidently_workspace_path, + cloud_workspace=cloud_workspace, + ) + + def _init_iris_data(self) -> None: + iris = load_iris() + self.columns = [norm_column_name(col) for col in iris.feature_names] + self.train_set = pd.DataFrame(iris.data, columns=self.columns) + + def do_tracking( + self, monitoring_context: mm_context.MonitoringApplicationContext + ) -> ModelMonitoringApplicationResult: + monitoring_context.logger.info("Running evidently app") + + sample_df = monitoring_context.sample_df[self.columns] + + data_drift_report_run = self.create_report_run( + sample_df, monitoring_context.end_infer_time + ) + self.evidently_workspace.add_run( + self.evidently_project_id, data_drift_report_run + ) + + self.log_evidently_object( + monitoring_context, data_drift_report_run, "evidently_report" + ) + monitoring_context.logger.info("Logged evidently object") + + return ModelMonitoringApplicationResult( + name="data_drift_test", + value=0.5, + kind=ResultKindApp.data_drift, + status=ResultStatusApp.potential_detection, + ) + + def create_report_run( + self, sample_df: pd.DataFrame, schedule_time: pd.Timestamp + ) -> "Snapshot": + metrics = [ + DataDriftPreset(), + DatasetMissingValueCount(), + DataSummaryPreset(), + ] + metrics.extend( + [ + ValueDrift(column=col_name, method="wasserstein") + for col_name in self.columns + ] + ) + + data_drift_report = Report( + metrics=metrics, + metadata={"timestamp": str(schedule_time)}, + include_tests=True, + ) + + return data_drift_report.run( + current_data=sample_df, reference_data=self.train_set + ) diff --git a/modules/src/evidently/item.yaml b/modules/src/evidently/item.yaml new file mode 100644 index 000000000..c6a2abc2c --- /dev/null +++ b/modules/src/evidently/item.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +categories: +- model-serving +- structured-ML +description: Demonstrates Evidently integration in MLRun for data quality and drift monitoring using the Iris dataset +example: evidently_iris.ipynb +generationDate: 2025-11-09 +hidden: false +labels: + author: Iguazio +mlrunVersion: 1.10.0-rc41 +name: evidently_iris +spec: + filename: evidently_iris.py + image: mlrun/mlrun + kind: monitoring_application + requirements: + - scikit-learn~=1.5.2 + - evidently~=0.7.6 + - pandas +version: 1.0.0 \ No newline at end of file diff --git a/modules/src/evidently/requirements.txt b/modules/src/evidently/requirements.txt new file mode 100644 index 000000000..bd4abb36f --- /dev/null +++ b/modules/src/evidently/requirements.txt @@ -0,0 +1,3 @@ +scikit-learn~=1.5.2 +evidently~=0.7.6 +pandas \ No newline at end of file diff --git a/modules/src/evidently/test_evidently_iris.py b/modules/src/evidently/test_evidently_iris.py new file mode 100644 index 000000000..6488768fd --- /dev/null +++ b/modules/src/evidently/test_evidently_iris.py @@ -0,0 +1,72 @@ +# Copyright 2023 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from contextlib import AbstractContextManager +from contextlib import nullcontext as does_not_raise +from pathlib import Path +from uuid import uuid4 + +import pytest +import semver + +from mlrun.errors import MLRunIncompatibleVersionError +from mlrun.model_monitoring.applications.evidently.base import ( + _check_evidently_version, +) + +from evidently_iris import EvidentlyIrisMonitoringApp + + +@pytest.mark.parametrize( + ("cur", "ref", "expectation"), + [ + ("0.4.11", "0.4.11", does_not_raise()), + ("0.4.12", "0.4.11", does_not_raise()), + ("1.23.0", "1.1.32", does_not_raise()), + ("0.4.11", "0.4.12", pytest.raises(MLRunIncompatibleVersionError)), + ("0.4.11", "0.4.12", pytest.raises(MLRunIncompatibleVersionError)), + ("1.0.3", "0.9.9", pytest.raises(MLRunIncompatibleVersionError)), + ("0.6.0", "0.3.0", pytest.warns(UserWarning)), + pytest.param("0.6.0", "0.3.0", does_not_raise(), marks=pytest.mark.xfail), + ], +) +def test_version_check( + cur: str, + ref: str, + expectation: AbstractContextManager, +) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error") + with expectation: + _check_evidently_version( + cur=semver.Version.parse(cur), ref=semver.Version.parse(ref) + ) + + +def test_demo_evidently_app(tmpdir: Path) -> None: + """Test that the workspace and the project's dashboards are created""" + evidently_app = EvidentlyIrisMonitoringApp( + evidently_project_id=uuid4(), evidently_workspace_path=str(tmpdir) + ) + run = evidently_app.create_report_run( + sample_df=evidently_app.train_set, schedule_time=None + ) + added_run_uid = evidently_app.evidently_workspace.add_run( + project_id=evidently_app.evidently_project_id, + run=run, + ).id + assert evidently_app.evidently_workspace.list_runs( + project_id=evidently_app.evidently_project_id + ) == [added_run_uid], "Different project runs than expected" From 3e2e7db106ecda04675a74f6722ba7762df8843f Mon Sep 17 00:00:00 2001 From: danielperezz Date: Tue, 11 Nov 2025 15:21:01 +0200 Subject: [PATCH 3/3] post review changes --- modules/src/evidently/evidently_iris.py | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/src/evidently/evidently_iris.py b/modules/src/evidently/evidently_iris.py index 3d8896107..e7a9f3ef9 100644 --- a/modules/src/evidently/evidently_iris.py +++ b/modules/src/evidently/evidently_iris.py @@ -24,25 +24,25 @@ ) from mlrun.feature_store.api import norm_column_name from mlrun.model_monitoring.applications import ModelMonitoringApplicationResult -from mlrun.model_monitoring.applications.evidently import ( - _HAS_EVIDENTLY, - EvidentlyModelMonitoringApplicationBase, +from mlrun.model_monitoring.applications.evidently import EvidentlyModelMonitoringApplicationBase + +from evidently.core.report import Report, Snapshot +from evidently.metrics import DatasetMissingValueCount, ValueDrift +from evidently.presets import DataDriftPreset, DataSummaryPreset +from evidently.ui.workspace import ( + STR_UUID, + OrgID, ) -if _HAS_EVIDENTLY: - from evidently.core.report import Report, Snapshot - from evidently.metrics import DatasetMissingValueCount, ValueDrift - from evidently.presets import DataDriftPreset, DataSummaryPreset - from evidently.ui.workspace import ( - STR_UUID, - OrgID, - ) - - _PROJECT_NAME = "Iris Monitoring" - _PROJECT_DESCRIPTION = "Test project using iris dataset" +_PROJECT_NAME = "Iris Monitoring" +_PROJECT_DESCRIPTION = "Test project using iris dataset" class EvidentlyIrisMonitoringApp(EvidentlyModelMonitoringApplicationBase): + """ + This model monitoring application is a simple example of integrating MLRun with Evidently for data monitoring, + which you can adapt to fit your own project needs or use as a reference implementation. + """ NAME = "Evidently-App-Test" def __init__(