diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d6b05ceebf..73e51b3de2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.4-dev0 +current_version = 0.4.5-dev1 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/docs/conf.py b/docs/conf.py index 2ce07541f3..c9fa4c05f7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -101,7 +101,7 @@ # built documents. # # The short X.Y version. -version = "0.4.4-dev0" +version = "0.4.5-dev1" # The full version, including alpha/beta/rc tags. release = "" # Is set by calling `setup.py docs` diff --git a/setup.cfg b/setup.cfg index 0808212ed8..1536024195 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ [metadata] name = whylogs -version = 0.4.4-dev0 +version = 0.4.5-dev1 description = Profile and monitor your ML data pipeline end-to-end author = WhyLabs.ai author-email = support@whylabs.ai diff --git a/src/whylogs/_version.py b/src/whylogs/_version.py index 489d85aff6..e541d45c51 100644 --- a/src/whylogs/_version.py +++ b/src/whylogs/_version.py @@ -1,3 +1,3 @@ """WhyLabs version number.""" -__version__ = "0.4.4-dev0" +__version__ = "0.4.5-dev1" diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 5837384e96..0a7a3f2677 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -20,7 +20,8 @@ def __init__(self, confusion_matrix: ConfusionMatrix = None, self.model_type = model_type if confusion_matrix is not None and regression_metrics is not None: - raise NotImplementedError("Regression Metrics together with Confusion Matrix not implemented yet") + raise NotImplementedError( + "Regression Metrics together with Confusion Matrix not implemented yet") if confusion_matrix is not None: if (self.model_type == ModelType.REGRESSION): @@ -38,14 +39,17 @@ def __init__(self, confusion_matrix: ConfusionMatrix = None, def to_protobuf(self, ) -> ModelMetricsMessage: return ModelMetricsMessage( scoreMatrix=self.confusion_matrix.to_protobuf() if self.confusion_matrix else None, - regressionMetrics=self.regression_metrics.to_protobuf() if self.regression_metrics else None, + regressionMetrics=self.regression_metrics.to_protobuf( + ) if self.regression_metrics else None, modelType=self.model_type) @classmethod def from_protobuf(cls, message, ): return ModelMetrics( - confusion_matrix=ConfusionMatrix.from_protobuf(message.scoreMatrix), - regression_metrics=RegressionMetrics.from_protobuf(message.regressionMetrics), + confusion_matrix=ConfusionMatrix.from_protobuf( + message.scoreMatrix), + regression_metrics=RegressionMetrics.from_protobuf( + message.regressionMetrics), model_type=message.modelType) def compute_confusion_matrix(self, predictions: List[Union[str, int, bool, float]], @@ -82,10 +86,12 @@ def compute_regression_metrics(self, predictions: List[Union[float, int]], targets: List[Union[float, int]], target_field: str = None, prediction_field: str = None): - regression_metrics = RegressionMetrics(target_field=target_field, prediction_field=prediction_field) + regression_metrics = RegressionMetrics( + target_field=target_field, prediction_field=prediction_field) regression_metrics.add(predictions, targets) if self.regression_metrics: - self.regression_metrics = self.regression_metrics.merge(regression_metrics) + self.regression_metrics = self.regression_metrics.merge( + regression_metrics) else: self.regression_metrics = regression_metrics @@ -96,20 +102,16 @@ def merge(self, other): """ if other is None: return self - if other.confusion_matrix is None and other.regression_metrics is None: - # TODO: return a copy instead - return self - if self.confusion_matrix is None and self.regression_metrics is None: - return other - if self.model_type is None or other.model_type is None: - model_type = ModelType.UNKNOWN - elif other.model_type != self.model_type: - model_type = ModelType.UNKNOWN - else: - model_type = self.model_type + model_type = self.model_type + if model_type not in (ModelType.REGRESSION, ModelType.CLASSIFICATION): + model_type = other.model_type + if model_type not in (ModelType.REGRESSION, ModelType.CLASSIFICATION): + model_type = ModelType.UNKNOWN return ModelMetrics( - confusion_matrix=self.confusion_matrix.merge(other.confusion_matrix) if self.confusion_matrix else None, - regression_metrics=self.regression_metrics.merge(other.regression_metrics)if self.regression_metrics else None, + confusion_matrix=self.confusion_matrix.merge( + other.confusion_matrix) if self.confusion_matrix else None, + regression_metrics=self.regression_metrics.merge( + other.regression_metrics)if self.regression_metrics else None, model_type=model_type) diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py index 95a98c199c..e93bb11ac1 100644 --- a/src/whylogs/core/metrics/regression_metrics.py +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -74,6 +74,8 @@ def merge(self, other): Returns: RegressionMetrics: merged regression metrics """ + if other is None: + return self if self.count == 0: return other diff --git a/tests/unit/core/metrics/test_model_metrics.py b/tests/unit/core/metrics/test_model_metrics.py index 3384dcf14e..ce48b927da 100644 --- a/tests/unit/core/metrics/test_model_metrics.py +++ b/tests/unit/core/metrics/test_model_metrics.py @@ -26,7 +26,7 @@ def tests_model_metrics(): jdx].floats.count == expected_1[idx][jdx] -def tests_model_metrics_to_protobuf(): +def tests_model_metrics_to_protobuf_classification(): mod_met = ModelMetrics(model_type=ModelType.CLASSIFICATION) targets_1 = ["cat", "dog", "pig"] @@ -39,12 +39,45 @@ def tests_model_metrics_to_protobuf(): message = mod_met.to_protobuf() - ModelMetrics.from_protobuf(message) + model_metrics = ModelMetrics.from_protobuf(message) + assert model_metrics.model_type == ModelType.CLASSIFICATION + assert model_metrics.confusion_matrix.labels == ["cat", "dog", "pig"] + + +def tests_no_metrics_to_protobuf_classification(): + mod_met = ModelMetrics(model_type=ModelType.CLASSIFICATION) + + assert mod_met.model_type == ModelType.CLASSIFICATION + message = mod_met.to_protobuf() + + model_metrics = ModelMetrics.from_protobuf(message) + assert model_metrics.model_type == ModelType.CLASSIFICATION + +def tests_no_metrics_to_protobuf_regression(): + + mod_met = ModelMetrics(model_type=ModelType.REGRESSION) + assert mod_met.model_type == ModelType.REGRESSION + message = mod_met.to_protobuf() + + + model_metrics = ModelMetrics.from_protobuf(message) + assert model_metrics.model_type == ModelType.REGRESSION + +def tests_model_metrics_to_protobuf_regression(): + regression_model = ModelMetrics(model_type=ModelType.REGRESSION) + + targets_1 = [0.1, 0.3, 0.4] + predictions_1 = [0.5, 0.5, 0.5] + regression_model.compute_regression_metrics(predictions_1, targets_1) + regression_message = regression_model.to_protobuf() + model_metrics_from_message = ModelMetrics.from_protobuf(regression_message) + assert model_metrics_from_message.model_type == ModelType.REGRESSION + def test_merge_none(): metrics = ModelMetrics() - metrics.merge(None) + assert metrics.merge(None) == metrics def test_merge_metrics_with_none_confusion_matrix(): @@ -54,11 +87,28 @@ def test_merge_metrics_with_none_confusion_matrix(): new_metrics = metrics.merge(other) +def test_merge_metrics_model(): + metrics = ModelMetrics() + other = ModelMetrics(model_type=ModelType.REGRESSION) + other.regression_metrics = None + new_metrics = metrics.merge(other) + assert new_metrics.model_type==ModelType.REGRESSION + assert new_metrics.confusion_matrix is None + + # keep initial model type during merge + metrics = ModelMetrics(model_type=ModelType.REGRESSION) + other = ModelMetrics(model_type=ModelType.CLASSIFICATION) + other.regression_metrics = None + new_metrics = metrics.merge(other) + assert new_metrics.model_type==ModelType.REGRESSION + assert new_metrics.confusion_matrix is None + def test_merge_metrics_with_none_regression_matrix(): metrics = ModelMetrics() - other = ModelMetrics() + other = ModelMetrics(model_type=ModelType.REGRESSION) other.regression_metrics = None - new_metrics= metrics.merge(other) + new_metrics = metrics.merge(other) + assert new_metrics.model_type==ModelType.REGRESSION def test_merge_metrics_with_none_confusion_matrix(): metrics = ModelMetrics() @@ -67,10 +117,11 @@ def test_merge_metrics_with_none_confusion_matrix(): other.regression_metrics = None new_metrics = metrics.merge(other) + assert new_metrics.model_type == ModelType.UNKNOWN + def test_model_metrics_init(): reg_met = RegressionMetrics() - conf_ma= ConfusionMatrix() + conf_ma = ConfusionMatrix() with pytest.raises(NotImplementedError): metrics = ModelMetrics(confusion_matrix=conf_ma, regression_metrics=reg_met) -