diff --git a/python/chronos/src/bigdl/chronos/metric/__init__.py b/python/chronos/src/bigdl/chronos/metric/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/chronos/src/bigdl/chronos/metric/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/chronos/src/bigdl/chronos/metric/forecast_metrics.py b/python/chronos/src/bigdl/chronos/metric/forecast_metrics.py new file mode 100644 index 00000000000..3e5e1667d8b --- /dev/null +++ b/python/chronos/src/bigdl/chronos/metric/forecast_metrics.py @@ -0,0 +1,78 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from functools import partial +from torchmetrics.functional import mean_squared_error, mean_absolute_error,\ + mean_absolute_percentage_error, r2_score, symmetric_mean_absolute_percentage_error + + +TORCHMETRICS_REGRESSION_MAP = { + 'mae': mean_absolute_error, + 'mse': mean_squared_error, + 'rmse': partial(mean_squared_error, squared=False), + 'mape': mean_absolute_percentage_error, + 'smape': symmetric_mean_absolute_percentage_error, + 'r2': r2_score, +} + + +class Evaluator(object): + """ + Evaluate metrics for y_true and y_pred. + """ + + @staticmethod + def evaluate(metrics, y_true, y_pred, aggregate='mean'): + """ + Evaluate a specific metrics for y_true and y_pred. + + :param metrics: String or list in ['mae', 'mse', 'rmse', 'r2', 'mape', 'smape'] + :param y_true: Array-like of shape = (n_samples, \*). Ground truth (correct) target values. + :param y_pred: Array-like of shape = (n_samples, \*). Estimated target values. + :param aggregate: aggregation method. Currently, "mean" and None are supported, + 'mean' represents aggregating by mean, while None will return the element-wise + result. The value defaults to 'mean'. + + :return: Float or ndarray of floats. + A floating point value, or an + array of floating point values, one for each individual target. + """ + if isinstance(metrics, str): + metrics = [metrics] + metrics = list(map(lambda x: x.lower(), metrics)) + assert all([metric in TORCHMETRICS_REGRESSION_MAP.keys() for metric in metrics]),\ + f"metric should be one of {TORCHMETRICS_REGRESSION_MAP.keys()}, "\ + f"but get {metrics}." + assert y_true.shape == y_pred.shape,\ + "y_true and y_pred should have the same shape, "\ + f"but get {y_true.shape} and {y_pred.shape}." + y_true, y_pred = torch.from_numpy(y_true), torch.from_numpy(y_pred) + + res_list = [] + for metric in metrics: + res = None + if aggregate is None: # TODO: not only support 3-dim data + res = torch.zeros(y_true.shape[1], y_true.shape[2]) + for i in range(y_true.shape[1]): + for j in range(y_true.shape[2]): + res[i, j] = TORCHMETRICS_REGRESSION_MAP[metric](y_pred[:, i, j], + y_true[:, i, j]) + else: # currently this is only for 'mean' + res = TORCHMETRICS_REGRESSION_MAP[metric](y_pred, y_true) + res_list.append(res.numpy()) + + return res_list diff --git a/python/chronos/test/bigdl/chronos/metric/__init__.py b/python/chronos/test/bigdl/chronos/metric/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/chronos/test/bigdl/chronos/metric/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/chronos/test/bigdl/chronos/metric/test_forecast_metrics.py b/python/chronos/test/bigdl/chronos/metric/test_forecast_metrics.py new file mode 100644 index 00000000000..fd3179fb8f5 --- /dev/null +++ b/python/chronos/test/bigdl/chronos/metric/test_forecast_metrics.py @@ -0,0 +1,59 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np +from unittest import TestCase +from numpy.testing import assert_almost_equal +from numpy.testing import assert_array_almost_equal + +from bigdl.chronos.metric.forecast_metrics import Evaluator + +class TestChronosForecastMetrics(TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_forecast_metric(self): + n_samples = 50 + y_true = np.arange(n_samples) + 1 + y_pred = y_true + 1 + + assert_almost_equal(Evaluator.evaluate("mse", y_true, y_pred, aggregate="mean")[0], 1.) + assert_almost_equal(Evaluator.evaluate("mae", y_true, y_pred, aggregate="mean")[0], 1.) + assert_almost_equal(Evaluator.evaluate("r2", y_true, y_pred, aggregate="mean")[0], 0.995, 2) + assert_almost_equal(Evaluator.evaluate("smape", y_true, y_pred, aggregate="mean")[0], 3.89*2/100, 2) + + y_true = np.array([3, -0.5, 2, 7]) + y_pred = np.array([2.5, -0.3, 2, 8]) + + assert_almost_equal(Evaluator.evaluate("mape", y_true, y_pred, aggregate="mean")[0], 17.74/100, 2) + assert_almost_equal(Evaluator.evaluate("RMSE", y_true, y_pred, aggregate="mean")[0], 0.57, 2) + + def test_highdim_array_metrics(self): + y_true = np.array([[[3, -0.5], [2, 7]], [[3, -0.5], [2, 7]], [[3, -0.5], [2, 7]]]) + y_pred = np.array([[[2.5, -0.3], [2, 8]], [[2.5, -0.3], [2, 8]], [[2.5, -0.3], [2, 8]]]) + + assert_almost_equal(Evaluator.evaluate("smape", y_true, y_pred, aggregate=None)[0], + [[9.09*2/100, 25*2/100], [0*2/100, 6.67*2/100]], 2) + assert_almost_equal(Evaluator.evaluate("mape", y_true, y_pred, aggregate=None)[0], + [[16.67/100, 40.00/100], [0/100, 14.29/100]], 2) + assert_almost_equal(Evaluator.evaluate("rmse", y_true, y_pred, aggregate=None)[0], + [[0.5, 0.2], [0, 1]], 2) + assert_almost_equal(Evaluator.evaluate("mse", y_true, y_pred, aggregate=None)[0], + [[0.25, 0.04], [0, 1]], 2)