From f3b3718c7d87af3f1426aa7d31100ba19f80cd58 Mon Sep 17 00:00:00 2001 From: Meraj Date: Sun, 28 Mar 2021 08:40:28 -0400 Subject: [PATCH 1/4] Support multi-dimensional output for cp regression --- tensorly/regression/cp_regression.py | 52 ++++++++++++++++++---------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/tensorly/regression/cp_regression.py b/tensorly/regression/cp_regression.py index 42a7a692c..3f9855aba 100644 --- a/tensorly/regression/cp_regression.py +++ b/tensorly/regression/cp_regression.py @@ -1,16 +1,18 @@ import numpy as np + +from .. import backend as T from ..base import partial_tensor_to_vec, partial_unfold -from ..tenalg import khatri_rao from ..cp_tensor import cp_to_tensor, cp_to_vec -from .. import backend as T +from ..tenalg import khatri_rao from ..utils import DefineDeprecated + # Author: Jean Kossaifi # License: BSD 3 clause -class CPRegressor(): +class CPRegressor: """CP tensor regression Learns a low rank CP tensor weight @@ -22,7 +24,7 @@ class CPRegressor(): tol : float convergence value reg_W : int, optional, default is 1 - regularisation on the weights + l2 regularisation constant for the regression weights (:math:`reg_W * \sum_i ||factors[i]||_F^2`) n_iter_max : int, optional, default is 100 maximum number of iteration random_state : None, int or RandomState, optional, default is None @@ -55,9 +57,9 @@ def fit(self, X, y): Parameters ---------- - X : ndarray - tensor data of shape (n_samples, N1, ..., NS) - y : 1D-array of shape (n_samples, ) + X : tensor of shape (n_samples, I_1, ..., I_p) + tensor data + y : tensor of shape (n_samples, O_1, ..., O_q) labels associated with each sample Returns @@ -68,8 +70,10 @@ def fit(self, X, y): # Initialise randomly the weights W = [] - for i in range(1, T.ndim(X)): # The first dimension of X is the number of samples + for i in range(1, T.ndim(X)): # The first dimension is the number of samples W.append(T.tensor(rng.randn(X.shape[i], self.weight_rank), **T.context(X))) + for i in range(1, T.ndim(y)): + W.append(T.tensor(rng.randn(y.shape[i], self.weight_rank), **T.context(X))) # Norm of the weight tensor at each iteration norm_W = [] @@ -79,12 +83,22 @@ def fit(self, X, y): # Optimise each factor of W for i in range(len(W)): - phi = T.reshape( - T.dot(partial_unfold(X, i, skip_begin=1), - khatri_rao(W, skip_matrix=i)), - (X.shape[0], -1)) - inv_term = T.dot(T.transpose(phi), phi) + self.reg_W*T.tensor(np.eye(phi.shape[1]), **T.context(X)) - W[i] = T.reshape(T.solve(inv_term, T.dot(T.transpose(phi), y)), (X.shape[i + 1], self.weight_rank)) + if i < T.ndim(X) - 1: + X_unfolded = partial_unfold(X, i, skip_begin=1) + phi = T.dot(X_unfolded, T.reshape(khatri_rao(W, skip_matrix=i), (X_unfolded.shape[-1], -1))) + phi = T.transpose(T.reshape(phi, (X.shape[0], X.shape[i + 1], -1, self.weight_rank)), (0, 2, 1, 3)) + phi = T.reshape(phi, (-1, X.shape[i + 1] * self.weight_rank)) + y_reshaped = T.reshape(y, (-1,)) + else: + X_unfolded = partial_tensor_to_vec(X, skip_begin=1) + phi = T.dot(X_unfolded, T.reshape(khatri_rao(W, skip_matrix=i), (X_unfolded.shape[-1], -1))) + phi = T.reshape(phi, (-1, self.weight_rank)) + y_reshaped = T.reshape(T.moveaxis(y, i - T.ndim(X), -1), (-1, y.shape[i - T.ndim(X)])) + + inv_term = T.dot(T.transpose(phi), phi) + self.reg_W * T.tensor(np.eye(phi.shape[1]), **T.context(X)) + W[i] = T.reshape( + T.solve(inv_term, T.dot(T.transpose(phi), y_reshaped)), + (X.shape[i + 1], self.weight_rank)) weight_tensor_ = cp_to_tensor((weights, W)) norm_W.append(T.norm(weight_tensor_, 2)) @@ -93,7 +107,7 @@ def fit(self, X, y): if iteration > 1: weight_evolution = abs(norm_W[-1] - norm_W[-2]) / norm_W[-1] - if (weight_evolution <= self.tol): + if weight_evolution <= self.tol: if self.verbose: print('\nConverged in {} iterations'.format(iteration)) break @@ -113,8 +127,10 @@ def predict(self, X): Parameters ---------- X : ndarray - tensor data of shape (n_samples, N1, ..., NS) + tensor data of shape (n_samples, I_1, ..., I_p) """ - return T.dot(partial_tensor_to_vec(X), self.vec_W_) + new_shape = (-1, *self.weight_tensor_.shape[T.ndim(X) - 1:]) + return T.dot(partial_tensor_to_vec(X), T.reshape(self.weight_tensor_, new_shape)) + -KruskalRegressor = DefineDeprecated('KruskalRegressor', CPRegressor) \ No newline at end of file +KruskalRegressor = DefineDeprecated('KruskalRegressor', CPRegressor) From beca90c9b759d257ab27d54d2c9661c74d6a2f1e Mon Sep 17 00:00:00 2001 From: Meraj Date: Tue, 13 Jul 2021 08:47:42 -0400 Subject: [PATCH 2/4] Fix predict method --- tensorly/regression/cp_regression.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tensorly/regression/cp_regression.py b/tensorly/regression/cp_regression.py index 3f9855aba..9f50fe1a2 100644 --- a/tensorly/regression/cp_regression.py +++ b/tensorly/regression/cp_regression.py @@ -23,7 +23,7 @@ class CPRegressor: rank of the CP decomposition of the regression weights tol : float convergence value - reg_W : int, optional, default is 1 + reg_W : float, optional, default is 1 l2 regularisation constant for the regression weights (:math:`reg_W * \sum_i ||factors[i]||_F^2`) n_iter_max : int, optional, default is 100 maximum number of iteration @@ -89,16 +89,17 @@ def fit(self, X, y): phi = T.transpose(T.reshape(phi, (X.shape[0], X.shape[i + 1], -1, self.weight_rank)), (0, 2, 1, 3)) phi = T.reshape(phi, (-1, X.shape[i + 1] * self.weight_rank)) y_reshaped = T.reshape(y, (-1,)) + inv_term = T.dot(T.transpose(phi), phi) + self.reg_W * T.tensor(np.eye(phi.shape[1]), **T.context(X)) + W[i] = T.reshape( + T.solve(inv_term, T.dot(T.transpose(phi), y_reshaped)), + (-1, self.weight_rank)) else: X_unfolded = partial_tensor_to_vec(X, skip_begin=1) phi = T.dot(X_unfolded, T.reshape(khatri_rao(W, skip_matrix=i), (X_unfolded.shape[-1], -1))) phi = T.reshape(phi, (-1, self.weight_rank)) - y_reshaped = T.reshape(T.moveaxis(y, i - T.ndim(X), -1), (-1, y.shape[i - T.ndim(X)])) - - inv_term = T.dot(T.transpose(phi), phi) + self.reg_W * T.tensor(np.eye(phi.shape[1]), **T.context(X)) - W[i] = T.reshape( - T.solve(inv_term, T.dot(T.transpose(phi), y_reshaped)), - (X.shape[i + 1], self.weight_rank)) + y_reshaped = T.reshape(T.moveaxis(y, i - T.ndim(X) + 2, -1), (-1, y.shape[i - T.ndim(X) + 2])) + inv_term = T.dot(T.transpose(phi), phi) + self.reg_W * T.tensor(np.eye(phi.shape[1]), **T.context(X)) + W[i] = T.transpose(T.solve(inv_term, T.dot(T.transpose(phi), y_reshaped))) weight_tensor_ = cp_to_tensor((weights, W)) norm_W.append(T.norm(weight_tensor_, 2)) @@ -129,8 +130,12 @@ def predict(self, X): X : ndarray tensor data of shape (n_samples, I_1, ..., I_p) """ - new_shape = (-1, *self.weight_tensor_.shape[T.ndim(X) - 1:]) - return T.dot(partial_tensor_to_vec(X), T.reshape(self.weight_tensor_, new_shape)) + out_shape = (-1, *self.weight_tensor_.shape[T.ndim(X) - 1:]) + if T.ndim(self.weight_tensor_) > T.ndim(X) - 1: + weight_shape = (-1, int(np.prod(self.weight_tensor_.shape[T.ndim(X) - 1:]))) + else: + weight_shape = (-1,) + return T.reshape(T.dot(partial_tensor_to_vec(X), T.reshape(self.weight_tensor_, weight_shape)), out_shape) KruskalRegressor = DefineDeprecated('KruskalRegressor', CPRegressor) From 02d57980d575c560b986f587a92c30010bcabf8c Mon Sep 17 00:00:00 2001 From: Meraj Date: Tue, 13 Jul 2021 08:48:29 -0400 Subject: [PATCH 3/4] Add test for mutltidim response --- .../regression/tests/test_cp_regression.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tensorly/regression/tests/test_cp_regression.py b/tensorly/regression/tests/test_cp_regression.py index efece7851..b941d2945 100644 --- a/tensorly/regression/tests/test_cp_regression.py +++ b/tensorly/regression/tests/test_cp_regression.py @@ -4,6 +4,7 @@ from ...base import tensor_to_vec, partial_tensor_to_vec from ...metrics.regression import RMSE from ... import backend as T +from ...random import random_cp from ...testing import assert_ @@ -42,3 +43,22 @@ def test_CPRegressor(): params['weight_rank'] = 5 estimator.set_params(**params) assert_(estimator.weight_rank == 5, msg='set_params did not correctly set the given parameters') + + +def test_multidim_CPRegressor(): + tol = 0.005 + rng = T.check_random_state(1234) + + regression_weights = random_cp(shape=(12, 5, 4, 3, 2), rank=4, full=True, random_state=rng) + X = T.randn((1200, 12, 5, 4), seed=rng) + y = T.reshape(T.dot(partial_tensor_to_vec(X), T.reshape(regression_weights, (-1, 3*2))), (-1, 3, 2)) + X_train = X[:1000] + X_test = X[1000:] + y_train = y[:1000] + y_test = y[1000:] + + estimator = CPRegressor(weight_rank=20, tol=1e-8, reg_W=0., n_iter_max=200, verbose=True) + estimator.fit(X_train, y_train) + y_pred = estimator.predict(X_test) + error = RMSE(y_test, y_pred) + assert_(error <= tol, msg='CP Regressor : RMSE is too large, {} > {}'.format(error, tol)) From 2dcee4036dca09274f3fb1d1fdd4de079995378d Mon Sep 17 00:00:00 2001 From: Aaron Meyer <2065146+aarmey@users.noreply.github.com> Date: Tue, 31 Jan 2023 09:32:27 -0800 Subject: [PATCH 4/4] Update cp_regression.py --- tensorly/regression/cp_regression.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorly/regression/cp_regression.py b/tensorly/regression/cp_regression.py index 860cfb843..bfcbdf7a1 100644 --- a/tensorly/regression/cp_regression.py +++ b/tensorly/regression/cp_regression.py @@ -88,8 +88,6 @@ def fit(self, X, y): for i in range( 1, T.ndim(X) ): # The first dimension of X is the number of samples - W.append(T.tensor(rng.randn(X.shape[i], self.weight_rank), **T.context(X))) - for i in range(1, T.ndim(y)): W.append(T.tensor(rng.randn(y.shape[i], self.weight_rank), **T.context(X))) # Norm of the weight tensor at each iteration