# [MRG+2] Neighborhood Components Analysis #10058

Merged
merged 89 commits into from Feb 28, 2019
+1,664 −23
Merged

# [MRG+2] Neighborhood Components Analysis#10058

Commits
Show all changes
89 commits
Select commit Hold shift + click to select a range
Filter file types
Failed to load files and symbols.

#### Just for now

wdevazelhes committed Jun 29, 2018
commit 85bd54fbcb501594fe6747a6cb4a18563cd98fe6
@@ -2,6 +2,7 @@
import sys
import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal
from sklearn import clone
from sklearn.exceptions import ConvergenceWarning
from sklearn.externals.six import StringIO
@@ -41,69 +42,25 @@ def test_simple_example():

def test_finite_differences():
Test if the gradient is correct by computing the relative difference
.. math::
PG = \mathbf d^{\top} \cdot \nabla
\mathcal L(\mathbf x)
and the finite differences FD:
.. math::
FD = \frac{\mathcal L(\mathbf x + \epsilon \mathbf d) -
\mathcal L(\mathbf x - \epsilon \mathbf d)}{2 \epsilon}
where :math:d is a random direction (random vector of shape n_features,
and norm 1), :math:\epsilon is a very small number, :math:\mathcal L is
the loss function and :math:\nabla \mathcal L is its gradient. This
relative difference should be zero:
.. math ::
\frac{|PG -FD|}{|PG|} = 0
Assert that the gradient is almost equal to its finite differences
approximation.
"""
# Initialize transformation, X and y and NCA
X = iris_data
y = iris_target
point = rng.randn(rng.randint(1, X.shape + 1), X.shape)
nca = NeighborhoodComponentsAnalysis(init=point)

X, y, init = nca._validate_params(X, y)
mask = y[:, np.newaxis] == y[np.newaxis, :] # (n_samples, n_samples)
# Initialize the transformation M, as well as X and y and NCA
X, y = make_classification()
M = rng.randn(rng.randint(1, X.shape + 1), X.shape)
nca = NeighborhoodComponentsAnalysis()
nca.n_iter_ = 0
mask = y[:, np.newaxis] == y[np.newaxis, :]

point = nca._initialize(X, y, init)
# compute the gradient at point

# create a random direction of norm 1
random_direction = rng.randn(*point.shape)
random_direction /= np.linalg.norm(random_direction)

#### GaelVaroquaux Feb 25, 2019

Member

It's a total nitpick (feel free to ignore), but I find that avoiding inlined function definition is better (same comment for right below).

#### wdevazelhes Feb 25, 2019

Author Contributor

I agree, will do

# compute finite differences
eps = 1e-5
right_loss, _ = nca._loss_grad_lbfgs(point + eps * random_direction, X,
left_loss, _ = nca._loss_grad_lbfgs(point - eps * random_direction, X,
finite_differences = 1 / (2 * eps) * (right_loss - left_loss)