Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'release-0.4'

  • Loading branch information...
commit 9a2e142f1e3121b0634158f014c2eeb4ba46ac3c 2 parents 928fe69 + e42a3e1
@nzhiltsov authored
View
7 README.md
@@ -8,7 +8,7 @@ Ext-RESCAL is a memory efficient implementation of [RESCAL](http://www.cip.ifi.l
Current Version
------------
-[0.3](https://github.com/nzhiltsov/Ext-RESCAL/archive/0.3.zip)
+[0.4](https://github.com/nzhiltsov/Ext-RESCAL/archive/0.4.zip)
Features
------------
@@ -56,6 +56,7 @@ Run the extended version of RESCAL algorithm to decompose a 3-D tensor and 2-D m
<pre>python extrescal.py --latent 2 --lmbda 0.001 --input tiny-mixed-example --outputentities entity.embeddings.csv --outputterms term.embeddings.csv --outputfactors latent.factors.csv --log extrescal.log</pre>
If we plot the resulting embeddings, we would get the following picture, which reveals the similarity of entities and words in the latent space:
+
![latent-space-visualization](tiny-mixed-example/TinyMixedExample.png)
Development and Contribution
@@ -66,6 +67,10 @@ This is a fork of the original code base provided by [Maximilian Nickel](http://
Release Notes
------------
+0.4 (March 14, 2013):
+
+* Add efficient computation of the exact objective value via trick with trace
+
0.3 (March 12, 2013):
* Fix random sampling for the basic task
View
18 commonFunctions.py
@@ -1,6 +1,8 @@
from numpy import dot
from numpy.random import randint
from numpy.random import random_integers
+from scipy.sparse import lil_matrix
+import numpy as np
def squareFrobeniusNormOfSparse(M):
"""
@@ -12,12 +14,20 @@ def squareFrobeniusNormOfSparse(M):
norm += M[rows[i],cols[i]] ** 2
return norm
-def fitNorm(row, col, Xi, ARk, A):
+def trace(M):
+ """ Compute the trace of a sparse matrix
"""
- Computes i,j element of the squared Frobenius norm of the fitting matrix
+ return sum(M.diagonal())
+
+def fitNorm(X, A, R):
+ """
+ Computes the squared Frobenius norm of the fitting matrix || X - A*R*A^T ||,
+ where X is a sparse matrix
"""
- ARAtValue = dot(ARk[row,:], A[col,:])
- return (Xi[row, col] - ARAtValue)**2
+ AtA = dot(A.T, A)
+ secondTerm = dot(A.T, dot(X.dot(A), R.T))
+ thirdTerm = dot(dot(AtA, R), dot(AtA, R.T))
+ return squareFrobeniusNormOfSparse(X) - 2 * trace(secondTerm) + np.trace(thirdTerm)
def reservoir(it, k):
ls = [next(it) for _ in range(k)]
View
8 commonFunctionsTest.py
@@ -17,19 +17,13 @@ def testSquareFrobeniusNorm():
def testFitNorm():
X = coo_matrix((ones(4),([0, 1, 2, 2], [1, 1, 0, 1])), shape=(3, 3), dtype=np.uint8).tolil()
- n = X.shape[0]
A = np.array([[0.9, 0.1],
[0.8, 0.2],
[0.1, 0.9]])
R = np.array([[0.9, 0.1],
[0.1, 0.9]])
expectedNorm = norm(X - dot(A,dot(R, A.T)))**2
- ARk = dot(A, R)
- fits = []
- for i in xrange(n):
- for j in xrange(n):
- fits.append(fitNorm(i, j, X, ARk, A))
- assert_almost_equal(sum(fits), expectedNorm)
+ assert_almost_equal(fitNorm(X, A, R), expectedNorm)
def testSampling():
xs = range(0, 3)
View
65 extrescal.py
@@ -7,8 +7,8 @@
import numpy as np
import os
import fnmatch
-from commonFunctions import squareFrobeniusNormOfSparse, fitNorm, checkingIndices
-from extrescalFunctions import updateA, updateV, matrixFitNormElement
+from commonFunctions import squareFrobeniusNormOfSparse, fitNorm
+from extrescalFunctions import updateA, updateV, matrixFitNorm
__DEF_MAXITER = 50
__DEF_PREHEATNUM = 1
@@ -16,9 +16,6 @@
__DEF_PROJ = True
__DEF_CONV = 1e-5
__DEF_LMBDA = 0
-__DEF_EXACT_FIT = False
-__DEF_MATRIX_FIT_SAMPLE_RATIO = 1
-__DEF_TENSOR_SLICE_FIT_SAMPLE_RATIO = 1
def rescal(X, D, rank, **kwargs):
"""
@@ -26,7 +23,7 @@ def rescal(X, D, rank, **kwargs):
Factors a three-way tensor X such that each frontal slice
X_k = A * R_k * A.T. The frontal slices of a tensor are
- N x N matrices that correspond to the adjecency matrices
+ N x N matrices that correspond to the adjacency matrices
of the relational graph for a particular relation.
For a full description of the algorithm see:
@@ -53,17 +50,14 @@ def rescal(X, D, rank, **kwargs):
Whether or not to use the QR decomposition when computing R_k.
True by default
maxIter : int, optional
- Maximium number of iterations of the ALS algorithm. 500 by default.
+ Maximium number of iterations of the ALS algorithm. 50 by default.
conv : float, optional
- Stop when residual of factorization is less than conv. 1e-5 by default
- exactfit: boolean, optional
- Whether or not to compute the exact fitting value
- False by default (i.e., approximate the norm by non-zero values of the targeting matrix/tensor)
+ Stop when residual of factorization is less than conv. 1e-5 by default
Returns
-------
A : ndarray
- array of shape ('N', 'rank') corresponding to the factor matrix A
+ matrix of latent embeddings for entities A
R : list
list of 'M' arrays of shape ('rank', 'rank') corresponding to the factor matrices R_k
f : float
@@ -72,6 +66,8 @@ def rescal(X, D, rank, **kwargs):
number of iterations until convergence
exectimes : ndarray
execution times to compute the updates in each iteration
+ V : ndarray
+ matrix of latent embeddings for words V
"""
# init options
@@ -81,26 +77,17 @@ def rescal(X, D, rank, **kwargs):
conv = kwargs.pop('conv', __DEF_CONV)
lmbda = kwargs.pop('lmbda', __DEF_LMBDA)
preheatnum = kwargs.pop('preheatnum', __DEF_PREHEATNUM)
- exactfit = kwargs.pop('exactfit', __DEF_EXACT_FIT)
- matrixSampleRatio = kwargs.pop('matrixSampleRatio', __DEF_MATRIX_FIT_SAMPLE_RATIO)
- tensorSliceSampleRatio = kwargs.pop('tensorSliceSampleRatio', __DEF_TENSOR_SLICE_FIT_SAMPLE_RATIO)
if not len(kwargs) == 0:
raise ValueError( 'Unknown keywords (%s)' % (kwargs.keys()) )
sz = X[0].shape
dtype = X[0].dtype
- n = sz[0]
- k = len(X)
+ n = sz[0]
_log.debug('[Config] rank: %d | maxIter: %d | conv: %7.1e | lmbda: %7.1e' % (rank,
maxIter, conv, lmbda))
_log.debug('[Config] dtype: %s' % dtype)
-
- if exactfit:
- _log.debug('[Config] The exact fit values will be computed during optimization.')
- else:
- _log.debug('[Config] The approximating fit values will be computed during optimization.')
# precompute norms of X
normX = [squareFrobeniusNormOfSparse(M) for M in X]
@@ -132,13 +119,6 @@ def rescal(X, D, rank, **kwargs):
exectimes = []
# prepare the checking indices to compute the fit
- if exactfit:
- matrixFitIndices = []
- tensorFitIndices = []
- else :
- matrixFitIndices = checkingIndices(D, ratio = matrixSampleRatio)
- tensorFitIndices = [checkingIndices(M, ratio = tensorSliceSampleRatio) for M in X]
- _log.debug('[Algorithm] Finished sampling of indices to compute the fit values.')
for iterNum in xrange(maxIter):
tic = time.clock()
@@ -150,7 +130,6 @@ def rescal(X, D, rank, **kwargs):
R = __updateR(X2, A2, lmbda)
else :
raise 'Projection via QR decomposition is required; pass proj=true'
-# R = __updateR(X, A, lmbda)
V = updateV(A, D, lmbda)
# compute fit values
@@ -160,30 +139,18 @@ def rescal(X, D, rank, **kwargs):
extRegularizedFit = 0
regRFit = 0
fitDAV = 0
- if iterNum > preheatnum:
+ if iterNum >= preheatnum:
if lmbda != 0:
for i in xrange(len(R)):
regRFit += norm(R[i])**2
regularizedFit = lmbda*(norm(A)**2) + lmbda*regRFit
if lmbda != 0:
extRegularizedFit = lmbda*(norm(V)**2)
- if exactfit:
- fitDAV = norm(D - dot(A,V))**2
- else :
- for ff in xrange(len(matrixFitIndices)):
- x, y = matrixFitIndices[ff]
- fitDAV += matrixFitNormElement(x, y, D, A, V)
-
- if exactfit:
- for i in xrange(len(R)):
- tensorFit += norm(X[i] - dot(A,dot(R[i], A.T)))**2
- else :
- for i in xrange(len(R)):
- ARk = dot(A, R[i])
- iTensorFitIndices = tensorFitIndices[i]
- for rr in xrange(len(iTensorFitIndices)):
- m, l = iTensorFitIndices[rr]
- tensorFit += fitNorm(m, l, X[i], ARk, A)
+
+ fitDAV = matrixFitNorm(D, A, V)
+
+ for i in xrange(len(R)):
+ tensorFit += fitNorm(X[i], A, R[i])
fit = 0.5*tensorFit
fit += regularizedFit
@@ -276,7 +243,7 @@ def __projectSlices(X, Q):
extDim = 0
with open('./%s/words' % inputDir) as words:
for line in words:
- extDim += 1
+ extDim += 1
print 'The number of words: %d' % extDim
extRow = loadtxt('./%s/ext-matrix-rows' % inputDir, dtype=np.int32)
View
11 extrescalFunctions.py
@@ -1,6 +1,8 @@
import numpy as np
from numpy import dot, zeros, eye, empty
from numpy.linalg import inv
+from commonFunctions import trace, squareFrobeniusNormOfSparse
+from scipy.sparse import lil_matrix
def updateA(X, A, R, V, D, lmbda):
n, rank = A.shape
@@ -26,11 +28,14 @@ def updateV(A, D, lmbda):
invPart = inv(dot(At, A) + lmbda * eye(rank))
return dot(invPart, At) * D
-def matrixFitNormElement(i, j, D, A, V):
+def matrixFitNorm(D, A, V):
"""
- Computes i,j element of the fitting matrix Frobenius norm ||D - A*V||
+ Computes the Frobenius norm of the fitting matrix ||D - A*V||,
+ where D is a sparse matrix
"""
- return (D[i,j] - dot(A[i,:], V[:, j]))**2
+ thirdTerm = dot(dot(V, V.T), dot(A.T, A))
+ secondTerm = dot(A.T, D.dot(V.T))
+ return squareFrobeniusNormOfSparse(D) - 2 * trace(secondTerm) + np.trace(thirdTerm)
View
25 extrescalFunctionsTest.py
@@ -1,7 +1,7 @@
from scipy.sparse import coo_matrix
from numpy import ones, dot, eye
import numpy as np
-from extrescalFunctions import updateA, updateV, matrixFitNormElement
+from extrescalFunctions import updateA, updateV, matrixFitNorm
from nose.tools import assert_almost_equal
from numpy.linalg import inv
from numpy.linalg.linalg import norm
@@ -63,21 +63,16 @@ def testUpdateV():
def testMatrixFitNorm():
A = np.array([[0.1, 0.1, 0.1],
- [0.1, 0.1, 0.1],
- [0.1, 0.1, 0.1],
- [0.1, 0.1, 0.1],
- [0.1, 0.1, 0.1],
- [0.1, 0.1, 0.1]])
- V = np.array([[0.1, 0.1, 0.1, 0.1],
- [0.1, 0.1, 0.1, 0.1],
- [0.1, 0.1, 0.1, 0.1]])
+ [0.1, 0.1, 0.001],
+ [0.2, 0.1, 0.1],
+ [0.1, 0.3, 0.1],
+ [0.4, 0.1, 0.1],
+ [0.001, 0.01, 0.1]])
+ V = np.array([[0.1, 0.4, 0.1, 0.1],
+ [0.01, 0.3, 0.1, 0.3],
+ [0.1, 0.01, 0.4, 0.001]])
D = coo_matrix((ones(6),([0, 1, 2, 3, 4, 5], [0, 1, 1, 2, 3, 3])), shape=(6, 4), dtype=np.uint8).tocsr()
- DrowNum, DcolNum = D.shape
expectedNorm = norm(D - dot(A,V))**2
- fit = 0
- for i in xrange(DrowNum):
- for j in xrange(DcolNum):
- fit += matrixFitNormElement(i, j, D, A, V)
- assert_almost_equal(fit, expectedNorm)
+ assert_almost_equal(matrixFitNorm(D, A, V), expectedNorm)
View
44 rescal.py
@@ -7,7 +7,7 @@
import numpy as np
import os
import fnmatch
-from commonFunctions import squareFrobeniusNormOfSparse, fitNorm, checkingIndices
+from commonFunctions import squareFrobeniusNormOfSparse, fitNorm
__DEF_MAXITER = 50
@@ -16,8 +16,6 @@
__DEF_PROJ = True
__DEF_CONV = 1e-5
__DEF_LMBDA = 0
-__DEF_EXACT_FIT = False
-__DEF_TENSOR_SLICE_FIT_SAMPLE_RATIO = 1
def rescal(X, rank, **kwargs):
"""
@@ -25,7 +23,7 @@ def rescal(X, rank, **kwargs):
Factors a three-way tensor X such that each frontal slice
X_k = A * R_k * A.T. The frontal slices of a tensor are
- N x N matrices that correspond to the adjecency matrices
+ N x N matrices that correspond to the adjacency matrices
of the relational graph for a particular relation.
For a full description of the algorithm see:
@@ -49,17 +47,14 @@ def rescal(X, rank, **kwargs):
Whether or not to use the QR decomposition when computing R_k.
True by default
maxIter : int, optional
- Maximium number of iterations of the ALS algorithm. 500 by default.
+ Maximium number of iterations of the ALS algorithm. 50 by default.
conv : float, optional
- Stop when residual of factorization is less than conv. 1e-5 by default
- exactfit: boolean, optional
- Whether or not to compute the exact fitting value
- False by default (i.e., approximate the norm by non-zero values of the targeting tensor)
+ Stop when residual of factorization is less than conv. 1e-5 by default
Returns
-------
A : ndarray
- array of shape ('N', 'rank') corresponding to the factor matrix A
+ matrix of latent embeddings A
R : list
list of 'M' arrays of shape ('rank', 'rank') corresponding to the factor matrices R_k
f : float
@@ -77,8 +72,6 @@ def rescal(X, rank, **kwargs):
conv = kwargs.pop('conv', __DEF_CONV)
lmbda = kwargs.pop('lmbda', __DEF_LMBDA)
preheatnum = kwargs.pop('preheatnum', __DEF_PREHEATNUM)
- exactfit = kwargs.pop('exactfit', __DEF_EXACT_FIT)
- tensorSliceSampleRatio = kwargs.pop('tensorSliceSampleRatio', __DEF_TENSOR_SLICE_FIT_SAMPLE_RATIO)
if not len(kwargs) == 0:
raise ValueError( 'Unknown keywords (%s)' % (kwargs.keys()) )
@@ -86,17 +79,11 @@ def rescal(X, rank, **kwargs):
sz = X[0].shape
dtype = X[0].dtype
n = sz[0]
- k = len(X)
_log.debug('[Config] rank: %d | maxIter: %d | conv: %7.1e | lmbda: %7.1e' % (rank,
maxIter, conv, lmbda))
_log.debug('[Config] dtype: %s' % dtype)
- if exactfit:
- _log.debug('[Config] The exact fit values will be computed during optimization.')
- else:
- _log.debug('[Config] The approximating fit values will be computed during optimization.')
-
# precompute norms of X
normX = [squareFrobeniusNormOfSparse(M) for M in X]
sumNormX = sum(normX)
@@ -120,12 +107,6 @@ def rescal(X, rank, **kwargs):
# compute factorization
fit = fitchange = fitold = 0
exectimes = []
-
- if exactfit:
- tensorFitIndices = []
- else :
- tensorFitIndices = [checkingIndices(M, ratio = tensorSliceSampleRatio) for M in X]
- _log.debug('[Algorithm] Finished sampling of indices to compute the fit values.')
for iterNum in xrange(maxIter):
tic = time.clock()
@@ -137,28 +118,19 @@ def rescal(X, rank, **kwargs):
R = __updateR(X2, A2, lmbda)
else :
raise 'Projection via QR decomposition is required; pass proj=true'
-# R = __updateR(X, A, lmbda)
# compute fit values
fit = 0
regularizedFit = 0
regRFit = 0
- if iterNum > preheatnum:
+ if iterNum >= preheatnum:
if lmbda != 0:
for i in xrange(len(R)):
regRFit += norm(R[i])**2
regularizedFit = lmbda*(norm(A)**2) + lmbda*regRFit
- if exactfit:
- for i in xrange(len(R)):
- fit = norm(X[i] - dot(A,dot(R[i], A.T)))**2
- else :
- for i in xrange(len(R)):
- ARk = dot(A, R[i])
- iTensorFitIndices = tensorFitIndices[i]
- for rr in xrange(len(iTensorFitIndices)):
- m, l = iTensorFitIndices[rr]
- fit += fitNorm(m, l, X[i], ARk, A)
+ for i in xrange(len(R)):
+ fit += fitNorm(X[i], A, R[i])
fit *= 0.5
fit += regularizedFit
fit /= sumNormX
Please sign in to comment.
Something went wrong with that request. Please try again.