-
Notifications
You must be signed in to change notification settings - Fork 0
/
multiclassLogReg.py
107 lines (73 loc) · 2.65 KB
/
multiclassLogReg.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
import numpy as np
# Function that evaluates f -> we are trying to minimize this
def eval_f(beta, X, Y):
numExam, numFeat = X.shape
numExam, numClasses = Y.shape
cost = 0.0
for i in range(numExam):
xi = X[i,:]
yi = Y[i]
dotProds = xi @ beta
terms = np.exp(dotProds)
probs = terms/np.sum(terms)
k = np.argmax(yi)
cost += np.log(probs[k])
return -cost
# Function that evaluates the full gradient for normal MCL regression
def eval_grad(beta, X, Y):
numExam, numFeat = X.shape
numExam, numClasses = Y.shape
grad = np.zeros((numFeat, numClasses))
probs = np.zeros(numClasses)
for i in range(numExam):
xi = X[i, :]
yi = Y[i]
gradi = np.zeros((numFeat,numClasses))
dotProds = xi @ beta
terms = np.exp(dotProds)
probs = terms/np.sum(terms)
for k in range(numClasses):
gradi[:, k] += (probs[k] - yi[k]) * xi
grad += gradi
return grad
# Function that performs MCL regression with the full gradient
def multiclassLogReg_grad(X, Y, t):
maxIter = 300
numExam, numFeat = X.shape
numExam, numClasses = Y.shape
beta = np.zeros((numFeat, numClasses))
costs = np.zeros(maxIter)
for idx in range(maxIter):
grad = eval_grad(beta, X, Y)
beta = beta - t * grad
costs[idx] = eval_f(beta, X, Y)
print("id: " + str(idx) + "cost: " + str(costs[idx]))
return(beta, costs)
# Function that only evaluates gradi and is used for SGD
def eval_gradi(beta, xi, yi):
numFeat = len(xi)
numClasses = len(yi)
gradi = np.zeros((numFeat,numClasses))
dotProds = xi @ beta
terms = np.exp(dotProds)
probs = terms/np.sum(terms)
for k in range(numClasses):
gradi[:, k] += (probs[k] - yi[k]) * xi
return gradi
# Function that evaluates MCL regression using SGD -> used on the MNIST dataset
def multiclassLogReg_SGD(X, Y, t):
numEpochs = 25
numExam, numFeat = X.shape
numExam, numClasses = Y.shape
beta = np.zeros((numFeat, numClasses))
costs = np.zeros(numEpochs)
for epoch in range(numEpochs):
for i in np.random.permutation(numExam):
xi = X[i, :]
yi = Y[i]
gradi = eval_gradi(beta, xi, yi)
beta = beta - t * gradi
cost = eval_f(beta, X, Y)
costs[epoch] = cost
# print("epoch: " + str(epoch) + "cost: " + str(cost))
return(beta, costs)