-
-
Notifications
You must be signed in to change notification settings - Fork 14
/
neural_net.py
401 lines (340 loc) · 15.7 KB
/
neural_net.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
import numpy as np
import matplotlib.pyplot as plt
import time
class TwoLayerNet(object):
"""
A two-layer fully-connected neural network. The net has an input dimension of
N, a hidden layer dimension of H, and performs classification over C classes.
We train the network with a softmax loss function and L2 regularization on the
weight matrices. The network uses a ReLU nonlinearity after the first fully
connected layer.
In other words, the network has the following architecture:
input - fully connected layer - ReLU - fully connected layer - softmax
The outputs of the second fully-connected layer are the scores for each class.
"""
def __init__(self, input_size, hidden_size, output_size, std=1e-4,
init_method="Normal"):
"""
Initialize the model. Weights are initialized to small random values and
biases are initialized to zero. Weights and biases are stored in the
variable self.params, which is a dictionary with the following keys:
W1: First layer weights; has shape (D, H)
b1: First layer biases; has shape (H,)
W2: Second layer weights; has shape (H, C)
b2: Second layer biases; has shape (C,)
Inputs:
- input_size: The dimension D of the input data.
- hidden_size: The number of neurons H in the hidden layer.
- output_size: The number of classes C.
"""
self.params = {}
self.params['W1'] = std * np.random.randn(input_size, hidden_size)
self.params['b1'] = np.zeros(hidden_size)
self.params['W2'] = std * np.random.randn(hidden_size, output_size)
self.params['b2'] = np.zeros(output_size)
#special initialization
if init_method=="i":
self.params['W1']=np.random.randn(input_size,hidden_size)/np.sqrt(input_size)
self.params['W2']=np.random.randn(hidden_size,output_size)/np.sqrt(hidden_size)
elif init_method=="io":
self.params['W1']=np.random.randn(input_size,hidden_size)*np.sqrt(2.0/(input_size+hidden_size))
self.params['W2']=np.random.randn(hidden_size,output_size)*np.sqrt(2.0/(hidden_size+output_size))
elif init_method=="ReLU":
self.params['W1']=np.random.randn(input_size,hidden_size)*np.sqrt(2.0/input_size)
self.params['W2']=np.random.randn(hidden_size,output_size)*np.sqrt(2.0/(hidden_size+output_size))
def loss(self, X, y=None, reg=0.0, dropout=0, dropMask=None,activation='Relu'):
"""
Compute the loss and gradients for a two layer fully connected neural
network.
Inputs:
- X: Input data of shape (N, D). Each X[i] is a training sample.
- y: Vector of training labels. y[i] is the label for X[i], and each y[i] is
an integer in the range 0 <= y[i] < C. This parameter is optional; if it
is not passed then we only return scores, and if it is passed then we
instead return the loss and gradients.
- reg: Regularization strength.
Returns:
If y is None, return a matrix scores of shape (N, C) where scores[i, c] is
the score for class c on input X[i].
If y is not None, instead return a tuple of:
- loss: Loss (data loss and regularization loss) for this batch of training
samples.
- grads: Dictionary mapping parameter names to gradients of those parameters
with respect to the loss function; has the same keys as self.params.
"""
# Unpack variables from the params dictionary
W1, b1 = self.params['W1'], self.params['b1']
W2, b2 = self.params['W2'], self.params['b2']
N, D = X.shape
# Compute the forward pass
scores = None
#############################################################################
# TODO: Perform the forward pass, computing the class scores for the input. #
# Store the result in the scores variable, which should be an array of #
# shape (N, C). #
#############################################################################
if activation=='leaky':
inp=X.dot(W1)+b1
a2=np.maximum(inp,.01*inp)
else:
a2=np.maximum(X.dot(W1)+b1,0)
if dropout != 0 and dropout<1:
a2*=(np.random.randn(*a2.shape)<dropout)/dropout
elif dropout>1:
W2*=dropMask['W2']/(dropout-1)
b2*=dropMask['b2']/(dropout-1)
# for convinient this is drop connect , drop rate= dropout-1
scores=a2.dot(W2)+b2 # z3
#############################################################################
# END OF YOUR CODE #
#############################################################################
# If the targets are not given then jump out, we're done
if y is None:
return scores
# Compute the loss
loss = None
#############################################################################
# TODO: Finish the forward pass, and compute the loss. This should include #
# both the data loss and L2 regularization for W1 and W2. Store the result #
# in the variable loss, which should be a scalar. Use the Softmax #
# classifier loss. So that your results match ours, multiply the #
# regularization loss by 0.5 #
#############################################################################
#do a softmax first
if dropout>1:
print dropMask['W2']
exp_scores=np.exp(scores)
a3=exp_scores/(np.sum(exp_scores,1))[:,None] #h(x)
loss=-np.sum(np.log(a3[range(len(a3)),y]))/len(a3)+\
0.5*reg*(np.sum(np.power(W1,2))+np.sum(np.power(W2,2)))
#############################################################################
# END OF YOUR CODE #
#############################################################################
# Backward pass: compute gradients
grads = {}
#############################################################################
# TODO: Compute the backward pass, computing the derivatives of the weights #
# and biases. Store the results in the grads dictionary. For example, #
# grads['W1'] should store the gradient on W1, and be a matrix of same size #
#############################################################################
delta_3=a3
delta_3[range(len(a3)),y]=a3[range(len(a3)),y]-1
delta_3/=len(a3)
grads['W2']=a2.T.dot(delta_3)+reg*W2
grads['b2']=np.sum(delta_3,0)
dF=np.ones(np.shape(a2))
if activation=='leaky':
dF[a2<0.0]=0.01
else:
dF[a2==0.0]=0 #activation res a2 has been ReLUed
delta_2=delta_3.dot(W2.T)*dF
grads['W1']=X.T.dot(delta_2)+reg*W1
grads['b1']=np.sum(delta_2,0)
#############################################################################
# END OF YOUR CODE #
#############################################################################
return loss, grads
def train(self, X, y, X_val, y_val,
learning_rate=1e-3, learning_rate_decay=0.95,
reg=1e-5, num_iters=100,
batch_size=200, verbose=False,
update="SGD",arg=.99,
dropout=0,
activation='ReLU'):
"""
Train this neural network using stochastic gradient descent.
Inputs:
- X: A numpy array of shape (N, D) giving training data.
- y: A numpy array f shape (N,) giving training labels; y[i] = c means that
X[i] has label c, where 0 <= c < C.
- X_val: A numpy array of shape (N_val, D) giving validation data.
- y_val: A numpy array of shape (N_val,) giving validation labels.
- learning_rate: Scalar giving learning rate for optimization.
- learning_rate_decay: Scalar giving factor used to decay the learning rate
after each epoch.
- reg: Scalar giving regularization strength.
- num_iters: Number of steps to take when optimizing.
- batch_size: Number of training examples to use per step.
- verbose: boolean; if true print progress during optimization.
"""
num_train = X.shape[0]
iterations_per_epoch = max(num_train / batch_size, 1)
# Use SGD to optimize the parameters in self.model
loss_history = []
train_acc_history = []
val_acc_history = []
#### for tracking top model
top_params=dict()
cache_params=dict()
top_acc=0
cache=dict()
dropMask=dict()
start_time=time.time()
####
for it in xrange(num_iters):
X_batch = None
y_batch = None
#########################################################################
# TODO: Create a random minibatch of training data and labels, storing #
# them in X_batch and y_batch respectively. #
#########################################################################
if num_train >= batch_size:
rand_idx=np.random.choice(num_train,batch_size)
else:
rand_idx=np.random.choice(num_train,batch_size,replace=True)
X_batch=X[rand_idx]
y_batch=y[rand_idx]
if dropout>1:
for param in ['W2','b2']:
dropMask[param]=np.random.randn(*self.params[param].shape)<(dropout-1)
#########################################################################
# END OF YOUR CODE #
#########################################################################
# Compute loss and gradients using the current minibatch
loss, grads = self.loss(X_batch, y=y_batch, reg=reg, dropout=dropout,dropMask=dropMask,activation=activation)
loss_history.append(loss)
#########################################################################
# TODO: Use the gradients in the grads dictionary to update the #
# parameters of the network (stored in the dictionary self.params) #
# using stochastic gradient descent. You'll need to use the gradients #
# stored in the grads dictionary defined above. #
#########################################################################
if np.isnan(grads['W1']).any() or np.isnan(grads['W2']).any() or \
np.isnan(grads['b1']).any() or np.isnan(grads['b2']).any():
continue
#cache_params=self.params.copy()
dx=None
for param in self.params:
if update=="SGD":
dx=learning_rate*grads[param]
#self.params[param]-=learning_rate*grads[param]
elif update=="momentum":
if not param in cache:
cache[param]=np.zeros(grads[param].shape)
cache[param]=arg*cache[param]-learning_rate*grads[param]
dx=-cache[param]
#self.params[param]+=cache[param]
elif update=="Nesterov momentum":
if not param in cache:
cache[param]=np.zeros(grads[param].shape)
v_prev = cache[param] # back this up
cache[param] = arg * cache[param] - learning_rate * grads[param] # velocity update stays the same
dx=arg * v_prev - (1 + arg) * cache[param]
#self.params[param] += -arg * v_prev + (1 + arg) * cache[param] # position update changes form
elif update=="rmsprop":
if not param in cache:
cache[param]=np.zeros(grads[param].shape)
cache[param]=arg*cache[param]+(1-arg)*np.power(grads[param],2)
dx=learning_rate*grads[param]/np.sqrt(cache[param]+1e-8)
#self.params[param]-=learning_rate*grads[param]/np.sqrt(cache[param]+1e-8)
elif update=="Adam":
print "update error"
elif update=="Adagrad":
print "update error"
else:
# if have time try more update methods
print "choose update method!"
if dropout>1:
if param == 'W2' or param == 'b2':
dx*=dropMask[param]
self.params[param]-=dx
#Bug: learning rate should not decay at first epoch
it+=1
#########################################################################
# END OF YOUR CODE #
#########################################################################
if verbose and it % 100 == 0:
print 'iteration %d / %d: loss %f' % (it, num_iters, loss)
# Every epoch, check train and val accuracy and decay learning rate.
if it % iterations_per_epoch == 0:
# Check accuracy
train_acc = (self.predict(X_batch) == y_batch).mean()
val_acc = (self.predict(X_val) == y_val).mean()
train_acc_history.append(train_acc)
val_acc_history.append(val_acc)
# Decay learning rate
learning_rate *= learning_rate_decay
### update top model
if val_acc > top_acc:
top_acc = val_acc
top_params=self.params.copy()
if verbose:
print ('train_acc %f, val_acc %f, time %d' % (train_acc, val_acc,(time.time()-start_time)/60.0))
self.params=top_params.copy()
### update params to top params finally
return {
'loss_history': loss_history,
'train_acc_history': train_acc_history,
'val_acc_history': val_acc_history,
}
def predict(self, X):
"""
Use the trained weights of this two-layer network to predict labels for
data points. For each data point we predict scores for each of the C
classes, and assign each data point to the class with the highest score.
Inputs:
- X: A numpy array of shape (N, D) giving N D-dimensional data points to
classify.
Returns:
- y_pred: A numpy array of shape (N,) giving predicted labels for each of
the elements of X. For all i, y_pred[i] = c means that X[i] is predicted
to have class c, where 0 <= c < C.
"""
y_pred = None
###########################################################################
# TODO: Implement this function; it should be VERY simple! #
###########################################################################
y_pred=np.argmax(np.maximum(0,(X.dot(self.params["W1"])+self.params['b1']))\
.dot(self.params['W2'])+self.params['b2'],1) # z3
###########################################################################
# END OF YOUR CODE #
###########################################################################
return y_pred
def accuracy(self,X,y):
"""
Use the trained model to predict labels for X, and compute the accuracy.
Inputs:
- X: A numpy array of shape (N, D) giving N D-dimensional data points to
classify.
- y: A numpy array of shape (N,) giving the correct labels.
Returns:
- acc: Accuracy
"""
acc = (self.predict(X) == y).mean()
return acc
def gradient_check(self,X,y):
realGrads=dict()
_,grads=self.loss(X,y)
keys=['W1','b1',
'W2','b2']
for key in keys:
W1=self.params[key]
W1_grad=[]
delta=1e-4
if len(np.shape(W1))==2:
for i in range(np.shape(W1)[0]):
grad=[]
for j in range(np.shape(W1)[1]):
W1[i,j]+=delta
self.params[key]=W1
l_plus,_=self.loss(X,y)
W1[i,j]-=2*delta
self.params[key]=W1
l_minus,_=self.loss(X,y)
grad.append((l_plus-l_minus)/2.0/delta)
W1[i,j]+=delta
W1_grad.append(grad)
else:
for i in range(len(W1)):
W1[i]+=delta
self.params[key]=W1
l_plus,_=self.loss(X,y)
W1[i]-=2*delta
self.params[key]=W1
l_minus,_=self.loss(X,y)
W1_grad.append((l_plus-l_minus)/2.0/delta)
W1[i]+=delta
print(W1_grad)
print(grads[key])
print key,"error",np.mean(np.sum(np.power((W1_grad-grads[key]),2),len(np.shape(W1))-1)\
/np.sum(np.power((W1_grad+grads[key]),2),len(np.shape(W1))-1))