-
Notifications
You must be signed in to change notification settings - Fork 1
/
GRADIENT DESC RELU.py
173 lines (119 loc) · 5.85 KB
/
GRADIENT DESC RELU.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
'''
This Example shows the Mnist Data Set classification using Gradient Descent Algorithm which was
coded in Python
If You like my work please consider supoorting me by following me on github and Youtube
Thanking you,
Soumil Nitin Shah
Bachelor in Electronic Engineering
Master in Electrical Engineering
Master in Computer Engineering
Graduate Teaching/Research Assistant
Python Developer
soushah@my.bridgeport.edu
——————————————————----------------------------------
Linkedin: https://www.linkedin.com/in/shah-soumil
Github
https://github.com/soumilshah1995
Youtube channel
https://www.youtube.com/channel/UC_eOodxvwS_H7x2uLQa-svw
------------------------------------------------------
'''
# Import library
try:
import os
import sys
import cv2
import numpy as np
from sklearn.utils import shuffle
import matplotlib.pyplot as plt
import time
import datetime
except:
print("Library not found ")
now_time= datetime.datetime.now()
counter = 0
x_data_epoch = []
y_data_error = []
train = np.empty((1000,28,28), dtype='float64')
trainY = np.zeros((1000,10,1))
test = np.empty((10000, 28, 28), dtype='float64')
testY = np.zeros((10000, 10, 1))
# --------------------------------------------------------Load in Image--------------------------------------
i = 0
for filename in os.listdir('/Users/soumilshah/IdeaProjects/Deep Learning/MNIST /Training1000'):
y = int(filename[0])
trainY[i, y] = 1.0
train[i] = cv2.imread('/Users/soumilshah/IdeaProjects/Deep Learning/MNIST /Training1000/{0}'.format(filename), 0)/255.0
i = i+1
# -------------------------------------------------LOAD TEST IMAGE ------------------------------------------------
i = 0
# read test data
for filename in os.listdir('/Users/soumilshah/IdeaProjects/Deep Learning/MNIST /Test10000'):
y = int(filename[0])
testY[i, y] = 1.0
test[i] = cv2.imread('/Users/soumilshah/IdeaProjects/Deep Learning/MNIST /Test10000/{0}'.format(filename), 0)/255.0
i = i+1
# ---------------------------------------------------------------------------------------------------------------------
trainX = train.reshape(train.shape[0],train.shape[1]*train.shape[2], 1)
testX = test.reshape(test.shape[0], test.shape[1] * test.shape[2], 1)
# --------------------------------- Neural Network ---------------------------------------
numNeuronsLayer1 = 200 # Number of Neuron in Layer 1
numNeuronsLayer2 = 10 # Expected output
numEpochs = 50 # Epoch
learningRate = 0.1 # Learning Rate
w1 = np.random.uniform(low=-0.1,high=0.1,size=(numNeuronsLayer1,784))
b1 = np.random.uniform(low=-1,high=1,size=(numNeuronsLayer1, 1))
w2 = np.random.uniform(low=- 0.1,high=0.1,size=(numNeuronsLayer2, numNeuronsLayer1))
b2 = np.random.uniform(low=-0.1,high=0.1,size=(numNeuronsLayer2,1))
for n in range(0, numEpochs):
loss = 0
trainX, trainY = shuffle(trainX, trainY)
gradw2 = np.zeros((numNeuronsLayer2, numNeuronsLayer1))
gradw1 = np.zeros((numNeuronsLayer1, 784))
gradb1 = np.zeros((numNeuronsLayer1, 1))
gradb2 = np.zeros((numNeuronsLayer2, 1))
for i in range(trainX.shape[0]):
s1 = np.dot(w1,trainX[i]) + b1 # S1 = W1.X + B1
a1 = 1 / (1 + np.exp(-s1)) # A1 = 1/ 1 + exp(-s1)
s2 = np.dot(w2, a1) + b2 # S2 = W2.A1+ B1
a2 = s2
loss += (0.5 * ((a2-trainY[i])*(a2-trainY[i]))).sum()
# -------------------------------------- BACK PROPOGATE --------------------------------------
error = -1 * (trainY[i] - a2) # 10x1
a2_act = np.ones(shape=(10, 1))
delta2 = error * a2_act
derv_act_1 =np.multiply(a1 , (1 - a1)) # derivative of act = A1.(1-A1)
error_2 = np.dot(w2.T, delta2) # e_2 = Delta2 . W2
delta1 = np.multiply( error_2, derv_act_1) # Delta1 = Delta2 . A1(1-A1).W2
gradw2 += np.dot(delta2,a1.T) # Grad = Delta.Output of previous Layer
gradw1 += np.dot(delta1, trainX[i].T) # Grad = Delta.Output of previous Layer
gradb2 += delta2 # GradB2 = Delta2
gradb1 += delta1 # GradB1 = Delta1
w2 = w2 - learningRate * (gradw2/1000) # Accumulate all Gradient and Divide by 1000
b2 = b2 - learningRate * gradb2/1000 # Accumulate all Gradient and Divide by 1000
w1 = w1 - learningRate * (gradw1/1000) # Accumulate all Gradient and Divide by 1000
b1 = b1 - learningRate * (gradb1/1000) # Accumulate all Gradient and Divide by 1000
x_data_epoch.append(n) # Append error and loss
y_data_error.append(loss)
print("epoch = " + str(n) + " loss = " + (str(loss)))
print("done training , starting testing..")
accuracyCount = 0
# --------------------- Testing Data ------------------------------------------------
for i in range(testY.shape[0]):
s1 = np.dot(w1,testX[i]) + b1 # S1 = W.X + B1
a1 = 1/(1+np.exp(-1*s1)) # A1 = 1/ 1 + exp(-S1)
s2 = np.dot(w2,a1) + b2 # S2 = W2.A1 + B2
a2 = 1/(1+np.exp(-1*s2)) # A2 = 1/ 1 + exp(-S2)
a2index = a2.argmax(axis=0) # Select Max from 10 Neuron
if testY[i, a2index] == 1: # calculate The Accuracy
accuracyCount = accuracyCount + 1
print("Accuracy count = " + str(accuracyCount/10000.0))
end = datetime.datetime.now()
t = end-now_time
# Plot The Graph
print('time{}'.format(t))
plt.plot(x_data_epoch,y_data_error)
plt.xlabel('X axis Neuron {}'.format(numNeuronsLayer1))
plt.ylabel('Loss')
plt.title('\n Gradient Descent \n Execution Time:{} \n Accuracy Count {}\n Loss :{}'.format(t,accuracyCount,loss))
plt.show()