-
Notifications
You must be signed in to change notification settings - Fork 0
/
softmax.py
53 lines (39 loc) · 1.2 KB
/
softmax.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
import numpy as np
from utilities import initalize
class Softmax(object):
def __init__(self, input_size: int,
output_size: int,
init_range: float = 1.0):
self.input_size = input_size
self.output_size = output_size
self.W = initalize((output_size, input_size), init_range)
self.dW = np.zeros((output_size, input_size))
self.params = [
('W', self.W, self.dW)
]
def init_sequence(self):
self.pred = []
self.x = []
self.targets = []
self.t = 0
self.dW[:] = 0
def forward(self, x: np.float64):
self.t += 1
y = self.W.dot(x)
y = np.exp(y - y.max())
y /= y.sum()
self.pred.append(y)
self.x.append(x)
return y
def backward(self, target):
self.t -= 1
self.targets.append(target)
x = self.x[self.t]
d = self.pred[self.t].copy()
d[target] -= 1
self.dW += np.outer(d, x)
delta = np.dot(self.W.T, d)
return delta
def get_cost(self):
combined = zip(self.targets, reversed(self.pred))
return sum(-np.log(y[target]) for target, y in combined)