-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathlayers.py
92 lines (56 loc) · 3.16 KB
/
layers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
import torch
import torch.nn as nn
class HighwayMLP(nn.Module):
def __init__(self, input_size, gate_bias=-2,
activation_function=nn.functional.relu,
gate_activation =nn.functional.softmax):
super(HighwayMLP, self).__init__()
self.activation_function = activation_function
self.gate_activation = gate_activation
self.normal_layer = nn.Linear(input_size, input_size)
self.gate_layer = nn.Linear(input_size, input_size)
self.gate_layer.bias.data.fill_(gate_bias)
def forward(self, x):
normal_layer_result = self.activation_function(self.normal_layer(x))
gate_layer_result = self.gate_activation(self.gate_layer(x))
multiplyed_gate_and_normal = torch.mul(normal_layer_result, gate_layer_result)
multiplyed_gate_and_input = torch.mul((1 - gate_layer_result), x)
return torch.add(multiplyed_gate_and_normal,
multiplyed_gate_and_input)
class HighwayCNN(nn.Module):
def __init__(self, input_size, gate_bias=-1,
activation_function=nn.functional.relu,
gate_activation=nn.functional.softmax):
super(HighwayCNN, self).__init__()
self.activation_function = activation_function
self.gate_activation = gate_activation
self.normal_layer = nn.Linear(input_size, input_size)
self.gate_layer = nn.Linear(input_size, input_size)
self.gate_layer.bias.data.fill_(gate_bias)
def forward(self, x):
normal_layer_result = self.activation_function(self.normal_layer(x))
gate_layer_result = self.gate_activation(self.gate_layer(x))
multiplyed_gate_and_normal = torch.mul(normal_layer_result, gate_layer_result)
multiplyed_gate_and_input = torch.mul((1 - gate_layer_result), x)
return torch.add(multiplyed_gate_and_normal,
multiplyed_gate_and_input)
class Cgnn(nn.Module):
def __init__(self, input_size, gate1_bias=-2, gate2_bias=-3,
activation_function=nn.functional.tanh,
gate_activation =nn.functional.sigmoid):
super(Cgnn, self).__init__()
self.activation_function = activation_function
self.gate_activation = gate_activation
self.normal_layer = nn.Linear(input_size, input_size)
self.gate_layer1 = nn.Linear(input_size, input_size)
self.gate_layer1.bias.data.fill_(gate1_bias)
self.gate_layer2 = nn.Linear(input_size, input_size)
self.gate_layer2.bias.data.fill_(gate2_bias)
def forward(self, x):
normal_layer_result = self.activation_function(self.normal_layer(x))
gate_layer1_result = self.gate_activation(self.gate_layer1(x))
gate_layer2_result = self.gate_activation(self.gate_layer2(x))
multiplyed_gate1_and_normal = torch.mul(normal_layer_result, gate_layer1_result)
activate_gate1_and_normal = self.activation_function(multiplyed_gate1_and_normal)
output_vector = torch.mul(activate_gate1_and_normal, gate_layer2_result)
return output_vector