/
normal.py
131 lines (94 loc) · 4.04 KB
/
normal.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
import numpy
from chainer.backends import cuda
from chainer import initializer
# Original code forked from MIT licensed keras project
# https://github.com/fchollet/keras/blob/master/keras/initializations.py
class Normal(initializer.Initializer):
"""Initializes array with a normal distribution.
Each element of the array is initialized by the value drawn
independently from Gaussian distribution whose mean is 0,
and standard deviation is ``scale``.
Args:
scale(float): Standard deviation of Gaussian distribution.
dtype: Data type specifier.
"""
def __init__(self, scale=0.05, dtype=None):
self.scale = scale
super(Normal, self).__init__(dtype)
def __call__(self, array):
xp = cuda.get_array_module(array)
args = {'loc': 0.0, 'scale': self.scale, 'size': array.shape}
if xp is not numpy:
# Only CuPy supports dtype option
if self.dtype == numpy.float32 or self.dtype == numpy.float16:
# float16 is not supported in cuRAND
args['dtype'] = numpy.float32
array[...] = xp.random.normal(**args)
class LeCunNormal(initializer.Initializer):
"""Initializes array with scaled Gaussian distribution.
Each element of the array is initialized by the value drawn
independently from Gaussian distribution whose mean is 0,
and standard deviation is
:math:`scale \\times \\sqrt{\\frac{1}{fan_{in}}}`,
where :math:`fan_{in}` is the number of input units.
Reference: LeCun 98, Efficient Backprop
http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf
Args:
scale (float): A constant that determines the scale
of the standard deviation.
dtype: Data type specifier.
"""
def __init__(self, scale=1.0, dtype=None):
self.scale = scale
super(LeCunNormal, self).__init__(dtype)
def __call__(self, array):
if self.dtype is not None:
assert array.dtype == self.dtype
fan_in, fan_out = initializer.get_fans(array.shape)
s = self.scale * numpy.sqrt(1. / fan_in)
Normal(s)(array)
class GlorotNormal(initializer.Initializer):
"""Initializes array with scaled Gaussian distribution.
Each element of the array is initialized by the value drawn
independently from Gaussian distribution whose mean is 0,
and standard deviation is
:math:`scale \\times \\sqrt{\\frac{2}{fan_{in} + fan_{out}}}`,
where :math:`fan_{in}` and :math:`fan_{out}` are the number of
input and output units, respectively.
Reference: Glorot & Bengio, AISTATS 2010
Args:
scale (float): A constant that determines the scale
of the standard deviation.
dtype: Data type specifier.
"""
def __init__(self, scale=1.0, dtype=None):
self.scale = scale
super(GlorotNormal, self).__init__(dtype)
def __call__(self, array):
if self.dtype is not None:
assert array.dtype == self.dtype
fan_in, fan_out = initializer.get_fans(array.shape)
s = self.scale * numpy.sqrt(2. / (fan_in + fan_out))
Normal(s)(array)
class HeNormal(initializer.Initializer):
"""Initializes array with scaled Gaussian distribution.
Each element of the array is initialized by the value drawn
independently from Gaussian distribution whose mean is 0,
and standard deviation is
:math:`scale \\times \\sqrt{\\frac{2}{fan_{in}}}`,
where :math:`fan_{in}` is the number of input units.
Reference: He et al., https://arxiv.org/abs/1502.01852
Args:
scale (float): A constant that determines the scale
of the standard deviation.
dtype: Data type specifier.
"""
def __init__(self, scale=1.0, dtype=None):
self.scale = scale
super(HeNormal, self).__init__(dtype)
def __call__(self, array):
if self.dtype is not None:
assert array.dtype == self.dtype
fan_in, fan_out = initializer.get_fans(array.shape)
s = self.scale * numpy.sqrt(2. / fan_in)
Normal(s)(array)