/
normal.py
99 lines (71 loc) · 3.05 KB
/
normal.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import numpy
from chainer import cuda
from chainer import initializer
# Original code forked from MIT licensed keras project
# https://github.com/fchollet/keras/blob/master/keras/initializations.py
class Normal(initializer.Initializer):
"""Initializes array with a normal distribution.
Each element of the array is initialized by the value drawn
independently from Gaussian distribution whose mean is 0,
and standard deviation is ``scale``.
Args:
scale(float): Standard deviation of Gaussian distribution.
dtype: Data type specifier.
"""
def __init__(self, scale=0.05, dtype=None):
self.scale = scale
super(Normal, self).__init__(dtype)
def __call__(self, array):
xp = cuda.get_array_module(array)
args = {'loc': 0.0, 'scale': self.scale, 'size': array.shape}
if xp is not numpy:
# Only CuPy supports dtype option
if self.dtype == numpy.float32 or self.dtype == numpy.float16:
# float16 is not supported in cuRAND
args['dtype'] = numpy.float32
array[...] = xp.random.normal(**args)
class GlorotNormal(initializer.Initializer):
"""Initializes array with scaled Gaussian distribution.
Each element of the array is initialized by the value drawn
independently from Gaussian distribution whose mean is 0,
and standard deviation is
:math:`scale \\times \\sqrt{\\frac{2}{fan_{in} + fan_{out}}}`,
where :math:`fan_{in}` and :math:`fan_{out}` are the number of
input and output units, respectively.
Reference: Glorot & Bengio, AISTATS 2010
Args:
scale (float): A constant that determines the scale
of the standard deviation.
dtype: Data type specifier.
"""
def __init__(self, scale=1.0, dtype=None):
self.scale = scale
super(GlorotNormal, self).__init__(dtype)
def __call__(self, array):
if self.dtype is not None:
assert array.dtype == self.dtype
fan_in, fan_out = initializer.get_fans(array.shape)
s = self.scale * numpy.sqrt(2. / (fan_in + fan_out))
Normal(s)(array)
class HeNormal(initializer.Initializer):
"""Initializes array with scaled Gaussian distribution.
Each element of the array is initialized by the value drawn
independently from Gaussian distribution whose mean is 0,
and standard deviation is
:math:`scale \\times \\sqrt{\\frac{2}{fan_{in}}}`,
where :math:`fan_{in}` is the number of input units.
Reference: He et al., https://arxiv.org/abs/1502.01852
Args:
scale (float): A constant that determines the scale
of the standard deviation.
dtype: Data type specifier.
"""
def __init__(self, scale=1.0, dtype=None):
self.scale = scale
super(HeNormal, self).__init__(dtype)
def __call__(self, array):
if self.dtype is not None:
assert array.dtype == self.dtype
fan_in, fan_out = initializer.get_fans(array.shape)
s = self.scale * numpy.sqrt(2. / fan_in)
Normal(s)(array)