/
nonlinear.jl
106 lines (86 loc) · 2.43 KB
/
nonlinear.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
"""
ReLU()
(l::ReLU)(x) = max(0,x)
Rectified Linear Unit function.
"""
struct ReLU <: Activation
end
(l::ReLU)(x) = relu.(x)
"""
Sigm()
(l::Sigm)(x) = sigm(x)
Sigmoid function
"""
struct Sigm <: Activation
end
(l::Sigm)(x) = sigm.(x)
"""
Tanh()
(l::Tanh)(x) = tanh(x)
Tangent hyperbolic function
"""
struct Tanh <: Activation
end
(l::Tanh)(x) = tanh.(x)
"""
ELU()
(l::ELU)(x) = elu(x) -> Computes x < 0 ? exp(x) - 1 : x
Exponential Linear Unit nonlineariy.
"""
struct ELU <: Activation
end
(l::ELU)(x) = elu.(x)
"""
LeakyReLU(α=0.2)
(l::LeakyReLU)(x) -> Computes x < 0 ? α*x : x
"""
struct LeakyReLU <: Activation
α::AbstractFloat
LeakyReLU(alpha::AbstractFloat=0.2) = new(alpha)
end
(l::LeakyReLU)(x) = relu.(x) .+ l.α*min.(0,x)
"""
Dropout(p=0)
Dropout Layer. `p` is the droput probability.
"""
mutable struct Dropout <: Activation
p::Real
end
Dropout(;p=0) = Dropout(p)
(l::Dropout)(x) = dropout(x,l.p)
"""
LogSoftMax(dims=:)
(l::LogSoftMax)(x)
Treat entries in x as as unnormalized log probabilities and return normalized log probabilities.
dims is an optional argument, if not specified the normalization is over the whole x, otherwise the normalization is performed over the given dimensions. In
particular, if x is a matrix, dims=1 normalizes columns of x and dims=2 normalizes rows of x.
"""
struct LogSoftMax <: Activation
dims
end
LogSoftMax(;dims=:) = LogSoftMax(dims)
(l::LogSoftMax)(x) = logp(x;dims=l.dims)
"""
SoftMax(dims=:)
(l::SoftMax)(x)
Treat entries in x as as unnormalized scores and return softmax probabilities.
dims is an optional argument, if not specified the normalization is over the whole x, otherwise the normalization is performed over the given dimensions. In
particular, if x is a matrix, dims=1 normalizes columns of x and dims=2 normalizes rows of x.
"""
struct SoftMax <: Activation
dims
end
SoftMax(;dims=:) = SoftMax(dims)
(l::SoftMax)(x) = exp.(logp(x;dims=l.dims))
"""
LogSumExp(dims=:)
(l::LogSumExp)(x)
Compute log(sum(exp(x);dims)) in a numerically stable manner.
dims is an optional argument, if not specified the summation is over the whole x, otherwise the summation is performed over the given dimensions. In particular if x
is a matrix, dims=1 sums columns of x and dims=2 sums rows of x.
"""
struct LogSumExp <: Activation
dims
end
LogSumExp(;dims=:) = LogSumExp(dims)
(l::LogSumExp)(x) = logsumexp(x;dims=l.dims)