revert back kuma and make log1pexp safe

xukai92 · Apr 28, 2019 · 653f4b3 · 653f4b3
1 parent fd22faa
commit 653f4b3
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/src/neural/activations.jl b/src/neural/activations.jl
@@ -1,6 +1,6 @@
 # TODO: implement CUDA kernels for below
 
-log1pexp(x) = log1p(exp(x))
+log1pexp(x) = log1p(exp(-abs(x))) + max(x, 0)
 logexpm1(x) = log(expm1(x))
 
 const softplus = log1pexp

diff --git a/src/neural/stochastic_layers.jl b/src/neural/stochastic_layers.jl
@@ -33,8 +33,8 @@ const BINARY_DIST_DICT = Dict(
                                 :Σ, :softplus),
     :GaussianLogVar => (:BatchNormalLogVar, :μ, :identity,
                                             :logΣ, :identity),
-    :Kumaraswamy => (:BatchKumaraswamy, :a, :(Knet.relu),
-                                        :b, :(Knet.relu)),
+    :Kumaraswamy => (:BatchKumaraswamy, :a, :softplus,
+                                        :b, :softplus),
 )
 
 for dist_sym in keys(BINARY_DIST_DICT)