stablized alpha softmax

orhanf · Nov 5, 2015 · b0f0627 · b0f0627
1 parent 9787eb2
commit b0f0627
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/session2/nmt.py b/session2/nmt.py
@@ -428,7 +428,8 @@ def _step_slice(m_, x_, xx_, xc_, h_, ctx_, alpha_, pctx_, cc_,
         # compute alignment weights
         alpha = tensor.dot(pctx__, U_att)+c_tt
         alpha = alpha.reshape([alpha.shape[0], alpha.shape[1]])
-        alpha = tensor.exp(alpha)
+        alpha = tensor.exp(alpha - alpha.max(0, keepdims=True))
+
         if context_mask:
             alpha = alpha * context_mask
         alpha = alpha / alpha.sum(0, keepdims=True)

diff --git a/session3/nmt.py b/session3/nmt.py
@@ -428,7 +428,8 @@ def _step_slice(m_, x_, xx_, xc_, h_, ctx_, alpha_, pctx_, cc_,
         # compute alignment weights
         alpha = tensor.dot(pctx__, U_att)+c_tt
         alpha = alpha.reshape([alpha.shape[0], alpha.shape[1]])
-        alpha = tensor.exp(alpha)
+        alpha = tensor.exp(alpha - alpha.max(0, keepdims=True))
+
         if context_mask:
             alpha = alpha * context_mask
         alpha = alpha / alpha.sum(0, keepdims=True)