Skip to content

update out_prob of dropout in several RNN layers #357

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 23, 2018
Merged

update out_prob of dropout in several RNN layers #357

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fa1aae
update out_prob of dropout in several RNN layers
nebulaV Feb 23, 2018
3ec8d3f
Merge branch 'master' into rnn
luomai Feb 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions tensorlayer/layers/recurrent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,10 @@ def __init__(
DropoutWrapper_fn = tf.contrib.rnn.DropoutWrapper
except Exception:
DropoutWrapper_fn = tf.nn.rnn_cell.DropoutWrapper
cell_creator = lambda: DropoutWrapper_fn(rnn_creator(), input_keep_prob=in_keep_prob, output_keep_prob=1.0) # out_keep_prob)
cell_creator = lambda is_last=True: \
DropoutWrapper_fn(rnn_creator(),
input_keep_prob=in_keep_prob,
output_keep_prob=out_keep_prob if is_last else 1.0)
else:
cell_creator = rnn_creator
self.fw_cell = cell_creator()
Expand All @@ -392,11 +395,11 @@ def __init__(
MultiRNNCell_fn = tf.nn.rnn_cell.MultiRNNCell

try:
self.fw_cell = MultiRNNCell_fn([cell_creator() for _ in range(n_layer)], state_is_tuple=True)
self.bw_cell = MultiRNNCell_fn([cell_creator() for _ in range(n_layer)], state_is_tuple=True)
self.fw_cell = MultiRNNCell_fn([cell_creator(is_last=i == n_layer - 1) for i in range(n_layer)], state_is_tuple=True)
self.bw_cell = MultiRNNCell_fn([cell_creator(is_last=i == n_layer - 1) for i in range(n_layer)], state_is_tuple=True)
except Exception:
self.fw_cell = MultiRNNCell_fn([cell_creator() for _ in range(n_layer)])
self.bw_cell = MultiRNNCell_fn([cell_creator() for _ in range(n_layer)])
self.fw_cell = MultiRNNCell_fn([cell_creator(is_last=i == n_layer - 1) for i in range(n_layer)])
self.bw_cell = MultiRNNCell_fn([cell_creator(is_last=i == n_layer - 1) for i in range(n_layer)])

# Initial state of RNN
if fw_initial_state is None:
Expand Down Expand Up @@ -1076,7 +1079,10 @@ def __init__(
# cell_instance_fn1(),
# input_keep_prob=in_keep_prob,
# output_keep_prob=out_keep_prob)
cell_creator = lambda: DropoutWrapper_fn(rnn_creator(), input_keep_prob=in_keep_prob, output_keep_prob=1.0)
cell_creator = lambda is_last=True: \
DropoutWrapper_fn(rnn_creator(),
input_keep_prob=in_keep_prob,
output_keep_prob=out_keep_prob if is_last else 1.0)
else:
cell_creator = rnn_creator
self.cell = cell_creator()
Expand All @@ -1090,10 +1096,10 @@ def __init__(
# cell_instance_fn2=cell_instance_fn # HanSheng
try:
# cell_instance_fn=lambda: MultiRNNCell_fn([cell_instance_fn2() for _ in range(n_layer)], state_is_tuple=True) # HanSheng
self.cell = MultiRNNCell_fn([cell_creator() for _ in range(n_layer)], state_is_tuple=True)
self.cell = MultiRNNCell_fn([cell_creator(is_last=i == n_layer - 1) for i in range(n_layer)], state_is_tuple=True)
except Exception: # when GRU
# cell_instance_fn=lambda: MultiRNNCell_fn([cell_instance_fn2() for _ in range(n_layer)]) # HanSheng
self.cell = MultiRNNCell_fn([cell_creator() for _ in range(n_layer)])
self.cell = MultiRNNCell_fn([cell_creator(is_last=i == n_layer - 1) for i in range(n_layer)])

# self.cell=cell_instance_fn() # HanSheng

Expand Down