-
Notifications
You must be signed in to change notification settings - Fork 0
/
lstm_seq2seq_edit_ferdosi.py
276 lines (214 loc) · 9.56 KB
/
lstm_seq2seq_edit_ferdosi.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
'''
Sequence to sequence example in Keras (character-level).
Edits by Morteza Zakeri:
- add support for UTF-8 inputs
- add save model
- add plot model
-
Date: 1396-10-15
Note: Run this script on GPU on massive train set
# Data download
./trainset/ferdosi_poem_1.txt
# References
- Sequence to Sequence Learning with Neural Networks
https://arxiv.org/abs/1409.3215
- Learning Phrase Representations using
RNN Encoder-Decoder for Statistical Machine Translation
https://arxiv.org/abs/1406.1078
'''
from __future__ import print_function
from keras.models import Model, load_model
from keras.layers import Input, LSTM, Dense
import numpy as np
from keras.utils import plot_model
import datetime
from random import randint
batch_size = 64 # Batch size for training.
epochs = 10 # Number of epochs to train for. # last time I try it!!!
latent_dim = 256 # Latent dimensionality of the encoding space.
num_samples = 100000 # Number of samples to train on. One element of a dataset.
# Path to the txt dataset file on disk, inside the code directory.
#data_path = './trainset/hafez_poem_1.txt' # 419594 character , 12824 mesra and 6412 beyt
data_path = './trainset/ferdosi_poem_3.txt' # 95884 mesra and 47942 beyt
# Vectorize the data.
input_texts = []
target_texts = []
input_characters = set()
target_characters = set()
# I add support to UTF-8 (for persian (not-enlish) inputs)
lines = open(data_path, encoding="utf8").read().split('\n')
for line in lines[: min(num_samples, len(lines) - 1)]:
input_text, target_text = line.split('\t')
# We use "tab" as the "start sequence" character
# for the targets, and "\n" as "end sequence" character.
target_text = '\t' + target_text + '\n'
input_texts.append(input_text)
target_texts.append(target_text)
for char in input_text:
if char not in input_characters:
input_characters.add(char)
for char in target_text:
if char not in target_characters:
target_characters.add(char)
input_characters = sorted(list(input_characters))
target_characters = sorted(list(target_characters))
num_encoder_tokens = len(input_characters)
num_decoder_tokens = len(target_characters)
max_encoder_seq_length = max([len(txt) for txt in input_texts])
max_decoder_seq_length = max([len(txt) for txt in target_texts])
print('Number of samples:', len(input_texts))
print('Number of unique input tokens:', num_encoder_tokens)
print('Number of unique output tokens:', num_decoder_tokens)
print('Max sequence length for inputs:', max_encoder_seq_length)
print('Max sequence length for outputs:', max_decoder_seq_length)
input_token_index = dict(
[(char, i) for i, char in enumerate(input_characters)])
target_token_index = dict(
[(char, i) for i, char in enumerate(target_characters)])
encoder_input_data = np.zeros(
(len(input_texts), max_encoder_seq_length, num_encoder_tokens),
dtype='float32')
decoder_input_data = np.zeros(
(len(input_texts), max_decoder_seq_length, num_decoder_tokens),
dtype='float32')
decoder_target_data = np.zeros(
(len(input_texts), max_decoder_seq_length, num_decoder_tokens),
dtype='float32')
# bulid one-hot vector
for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):
for t, char in enumerate(input_text):
encoder_input_data[i, t, input_token_index[char]] = 1.
for t, char in enumerate(target_text):
# decoder_target_data is ahead of decoder_input_data by one timestep
decoder_input_data[i, t, target_token_index[char]] = 1.
if t > 0:
# decoder_target_data will be ahead by one timestep
# and will not include the start character.
decoder_target_data[i, t - 1, target_token_index[char]] = 1.
# Define an input sequence and process it.
encoder_inputs = Input(shape=(None, num_encoder_tokens))
#print(type(encoder_inputs))
encoder = LSTM(latent_dim, return_state=True)
#print(type(encoder))
encoder_outputs, state_h, state_c = encoder(encoder_inputs)
#print(type(encoder_outputs))
#encoder_outputs, state_h, state_c = LSTM(latent_dim, return_state=True)(encoder_inputs)
# We discard `encoder_outputs` and only keep the states.
encoder_states = [state_h, state_c]
#input()
#quit()
# Set up the decoder, using `encoder_states` as initial state.
decoder_inputs = Input(shape=(None, num_decoder_tokens))
# We set up our decoder to return full output sequences,
# and to return internal states as well. We don't use the
# return states in the training model, but we will use them in inference.
decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
initial_state = encoder_states)
decoder_dense = Dense(num_decoder_tokens, activation = 'softmax')
decoder_outputs = decoder_dense(decoder_outputs)
# Define the model that will turn
# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
# Run training
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
#del model # deletes the existing model
# returns a compiled model
# identical to the previous one
#model = load_model('./model_ferdosi/s2s_model_2.h5')
model.fit([encoder_input_data, decoder_input_data], decoder_target_data,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2)
# Save model (requires HDF5 and h5py)
dt = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
#model.save('./model_ferdosi/s2s_'+ dt + '_edit_ferdosi_ep' + str(epochs) + '.h5')
model.save('./model_ferdosi/s2s_model_2.h5')
# plot model
#plot_model(model, to_file = './modelpic/seq2seq_model_' + dt + '.png', show_shapes=True, show_layer_names=True)
# ********************************
# Next: inference mode (sampling).
# Here's the drill:
# 1) encode input and retrieve initial decoder state
# 2) run one step of decoder with this initial state
# and a "start of sequence" token as target.
# Output will be the next target token
# 3) Repeat with the current target token and current states
# Define sampling models
## encoder model
encoder_model = Model(encoder_inputs, encoder_states)
## decoder model
decoder_state_input_h = Input(shape=(latent_dim,))
decoder_state_input_c = Input(shape=(latent_dim,))
decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
decoder_outputs, state_h, state_c = decoder_lstm(
decoder_inputs, initial_state = decoder_states_inputs)
decoder_states = [state_h, state_c]
decoder_outputs = decoder_dense(decoder_outputs)
decoder_model = Model(
[decoder_inputs] + decoder_states_inputs,
[decoder_outputs] + decoder_states)
## plot sampling models
#plot_model(encoder_model, to_file = './modelpic/sampling_encoder_model_' + dt + '.png', show_shapes=True, show_layer_names=True)
#plot_model(decoder_model, to_file = './modelpic/sampling_decoder_model_' + dt + '.png', show_shapes=True, show_layer_names=True)
# Reverse-lookup token index to decode sequences back to
# something readable.
reverse_input_char_index = dict(
(i, char) for char, i in input_token_index.items())
reverse_target_char_index = dict(
(i, char) for char, i in target_token_index.items())
def decode_sequence(input_seq):
# Encode the input as state vectors.
states_value = encoder_model.predict(input_seq)
# Generate empty target sequence of length 1.
target_seq = np.zeros((1, 1, num_decoder_tokens))
# Populate the first character of target sequence with the start character.
target_seq[0, 0, target_token_index['\t']] = 1.
# Sampling loop for a batch of sequences
# (to simplify, here we assume a batch of size 1).
stop_condition = False
decoded_sentence = ''
while not stop_condition:
output_tokens, h, c = decoder_model.predict(
[target_seq] + states_value)
# Sample a token
sampled_token_index = np.argmax(output_tokens[0, -1, :])
sampled_char = reverse_target_char_index[sampled_token_index]
decoded_sentence += sampled_char
# Exit condition: either hit max length
# or find stop character.
if (sampled_char == '\n' or
len(decoded_sentence) > max_decoder_seq_length-1):
stop_condition = True
# Update the target sequence (of length 1).
target_seq = np.zeros((1, 1, num_decoder_tokens))
target_seq[0, 0, sampled_token_index] = 1.
# Update states
states_value = [h, c]
return decoded_sentence
index = randint(0,len(input_texts)-1)
input_seq = encoder_input_data[index: index+1]
decoded_sentence = decode_sequence(input_seq)
gen_poem = ''
#print('-----')
print('Input verse:', input_texts[index])
gen_poem += input_texts[0] + '\n'
#print('Next verses:', decoded_sentence)
gen_poem += decoded_sentence + '\n'
encoder_input_data_test = np.zeros(
(1, max_encoder_seq_length, num_encoder_tokens), dtype='float32')
for seq_index in range(64):
print('Input verse:', decoded_sentence)
decoded_sentence = decoded_sentence.replace('\n', '')
decoded_sentence = decoded_sentence.replace('\t', '')
print('Input verse:', decoded_sentence)
for t, char in enumerate(decoded_sentence):
encoder_input_data_test[0, t, input_token_index[char]] = 1.
input_seq = encoder_input_data_test[0: 1]
decoded_sentence = decode_sequence(input_seq)
print('-----')
print('Next verses:', decoded_sentence)
gen_poem += decoded_sentence + '\n'
new_poem = './result_ferdosi/new_poem_ferdosi' + dt + '_ep' + str(epochs) + '.txt'
with open(new_poem, 'w', encoding='utf8') as nhp:
nhp.write(str(gen_poem))