-
Notifications
You must be signed in to change notification settings - Fork 0
/
train_model.py
77 lines (59 loc) · 2.31 KB
/
train_model.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
import numpy
import sys
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM
from keras.utils import np_utils
from keras.callbacks import ModelCheckpoint
file = open("victorhugo.txt").read()
file = file.replace('\n \n', '\n')
file = file.replace(' ', ' ')
def tokenize_words(input):
# lowercase everything to standardize it
# input = input.lower()
# instantiate the tokenizer
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(input)
# if the created token isn't in the stop words, make it part of "filtered"
# filtered = filter(lambda token: token not in stopwords.words('french'), tokens)
return " ".join(tokens)
# preprocess the input data, make tokens
processed_inputs = tokenize_words(file)
chars = sorted(list(set(processed_inputs)))
char_to_num = dict((c, i) for i, c in enumerate(chars))
input_len = len(processed_inputs)
vocab_len = len(chars)
print ("Total number of characters:", input_len)
print ("Total vocab:", vocab_len)
seq_length = 100
x_data = []
y_data = []
# loop through inputs, start at the beginning and go until we hit
# the final character we can create a sequence out of
for i in range(0, input_len - seq_length, 1):
# Define input and output sequences
# Input is the current character plus desired sequence length
in_seq = processed_inputs[i:i + seq_length]
# Out sequence is the initial character plus total sequence length
out_seq = processed_inputs[i + seq_length]
# We now convert list of characters to integers based on
# previously and add the values to our lists
x_data.append([char_to_num[char] for char in in_seq])
y_data.append(char_to_num[out_seq])
n_patterns = len(x_data)
print ("Total Patterns:", n_patterns)
X = numpy.reshape(x_data, (n_patterns, seq_length, 1))
X = X/float(vocab_len)
y = np_utils.to_categorical(y_data)
model = Sequential()
model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(128))
model.add(Dropout(0.2))
# model.add(LSTM(128))
# model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.fit(X, y, epochs=4, batch_size=32)
model.save('model')