def lstm_basic_api (xtrain, xtest, ytrain, ytest, nodesx, acty_first, lrx, name_pkl): ''' LSTM implemented with attention and the original architecture''' dim1 = xtrain.shape[1] dim2 = xtrain.shape[2] batch_size = 32 #Imput dimensions of the model inputs1 = Input(shape=(dim1, dim2)) ##Create the mask model_input= Masking(name='mask')(inputs1) #LSTM bidirectional lstm_l = Bidirectional(LSTM(nodesx, activation=acty_first, return_sequences=True, name= 'lstm_1'))(model_input) ##Create attention layer att = Attention( name='attention_weight')(lstm_l) ##create output layer dense_end = Dense(units=1, activation='sigmoid') (att) ##create the model model = Model(inputs=inputs1, outputs=dense_end) print(model.summary()) #plot_model(model, to_file='multilayer_perceptron_graph.png') model.compile(optimizer=Adam(lrx), loss = 'mse', metrics=['accuracy', mcc_metric]) output_dir = 'task_add_two_numbers' if not os.path.exists(output_dir): os.makedirs(output_dir) ##check point #filepath="weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5" #checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') #callbacks_list = [checkpoint,history] max_epoch = 100 filepath="weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') #checkpoint = ModelCheckpoint("best_model.hdf5", monitor='loss', verbose=1, save_best_only=True, mode='auto', period=1) class VisualiseAttentionMap(Callback): def on_epoch_end(self, epoch, logs=None): attention_map = get_activations(model, xtrain, layer_names='attention_weight')['attention_weight'] # top is attention map. # bottom is ground truth. plt.imshow(attention_map, cmap='coolwarm') plt.colorbar() iteration_no = str(epoch).zfill(3) #plt.axis('off') plt.title(f'Iteration {iteration_no} / {max_epoch}') plt.xlabel('Windows') plt.ylabel('Sample') plt.savefig(f'{output_dir}/epoch_{iteration_no}.png') plt.close() plt.clf() np.save(f'epoch_{iteration_no}',attention_map) #modelh5 = load_model('test_model.h5', custom_objects={'Attention': Attention, 'mcc_metric': mcc_metric}) fit_outcome = model.fit(xtrain, ytrain, batch_size=batch_size, epochs=100, shuffle=True, validation_data=(xtest, ytest), callbacks=[VisualiseAttentionMap(), checkpoint]) model.save('lstm_model.h5') evaluate_outcome = model.evaluate(xtest, ytest, batch_size=batch_size), fit_history = fit_outcome.history y_pred = model.predict(xtest) y_pred_c = (y_pred > 0.5).astype(int) y_trainP = model.predict(xtrain) y_trainP_C =(y_trainP > 0.5).astype(int) ##Save the history to a file pkl_creation = create_pickle(name_pkl, fit_history) ##Make sure that the MCC was correctly calculated mcc_sk = matthews_corrcoef(ytest, y_pred_c) mcc_skT = matthews_corrcoef(ytrain, y_trainP_C) #mcc_val= fit_history.get('val_accuracy') #mcc_last = (acc_val[-1]) #print ("MCC value calculated:", mcc_last, 'MCC Sklearn:', mcc_sk) #print ("MCC value calculated:", mse_last, 'MSE Sklearn:', mse_sk) model.reset_states() return fit_history, evaluate_outcome, y_pred, y_pred_c, mcc_sk, pkl_creation, mcc_skT