nan loss correction by reducing learning rate

materialsvirtuallab · Apr 29, 2019 · 34f64a9 · 34f64a9
1 parent b9fe36e
commit 34f64a9
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/megnet/models.py b/megnet/models.py
@@ -5,7 +5,7 @@
 from megnet.losses import mse_scale
 from keras.regularizers import l2
 from keras.models import Model
-from megnet.callbacks import ModelCheckpointMAE, ManualStop
+from megnet.callbacks import ModelCheckpointMAE, ManualStop, ReduceLRUponNan
 from megnet.data.graph import GraphBatchDistanceConvert, GraphBatchGenerator, GaussianDistance
 from megnet.data.crystal import CrystalGraph
 import numpy as np
@@ -106,6 +106,7 @@ def train_from_graphs(self,
         if callbacks is None:
             # with this call back you can stop the model training by `touch STOP`
             callbacks = [ManualStop()]
+        callbacks.append(ReduceLRUponNan())
         train_targets = np.array(train_targets).ravel()
         if validation_graphs is not None:
             filepath = pjoin(dirname, 'val_mae_{epoch:05d}_{%s:.6f}.hdf5' % monitor)