misc

Author: GreenWizard2015

NN/Utils.py

@@ -27,9 +27,11 @@ def call(self, x):
     coefs = tf.pow(self._base, powers)
     return tf.reduce_sum(x * coefs, axis=-1)
 ############################################
+SMLP_GLOBAL_DROPOUT = 0.01
 class sMLP(tf.keras.layers.Layer):
-  def __init__(self, sizes, activation='linear', dropout=0.01, **kwargs):
+  def __init__(self, sizes, activation='linear', dropout=None, **kwargs):
     super().__init__(**kwargs)
+    dropout = SMLP_GLOBAL_DROPOUT if dropout is None else dropout
     layers = []
     for i, sz in enumerate(sizes):
       if 0.0 < dropout:

app.py

@@ -342,9 +342,9 @@ def _modelFromArgs(args):
     stats = json.load(f)
 
   # My own ids hardcoded here for simplicity
-  userId = '98fdb9d9-14ef-9276-31e6-836e830acc19'
-  placeId = 'ce42c1a9-f4ef-42d6-a219-cf25fad912ed'
-  screenId = 'ce42c1a9-f4ef-42d6-a219-cf25fad912ed/29f35417-7bb7-3c94-124c-2ae16bda235d'
+  userId = 'ce42c1a9-f4ef-42d6-a219-cf25fad912ed'
+  placeId = '29ecaa6a-d3b5-784b-887e-f50a0c6533fa'
+  screenId = placeId + '/' + '29f35417-7bb7-3c94-124c-2ae16bda235d'
   return CModelWrapper(
     timesteps=args.steps,  
     user=dict(
@@ -353,7 +353,7 @@ def _modelFromArgs(args):
       screenId=screenId,
     ),
     stats=stats,
-    weights=dict(folder=args.folder, postfix=args.model)
+    weights=dict(folder=args.folder, postfix=args.model, embeddings=True)
   )
 
 def _predictorFromArgs(args):

scripts/train.py

@@ -73,25 +73,27 @@ def _eval(dataset, model, plotFilename, args):
 
 def evaluator(datasets, model, folder, args):
   losses = [np.inf] * len(datasets) # initialize with infinity
-  def evaluate():
+  def evaluate(onlyImproved=False):
     totalLoss = totalDist = 0.0
     for i, dataset in enumerate(datasets):
       loss, dist, T = _eval(dataset, model, os.path.join(folder, 'pred-%d.png' % i), args)
-      print('Test %d / %d | %.2f sec | Loss: %.5f (%.5f). Distance: %.5f' % (
-        i + 1, len(datasets), T, loss, losses[i], dist
-      ))
+      if not onlyImproved:
+        print('Test %d / %d | %.2f sec | Loss: %.5f (%.5f). Distance: %.5f' % (
+          i + 1, len(datasets), T, loss, losses[i], dist
+        ))
       if loss < losses[i]:
-        print('Improved %.5f => %.5f' % (losses[i], loss))
+        print('Test %d / %d | Improved %.5f => %.5f' % (i + 1, len(datasets), losses[i], loss))
         model.save(folder, postfix='best-%d' % i) # save the model separately
         losses[i] = loss
         pass
 
       totalLoss += loss
       totalDist += dist
       continue
-    print('Mean loss: %.5f | Mean distance: %.5f' % (
-      totalLoss / len(datasets), totalDist / len(datasets)
-    ))
+    if not onlyImproved:
+      print('Mean loss: %.5f | Mean distance: %.5f' % (
+        totalLoss / len(datasets), totalDist / len(datasets)
+      ))
     return totalLoss / len(datasets)
   return evaluate
 
@@ -268,10 +270,19 @@ def main(args):
         break
       if 'reset' == args.on_patience:
         print('Resetting the model to the average of the best models')
-        # and add some noise
-        averageModels(folder, model, noiseStd=0.01)
-        bestEpoch = epoch
-        continue
+        bestEpoch = epoch # reset the best epoch
+        for _ in range(args.restarts):
+          # and add some noise
+          averageModels(folder, model, noiseStd=args.noise)
+          # re-evaluate the model with the new weights
+          testLoss = eval(onlyImproved=True)
+          if testLoss < bestLoss:
+            print('Improved %.5f => %.5f' % (bestLoss, testLoss))
+            bestLoss = testLoss
+            bestEpoch = epoch
+            model.save(folder, postfix='best')
+            continue
+          continue
     continue
   return
 
@@ -299,6 +310,11 @@ def main(args):
     help='JSON file with the scheduler parameters for sampling the training dataset'
   )
   parser.add_argument('--debug', action='store_true')
+  parser.add_argument('--noise', type=float, default=1e-4)
+  parser.add_argument(
+    '--restarts', type=int, default=1,
+    help='Number of times to restart the model reinitializing the weights'
+  )
 
   main(parser.parse_args())
   pass