test: Evaluate all experiments

Author: jonnor

TODO.md

@@ -22,7 +22,6 @@ Report
 Experiments
 
 - Output STM32 stats to file
-- Write tool for running all experiments
 - Write uniform Keras model based on SB-CNN etc
 - Determine filter settings to make MACCs approx equal
 - Run each model on device, record inference time

microesc/report.py

@@ -6,7 +6,8 @@
 import seaborn as sns
 import matplotlib.pyplot as plt     
 
-from . import common, urbansound8k
+import common, urbansound8k
+#from . import common, urbansound8k
 
 groups = {
     'social_activity': [ 'street_music', 'children_playing', 'dog_bark' ],
@@ -78,6 +79,9 @@ def parse(args):
 
     common.add_arguments(parser)
 
+    a('--run', dest='run', default='',
+        help='%(default)s')
+
     a('--out', dest='results_dir', default='./data/results',
         help='%(default)s')
 
@@ -98,7 +102,7 @@ def main():
 
     args = parse(None)
 
-    cm = numpy.load(os.path.join(args.results_dir, '{}'.format(args.experiment), 'confusion.npz'))
+    cm = numpy.load(os.path.join(args.results_dir, args.run, '{}.confusion.npz'.format(1)))
     val, test = cm['val'], cm['test']
 
 
@@ -108,15 +112,14 @@ def main():
     val_fig.savefig('val.cm.png')
     test_fig.savefig('test.cm.png')
 
-    c_acc = cm_class_accuracy(numpy.mean(val, axis=0))
-    print_accuracies(c_acc, 'class_acc')
+    tests_acc = [ cm_accuracy(test[f]) for f in range(0, len(test)) ]
+    print_accuracies(tests_acc, 'test_acc') 
 
     folds_acc = [ cm_accuracy(val[f]) for f in range(0, len(val)) ]
     print_accuracies(folds_acc, 'val_acc')
 
-    tests_acc = [ cm_accuracy(test[f]) for f in range(0, len(test)) ]
-    print_accuracies(tests_acc, 'test_acc') 
-
+    c_acc = cm_class_accuracy(numpy.mean(val, axis=0))
+    print_accuracies(c_acc, 'class_acc')
 
     print('wrote')
 

microesc/test.py

@@ -13,14 +13,14 @@
 
 
 def load_model_info(jobs_dir, job_dir):
-    template, date, time, rnd, fold = job_dir.split('-')
+    experiment, date, time, rnd, fold = job_dir.split('-')
     hist_path = os.path.join(jobs_dir, job_dir, 'train.csv')
 
     df = pandas.read_csv(hist_path)
 
     df['epoch'] = df.epoch + 1
     df['fold'] = int(fold[-1])
-    df['template'] = template
+    df['experiment'] = experiment
     df['run'] = '-'.join([date, time, rnd])
 
     models = []
@@ -34,7 +34,7 @@ def load_model_info(jobs_dir, job_dir):
     expected_last = 'e{:02d}-'.format(len(models))
     assert last_model.startswith(expected_last), (last_model, expected_last)
 
-    df['model'] = [ os.path.join(jobs_dir, job_dir, m) for m in models ]
+    df['model_path'] = [ os.path.join(jobs_dir, job_dir, m) for m in models ]
     return df
 
 def load_train_history(jobs_dir, limit=None):
@@ -44,7 +44,6 @@ def load_train_history(jobs_dir, limit=None):
         matching = [ d for d in jobs if limit in d ]
     else:
         matching = jobs
-    #assert len(matching) == 9, "Expected 9 folds, found {} matching {}".format(len(matching), job_id)
 
     dataframes = []
 
@@ -72,51 +71,67 @@ def pick_best(history, n_best=1):
 
     def best_by_loss(df):
         return df.sort_values('voted_val_acc', ascending=False).head(n_best)
-    return history.groupby('fold').apply(best_by_loss)
+    return history.groupby(['experiment', 'fold']).apply(best_by_loss)
 
-
-def evaluate(models, folds, test, predictor):
+def evaluate_model(predictor, model_path, val_data, test_data):
 
     def score(model, data):
         y_true = data.classID
         p = predictor(model, data)
         y_pred = numpy.argmax(p, axis=1)
         # other metrics can be derived from confusion matrix
         acc = sklearn.metrics.accuracy_score(y_true, y_pred)
-        print('acc', acc)
         labels = list(range(len(urbansound8k.classnames)))
         confusion = sklearn.metrics.confusion_matrix(y_true, y_pred, labels=labels)
-        return confusion
+        return acc, confusion
 
-    # validation
-    out = {
-        'val_foreground': [],
-        'val_background': [],
-        'test_foreground': [],
-        'test_background': [],
-    }
+    model = keras.models.load_model(model_path)
 
     salience_info = { 'foreground': 1, 'background': 2 }
-
-    # val
-    for i, m in enumerate(models):
-        data = folds[i][1]
+    test_info = { 'val': val_data, 'test': test_data }
+    out = {}
+    for setname, data in test_info.items():
         for variant, salience in salience_info.items():   
-            s = score(m, data[data.salience == salience])
-            out['val_'+variant].append(s)
-
-    # test
-    for i, m in enumerate(models):
-        data = test
-        for variant, salience in salience_info.items():   
-            s = score(m, data[data.salience == salience])
-            out['test_'+variant].append(s)
-     
-    for k, v in out.items():
-        out[k] = numpy.stack(v)
+            key = '{}_{}'.format(setname, variant)
+            acc, confusion = score(model, data[data.salience == salience])
+            print('acc for ', key, acc) 
+            out[key] = confusion
 
     out['val'] = out['val_foreground'] + out['val_background']
     out['test'] = out['test_foreground'] + out['test_background']
+    return out
+
+def evaluate(models, folds, testset, predictor, out_dir, dry_run=False):
+
+    def eval_experiment(df):
+        results = {}
+        by_fold = df.sort_index(level="fold", ascending=True)
+
+        for idx, row in by_fold.iterrows():
+            print('Testing model {} fold={}'.format(row['experiment'], row['fold']))
+
+            model_path = row['model_path']
+            val = folds[row['fold']][1]
+            test = testset
+            if dry_run:
+                val = test[0:20]
+                test = test[0:20]
+            
+            result = evaluate_model(predictor, model_path, val, test)
+
+            # convert to dict-of-arrays
+            for k, v in result.items():
+                if results.get(k) is None:
+                    results[k] = []
+                results[k].append(v)
+
+        exname = df['experiment'].unique()[0]
+        results_path = os.path.join(out_dir, '{}.confusion.npz'.format(exname))
+        numpy.savez(results_path, **results)
+        print('Wrote', results_path)
+        return results_path
+
+    out = models.groupby(level='experiment').apply(eval_experiment)
 
     return out
 
@@ -132,8 +147,8 @@ def parse(args):
 
     a('--run', dest='run', default='',
         help='%(default)s')
-    a('--model', dest='model', default='',
-        help='%(default)s')
+    a('--check', action='store_true', default='',
+        help='Run a check pass, not actually evaluating')
 
     a('--out', dest='results_dir', default='./data/results',
         help='%(default)s')
@@ -146,10 +161,7 @@ def parse(args):
 def main():
 
     args = parse(sys.argv[1:])
-    if not args.run:
-        args.run = args.experiment
-
-    out_dir = os.path.join(args.results_dir, args.experiment)
+    out_dir = os.path.join(args.results_dir, args.run)
 
     common.ensure_directories(out_dir)
 
@@ -180,19 +192,16 @@ def predict(model, data):
                                         method=voting, overlap=overlap)
 
     history = load_train_history(args.models_dir, args.run)
-    best = pick_best(history)
+    n_folds = len(history.fold.unique())
+    n_experiments = len(history.experiment.unique())
+    print("Found {} experiments across {} folds", n_folds, n_experiments)
 
-    print('Loading models...')
-    models = best['model'].apply(lambda p: keras.models.load_model(p))
-    print('Best model', best.voted_val_acc)
+    best = pick_best(history)
+    print('Best models\n', best[['epoch', 'voted_val_acc']])
 
     print('Testing models...')
-    results = evaluate(models, folds, test, predictor=predict)
-
-    results_path = os.path.join(out_dir, 'confusion.npz')
-    numpy.savez(results_path, **results)
+    results = evaluate(best, folds, test, predictor=predict, out_dir=out_dir, dry_run=args.check)
 
-    print('Wrote to', results_path)
 
 if __name__ == '__main__':
     main()