train: Compute voted accuracy

Author: jonnor

microesc/features.py

@@ -5,6 +5,7 @@
 import os.path
 import urllib.request
 import zipfile
+import collections
 
 import pandas
 import numpy
@@ -166,3 +167,52 @@ def load_sample(sample, settings, feature_dir, window_frames,
     # add channel dimension
     data = numpy.expand_dims(padded, -1)
     return data
+
+
+Sample = collections.namedtuple('Sample', 'start end fold slice_file_name')
+
+def load_windows(sample, settings, loader, overlap):
+    sample_rate = settings['samplerate']
+    frame_samples = settings['hop_length']
+    window_frames = settings['frames']
+
+    windows = []
+
+    duration = sample.end - sample.start
+    length = int(sample_rate * duration)
+
+    for win in sample_windows(length, frame_samples, window_frames, overlap=overlap):
+        chunk = Sample(start=win[0]/sample_rate,
+                       end=win[1]/sample_rate,
+                       fold=sample.fold,
+                       slice_file_name=sample.slice_file_name)    
+        d = loader(chunk)
+        windows.append(d)
+
+    return windows
+
+def predict_voted(settings, model, samples, loader, method='mean', overlap=0.5):
+
+    out = []
+    for _, sample in samples.iterrows():
+        windows = load_windows(sample, settings, loader, overlap=overlap)
+        inputs = numpy.stack(windows)
+
+        predictions = model.predict(inputs)
+        if method == 'mean':
+            p = numpy.mean(predictions, axis=0)
+            assert len(p) == 10
+            out.append(p)
+        elif method == 'majority':
+            votes = numpy.argmax(predictions, axis=1)
+            p = numpy.bincount(votes, minlength=10) / len(votes)
+            out.append(p)
+
+    ret = numpy.stack(out)
+    assert len(ret.shape) == 2, ret.shape
+    assert ret.shape[0] == len(out), ret.shape
+    assert ret.shape[1] == 10, ret.shape # classes
+
+    return ret
+
+

microesc/test.py

@@ -2,7 +2,6 @@
 import math
 import os.path
 import sys
-import collections
 
 import keras
 import sklearn
@@ -12,50 +11,6 @@
 
 from . import urbansound8k, features, common
 
-Sample = collections.namedtuple('Sample', 'start end fold slice_file_name')
-
-def load_windows(sample, settings, loader, window_frames, overlap):
-    sample_rate = settings['samplerate']
-    frame_samples = settings['hop_length']
-
-    windows = []
-
-    duration = sample.end - sample.start
-    length = int(sample_rate * duration)
-
-    for win in features.sample_windows(length, frame_samples, window_frames, overlap=overlap):
-        chunk = Sample(start=win[0]/sample_rate,
-                       end=win[1]/sample_rate,
-                       fold=sample.fold,
-                       slice_file_name=sample.slice_file_name)    
-        d = loader(chunk)
-        windows.append(d)
-
-    return windows
-
-def predict_voted(settings, model, samples, loader, window_frames, method='mean', overlap=0.5):
-
-    out = []
-    for _, sample in samples.iterrows():
-        windows = load_windows(sample, settings, loader, window_frames, overlap=overlap)
-        inputs = numpy.stack(windows)
-
-        predictions = model.predict(inputs)
-        if method == 'mean':
-            p = numpy.mean(predictions, axis=0)
-            assert len(p) == 10
-            out.append(p)
-        elif method == 'majority':
-            votes = numpy.argmax(predictions, axis=1)
-            p = numpy.bincount(votes, minlength=10) / len(votes)
-            out.append(p)
-
-    ret = numpy.stack(out)
-    assert len(ret.shape) == 2, ret.shape
-    assert ret.shape[0] == len(out), ret.shape
-    assert ret.shape[1] == 10, ret.shape # classes
-
-    return ret
 
 def load_history(jobs_dir, job_id):
 
@@ -133,28 +88,6 @@ def score(model, data):
     return val_scores, test_scores
 
 
-def test_predict_windowed():
-
-    from sklearn.metrics import accuracy_score
-    t = test[0:10]
-
-    sbcnn16k32_settings = dict(
-        feature='mels',
-        samplerate=16000,
-        n_mels=32,
-        fmin=0,
-        fmax=8000,
-        n_fft=512,
-        hop_length=256,
-        augmentations=5,
-    )
-
-    def load_sample32(sample):
-        return features.load_sample(sample, sbcnn16k32_settings, window_frames=72, feature_dir='../../scratch/aug')
-
-    mean_m = predict_windowed(sbcnn16k32_settings, model, t, loader=load_sample32, method='mean')
-    accuracy_score(t.classID, mean_m)
-
 def parse(args):
 
     import argparse
@@ -190,11 +123,11 @@ def main():
     urbansound8k.maybe_download_dataset(args.datasets_dir)
     data = urbansound8k.load_dataset()
     folds, test = urbansound8k.folds(data)
-    settings = common.load_experiment(args.experiments_dir, args.experiment)
-    frames = settings['frames']
-    voting = settings['voting']
-    overlap = settings['voting_overlap']
-    settings = features.settings(settings)
+    exsettings = common.load_experiment(args.experiments_dir, args.experiment)
+    frames = exsettings['frames']
+    voting = exsettings['voting']
+    overlap = exsettings['voting_overlap']
+    settings = features.settings(exsettings)
 
 
     all_folds = pandas.concat([f[0] for f in folds])
@@ -210,8 +143,8 @@ def load_sample(sample):
                     window_frames=frames, feature_dir=args.features_dir)
 
     def predict(model, data):
-        return predict_voted(settings, model, data, loader=load_sample,
-                             window_frames=frames, method=voting, overlap=overlap)
+        return features.predict_voted(exsettings, model, data, loader=load_sample,
+                                        method=voting, overlap=overlap)
 
     if args.model:
         best = pandas.DataFrame({ 'model': [  args.model ] * 9})

microesc/train.py

@@ -7,11 +7,13 @@
 import json
 import functools
 import datetime
+import csv
 
 import pandas
 import numpy
 import keras
 import librosa
+import sklearn.metrics
 
 from . import features, urbansound8k, common
 from .models import sbcnn
@@ -38,28 +40,90 @@ def dataframe_generator(X, Y, loader, batchsize=10, n_classes=10):
         yield batch
 
 
+class LogCallback(keras.callbacks.Callback):
+    def __init__(self, log_path, score_epoch):
+        super().__init__()
+    
+        self.log_path = log_path
+        self.score = score_epoch   
+
+        self._log_file = None
+        self._csv_writer = None
+
+    def __del__(self):
+        if self._log_file:
+            self._log_file.close()
+       
+
+    def write_entry(self, epoch, data):
+        data = data.copy()
+
+        if not self._csv_writer:
+            # create writer when we know what fields
+            self._log_file = open(self.log_path, 'w')
+            fields = ['epoch'] + sorted(data.keys())
+            self._csv_writer = csv.DictWriter(self._log_file, fields)
+            self._csv_writer.writeheader()
+        
+        data['epoch'] = epoch
+        self._csv_writer.writerow(data)
+        self._log_file.flush() # ensure data hits disk
+
+    def on_epoch_end(self, epoch, logs):
+        logs = logs.copy()
+    
+        more = self.score() # uses current model
+        for k, v in more.items():
+            logs[k] = v
+
+        self.write_entry(epoch, logs)
+
+
+
+
 def train_model(out_dir, fold, builder,
-                loader, val_loader,
-                frame_samples, window_frames,
-                train_samples=12000, val_samples=3000,
-                batch_size=200, epochs=50, seed=1, learning_rate=3e-4):
+                loader, val_loader, settings, seed=1):
     """Train a single model"""    
 
+    frame_samples = settings['hop_length']
+    train_samples = settings['train_samples']
+    window_frames = settings['frames']
+    val_samples = settings['val_samples']
+    epochs = settings['epochs']
+    batch_size = settings['batch']
+    #learning_rate = settings['learning_rate']   
+
+    train, val = fold
+
+    def top3(y_true, y_pred):
+        return keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
+
     model = builder()
     model.compile(loss='categorical_crossentropy',
                   optimizer=keras.optimizers.SGD(lr=0.001, momentum=0.95, nesterov=True),
                   metrics=['accuracy'])
 
-
     model_path = os.path.join(out_dir, 'e{epoch:02d}-v{val_loss:.2f}.t{loss:.2f}.model.hdf5')
     checkpoint = keras.callbacks.ModelCheckpoint(model_path, monitor='val_acc', mode='max',
                                          period=1, verbose=1, save_best_only=False)
-    callbacks_list = [checkpoint]
 
-    train, val = fold
+    def voted_score():
+        y_pred = features.predict_voted(settings, model, val,
+                                loader=val_loader, method='mean', overlap=0.5)
+        class_pred = numpy.argmax(y_pred, axis=1)
+        acc = sklearn.metrics.accuracy_score(val.classID, class_pred)
+        d = {
+            'voted_val_acc': acc,
+        }
+        return d
+    log_path = os.path.join(out_dir, 'train.csv')
+    log = LogCallback(log_path, voted_score)
+
+
     train_gen = dataframe_generator(train, train.classID, loader=loader, batchsize=batch_size)
     val_gen = dataframe_generator(val, val.classID, loader=val_loader, batchsize=batch_size)
 
+    callbacks_list = [checkpoint, log]
     hist = model.fit_generator(train_gen, validation_data=val_gen,
                         steps_per_epoch=math.ceil(train_samples/batch_size),
                         validation_steps=math.ceil(val_samples/batch_size),
@@ -147,7 +211,6 @@ def settings(args):
     train_settings = {}
     for k in default_training_settings.keys():
         v = args.get(k, default_training_settings[k])
-        print('v', k, v, args.get(k))
         train_settings[k] = v
     return train_settings
 
@@ -217,12 +280,7 @@ def build_model():
                       builder=build_model,
                       loader=functools.partial(load, validation=False),
                       val_loader=functools.partial(load, validation=True),
-                      frame_samples=feature_settings['hop_length'],
-                      window_frames=model_settings['frames'],
-                      epochs=train_settings['epochs'],
-                      train_samples=train_settings['train_samples'],
-                      val_samples=train_settings['val_samples'],
-                      batch_size=train_settings['batch'])
+                      settings=exsettings)
 
 
 

test/test_training.py

@@ -56,3 +56,26 @@ def test_window_typical():
     w = list(features.sample_windows(int(length), frame_samples, window_frames))
     assert len(w) == 8, len(w) 
     assert w[-1][1] == length
+
+
+def _test_predict_windowed():
+
+    t = test[0:10]
+
+    sbcnn16k32_settings = dict(
+        feature='mels',
+        samplerate=16000,
+        n_mels=32,
+        fmin=0,
+        fmax=8000,
+        n_fft=512,
+        hop_length=256,
+        augmentations=5,
+    )
+
+    def load_sample32(sample):
+        return features.load_sample(sample, sbcnn16k32_settings, window_frames=72, feature_dir='../../scratch/aug')
+
+    mean_m = features.predict_voted(sbcnn16k32_settings, model, t, loader=load_sample32, method='mean')
+
+