Include all folds in cross-validation

There is no dedicated testset in recommended procedure

Author: jonnor

microesc/jobs.py

@@ -138,7 +138,7 @@ def parse(args):
         help='Only run a pre-flight check')
     a('--jobs', type=int, default=5,
         help='Number of parallel jobs')
-    a('--folds', type=int, default=9,
+    a('--folds', type=int, default=10,
         help='Number of folds to test')
 
     a('--start', type=int, default=0,
@@ -160,7 +160,8 @@ def main():
     experiments = experiments.loc[range(args.start, stop)]
 
     overrides = {}
-    folds = list(range(0, args.folds))
+    folds = list(range(1, args.folds+1))
+    assert max(folds) <= 10
     if args.check:
         batches = 2
         overrides['batch'] = 10

microesc/test.py

@@ -106,18 +106,35 @@ def score(model, data):
     out['test'] = out['test_foreground'] + out['test_background']
     return out
 
-def evaluate(models, folds, testset, predictor, out_dir, dry_run=False):
+def evaluate(models, folds_data, predictor, out_dir, dry_run=False):
 
     def eval_experiment(df):
         results = {}
         by_fold = df.sort_index(level="fold", ascending=True)
 
         for idx, row in by_fold.iterrows():
-            print('Testing model {} fold={}'.format(row['experiment'], row['fold']))
-
+            fold = row['fold']
+            assert fold > 0, 'fold number should be 1 indexed'
+            print('Testing model {} fold={}'.format(row['experiment'], fold))
+            
             model_path = row['model_path']
-            val = folds[row['fold']][1]
-            test = testset
+            val = folds_data[fold-1][1]
+            test = folds_data[fold-1][2]
+            test_folds = test.fold.unique()
+            assert len(test_folds) == 1 
+            assert test_folds[0] == fold
+            val_folds = val.fold.unique()
+            assert len(val_folds) == 1 
+            assert val_folds[0] != fold 
+
+            train_data = folds_data[fold-1][0]
+            train_files = set(train_data.slice_file_name.unique())
+            assert len(train_files) > 7000, len(train_files)
+            test_files = set(test.slice_file_name.unique())
+            assert len(test_files) > 700
+            common_files = train_files.intersection(test_files)
+            assert len(common_files) == 0, common_files
+
             if dry_run:
                 val = test[0:20]
                 test = test[0:20]
@@ -173,22 +190,14 @@ def main():
 
     urbansound8k.maybe_download_dataset(args.datasets_dir)
     data = urbansound8k.load_dataset()
-    folds, test = urbansound8k.folds(data)
+    folds = urbansound8k.folds(data)
     exsettings = common.load_settings_path(args.settings_path)
     frames = exsettings['frames']
     voting = exsettings['voting']
     overlap = exsettings['voting_overlap']
     settings = features.settings(exsettings)
 
 
-    all_folds = pandas.concat([f[0] for f in folds])
-    train_files = set(all_folds.slice_file_name.unique())
-    test_files = set(test.slice_file_name.unique())
-    assert len(train_files) > 7000
-    assert len(test_files) > 700
-    common_files = train_files.intersection(test_files)
-    assert len(common_files) == 0
-
     def load_sample(sample):
         return features.load_sample(sample, settings, start_time=sample.start,
                     window_frames=frames, feature_dir=args.features_dir)
@@ -219,7 +228,7 @@ def get_stats(row):
     model_stats.to_csv(os.path.join(out_dir, 'stm32stats.csv'))
 
     print('Testing models...')
-    results = evaluate(best, folds, test, predictor=predict, out_dir=out_dir, dry_run=args.check)
+    results = evaluate(best, folds, predictor=predict, out_dir=out_dir, dry_run=args.check)
 
 
 if __name__ == '__main__':

microesc/train.py

@@ -81,7 +81,7 @@ def on_epoch_end(self, epoch, logs):
 
 
 
-def train_model(out_dir, fold, builder,
+def train_model(out_dir, train, val, builder,
                 loader, val_loader, settings, seed=1):
     """Train a single model"""    
 
@@ -93,7 +93,7 @@ def train_model(out_dir, fold, builder,
     batch_size = settings['batch']
     learning_rate = settings.get('learning_rate', 0.01)
 
-    train, val = fold
+    assert len(train) > len(val) * 5, 'training data should be much larger than validation'
 
     def top3(y_true, y_pred):
         return keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
@@ -159,7 +159,7 @@ def parse(args):
     common.add_arguments(parser)
     Settings.add_arguments(parser)
 
-    a('--fold', type=int, default=0,
+    a('--fold', type=int, default=1,
         help='')
     a('--skip_model_check', action='store_true', default=False,
         help='Skip checking whether model fits on STM32 device')
@@ -182,6 +182,17 @@ def setup_keras():
     sess = tf.Session(config=session_config)
     B.set_session(sess)
 
+def load_training_data(data, fold):
+    assert fold >= 1 # should be 1 indexed
+    folds = urbansound8k.folds(data)
+    assert len(folds) == 10
+    train_data = folds[fold-1][0]
+    val_data = folds[fold-1][1]
+    test_folds = folds[fold-1][2].fold.unique()
+    assert len(test_folds) == 1
+    assert test_folds[0] == fold, (test_folds[0], '!=', fold) # by convention, test fold is fold number
+    return train_data, val_data
+
 def main():
     setup_keras()
 
@@ -216,10 +227,8 @@ def main():
 
     features.maybe_download(feature_settings, feature_dir)
 
-
     data = urbansound8k.load_dataset()
-    folds, test = urbansound8k.folds(data)
-    assert len(folds) == 9
+    train_data, val_data = load_training_data(data, fold)
 
     def load(sample, validation):
         augment = not validation and train_settings['augment'] != 0
@@ -245,7 +254,7 @@ def build_model():
     print('Training model', name)
     print('Settings', json.dumps(exsettings))
 
-    h = train_model(output_dir, folds[fold],
+    h = train_model(output_dir, train_data, val_data,
                       builder=build_model,
                       loader=functools.partial(load, validation=False),
                       val_loader=functools.partial(load, validation=True),

microesc/urbansound8k.py

@@ -3,6 +3,7 @@
 import urllib.request
 import tarfile
 
+import numpy
 import pandas
 
 here = os.path.dirname(__file__)
@@ -74,17 +75,70 @@ def sample_path(sample, dataset_path = None):
     return os.path.join(dataset_path, 'audio', 'fold'+str(sample.fold), sample.slice_file_name)
 
 
-# Use fold=10 for testing, as recommended by Urbansound8k dataset authors
+# Split the 10 folds into training, testing, and
 def folds(data):
-    test_fold = 10
-    train = data[data.fold != test_fold]
-    test = data[data.fold == test_fold]
+    fold_idxs = folds_idx(n_folds=10) 
+    assert len(fold_idxs) == 10
 
     folds = []
-    for fold in range(1, 10):
-        assert fold != test_fold
-        fold_train = train[train.fold != fold]
-        fold_val = train[train.fold == fold]
-        folds.append((fold_train, fold_val))
+    for fold in fold_idxs:
+        train, val, test = fold
+
+        # our folds are 1-indexed instead of 0...
+        train = numpy.array(train) + 1
+        val = numpy.array(val) + 1
+        test = numpy.array(test) + 1
+        fold_train = data[data.fold.isin(train)]
+        fold_val = data[data.fold.isin(val)]
+        fold_test = data[data.fold.isin(test)]
+
+        # post-condition
+        train_folds = set(fold_train.fold.unique())
+        val_folds = set(fold_val.fold.unique())
+        test_folds = set(fold_test.fold.unique())
+        assert len(train_folds) == 8, len(train_folds)
+        assert train_folds.intersection(val_folds) == set()
+        assert train_folds.intersection(test_folds) == set()
+        assert val_folds.intersection(test_folds) == set()
+
+        folds.append((fold_train, fold_val, fold_test))
 
-    return folds, test
+    return folds
+
+
+def ensure_valid_fold(fold, n_folds=10):
+    train, val, test = fold
+    assert len(train) == n_folds-2, len(train)
+    assert 0 <= train[0] < n_folds, train[0]
+    assert len(val) == 1, len(val)
+    assert 0 <= val[0] < n_folds, val[0]
+    assert len(test) == 1, len(test)
+    assert 0 <= test[0] < n_folds, test[0]
+    assert test[0] != val[0]
+    test_overlap = set(train).intersection(set(test))
+    val_overlap =  set(train).intersection(set(val))
+    assert test_overlap == set(), test_overlap
+    assert val_overlap == set(), val_overlap
+    assert sorted(train + val + test) == list(range(0, n_folds))
+    return True
+
+def folds_idx(n_folds):
+    """Generate fold indices for cross-validation.
+    Each fold has 1 validation, 1 test set and the remaining train"""
+    test_fold = 10
+
+    folds = []
+    all_folds = list(range(0, n_folds))
+    for idx in range(0, n_folds):
+        test = [ all_folds[idx] ]
+        # using Python negative index support for lists to wrap around at edges of array
+        val =  [ all_folds[idx-1] ]
+        train = list(set(all_folds).difference(set(test+val)))
+        fold = ( train, val, test )
+        ensure_valid_fold(fold)
+        folds.append(fold)
+
+    assert len(folds) == n_folds, len(folds)
+    return folds
+
+

test/test_urbansound.py

@@ -3,6 +3,7 @@
 import shutil
 
 import numpy
+import pytest
 
 from microesc import preprocess, urbansound8k, features, report
 
@@ -65,3 +66,39 @@ def test_grouped_confusion():
 
     # danger, only one class
     assert(gcm[3][3] == 82)
+
+
+folds = urbansound8k
+CORRECT_FOLDS={
+    '8val-9train': ((0,1,2,3,4,5,6,7), (8,), (9,)),
+}
+WRONG_FOLDS={
+    'train too short': ((0,1,2), (3,), (4,)),
+    'out-of-bounds train': ((0,1,2,3,4,5,6,7), (4,), (5,)),
+    'out-of-bounds val': ((0,2,3,4,5,6,7,8), (10,), (5,)),
+}
+
+@pytest.mark.parametrize('example', CORRECT_FOLDS.keys())
+def test_ensure_valid_fold_passes_correct(example):
+    fold = CORRECT_FOLDS[example]
+    folds.ensure_valid_fold(fold)
+
+@pytest.mark.parametrize('example', WRONG_FOLDS.keys())
+def test_ensure_valid_fold_detects_wrong(example):
+    fold = WRONG_FOLDS[example]
+    with pytest.raises(AssertionError) as e_info:
+        folds.ensure_valid_fold(fold)
+
+def test_folds_idx():
+    f = folds.folds_idx(10)
+    print('\n'+'\n'.join([ str(i) for i in f ]))
+    assert f[0][2][0] == 0, "first test fold should be 0" 
+    assert f[-1][2][0] == 9, "last test fold should be 9" 
+
+
+def test_folds():
+    data = urbansound8k.load_dataset()
+    f = urbansound8k.folds(data)
+    assert len(f) == 10
+
+