Coursework and results as submitted

Report/DNSDL_cw2.aux

@@ -0,0 +1,29 @@
+\relax 
+\select@language{english}
+\@writefile{toc}{\select@language{english}}
+\@writefile{lof}{\select@language{english}}
+\@writefile{lot}{\select@language{english}}
+\@writefile{toc}{\contentsline {section}{\numberline {1}Methodology}{1}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {1.1}Definitions and notations}{1}}
+\@writefile{toc}{\contentsline {section}{\numberline {2}Naive approach - grid search}{1}}
+\@writefile{toc}{\contentsline {section}{\numberline {3}Bayesian optimisation}{1}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Overview}{1}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {3.2}SMAC}{1}}
+\@writefile{toc}{\contentsline {paragraph}{Note on the reliability of the procedure}{2}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Vanilla gradient descent}{2}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {3.4}Training with regularization}{2}}
+\@writefile{toc}{\contentsline {paragraph}{Continuous values for $l$, $m$, and $w$}{2}}
+\@writefile{toc}{\contentsline {paragraph}{Continuous values for $l$, $m$, and $w$, two possible values for $n$, $e$, and $b$}{2}}
+\@writefile{toc}{\contentsline {paragraph}{Continuous values for $l$, $m$, and $w$, and integer ranges}{2}}
+\@writefile{toc}{\contentsline {paragraph}{Different range}{2}}
+\@writefile{toc}{\contentsline {paragraph}{Logarithmic scale for weight decay}{2}}
+\@writefile{toc}{\contentsline {section}{\numberline {4}Results}{2}}
+\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Best set of parameters as found by SMAC for the different spaces}}{3}}
+\@writefile{toc}{\contentsline {section}{\numberline {5}Statistical tests}{3}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Nature of the prediction error}{3}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Selection of the best set of parameters}{3}}
+\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Normal quantile-quantile plot for the standardized experimental data}}{3}}
+\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces Classification error on the test set on 10 runs for each parameter set. Average in bold. Standard deviation in italic.}}{3}}
+\@writefile{toc}{\contentsline {section}{\numberline {6}Conclusion}{4}}
+\@writefile{toc}{\contentsline {section}{\numberline {7}A note on the code}{4}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {7.1}Troubleshooting}{4}}

TrainAndTest.m

@@ -0,0 +1,41 @@
+load('caltech101_silhouettes_28_split1');
+run('medal/startLearning.m');
+
+% The parameters are from S5
+
+nHid = 983
+learning_rate = 0.09867863720310953
+nepochs = 188
+batch_size = 87
+momentum = 0.3595338313047923
+decay_weight = 0.0032118183957211485
+
+[~,nVis] = size(train_data);
+
+% DEFINE A MODEL ARCHITECTURE
+arch = struct('size', [nVis,nHid], 'classifier',true, 'inputType','binary');
+
+% GLOBAL OPTIONS
+arch.opts = {'verbose', 1, ...
+     'lRate', learning_rate, ...
+    'nEpoch', nepochs, ...
+    'batchSz', batch_size, ...
+    'beginAnneal', 10, ...
+    'nGibbs', 1, ...
+    'varyEta',7, ...
+    'displayEvery', 20, ...
+    'verbose', 1, ...
+    'momentum', momentum, ...
+    'wPenalty', decay_weight};
+
+% INITIALIZE RBM
+r = rbm(arch);
+
+% TRAIN THE RBM
+r = r.train(train_data,single(train_labels));
+
+% IMPORTANT: The parameters were found by using the data labeled
+% "test_data" as the validation set. The actual training set is "val_data".
+[~,error,~] = r.classify(val_data, single(val_labels));
+
+fprintf('Error: %f%%\n', 100*error);

code/RBM_grid_search.m

@@ -0,0 +1,42 @@
+
+% Find parameters using grid search on a small hyperparameter space
+
+clear;
+
+load('caltech101_silhouettes_28_split1');
+run('medal/startLearning.m');
+
+[~,nVis] = size(train_data);
+
+for nHid=[100,100]
+    for learning_rate=[0.01, 0.1]
+        for batch_size=[100,250]
+            for nepochs=[100,250]
+                % DEFINE A MODEL ARCHITECTURE
+                arch = struct('size', [nVis,nHid], 'classifier',true, 'inputType','binary');
+
+                % GLOBAL OPTIONS
+                arch.opts = {'verbose', 1, ...
+                     'lRate', learning_rate, ...
+                    'nEpoch', nepochs, ...
+                    'batchSz', batch_size, ...
+                    'beginAnneal', 10, ...
+                    'nGibbs', 1, ...
+                    'varyEta',7, ...
+                    'displayEvery', 20, ...
+                    'verbose', 1};
+
+                % INITIALIZE RBM
+                r = rbm(arch);
+
+                % TRAIN THE RBM
+                r = r.train(train_data,single(train_labels));
+
+                [~,error,~] = r.classify(test_data, single(test_labels));
+
+                disp([nHid, learning_rate, batch_size, nepochs]);
+                fprintf('Error: %f%%\n\n', 100*error);
+            end
+        end
+    end
+end

code/README.txt

@@ -0,0 +1,16 @@
+File TrainAndTest.m trains an RBM with the parameters from S5 (see documentation) and output the prediction error on the test set (data val).
+
+Files tenAveragedTestRBM.m and its noreg counterpart train and test ton RBMs for a given parameter set.
+
+SMAC must be run as follows: ./smac --scenario-file FILE.
+An optional parameter --seed allows to distinguish more easily between runs (used for naming the folders in smac-output).
+
+The file rbm.mat contains a pre-trained RBM using the parameters from S5.
+
+A few csv files are provided as evidence of the different experimentations.
+
+Troubleshooting:
+
+SMAC will fail to launch if the path to its script contains spaces.
+
+The MATLAB functions called by the wrappers SMAC uses require Medal to be in the same folder as them.