Experiment updates, and nicer stats tables.

ComputeBatchEntailmentCostAndGrad.m

@@ -242,6 +242,13 @@
         embGrad = [];
     end
 
+    if hyperParams.clipGradients
+        gradNorm = norm(grad);
+        if gradNorm > hyperParams.maxGradNorm
+            grad = grad .* (hyperParams.maxGradNorm ./ gradNorm);
+        end
+    end
+
     if sum(isnan(grad)) > 0
         [ mergeMatrices, mergeMatrix, ...
             softmaxMatrix, trainedWordFeatures, compositionMatrices,...

ComputeBatchSentenceClassificationCostAndGrad.m

@@ -189,11 +189,18 @@
         embGrad = [];
     end
 
+    if hyperParams.clipGradients
+        gradNorm = norm(grad);
+        if gradNorm > hyperParams.maxGradNorm
+            grad = grad .* (hyperParams.maxGradNorm ./ gradNorm);
+        end
+    end
+
     if sum(isnan(grad)) > 0
-    [ ~, ~, ...
-        softmaxMatrix, trainedWordFeatures, connectionMatrix, ...
-        compositionMatrix, scoringVector, classifierExtraMatrix, embeddingTransformMatrix ] ...
-            = stack2param(grad, decoder);
+        [ ~, ~, ...
+            softmaxMatrix, trainedWordFeatures, connectionMatrix, ...
+            compositionMatrix, scoringVector, classifierExtraMatrix, embeddingTransformMatrix ] ...
+                = stack2param(grad, decoder);
 
         softmaxMatrix, trainedWordFeatures, connectionMatrix, ...
         compositionMatrix, scoringVector, classifierExtraMatrix, embeddingTransformMatrix

ComputeSentenceClassificationExampleCostAndGrad.m

@@ -138,7 +138,7 @@
     if hyperParams.clipGradients
         gradNorm = norm(grad);
         if gradNorm > hyperParams.maxGradNorm
-            grad = grad ./ gradNorm;
+            grad = grad .* (hyperParams.maxGradNorm ./ gradNorm);
         end
     end
 end

RunExperiments/RunAndOrExperiments.sh

@@ -145,13 +145,13 @@ export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.0001; dim = 5
 % 0.0001 best, by far, still
 
 export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 2; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
-% 0.0003 best @ ~90, still waiting on rest
+% 0.0003 best @ ~90
 
 export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.003; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
 export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.0001; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
 export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
 % All garbage. Re-run.
-% 1e05 best so far at 57
+% 0.0003 best at ~90
 
 % To run
 export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.001; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-f'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
@@ -165,4 +165,19 @@ export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.00001; dim =
 
 export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.000001; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-f'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
 
+--
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.00003; dim = 50; td = 3; penult = 100; tot = 0; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.0001; dim = 50; td = 3; penult = 100; tot = 1; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 2; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-6'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+
+export MATLABCMD="cd quant; dataflag = 'and-or-deep'; lambda = 0.00003; dim = 50; td = 3; penult = 100; tot = 0; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep'; lambda = 0.0001; dim = 50; td = 3; penult = 100; tot = 1; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 2; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-3'; lambda = 0.00003; dim = 50; td = 3; penult = 100; tot = 0; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-3'; lambda = 0.0001; dim = 50; td = 3; penult = 100; tot = 1; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-3'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 2; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; dataflag = 'and-or-deep-3'; lambda = 0.0003; dim = 50; td = 3; penult = 100; tot = 3; name='/scr/sbowman/tuning6-c'; TrainModel(''\, 1\, @AndOr\, name\, dataflag\, dim\, penult\, td\, lambda\, tot\, 32);" ; qsub -v MATLABCMD quant/run.sh
 

RunExperiments/RunPragBankExperiments.sh

@@ -1,3 +1,27 @@
-export MATLABCMD="cd quant; lambda = 0.0001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.9\, 0.9]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt2-'; TrainModel('snlirc-snlirc2-only-l1e-07-dim50-ed200-td3-pen100-do0.9-0.9-co1-comp2-dp1/ckpt-best-tr150517185056@353375.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh 
-export MATLABCMD="cd quant; lambda = 0.0001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.9\, 0.9]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt2-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh
+export MATLABCMD="cd quant; lambda = 0.0001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [1\, 1]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.0001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [1\, 1]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.0003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [1\, 1]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.0003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [1\, 1]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
 
+export MATLABCMD="cd quant; lambda = 0.0003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.5\, 0.5]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.0003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.5\, 0.5]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+
+export MATLABCMD="cd quant; lambda = 0.0001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.75\, 0.75]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.0001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.75\, 0.75]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+
+export MATLABCMD="cd quant; lambda = 0.003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.75\, 0.75]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.75\, 0.75]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+
+export MATLABCMD="cd quant; lambda = 0.001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.75\, 0.75]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.75\, 0.75]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+
+export MATLABCMD="cd quant; lambda = 0.01; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.5\, 0.5]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.01; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.5\, 0.5]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+
+% Next round
+
+export MATLABCMD="cd quant; lambda = 0.0003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.25\, 0.25]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.0003; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.25\, 0.25]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+
+export MATLABCMD="cd quant; lambda = 0.001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.25\, 0.25]; tot = 2; collo = 1; dataflag='pragbank-transfer'; name='/scr/sbowman/pbt3-'; TrainModel('/scr/sbowman/snlirc2-snlirc2-only-l1e-05-dim50-ed200-td3-pen100-do1-1-co1-comp2-dp1-gc0/ckpt-best-tr150522185602@435250.mat'\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp
+export MATLABCMD="cd quant; lambda = 0.001; dim = 50; ed = 200; td = 3; penult = 100; dropout = [0.25\, 0.25]; tot = 2; collo = 1; dataflag='pragbank'; name='/scr/sbowman/pbt3-'; TrainModel(''\, 1\, @PragBank\, name\, dataflag\, ed\, dim\, td\, penult\, lambda\, tot\, dropout(1)\, dropout(2)\, collo);" ; qsub -v MATLABCMD quant/run.sh -q nlp