Merge pull request #3985 from deeplearning4j/ab_dropout

Schedules and Dropout

deeplearning4j-core/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIterator.java

@@ -31,7 +31,6 @@
 import org.datavec.api.util.ndarray.RecordConverter;
 import org.datavec.api.writable.NDArrayWritable;
 import org.datavec.api.writable.Writable;
-import org.nd4j.linalg.primitives.Pair;
 import org.deeplearning4j.datasets.datavec.exception.ZeroLengthSequenceException;
 import org.deeplearning4j.exception.DL4JException;
 import org.nd4j.linalg.api.ndarray.INDArray;
@@ -41,6 +40,7 @@
 import org.nd4j.linalg.factory.Nd4j;
 import org.nd4j.linalg.indexing.INDArrayIndex;
 import org.nd4j.linalg.indexing.NDArrayIndex;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.io.IOException;
 import java.io.Serializable;

deeplearning4j-core/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java

@@ -19,21 +19,12 @@
 package org.deeplearning4j.datasets.iterator.impl;
 
 import lombok.Getter;
-import org.datavec.api.io.labels.ParentPathLabelGenerator;
-import org.datavec.api.io.labels.PathLabelGenerator;
-import org.datavec.api.io.labels.PatternPathLabelGenerator;
-import org.datavec.image.recordreader.ImageRecordReader;
 import org.datavec.image.transform.ImageTransform;
 import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
-import org.deeplearning4j.datasets.fetchers.*;
-import org.deeplearning4j.datasets.iterator.BaseDatasetIterator;
+import org.deeplearning4j.datasets.fetchers.DataSetType;
+import org.deeplearning4j.datasets.fetchers.TinyImageNetFetcher;
 import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
 /**
  * Tiny ImageNet is a subset of the ImageNet database. TinyImageNet is the default course challenge for CS321n
  * at Stanford University.

deeplearning4j-core/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java

@@ -23,7 +23,6 @@
 import com.google.common.util.concurrent.AtomicDouble;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.math3.util.FastMath;
-import org.nd4j.linalg.primitives.Pair;
 import org.deeplearning4j.clustering.sptree.DataPoint;
 import org.deeplearning4j.clustering.sptree.SpTree;
 import org.deeplearning4j.clustering.vptree.VPTree;
@@ -39,6 +38,7 @@
 import org.nd4j.linalg.indexing.conditions.Conditions;
 import org.nd4j.linalg.indexing.functions.Value;
 import org.nd4j.linalg.learning.legacy.AdaGrad;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -671,11 +671,6 @@ public void setBackpropGradientsViewArray(INDArray gradients) {
         throw new UnsupportedOperationException();
     }
 
-    @Override
-    public void applyLearningRateScoreDecay() {
-        throw new UnsupportedOperationException("Not yet implemented");
-    }
-
     @Override
     public void fit(INDArray data) {
         this.x = data;

deeplearning4j-core/src/main/java/org/deeplearning4j/plot/Tsne.java

@@ -2,7 +2,6 @@
 
 import com.google.common.primitives.Ints;
 import org.apache.commons.math3.util.FastMath;
-import org.nd4j.linalg.primitives.Pair;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.dimensionalityreduction.PCA;
 import org.nd4j.linalg.factory.Nd4j;
@@ -12,6 +11,7 @@
 import org.nd4j.linalg.indexing.conditions.Conditions;
 import org.nd4j.linalg.indexing.functions.Value;
 import org.nd4j.linalg.learning.legacy.AdaGrad;
+import org.nd4j.linalg.primitives.Pair;
 import org.nd4j.linalg.util.ArrayUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

deeplearning4j-core/src/test/java/org/deeplearning4j/TestUtils.java

@@ -0,0 +1,55 @@
+package org.deeplearning4j;
+
+import org.deeplearning4j.nn.graph.ComputationGraph;
+import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
+import org.deeplearning4j.util.ModelSerializer;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestUtils {
+
+    public static MultiLayerNetwork testModelSerialization(MultiLayerNetwork net){
+
+        try {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            ModelSerializer.writeModel(net, baos, true);
+            byte[] bytes = baos.toByteArray();
+
+            ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+            MultiLayerNetwork restored = ModelSerializer.restoreMultiLayerNetwork(bais, true);
+
+            assertEquals(net.getLayerWiseConfigurations(), restored.getLayerWiseConfigurations());
+            assertEquals(net.params(), restored.params());
+
+            return restored;
+        } catch (IOException e){
+            //Should never happen
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static ComputationGraph testModelSerialization(ComputationGraph net){
+
+        try {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            ModelSerializer.writeModel(net, baos, true);
+            byte[] bytes = baos.toByteArray();
+
+            ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+            ComputationGraph restored = ModelSerializer.restoreComputationGraph(bais, true);
+
+            assertEquals(net.getConfiguration(), restored.getConfiguration());
+            assertEquals(net.params(), restored.params());
+
+            return restored;
+        } catch (IOException e){
+            //Should never happen
+            throw new RuntimeException(e);
+        }
+    }
+
+}

deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIteratorTest.java

@@ -1,10 +1,10 @@
 package org.deeplearning4j.datasets.iterator;
 
 import org.apache.commons.lang3.RandomUtils;
-import org.nd4j.linalg.primitives.Pair;
 import org.junit.Test;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicInteger;

deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIteratorTest.java

@@ -98,7 +98,7 @@ public void testLoadBatchDataSet() throws Exception {
     public void testCifarDataSetIteratorReset() {
         int epochs = 2;
         Nd4j.getRandom().setSeed(12345);
-        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).learningRate(1.0)
+        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                         .weightInit(WeightInit.XAVIER).seed(12345L).list()
                         .layer(0, new DenseLayer.Builder().nIn(400).nOut(50).activation(Activation.RELU).build())
                         .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)

deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java

@@ -15,7 +15,6 @@
 import org.deeplearning4j.nn.api.OptimizationAlgorithm;
 import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
 import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
-import org.deeplearning4j.nn.conf.Updater;
 import org.deeplearning4j.nn.conf.layers.BaseLayer;
 import org.deeplearning4j.nn.conf.layers.DenseLayer;
 import org.deeplearning4j.nn.conf.layers.OutputLayer;
@@ -29,6 +28,8 @@
 import org.nd4j.linalg.dataset.DataSet;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
 import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.learning.config.Nesterovs;
+import org.nd4j.linalg.learning.config.Sgd;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -47,7 +48,7 @@ public class TestEarlyStopping {
     public void testEarlyStoppingIris() {
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).weightInit(WeightInit.XAVIER).list()
+                        .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                         .pretrain(false).backprop(true).build();
@@ -90,7 +91,7 @@ public void testEarlyStoppingIris() {
     public void testEarlyStoppingEveryNEpoch() {
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).weightInit(WeightInit.XAVIER).list()
+                        .updater(new Sgd(0.01)).weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                         .pretrain(false).backprop(true).build();
@@ -118,7 +119,7 @@ public void testEarlyStoppingEveryNEpoch() {
     public void testEarlyStoppingIrisMultiEpoch() {
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).weightInit(WeightInit.XAVIER).list()
+                        .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                         .pretrain(false).backprop(true).build();
@@ -166,7 +167,7 @@ public void testBadTuning() {
         Nd4j.getRandom().setSeed(12345);
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).learningRate(5.0) //Intentionally huge LR
+                        .updater(new Sgd(5.0)) //Intentionally huge LR
                         .weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
@@ -205,7 +206,7 @@ public void testTimeTermination() {
         Nd4j.getRandom().setSeed(12345);
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).learningRate(1e-6).weightInit(WeightInit.XAVIER).list()
+                        .updater(new Sgd(1e-6)).weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                         .pretrain(false).backprop(true).build();
@@ -247,7 +248,7 @@ public void testNoImprovementNEpochsTermination() {
         Nd4j.getRandom().setSeed(12345);
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).learningRate(0.0).weightInit(WeightInit.XAVIER).list()
+                        .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                         .pretrain(false).backprop(true).build();
@@ -285,8 +286,8 @@ public void testMinImprovementNEpochsTermination() {
         Random rng = new Random(123);
         Nd4j.getRandom().setSeed(12345);
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).iterations(10)
-                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(0.0)
-                        .updater(Updater.NESTEROVS).momentum(0.9).list()
+                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
+                        .updater(new Nesterovs(0.0,0.9)).list()
                         .layer(0, new DenseLayer.Builder().nIn(1).nOut(20)
                                         .weightInit(WeightInit.XAVIER).activation(
                                                         Activation.TANH)
@@ -333,7 +334,7 @@ public void testMinImprovementNEpochsTermination() {
     public void testEarlyStoppingGetBestModel() {
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).weightInit(WeightInit.XAVIER).list()
+                        .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                         .pretrain(false).backprop(true).build();
@@ -364,14 +365,14 @@ public void testEarlyStoppingGetBestModel() {
         assertEquals(net.conf().getOptimizationAlgo(), mln.conf().getOptimizationAlgo());
         BaseLayer bl = (BaseLayer) net.conf().getLayer();
         assertEquals(bl.getActivationFn().toString(), ((BaseLayer) mln.conf().getLayer()).getActivationFn().toString());
-        assertEquals(bl.getUpdater(), ((BaseLayer) mln.conf().getLayer()).getUpdater());
+        assertEquals(bl.getIUpdater(), ((BaseLayer) mln.conf().getLayer()).getIUpdater());
     }
 
     @Test
     public void testListeners() {
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).weightInit(WeightInit.XAVIER).list()
+                        .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).list()
                         .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                         .pretrain(false).backprop(true).build();

deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStoppingCompGraph.java

@@ -31,7 +31,6 @@
 import org.deeplearning4j.nn.api.OptimizationAlgorithm;
 import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
 import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
-import org.deeplearning4j.nn.conf.Updater;
 import org.deeplearning4j.nn.conf.layers.OutputLayer;
 import org.deeplearning4j.nn.graph.ComputationGraph;
 import org.deeplearning4j.nn.weights.WeightInit;
@@ -40,6 +39,7 @@
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
 import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.learning.config.Sgd;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,7 +55,7 @@ public class TestEarlyStoppingCompGraph {
     public void testEarlyStoppingIris() {
         ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
+                        .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                         .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                         .setOutputs("0").pretrain(false).backprop(true).build();
@@ -98,7 +98,7 @@ public void testBadTuning() {
         Nd4j.getRandom().setSeed(12345);
         ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).learningRate(5.0) //Intentionally huge LR
+                        .updater(new Sgd(5.0)) //Intentionally huge LR
                         .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                         .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
@@ -134,7 +134,7 @@ public void testTimeTermination() {
         Nd4j.getRandom().setSeed(12345);
         ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).learningRate(1e-6).weightInit(WeightInit.XAVIER).graphBuilder()
+                        .updater(new Sgd(1e-6)).weightInit(WeightInit.XAVIER).graphBuilder()
                         .addInputs("in")
                         .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
@@ -175,7 +175,7 @@ public void testNoImprovementNEpochsTermination() {
         Nd4j.getRandom().setSeed(12345);
         ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).learningRate(0.0).weightInit(WeightInit.XAVIER).graphBuilder()
+                        .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).graphBuilder()
                         .addInputs("in")
                         .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
@@ -209,7 +209,7 @@ public void testNoImprovementNEpochsTermination() {
     public void testListeners() {
         ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                         .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
-                        .updater(Updater.SGD).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
+                        .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                         .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                                         .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                         .setOutputs("0").pretrain(false).backprop(true).build();