Misc DL4J/ND4J fixes

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalTest.java

@@ -537,4 +537,72 @@ public void testMultiOutputEvalCG(){
 
         cg.evaluate(new SingletonMultiDataSetIterator(mds), m);
     }
+
+    @Test
+    public void testInvalidEvaluation(){
+
+        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
+
+                .list()
+                .layer(new DenseLayer.Builder().nIn(4).nOut(10).build())
+                .layer(new OutputLayer.Builder().nIn(10).nOut(3).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.RELU).build())
+                .build();
+
+        MultiLayerNetwork net = new MultiLayerNetwork(conf);
+        net.init();
+
+        DataSetIterator iter = new IrisDataSetIterator(150, 150);
+        try {
+            net.evaluate(iter);
+            fail("Expected exception");
+        } catch (IllegalStateException e){
+            assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("Evaluation"));
+        }
+
+        try {
+            net.evaluateROC(iter);
+            fail("Expected exception");
+        } catch (IllegalStateException e){
+            assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC"));
+        }
+
+        try {
+            net.evaluateROCMultiClass(iter);
+            fail("Expected exception");
+        } catch (IllegalStateException e){
+            assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass"));
+        }
+
+        ComputationGraph cg = net.toComputationGraph();
+        try {
+            cg.evaluate(iter);
+            fail("Expected exception");
+        } catch (IllegalStateException e){
+            assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("Evaluation"));
+        }
+
+        try {
+            cg.evaluateROC(iter);
+            fail("Expected exception");
+        } catch (IllegalStateException e){
+            assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC"));
+        }
+
+        try {
+            cg.evaluateROCMultiClass(iter);
+            fail("Expected exception");
+        } catch (IllegalStateException e){
+            assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass"));
+        }
+
+
+        //Disable validation, and check same thing:
+        net.getLayerWiseConfigurations().setValidateOutputLayerConfig(false);
+        net.evaluate(iter);
+        net.evaluateROCMultiClass(iter);
+
+        cg.getConfiguration().setValidateOutputLayerConfig(false);
+        cg.evaluate(iter);
+        cg.evaluateROCMultiClass(iter);
+    }
 }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java

@@ -1819,4 +1819,59 @@ public void testAddRemoveVertex() {
                 .addVertex("test", new ScaleVertex(0), "toRemove")
                 .removeVertex("toRemove", true);
     }
+
+
+    @Test
+    public void testGetSetParamUnderscores(){
+        //Test get/set param with underscores in layer nome
+        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
+                .graphBuilder()
+                .addInputs("in")
+                .layer("layer_zero", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in")
+                .layer("layer_one", new OutputLayer.Builder().nIn(10).nOut(10).build(), "layer_zero")
+                .setOutputs("layer_one")
+                .build();
+
+        ComputationGraph cg = new ComputationGraph(conf);
+        cg.init();
+        cg.params().assign(Nd4j.linspace(1, 220, 220).reshape(1, -11));
+
+        INDArray p0w = cg.getParam("layer_zero_W");
+        assertEquals(Nd4j.linspace(1, 100, 100).reshape('f', 10, 10), p0w);
+
+        INDArray p1b = cg.getParam("layer_one_b");
+        assertEquals(Nd4j.linspace(211, 220, 10).reshape(1,10), p1b);
+
+        INDArray newP1b = Nd4j.valueArrayOf(new long[]{1,10}, -1);
+        cg.setParam("layer_one_b", newP1b);
+
+        assertEquals(newP1b, p1b);
+    }
+
+    @Test
+    public void testOutputSpecificLayers(){
+        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
+                .seed(12345)
+                .graphBuilder()
+                .addInputs("in")
+                .layer("0", new DenseLayer.Builder().nIn(10).nOut(9).build(), "in")
+                .layer("1", new DenseLayer.Builder().nIn(9).nOut(8).build(), "0")
+                .layer("2", new DenseLayer.Builder().nIn(8).nOut(7).build(), "1")
+                .layer("3", new OutputLayer.Builder().nIn(7).nOut(6).build(), "2")
+                .setOutputs("3")
+                .build();
+
+        ComputationGraph cg = new ComputationGraph(conf);
+        cg.init();
+
+        INDArray in = Nd4j.rand(1, 10);
+
+        Map<String,INDArray> outMap = cg.feedForward(in, false);
+
+        INDArray[] outSpecific = cg.output(Arrays.asList("1", "3"), false, new INDArray[]{in}, null);
+        assertEquals(2, outSpecific.length);
+
+        assertEquals(outMap.get("1"), outSpecific[0]);
+        assertEquals(outMap.get("3"), outSpecific[1]);
+    }
 }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java

@@ -32,6 +32,7 @@
 import org.deeplearning4j.nn.conf.distribution.NormalDistribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.*;
+import org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer;
 import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder;
 import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor;
 import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor;
@@ -1397,6 +1398,38 @@ public void testPretrainFitMethods(){
         assertEquals(exp, listener.getModelClasses());
     }
 
+    @Test
+    public void testINDArrayConfigCloning(){
+        //INDArrays in config should be cloned to avoid threading issues
+
+        int mb = 3;
+        int b = 4;
+        int c = 3;
+        int depth = b * (5 + c);
+        int w = 6;
+        int h = 6;
+
+        INDArray bbPrior = Nd4j.rand(b, 2).muliRowVector(Nd4j.create(new double[]{w, h}));
+
+
+        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
+                .l2(0.01)
+                .list()
+                .layer(new ConvolutionLayer.Builder().nIn(depth).nOut(depth).kernelSize(1,1).build())
+                .layer(new Yolo2OutputLayer.Builder()
+                        .boundingBoxPriors(bbPrior)
+                        .build())
+                .build();
+
+        MultiLayerConfiguration conf2 = conf.clone();
+
+        INDArray bb1 = ((Yolo2OutputLayer)conf.getConf(1).getLayer()).getBoundingBoxes();
+        INDArray bb2 = ((Yolo2OutputLayer)conf2.getConf(1).getLayer()).getBoundingBoxes();
+        assertFalse(bb1 == bb2);
+
+        assertEquals(bb1, bb2);
+    }
+
     @Data
     public static class CheckModelsListener extends BaseTrainingListener {
 

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ComputationGraphConfiguration.java

@@ -76,6 +76,8 @@ public class ComputationGraphConfiguration implements Serializable, Cloneable {
     @Setter
     protected CacheMode cacheMode;
 
+    protected boolean validateOutputLayerConfig = true;     //Default for 10.0.-beta3 and earlier nets
+
     /**
      * List of inputs to the network, by name
      */
@@ -266,6 +268,7 @@ public ComputationGraphConfiguration clone() {
         conf.cacheMode = this.cacheMode;
         conf.defaultConfiguration.cacheMode = this.cacheMode;
         conf.legacyBatchScaledL2 = this.legacyBatchScaledL2;
+        conf.validateOutputLayerConfig = this.validateOutputLayerConfig;
 
         return conf;
     }
@@ -1034,6 +1037,7 @@ private ComputationGraphConfiguration buildConfig(){
             conf.trainingWorkspaceMode = globalConfiguration.trainingWorkspaceMode;
             conf.inferenceWorkspaceMode = globalConfiguration.inferenceWorkspaceMode;
             conf.cacheMode = globalConfiguration.cacheMode;
+            conf.validateOutputLayerConfig = validateOutputConfig;
 
             conf.defaultConfiguration = globalConfiguration.build();
             conf.getDefaultConfiguration().setPretrain(pretrain);

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/MultiLayerConfiguration.java

@@ -62,6 +62,7 @@ public class MultiLayerConfiguration implements Serializable, Cloneable {
     protected int tbpttBackLength = 20;
     @Getter @Setter
     protected boolean legacyBatchScaledL2 = true;   //Default to legacy for pre 1.0.0-beta3 networks on deserialization
+    protected boolean validateOutputLayerConfig = true; //Default to legacy for pre 1.0.0-beta3 networks on deserialization
 
     @Getter
     @Setter
@@ -318,6 +319,7 @@ public MultiLayerConfiguration clone() {
             clone.trainingWorkspaceMode = this.trainingWorkspaceMode;
             clone.cacheMode = this.cacheMode;
             clone.legacyBatchScaledL2 = legacyBatchScaledL2;
+            clone.validateOutputLayerConfig = this.validateOutputLayerConfig;
 
             return clone;
 

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java

@@ -34,6 +34,7 @@
 import org.nd4j.shade.jackson.annotation.JsonTypeInfo;
 
 import java.io.Serializable;
+import java.lang.reflect.Field;
 import java.util.*;
 
 /**
@@ -107,7 +108,35 @@ public void resetLayerDefaultConfig() {
     @Override
     public Layer clone() {
         try {
-            return (Layer) super.clone();
+            Layer ret = (Layer) super.clone();
+            //Let's check for any INDArray fields and dup them (in case cloned layer will be used in different threads on CUDA...
+            // we don't want it being relocated contantly between devices)
+            Class<?> c = getClass();
+            while(c != Object.class){
+                Field[] fields = c.getDeclaredFields();
+                for(Field f : fields){
+                    if(f.getType() == INDArray.class){
+                        f.setAccessible(true);
+                        INDArray toClone;
+                        try{
+                            toClone = (INDArray) f.get(this);
+                        } catch (Exception e){
+                            throw new RuntimeException(e);
+                        }
+                        if(toClone != null){
+                            try {
+                                f.set(this, toClone.dup());
+                            } catch (Exception e){
+                                throw new RuntimeException(e);
+                            }
+                        }
+                    }
+                }
+
+                c = c.getSuperclass();
+            }
+
+            return ret;
         } catch (CloneNotSupportedException e) {
             throw new RuntimeException(e);
         }

deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java

@@ -54,6 +54,7 @@
 import org.deeplearning4j.util.CrashReportingUtil;
 import org.deeplearning4j.util.ModelSerializer;
 import org.deeplearning4j.util.NetworkUtils;
+import org.deeplearning4j.util.OutputLayerUtil;
 import org.nd4j.base.Preconditions;
 import org.nd4j.evaluation.EvaluationUtils;
 import org.nd4j.evaluation.IEvaluation;
@@ -1838,6 +1839,27 @@ public INDArray outputSingle(MultiDataSetIterator iterator){
         return output(iterator)[0];
     }
 
+    /**
+     * Get the activations for the specific layers only
+     * @param layers       Layers to get the specified activations for
+     * @param train        If true: train mode. False: test (inference) mode
+     * @param features     Features array
+     * @param featureMasks Feature masks array. May be null
+     * @return Activations of the selected layers, in the same order as the "layers" arg/list
+     */
+    public INDArray[] output(List<String> layers, boolean train, INDArray[] features, INDArray[] featureMasks){
+        Preconditions.checkState(layers != null && layers.size() > 0, "Layers must not be null: got later names %s", layers);
+        int[] layerNums = new int[layers.size()];
+        for( int i=0; i<layers.size(); i++ ){
+            String n = layers.get(i);
+            Preconditions.checkState(verticesMap.containsKey(n), "Layer with name %s not found in network", n);
+            layerNums[i] = verticesMap.get(n).getVertexIndex();
+        }
+        INDArray[] out = outputOfLayersDetached(train, FwdPassType.STANDARD, layerNums, features, featureMasks, null, true,
+                false, null);
+        return out;
+    }
+
 
     protected void validateArrayWorkspaces(LayerWorkspaceMgr mgr, INDArray array, ArrayType arrayType, String vertexName, boolean isInputVertex, String op){
         try{
@@ -3123,7 +3145,7 @@ public void update(Gradient gradient) {
         for (Map.Entry<String, INDArray> entry : gradient.gradientForVariable().entrySet()) {
             String key = entry.getKey();
             INDArray val = entry.getValue();
-            int idx = key.indexOf('_');
+            int idx = key.lastIndexOf('_');
             if (idx == -1)
                 throw new IllegalStateException("Invalid param key: not have layer separator: \"" + key + "\"");
             String layerName = key.substring(0, idx);
@@ -3280,7 +3302,7 @@ public ConvexOptimizer getOptimizer() {
     @Override
     public INDArray getParam(String paramName) {
         //        throw new UnsupportedOperationException("Not implemented");
-        int idx = paramName.indexOf('_');
+        int idx = paramName.lastIndexOf('_');
         if (idx == -1)
             throw new IllegalStateException("Invalid param key: not have layer separator: \"" + paramName + "\"");
         String layerName = paramName.substring(0, idx);
@@ -3331,7 +3353,7 @@ public void setParamTable(@NonNull Map<String, INDArray> paramTable) {
     @Override
     public void setParam(String key, INDArray val) {
         //        throw new UnsupportedOperationException("Not implemented");
-        int idx = key.indexOf('_');
+        int idx = key.lastIndexOf('_');
         if (idx == -1)
             throw new IllegalStateException("Invalid param key: not have layer separator: \"" + key + "\"");
         String layerName = key.substring(0, idx);
@@ -3816,6 +3838,11 @@ public <T extends Evaluation> T evaluate(DataSetIterator iterator, List<String>
         if (labelsList == null)
             labelsList = iterator.getLabels();
 
+        Layer outputLayer = getOutputLayer(0);
+        if(getConfiguration().isValidateOutputLayerConfig()){
+            OutputLayerUtil.validateOutputLayerForClassifierEvaluation(outputLayer.conf().getLayer(), Evaluation.class);
+        }
+
         return (T)doEvaluation(iterator, new org.deeplearning4j.eval.Evaluation(labelsList, topN))[0];
     }
 
@@ -3829,6 +3856,10 @@ public <T extends Evaluation> T evaluate(DataSetIterator iterator, List<String>
      * @return Evaluation object, summarizing the results of the evaluation on the provided DataSetIterator
      */
     public <T extends Evaluation> T evaluate(MultiDataSetIterator iterator, List<String> labelsList, int topN) {
+        Layer outputLayer = getOutputLayer(0);
+        if(getConfiguration().isValidateOutputLayerConfig()){
+            OutputLayerUtil.validateOutputLayerForClassifierEvaluation(outputLayer.conf().getLayer(), Evaluation.class);
+        }
         return (T)doEvaluation(iterator, new org.deeplearning4j.eval.Evaluation(labelsList, topN))[0];
     }
 
@@ -3892,6 +3923,10 @@ public <T extends ROC> T evaluateROC(DataSetIterator iterator) {
      * @return ROC evaluation on the given dataset
      */
     public <T extends ROC> T evaluateROC(DataSetIterator iterator, int rocThresholdSteps) {
+        Layer outputLayer = getOutputLayer(0);
+        if(getConfiguration().isValidateOutputLayerConfig()){
+            OutputLayerUtil.validateOutputLayerForClassifierEvaluation(outputLayer.conf().getLayer(), ROC.class);
+        }
         return (T)doEvaluation(iterator, new org.deeplearning4j.eval.ROC(rocThresholdSteps))[0];
     }
 
@@ -3914,6 +3949,10 @@ public <T extends ROC> T evaluateROC(MultiDataSetIterator iterator) {
      * @return ROC evaluation on the given dataset
      */
     public <T extends ROC> T evaluateROC(MultiDataSetIterator iterator, int rocThresholdSteps) {
+        Layer outputLayer = getOutputLayer(0);
+        if(getConfiguration().isValidateOutputLayerConfig()){
+            OutputLayerUtil.validateOutputLayerForClassifierEvaluation(outputLayer.conf().getLayer(), ROC.class);
+        }
         return (T)doEvaluation(iterator, new org.deeplearning4j.eval.ROC(rocThresholdSteps))[0];
     }
 
@@ -3936,6 +3975,10 @@ public <T extends ROCMultiClass> T evaluateROCMultiClass(DataSetIterator iterato
      * @return Multi-class ROC evaluation on the given dataset
      */
     public <T extends ROCMultiClass> T evaluateROCMultiClass(DataSetIterator iterator, int rocThresholdSteps) {
+        Layer outputLayer = getOutputLayer(0);
+        if(getConfiguration().isValidateOutputLayerConfig()){
+            OutputLayerUtil.validateOutputLayerForClassifierEvaluation(outputLayer.conf().getLayer(), ROCMultiClass.class);
+        }
         return (T)doEvaluation(iterator, new org.deeplearning4j.eval.ROCMultiClass(rocThresholdSteps))[0];
     }
 
@@ -3947,6 +3990,10 @@ public <T extends ROCMultiClass> T evaluateROCMultiClass(DataSetIterator iterato
      * @return Multi-class ROC evaluation on the given dataset
      */
     public <T extends ROCMultiClass> T evaluateROCMultiClass(MultiDataSetIterator iterator, int rocThresholdSteps) {
+        Layer outputLayer = getOutputLayer(0);
+        if(getConfiguration().isValidateOutputLayerConfig()){
+            OutputLayerUtil.validateOutputLayerForClassifierEvaluation(outputLayer.conf().getLayer(), ROCMultiClass.class);
+        }
         return (T)doEvaluation(iterator, new org.deeplearning4j.eval.ROCMultiClass(rocThresholdSteps))[0];
     }