From 707b803ba7da9fc2168359867e658fb9da99d97f Mon Sep 17 00:00:00 2001 From: Alex Black Date: Thu, 21 Dec 2017 11:04:20 +1100 Subject: [PATCH] More tests/implementation --- deeplearning4j-core/pom.xml | 10 ++ .../deeplearning4j/samediff/SameDiffTest.java | 151 ++++++++++++++++-- .../samediff/testlayers/SameDiffDense.java | 4 +- .../layers/samediff/BaseSameDiffLayer.java | 2 +- .../nn/layers/samediff/SameDiffLayer.java | 38 +++-- 5 files changed, 179 insertions(+), 26 deletions(-) diff --git a/deeplearning4j-core/pom.xml b/deeplearning4j-core/pom.xml index 31da41ef3b7..ecd3a63dc53 100644 --- a/deeplearning4j-core/pom.xml +++ b/deeplearning4j-core/pom.xml @@ -36,6 +36,16 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/SameDiffTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/SameDiffTest.java index b7daccb85de..29de9219463 100644 --- a/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/SameDiffTest.java +++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/SameDiffTest.java @@ -1,5 +1,6 @@ package org.deeplearning4j.samediff; +import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.DenseLayer; @@ -7,19 +8,24 @@ import org.deeplearning4j.nn.params.DefaultParamInitializer; import org.deeplearning4j.samediff.testlayers.SameDiffDense; import org.junit.Test; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +@Slf4j public class SameDiffTest { @Test - public void testSameDiffDenseBasic(){ + public void testSameDiffDenseBasic() { int nIn = 3; int nOut = 4; @@ -32,7 +38,7 @@ public void testSameDiffDenseBasic(){ MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); - Map pt1 = net.getLayer(0).paramTable(); + Map pt1 = net.getLayer(0).paramTable(); assertNotNull(pt1); assertEquals(2, pt1.size()); assertNotNull(pt1.get(DefaultParamInitializer.WEIGHT_KEY)); @@ -43,8 +49,9 @@ public void testSameDiffDenseBasic(){ } @Test - public void testSameDiffDenseForward(){ + public void testSameDiffDenseForward() { + int minibatch = 5; int nIn = 3; int nOut = 4; @@ -56,21 +63,143 @@ public void testSameDiffDenseForward(){ MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); - Map pt1 = net.paramTable(); + Map pt1 = net.paramTable(); assertNotNull(pt1); System.out.println(pt1); -// MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() -// .list() -// .layer(new DenseLayer.Builder().activation(Activation.SIGMOID).nIn(nIn).nOut(nOut).build()) -// .build(); -// -// MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); -// net2.init(); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .list() + .layer(new DenseLayer.Builder().activation(Activation.SIGMOID).nIn(nIn).nOut(nOut).build()) + .build(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + net.params().assign(net2.params()); + INDArray in = Nd4j.rand(minibatch, nIn); + INDArray out = net.output(in); + INDArray outExp = net2.output(in); + assertEquals(outExp, out); } + @Test + public void testShapeResolutionMinus1() { + + int nIn = 3; + int nOut = 4; + + int minibatch = 3; + +// for(boolean useMinus1 : new boolean[]{false, true}) { + for (boolean useMinus1 : new boolean[]{true}) { + log.info("Starting: {}", (useMinus1 ? "minibatch -1" : "minibatch 3")); + + int[] inShape; + if (useMinus1) { + inShape = new int[]{-1, nIn}; + } else { + inShape = new int[]{minibatch, nIn}; + } + int[] wShape = new int[]{nIn, nOut}; + int[] bShape = new int[]{1, nOut}; + + SameDiff sd = SameDiff.create(); + SDVariable layerInput = sd.var("in", inShape); + SDVariable weights = sd.var("W", wShape); + SDVariable bias = sd.var("b", bShape); + + SDVariable mmul = sd.mmul("mmul", layerInput, weights); + SDVariable z = mmul.add("z", bias); + SDVariable out = sd.sigmoid("out", z); + + INDArray in = Nd4j.rand(new int[]{minibatch, nIn}); + INDArray w = Nd4j.rand(wShape); + INDArray b = Nd4j.rand(bShape); + + Map m = new HashMap<>(); + m.put("in", in); + m.put("W", w); + m.put("b", b); + + sd.associateArrayWithVariable(in, sd.getVariable("in")); + sd.associateArrayWithVariable(w, sd.getVariable("W")); + sd.associateArrayWithVariable(b, sd.getVariable("b")); + +// INDArray outArr = sd.execAndEndResult(); + + sd.addAsPlaceHolder("in"); + sd.addAsPlaceHolder("W"); + sd.addAsPlaceHolder("b"); + + sd.execWithPlaceHolder(m); + + INDArray outArr = sd.getVariable("out").getArr(); + + assertArrayEquals(new int[]{minibatch, nOut}, outArr.shape()); + } + } + + @Test + public void debug() { + + int nIn = 3; + int nOut = 4; + + int minibatch = 3; + + int[] inShape = new int[]{-1, nIn}; + int[] wShape = new int[]{nIn, nOut}; + int[] bShape = new int[]{1, nOut}; + + SameDiff sd = SameDiff.create(); + SDVariable layerInput = sd.var("in", inShape); + SDVariable weights = sd.var("W", wShape); + SDVariable bias = sd.var("b", bShape); + + assertArrayEquals(inShape, layerInput.getShape()); + assertArrayEquals(wShape, weights.getShape()); + + SDVariable mmul = sd.mmul("mmul", layerInput, weights); + SDVariable z = mmul.add("z", bias); + SDVariable out = sd.sigmoid("out", z); + + INDArray in = Nd4j.rand(new int[]{minibatch, nIn}); + INDArray w = Nd4j.rand(wShape); + INDArray b = Nd4j.rand(bShape); + + Map m = new HashMap<>(); + m.put("in", in); + m.put("W", w); + m.put("b", b); + + sd.associateArrayWithVariable(in, sd.getVariable("in")); + sd.associateArrayWithVariable(w, sd.getVariable("W")); + sd.associateArrayWithVariable(b, sd.getVariable("b")); + +// INDArray outArr = sd.execAndEndResult(); + + sd.addAsPlaceHolder("in"); + sd.addAsPlaceHolder("W"); + sd.addAsPlaceHolder("b"); + + sd.execWithPlaceHolder(m); + + INDArray outArr = sd.getVariable("out").getArr(); + + assertArrayEquals(new int[]{minibatch, nOut}, outArr.shape()); + } + + @Test + public void debug2() { + int[] inShape = new int[]{-1, 3}; + + SameDiff sd = SameDiff.create(); + SDVariable layerInput = sd.var("in", inShape); + + int[] actShape = layerInput.getShape(); //Getting: [1,3] + assertArrayEquals(inShape, actShape); + } } diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/testlayers/SameDiffDense.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/testlayers/SameDiffDense.java index 7055d277b44..56b00d66ac6 100644 --- a/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/testlayers/SameDiffDense.java +++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/testlayers/SameDiffDense.java @@ -66,13 +66,15 @@ public Map paramShapes() { } @Override - public void defineLayer(SameDiff sd, SDVariable layerInput, Map paramTable) { + public List defineLayer(SameDiff sd, SDVariable layerInput, Map paramTable) { SDVariable weights = paramTable.get(DefaultParamInitializer.WEIGHT_KEY); SDVariable bias = paramTable.get(DefaultParamInitializer.BIAS_KEY); SDVariable mmul = sd.mmul("mmul", layerInput, weights); SDVariable z = mmul.add("z", bias); SDVariable out = sd.sigmoid("out", z); + + return Collections.singletonList("out"); } public static class Builder extends BaseSameDiffLayer.Builder { diff --git a/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/BaseSameDiffLayer.java b/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/BaseSameDiffLayer.java index bc8f89ef419..1e181ae62b3 100644 --- a/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/BaseSameDiffLayer.java +++ b/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/BaseSameDiffLayer.java @@ -56,7 +56,7 @@ protected BaseSameDiffLayer(Builder builder){ public abstract Map paramShapes(); - public abstract void defineLayer(SameDiff sameDiff, SDVariable layerInput, Map paramTable); + public abstract List defineLayer(SameDiff sameDiff, SDVariable layerInput, Map paramTable); //================================================================================================================== diff --git a/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java b/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java index f40e64ee2ca..bcd64b1fed4 100644 --- a/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java +++ b/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java @@ -15,6 +15,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; public class SameDiffLayer extends AbstractLayer { @@ -22,7 +23,7 @@ public class SameDiffLayer extends AbstractLayer { private static final String INPUT_KEY = "input"; protected SameDiff sameDiff; - protected String outputKey; + protected List outputKeys; protected INDArray params; protected INDArray gradients; @@ -56,13 +57,15 @@ public INDArray activate(boolean training) { doInit(); } - SameDiff sd = sameDiff.getFunction(outputKey); //Build map: - Map map = new HashMap<>(paramTable()); - map.put(INPUT_KEY, input); +// Map map = new HashMap<>(paramTable()); +// map.put(INPUT_KEY, input); + + sameDiff.associateArrayWithVariable(input, sameDiff.getVariable(INPUT_KEY)); try(MemoryWorkspace ws = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) { - return sd.execAndEndResult(); + INDArray result = sameDiff.execAndEndResult(); + return result; } } @@ -76,16 +79,15 @@ public INDArray preOutput(boolean training) { public Pair backpropGradient(INDArray epsilon) { Gradient g = new DefaultGradient(); - SameDiff sd = sameDiff.getFunction(outputKey); INDArray dLdIn; try(MemoryWorkspace ws = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()){ - sd.execBackwards(); + sameDiff.execBackwards(); for(String s : layerConf().paramKeys() ){ - INDArray pg = sd.grad(s).getArr(); + INDArray pg = sameDiff.grad(s).getArr(); g.gradientForVariable().put(s, pg); } - dLdIn = sd.grad(INPUT_KEY).getArr(); + dLdIn = sameDiff.grad(INPUT_KEY).getArr(); } return new Pair<>(g, dLdIn); @@ -183,14 +185,24 @@ protected void doInit(){ Map p = paramTable(); int[] inputShape = input.shape().clone(); - inputShape[0] = -1; - SDVariable inputVar = sameDiff.var(INPUT_KEY, inputShape); //TODO WHAT ABOUT VARIABLE SIZES? +// inputShape[0] = -1; //TODO THIS DOESN'T ENABLE VARIABLE SIZE MINIBATCHES + SDVariable inputVar = sameDiff.var(INPUT_KEY, inputShape); Map paramShapes = layerConf().paramShapes(); Map params = new LinkedHashMap<>(); for(String s : layerConf().paramKeys()){ int[] ps = paramShapes.get(s); - params.put(s, sameDiff.var(s, ps)); + SDVariable v = sameDiff.var(s, ps); + params.put(s, v); + } + List outputKeys = layerConf().defineLayer(sameDiff, inputVar, params); + if(outputKeys == null || outputKeys.size() != 1){ + throw new IllegalStateException("Invalid output keys: " + outputKeys); } - layerConf().defineLayer(sameDiff, inputVar, params); + + for(Map.Entry e : p.entrySet()){ + sameDiff.associateArrayWithVariable(e.getValue(), sameDiff.getVariable(e.getKey())); + } + + this.outputKeys = outputKeys; } }