Skip to content

Commit

Permalink
SameDiff loss layer: implement + test computeScoreForExamples
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexDBlack
AlexDBlack committed Feb 19, 2018
1 parent 78eaf33 commit 43f85ef
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 67 deletions.
2 changes: 1 addition & 1 deletion deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/TestSameDiffConv.java
View file
Expand Up @@ -76,7 +76,7 @@ public void testSameDiffConvForward() {
//With n=1, m=3 this is 1 out of every 3 tests (on average)
Random r = new Random(12345);
int n = 1;
int m = 3;
int m = 5;
for (int minibatch : new int[]{5, 1}) {

Activation[] afns = new Activation[]{
Expand Down
150 changes: 86 additions & 64 deletions deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/TestSameDiffLoss.java
View file
Expand Up @@ -16,6 +16,7 @@
import org.nd4j.autodiff.samediff.SameDiff;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import org.nd4j.linalg.primitives.Pair;
Expand Down Expand Up @@ -194,6 +195,9 @@ public void testReductionsBackwards2() {
@Test
public void testSameDiffLossVsDl4j() {

double[] l1s = new double[]{0.0, 0.0, 0.4, 0.4};
double[] l2s = new double[]{0.0, 0.3, 0.0, 0.3};

for (int minibatch : new int[]{5, 1}) {
int nIn = 3;
int nOut = 4;
Expand All @@ -211,70 +215,88 @@ public void testSameDiffLossVsDl4j() {
};

for (int i = 0; i < lossFns.length; i++) {
LossFunctions.LossFunction lf = lossFns[i];
log.info("Starting test - " + lf);
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.list()
.layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.TANH).build())
.layer(new SameDiffLoss.Builder()
.lossFunction(lf)
.build())
.build();

MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();

assertNotNull(net.paramTable());

MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
.list()
.layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.TANH).build())
.layer(new LossLayer.Builder()
.lossFunction(lf)
.activation(Activation.IDENTITY)
.build())
.build();

MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
net2.init();

net.params().assign(net2.params());

//Check params:
assertEquals(net2.params(), net.params());
Map<String, INDArray> params1 = net.paramTable();
Map<String, INDArray> params2 = net2.paramTable();
assertEquals(params2, params1);

INDArray in = Nd4j.rand(minibatch, nIn);
INDArray out = net.output(in);
INDArray outExp = net2.output(in);

assertEquals(outExp, out);

//Check scores:
INDArray label = Nd4j.rand(minibatch, nOut);
net.setLabels(label);
net2.setLabels(label);

net.computeGradientAndScore();
net2.computeGradientAndScore();

double scoreExp = net2.score();
double scoreAct = net.score();
assertTrue(scoreExp > 0);
assertEquals(scoreExp, scoreAct, 1e-6);

INDArray gradExp = net2.getFlattenedGradients();
INDArray gradAct = net.getFlattenedGradients();

assertEquals(gradExp, gradAct);

//Also check serialization:
MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
INDArray outLoaded = netLoaded.output(in);

assertEquals(outExp, outLoaded);

for( int j=0; j<l1s.length; j++ ) {
double l1 = l1s[j];
double l2 = l2s[j];

LossFunctions.LossFunction lf = lossFns[i];
String msg = "Starting test - " + lf + ", minibatch=" + minibatch + ", l1=" + l1 + ", l2=" + l2;
log.info(msg);

MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.list()
.layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.TANH).build())
.layer(new SameDiffLoss.Builder()
.lossFunction(lf)
.build())
.build();

MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();

assertNotNull(msg, net.paramTable());

MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
.list()
.layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.TANH).build())
.layer(new LossLayer.Builder()
.lossFunction(lf)
.activation(Activation.IDENTITY)
.build())
.build();

MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
net2.init();

net.params().assign(net2.params());

//Check params:
assertEquals(msg, net2.params(), net.params());
Map<String, INDArray> params1 = net.paramTable();
Map<String, INDArray> params2 = net2.paramTable();
assertEquals(msg, params2, params1);

INDArray in = Nd4j.rand(minibatch, nIn);
INDArray out = net.output(in);
INDArray outExp = net2.output(in);

assertEquals(msg, outExp, out);

//Check scores:
INDArray label = Nd4j.rand(minibatch, nOut);
net.setLabels(label);
net2.setLabels(label);

net.computeGradientAndScore();
net2.computeGradientAndScore();

double scoreExp = net2.score();
double scoreAct = net.score();
assertTrue(msg, scoreExp > 0);
assertEquals(msg, scoreExp, scoreAct, 1e-6);

//Test: computeScoreForExamples
for(boolean includeReg : new boolean[]{true, false}) {
INDArray expScoreForEx = net2.scoreExamples(new DataSet(in, label), true);
INDArray actScoreForEx = net.scoreExamples(new DataSet(in, label), true);

String msg2 = msg + ", addRegTerms=" + includeReg;
assertEquals(msg2, expScoreForEx, actScoreForEx);
}

//TODO GRADIENTS NEED FIXING - maybe related: https://github.com/deeplearning4j/nd4j/issues/2485
INDArray gradExp = net2.getFlattenedGradients();
INDArray gradAct = net.getFlattenedGradients();

assertEquals(gradExp, gradAct);

//Also check serialization:
MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
INDArray outLoaded = netLoaded.output(in);

assertEquals(outExp, outLoaded);
}
}
}
}
Expand Down
7 changes: 6 additions & 1 deletion deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/testlayers/SameDiffLoss.java
View file
Expand Up @@ -35,7 +35,12 @@ private SameDiffLoss() {
}

public Pair<String,String> lossKeys() {
return new Pair<>("lossPerEx", "score");
return new Pair<>(lossPerExampleVar(), "score");
}

@Override
public String lossPerExampleVar(){
return "lossPerEx";
}

@Override
Expand Down
2 changes: 2 additions & 0 deletions ...4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/BaseSameDiffLossLayer.java
View file
Expand Up @@ -23,6 +23,8 @@ protected BaseSameDiffLossLayer(){

public abstract void defineLayer(SameDiff sameDiff, SDVariable layerInput, SDVariable label);

public abstract String lossPerExampleVar();

@Override
public Layer instantiate(NeuralNetConfiguration conf, Collection<IterationListener> iterationListeners, int layerIndex, INDArray layerParamsView, boolean initializeParams) {
SameDiffLossLayer ret = new SameDiffLossLayer(conf);
Expand Down
17 changes: 16 additions & 1 deletion deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLossLayer.java
View file
@@ -1,5 +1,6 @@
package org.deeplearning4j.nn.layers.samediff;

import com.google.common.base.Preconditions;
import lombok.Getter;
import lombok.Setter;
import org.deeplearning4j.nn.api.Layer;
Expand Down Expand Up @@ -60,7 +61,21 @@ public double computeScore(double fullNetworkL1, double fullNetworkL2, boolean t

@Override
public INDArray computeScoreForExamples(double fullNetworkL1, double fullNetworkL2) {
throw new UnsupportedOperationException("Not yet implemented");
Preconditions.checkNotNull(input, "Input cannot be null when calculating score for examples");
Preconditions.checkNotNull(labels, "Labels cannot be null when calculating score for exapmles");

sameDiff.associateArrayWithVariable(input, inVar);
sameDiff.associateArrayWithVariable(labels, labelVar);

sameDiff.exec();

//Next: need to determine name of the "score for each example" component...
String key = ((BaseSameDiffLossLayer)conf().getLayer()).lossPerExampleVar();
INDArray out = sameDiff.getVariable(key).getArr();
if(fullNetworkL1 > 0 || fullNetworkL2 > 0){
out.addi(fullNetworkL1 + fullNetworkL2);
}
return out;
}


Expand Down

0 comments on commit 43f85ef

Please sign in to comment.