Skip to content

Commit

Permalink
Start plugging new dropout in to layers/config
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexDBlack
AlexDBlack committed Aug 31, 2017
1 parent 8358b8a commit d2ce0ae
Show file tree
Hide file tree
Showing 7 changed files with 66 additions and 41 deletions.
16 changes: 11 additions & 5 deletions deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java
View file
Expand Up @@ -27,6 +27,8 @@
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.api.layers.LayerConstraint;
import org.deeplearning4j.nn.conf.distribution.Distribution;
import org.deeplearning4j.nn.conf.dropout.Dropout;
import org.deeplearning4j.nn.conf.dropout.IDropout;
import org.deeplearning4j.nn.conf.graph.GraphVertex;
import org.deeplearning4j.nn.conf.inputs.InputType;
import org.deeplearning4j.nn.conf.layers.*;
Expand Down Expand Up @@ -599,7 +601,7 @@ public static class Builder implements Cloneable {
protected double l2 = Double.NaN;
protected double l1Bias = Double.NaN;
protected double l2Bias = Double.NaN;
protected double dropOut = 0;
protected IDropout idropOut = null;
@Deprecated
protected Updater updater = Updater.SGD;
protected IUpdater iUpdater = new Sgd();
Expand Down Expand Up @@ -1036,7 +1038,11 @@ public Builder l2Bias(double l2Bias) {
* @param inputRetainProbability Dropout probability (probability of retaining each input activation value for a layer)
*/
public Builder dropOut(double inputRetainProbability) {
this.dropOut = inputRetainProbability;
return dropOut(new Dropout(inputRetainProbability));
}

public Builder dropOut(IDropout dropout){
this.idropOut = dropout;
return this;
}

Expand Down Expand Up @@ -1367,14 +1373,14 @@ private void configureLayer(Layer layer) {
sl.setConvolutionMode(convolutionMode);
}
}
LayerValidation.generalValidation(layerName, layer, useDropConnect, dropOut, l2, l2Bias,
LayerValidation.generalValidation(layerName, layer, useDropConnect, idropOut, l2, l2Bias,
l1, l1Bias, dist, constraints);
}

private void copyConfigToLayer(String layerName, Layer layer) {

if (Double.isNaN(layer.getDropOut()))
layer.setDropOut(dropOut);
if (layer.getIDropout() == null)
layer.setIDropout(idropOut);

if (layer instanceof BaseLayer) {
BaseLayer bLayer = (BaseLayer) layer;
Expand Down
6 changes: 4 additions & 2 deletions deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DropoutLayer.java
View file
Expand Up @@ -7,6 +7,7 @@
import org.deeplearning4j.nn.api.ParamInitializer;
import org.deeplearning4j.nn.conf.InputPreProcessor;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.dropout.Dropout;
import org.deeplearning4j.nn.conf.inputs.InputType;
import org.deeplearning4j.nn.conf.memory.LayerMemoryReport;
import org.deeplearning4j.nn.conf.memory.MemoryReport;
Expand Down Expand Up @@ -107,8 +108,9 @@ public LayerMemoryReport getMemoryReport(InputType inputType) {

@NoArgsConstructor
public static class Builder extends FeedForwardLayer.Builder<DropoutLayer.Builder> {
public Builder(double dropOut) {
this.dropOut = dropOut;

public Builder(double dropout){
this.dropOut(new Dropout(dropout));
}

@Override
Expand Down
18 changes: 12 additions & 6 deletions deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java
View file
Expand Up @@ -25,6 +25,8 @@
import org.deeplearning4j.nn.conf.InputPreProcessor;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.dropout.Dropout;
import org.deeplearning4j.nn.conf.dropout.IDropout;
import org.deeplearning4j.nn.conf.inputs.InputType;
import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer;
import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder;
Expand Down Expand Up @@ -73,13 +75,13 @@
@NoArgsConstructor
public abstract class Layer implements Serializable, Cloneable {
protected String layerName;
protected double dropOut;
protected IDropout iDropout;
protected List<LayerConstraint> constraints;


public Layer(Builder builder) {
this.layerName = builder.layerName;
this.dropOut = builder.dropOut;
this.iDropout = builder.iDropout;
this.constraints = builder.constraints;
}

Expand All @@ -90,7 +92,7 @@ public Layer(Builder builder) {
*/
public void resetLayerDefaultConfig() {
//clear the learning related params for all layers in the origConf and set to defaults
this.setDropOut(Double.NaN);
this.iDropout = null;
}

@Override
Expand Down Expand Up @@ -218,7 +220,7 @@ public IUpdater getIUpdaterByParam(String paramName) {
@SuppressWarnings("unchecked")
public abstract static class Builder<T extends Builder<T>> {
protected String layerName = null;
protected double dropOut = Double.NaN;
protected IDropout iDropout;
protected List<LayerConstraint> constraints = null;

/**
Expand Down Expand Up @@ -253,8 +255,12 @@ public T name(String layerName) {
* @param inputRetainProbability Dropout probability (probability of retaining each input activation value for a layer)
*/
public T dropOut(double inputRetainProbability) {
this.dropOut = inputRetainProbability;
return (T) this;
return dropOut(new Dropout(inputRetainProbability));
}

public T dropOut(IDropout dropout){
this.iDropout = dropout;
return (T)this;
}

/**
Expand Down
29 changes: 17 additions & 12 deletions deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerValidation.java
View file
Expand Up @@ -5,6 +5,7 @@
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.distribution.Distribution;
import org.deeplearning4j.nn.conf.distribution.NormalDistribution;
import org.deeplearning4j.nn.conf.dropout.IDropout;
import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.util.OneTimeLogger;
Expand Down Expand Up @@ -290,34 +291,34 @@ else if (momentumSchedule == null && layer.getMomentumSchedule() == null)
}
}

public static void generalValidation(String layerName, Layer layer, boolean useDropConnect, Double dropOut,
public static void generalValidation(String layerName, Layer layer, boolean useDropConnect, IDropout iDropOut,
Double l2, Double l2Bias, Double l1, Double l1Bias,
Distribution dist, List<LayerConstraint> constraints) {
generalValidation(layerName, layer, useDropConnect, dropOut == null ? 0.0 : dropOut,
generalValidation(layerName, layer, useDropConnect, iDropOut,
l2 == null ? Double.NaN : l2, l2Bias == null ? Double.NaN : l2Bias,
l1 == null ? Double.NaN : l1, l1Bias == null ? Double.NaN : l1Bias, dist, constraints);
}

public static void generalValidation(String layerName, Layer layer, boolean useDropConnect, double dropOut,
public static void generalValidation(String layerName, Layer layer, boolean useDropConnect, IDropout iDropout,
double l2, double l2Bias, double l1, double l1Bias,
Distribution dist, List<LayerConstraint> constraints) {

if (layer != null) {

if (useDropConnect && (Double.isNaN(dropOut) && (Double.isNaN(layer.getDropOut()))))
OneTimeLogger.warn(log, "Layer \"" + layerName
+ "\" dropConnect is set to true but dropout rate has not been added to configuration.");
if (useDropConnect && layer.getDropOut() == 0.0)
OneTimeLogger.warn(log,
"Layer \"" + layerName + " dropConnect is set to true but dropout rate is set to 0.0");
// if (useDropConnect && (Double.isNaN(dropOut) && (Double.isNaN(layer.getDropOut()))))
// OneTimeLogger.warn(log, "Layer \"" + layerName
// + "\" dropConnect is set to true but dropout rate has not been added to configuration.");
// if (useDropConnect && layer.getDropOut() == 0.0)
// OneTimeLogger.warn(log,
// "Layer \"" + layerName + " dropConnect is set to true but dropout rate is set to 0.0");

if (layer instanceof BaseLayer) {
BaseLayer bLayer = (BaseLayer) layer;
configureBaseLayer(layerName, bLayer, useDropConnect, dropOut, l2, l2Bias, l1,
configureBaseLayer(layerName, bLayer, useDropConnect, iDropout, l2, l2Bias, l1,
l1Bias, dist);
} else if (layer instanceof FrozenLayer && ((FrozenLayer) layer).getLayer() instanceof BaseLayer) {
BaseLayer bLayer = (BaseLayer) ((FrozenLayer) layer).getLayer();
configureBaseLayer(layerName, bLayer, useDropConnect, dropOut, l2, l2Bias, l1,
configureBaseLayer(layerName, bLayer, useDropConnect, iDropout, l2, l2Bias, l1,
l1Bias, dist);
}

Expand All @@ -330,7 +331,7 @@ public static void generalValidation(String layerName, Layer layer, boolean useD
}

private static void configureBaseLayer(String layerName, BaseLayer bLayer, boolean useDropConnect,
Double dropOut, Double l2, Double l2Bias, Double l1, Double l1Bias,
IDropout iDropout, Double l2, Double l2Bias, Double l1, Double l1Bias,
Distribution dist) {

if (!Double.isNaN(l1) && Double.isNaN(bLayer.getL1())) {
Expand Down Expand Up @@ -359,6 +360,10 @@ private static void configureBaseLayer(String layerName, BaseLayer bLayer, bool
bLayer.setL1Bias(0.0);
}

if(bLayer.getIDropout() == null){
bLayer.setIDropout(iDropout);
}


if (bLayer.getWeightInit() == WeightInit.DISTRIBUTION) {
if (dist != null && bLayer.getDist() == null)
Expand Down
5 changes: 5 additions & 0 deletions deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/schedule/MapSchedule.java
View file
Expand Up @@ -30,6 +30,11 @@ public double valueAt(double currentValue, int iteration, int epoch) {
}
}

@Override
public ISchedule clone() {
return new MapSchedule(scheduleType, values);
}

public static class Builder {

private ScheduleType scheduleType;
Expand Down
18 changes: 9 additions & 9 deletions deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java
View file
Expand Up @@ -355,18 +355,18 @@ public void clear() {
input = null;
}

protected void applyDropOutIfNecessary(boolean training) {
if (layerConf().getDropOut() > 0 && !conf.isUseDropConnect() && training && !dropoutApplied) {
protected void applyDropOutIfNecessary(boolean training, int iteration, int epoch) {
if(layerConf().getIDropout() != null && training ){
//TODO: Epoch + iteration counters...
if (Nd4j.getWorkspaceManager().checkIfWorkspaceExists(ComputationGraph.workspaceExternal)) {
try (MemoryWorkspace ws = Nd4j.getWorkspaceManager()
.getWorkspaceForCurrentThread(ComputationGraph.workspaceExternal)
.notifyScopeBorrowed()) {
input = input.isView() ? input.dup() : input.unsafeDuplication();
.getWorkspaceForCurrentThread(ComputationGraph.workspaceExternal)
.notifyScopeBorrowed()) {
input = layerConf().getIDropout().applyDropout(input, iteration, epoch, false);
}
} else
input = input.isView() ? input.dup() : input.unsafeDuplication();

Dropout.applyDropout(input, layerConf().getDropOut());
} else {
input = layerConf().getIDropout().applyDropout(input, iteration, epoch, false);
}
dropoutApplied = true;
}
}
Expand Down
15 changes: 8 additions & 7 deletions ...ning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java
View file
Expand Up @@ -8,6 +8,7 @@
import org.deeplearning4j.nn.api.layers.LayerConstraint;
import org.deeplearning4j.nn.conf.*;
import org.deeplearning4j.nn.conf.distribution.Distribution;
import org.deeplearning4j.nn.conf.dropout.IDropout;
import org.deeplearning4j.nn.conf.layers.*;
import org.deeplearning4j.nn.conf.stepfunctions.StepFunction;
import org.deeplearning4j.nn.weights.WeightInit;
Expand Down Expand Up @@ -45,7 +46,8 @@ public class FineTuneConfiguration {
protected Double l2;
protected Double l1Bias;
protected Double l2Bias;
protected Double dropOut;
// protected Double dropOut;
protected IDropout iDropout;
@Deprecated
protected Updater updater;
protected IUpdater iUpdater;
Expand Down Expand Up @@ -145,8 +147,8 @@ public void applyToNeuralNetConfiguration(NeuralNetConfiguration nnc) {
WeightInit origWeightInit = null;

if (l != null) {
if (dropOut != null)
l.setDropOut(dropOut);
if (iDropout != null)
l.setIDropout(iDropout);
}

if (l != null && l instanceof BaseLayer) {
Expand Down Expand Up @@ -285,9 +287,8 @@ public void applyToNeuralNetConfiguration(NeuralNetConfiguration nnc) {
if (l != null) {
LayerValidation.updaterValidation(l.getLayerName(), l, learningRate, momentum, momentumSchedule,
adamMeanDecay, adamVarDecay, rho, rmsDecay, epsilon);

boolean useDropCon = (useDropConnect == null ? nnc.isUseDropConnect() : useDropConnect);
LayerValidation.generalValidation(l.getLayerName(), l, useDropCon, dropOut, l2, l2Bias, l1, l1Bias, dist, constraints);
LayerValidation.generalValidation(l.getLayerName(), l, useDropCon, iDropout, l2, l2Bias, l1, l1Bias, dist, constraints);
}

//Also: update the LR, L1 and L2 maps, based on current config (which might be different to original config)
Expand Down Expand Up @@ -352,8 +353,8 @@ public NeuralNetConfiguration.Builder appliedNeuralNetConfigurationBuilder() {
confBuilder.setL1Bias(l1Bias);
if (l2Bias != null)
confBuilder.setL2Bias(l2Bias);
if (dropOut != null)
confBuilder.setDropOut(dropOut);
if (iDropout != null)
confBuilder.setIdropOut(iDropout);
if (iUpdater != null)
confBuilder.updater(iUpdater);
if (updater != null)
Expand Down

0 comments on commit d2ce0ae

Please sign in to comment.