Arbiter enhancements

arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/BooleanSpace.java

@@ -0,0 +1,60 @@
+package org.deeplearning4j.arbiter.optimize.parameter;
+
+import lombok.EqualsAndHashCode;
+import org.deeplearning4j.arbiter.optimize.api.ParameterSpace;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * BooleanParameterSpace is a {@code ParameterSpace<Boolean>}; Defines {True, False} as a parameter space
+ * If argument to setValue is less than or equal to 0.5 it will return True else False
+ *
+ * @author susaneraly
+ */
+@EqualsAndHashCode
+public class BooleanSpace implements ParameterSpace<Boolean> {
+    private int index = -1;
+
+    @Override
+    public Boolean getValue(double[] input) {
+        if (index == -1) {
+            throw new IllegalStateException("Cannot get value: ParameterSpace index has not been set");
+        }
+        if (input[index] <= 0.5) return Boolean.TRUE;
+        else return Boolean.FALSE;
+    }
+
+    @Override
+    public int numParameters() {
+        return 1;
+    }
+
+    @Override
+    public List<ParameterSpace> collectLeaves() {
+        return Collections.singletonList((ParameterSpace) this);
+    }
+
+    @Override
+    public Map<String, ParameterSpace> getNestedSpaces() {
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public boolean isLeaf() {
+        return true;
+    }
+
+    @Override
+    public void setIndices(int... indices) {
+        if (indices == null || indices.length != 1)
+            throw new IllegalArgumentException("Invalid index");
+        this.index = indices[0];
+    }
+
+    @Override
+    public String toString() {
+        return "BooleanSpace()";
+    }
+}

arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestJson.java

@@ -24,6 +24,7 @@
 import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider;
 import org.deeplearning4j.arbiter.optimize.generator.GridSearchCandidateGenerator;
 import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator;
+import org.deeplearning4j.arbiter.optimize.parameter.BooleanSpace;
 import org.deeplearning4j.arbiter.optimize.parameter.FixedValue;
 import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace;
 import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace;
@@ -79,6 +80,7 @@ public void testParameterSpaceJson() throws Exception {
         l.add(new DiscreteParameterSpace<>("first", "second", "third"));
         l.add(new IntegerParameterSpace(0, 10));
         l.add(new IntegerParameterSpace(new UniformIntegerDistribution(0, 50)));
+        l.add(new BooleanSpace());
 
         for (ParameterSpace<?> ps : l) {
             String strJson = jsonMapper.writeValueAsString(ps);

arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/parameter/TestParameterSpaces.java

@@ -36,7 +36,7 @@ public void testContinuousParameterSpace() {
 
         for (int i = 0; i < 10; i++) {
             double d = i / 10.0;
-            assertEquals(d, cps.getValue(new double[] {d}), 0.0);
+            assertEquals(d, cps.getValue(new double[]{d}), 0.0);
         }
 
         cps = new ContinuousParameterSpace(10, 20);
@@ -45,7 +45,7 @@ public void testContinuousParameterSpace() {
         for (int i = 0; i < 10; i++) {
             double d = i / 10.0;
             double exp = d * 10 + 10;
-            assertEquals(exp, cps.getValue(new double[] {d}), 0.0);
+            assertEquals(exp, cps.getValue(new double[]{d}), 0.0);
         }
 
 
@@ -54,7 +54,7 @@ public void testContinuousParameterSpace() {
         cps.setIndices(0);
         for (int i = 0; i < 11; i++) {
             double d = i / 10.0;
-            assertEquals(nd.inverseCumulativeProbability(d), cps.getValue(new double[] {d}), 1e-4);
+            assertEquals(nd.inverseCumulativeProbability(d), cps.getValue(new double[]{d}), 1e-4);
         }
     }
 
@@ -67,9 +67,9 @@ public void testDiscreteParameterSpace() {
             double d = i / 5.0 + 0.1; //Center
             double dEdgeLower = i / 5.0 + 1e-8; //Edge case: just above split threshold
             double dEdgeUpper = (i + 1) / 5.0 - 1e-8; //Edge case: just below split threshold
-            assertEquals(i, (int) dps.getValue(new double[] {d}));
-            assertEquals(i, (int) dps.getValue(new double[] {dEdgeLower}));
-            assertEquals(i, (int) dps.getValue(new double[] {dEdgeUpper}));
+            assertEquals(i, (int) dps.getValue(new double[]{d}));
+            assertEquals(i, (int) dps.getValue(new double[]{dEdgeLower}));
+            assertEquals(i, (int) dps.getValue(new double[]{dEdgeUpper}));
         }
     }
 
@@ -82,10 +82,21 @@ public void testIntegerParameterSpace() {
             double d = i / 5.0 + 0.1; //Center
             double dEdgeLower = i / 5.0 + 1e-8; //Edge case: just above split threshold
             double dEdgeUpper = (i + 1) / 5.0 - 1e-8; //Edge case: just below split threshold
-            assertEquals(i, (int) ips.getValue(new double[] {d}));
-            assertEquals(i, (int) ips.getValue(new double[] {dEdgeLower}));
-            assertEquals(i, (int) ips.getValue(new double[] {dEdgeUpper}));
+            assertEquals(i, (int) ips.getValue(new double[]{d}));
+            assertEquals(i, (int) ips.getValue(new double[]{dEdgeLower}));
+            assertEquals(i, (int) ips.getValue(new double[]{dEdgeUpper}));
         }
     }
 
+    @Test
+    public void testBooleanSpace() {
+        ParameterSpace<Boolean> bSpace = new BooleanSpace();
+        bSpace.setIndices(1); //randomly setting to non zero
+
+        assertEquals(true, (boolean) bSpace.getValue(new double[]{0.0, 0.0}));
+        assertEquals(true, (boolean) bSpace.getValue(new double[]{0.1, 0.5}));
+        assertEquals(false, (boolean) bSpace.getValue(new double[]{0.2, 0.7}));
+        assertEquals(false, (boolean) bSpace.getValue(new double[]{0.3, 1.0}));
+    }
+
 }

arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/BaseNetworkSpace.java

@@ -30,6 +30,7 @@
 import org.deeplearning4j.arbiter.optimize.serde.jackson.YamlMapper;
 import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration;
 import org.deeplearning4j.nn.api.OptimizationAlgorithm;
+import org.deeplearning4j.nn.api.layers.LayerConstraint;
 import org.deeplearning4j.nn.conf.*;
 import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.dropout.Dropout;
@@ -44,6 +45,7 @@
 import org.nd4j.shade.jackson.core.JsonProcessingException;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -72,6 +74,8 @@ public abstract class BaseNetworkSpace<T> extends AbstractParameterSpace<T> {
     protected ParameterSpace<StepFunction> stepFunction;
     protected ParameterSpace<Double> l1;
     protected ParameterSpace<Double> l2;
+    protected ParameterSpace<Double> l1Bias;
+    protected ParameterSpace<Double> l2Bias;
     protected ParameterSpace<IUpdater> updater;
     protected ParameterSpace<IUpdater> biasUpdater;
     protected ParameterSpace<IWeightNoise> weightNoise;
@@ -89,6 +93,10 @@ public abstract class BaseNetworkSpace<T> extends AbstractParameterSpace<T> {
     protected ParameterSpace<Integer> tbpttFwdLength;
     protected ParameterSpace<Integer> tbpttBwdLength;
 
+    protected ParameterSpace<List<LayerConstraint>> allParamConstraints;
+    protected ParameterSpace<List<LayerConstraint>> weightConstraints;
+    protected ParameterSpace<List<LayerConstraint>> biasConstraints;
+
     protected int numEpochs = 1;
 
 
@@ -111,13 +119,18 @@ protected BaseNetworkSpace(Builder builder) {
         this.stepFunction = builder.stepFunction;
         this.l1 = builder.l1;
         this.l2 = builder.l2;
+        this.l1Bias = builder.l1Bias;
+        this.l2Bias = builder.l2Bias;
         this.updater = builder.updater;
         this.biasUpdater = builder.biasUpdater;
         this.weightNoise = builder.weightNoise;
         this.dropout = builder.dropout;
         this.gradientNormalization = builder.gradientNormalization;
         this.gradientNormalizationThreshold = builder.gradientNormalizationThreshold;
         this.convolutionMode = builder.convolutionMode;
+        this.allParamConstraints = builder.allParamConstraints;
+        this.weightConstraints = builder.weightConstraints;
+        this.biasConstraints = builder.biasConstraints;
 
 
         this.backprop = builder.backprop;
@@ -161,6 +174,10 @@ protected NeuralNetConfiguration.Builder randomGlobalConf(double[] values) {
             builder.l1(l1.getValue(values));
         if (l2 != null)
             builder.l2(l2.getValue(values));
+        if (l1Bias != null)
+            builder.l1Bias(l1Bias.getValue(values));
+        if (l2Bias != null)
+            builder.l2Bias(l2Bias.getValue(values));
         if (updater != null)
             builder.updater(updater.getValue(values));
         if (biasUpdater != null)
@@ -175,6 +192,24 @@ protected NeuralNetConfiguration.Builder randomGlobalConf(double[] values) {
             builder.gradientNormalizationThreshold(gradientNormalizationThreshold.getValue(values));
         if (convolutionMode != null)
             builder.convolutionMode(convolutionMode.getValue(values));
+        if (allParamConstraints != null){
+            List<LayerConstraint> c = allParamConstraints.getValue(values);
+            if(c != null){
+                builder.constrainAllParameters(c.toArray(new LayerConstraint[c.size()]));
+            }
+        }
+        if (weightConstraints != null){
+            List<LayerConstraint> c = weightConstraints.getValue(values);
+            if(c != null){
+                builder.constrainWeights(c.toArray(new LayerConstraint[c.size()]));
+            }
+        }
+        if (biasConstraints != null){
+            List<LayerConstraint> c = biasConstraints.getValue(values);
+            if(c != null){
+                builder.constrainBias(c.toArray(new LayerConstraint[c.size()]));
+            }
+        }
 
         return builder;
     }
@@ -253,6 +288,8 @@ protected abstract static class Builder<T extends Builder<T>> {
         private ParameterSpace<StepFunction> stepFunction;
         private ParameterSpace<Double> l1;
         private ParameterSpace<Double> l2;
+        private ParameterSpace<Double> l1Bias;
+        private ParameterSpace<Double> l2Bias;
         private ParameterSpace<IUpdater> updater;
         private ParameterSpace<IUpdater> biasUpdater;
         private ParameterSpace<IWeightNoise> weightNoise;
@@ -261,6 +298,10 @@ protected abstract static class Builder<T extends Builder<T>> {
         private ParameterSpace<Double> gradientNormalizationThreshold;
         private ParameterSpace<ConvolutionMode> convolutionMode;
 
+        private ParameterSpace<List<LayerConstraint>> allParamConstraints;
+        private ParameterSpace<List<LayerConstraint>> weightConstraints;
+        private ParameterSpace<List<LayerConstraint>> biasConstraints;
+
         //NeuralNetConfiguration.ListBuilder/MultiLayerConfiguration.Builder<T> options:
         private ParameterSpace<Boolean> backprop;
         private ParameterSpace<Boolean> pretrain;
@@ -380,6 +421,23 @@ public T l2(ParameterSpace<Double> l2) {
             this.l2 = l2;
             return (T) this;
         }
+        public T l1Bias(double l1Bias) {
+            return l1Bias(new FixedValue<>(l1Bias));
+        }
+
+        public T l1Bias(ParameterSpace<Double> l1Bias) {
+            this.l1Bias = l1Bias;
+            return (T) this;
+        }
+
+        public T l2Bias(double l2Bias) {
+            return l2Bias(new FixedValue<>(l2Bias));
+        }
+
+        public T l2Bias(ParameterSpace<Double> l2Bias) {
+            this.l2Bias = l2Bias;
+            return (T) this;
+        }
 
         public T updater(IUpdater updater){
             return updater(new FixedValue<>(updater));
@@ -497,6 +555,33 @@ public T tbpttBwdLength(ParameterSpace<Integer> tbpttBwdLength) {
             return (T) this;
         }
 
+        public T constrainWeights(LayerConstraint... constraints){
+            return constrainWeights(new FixedValue<List<LayerConstraint>>(Arrays.asList(constraints)));
+        }
+
+        public T constrainWeights(ParameterSpace<List<LayerConstraint>> constraints){
+            this.weightConstraints = constraints;
+            return (T) this;
+        }
+
+        public T constrainBias(LayerConstraint... constraints){
+            return constrainBias(new FixedValue<List<LayerConstraint>>(Arrays.asList(constraints)));
+        }
+
+        public T constrainBias(ParameterSpace<List<LayerConstraint>> constraints){
+            this.biasConstraints = constraints;
+            return (T) this;
+        }
+
+        public T constrainAllParams(LayerConstraint... constraints){
+            return constrainAllParams(new FixedValue<List<LayerConstraint>>(Arrays.asList(constraints)));
+        }
+
+        public T constrainAllParams(ParameterSpace<List<LayerConstraint>> constraints){
+            this.allParamConstraints = constraints;
+            return (T) this;
+        }
+
         /**
          * Fixed number of training epochs. Default: 1
          * Note if both EarlyStoppingConfiguration and number of epochs is present, early stopping will be used in preference.

arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/MultiLayerSpace.java

@@ -219,28 +219,41 @@ public Builder layer(Layer layer){
         }
 
         public Builder layer(LayerSpace<?> layerSpace) {
-            return layer(layerSpace, new FixedValue<>(1), true);
+            return layer(layerSpace, new FixedValue<>(1));
         }
 
-        public Builder layer(LayerSpace<? extends Layer> layerSpace, ParameterSpace<Integer> numLayersDistribution,
-                                boolean duplicateConfig) {
-            return addLayer(layerSpace, numLayersDistribution, duplicateConfig);
+        public Builder layer(LayerSpace<? extends Layer> layerSpace, ParameterSpace<Integer> numLayersDistribution) {
+            return addLayer(layerSpace, numLayersDistribution);
         }
 
 
         public Builder addLayer(LayerSpace<?> layerSpace) {
-            return addLayer(layerSpace, new FixedValue<>(1), true);
+            return addLayer(layerSpace, new FixedValue<>(1));
+        }
+
+        /**
+         * duplicateConfig not supported. Will always be true
+         * @param layerSpace
+         * @param numLayersDistribution
+         * @param duplicateConfig
+         * @return
+         */
+        @Deprecated
+        public Builder addLayer(LayerSpace<? extends Layer> layerSpace, ParameterSpace<Integer> numLayersDistribution, boolean duplicateConfig) {
+            if (!duplicateConfig) throw new IllegalArgumentException("Duplicate Config false not supported");
+            String layerName = "layer_" + layerSpaces.size();
+            duplicateConfig = true; //hard coded to always duplicate layers
+            layerSpaces.add(new LayerConf(layerSpace, layerName, null, numLayersDistribution, duplicateConfig, null));
+            return this;
         }
 
         /**
          * @param layerSpace
          * @param numLayersDistribution Distribution for number of layers to generate
-         * @param duplicateConfig       Only used if more than 1 layer can be generated. If true: generate N identical (stacked) layers.
-         *                              If false: generate N independent layers
          */
-        public Builder addLayer(LayerSpace<? extends Layer> layerSpace, ParameterSpace<Integer> numLayersDistribution,
-                        boolean duplicateConfig) {
+        public Builder addLayer(LayerSpace<? extends Layer> layerSpace, ParameterSpace<Integer> numLayersDistribution) {
             String layerName = "layer_" + layerSpaces.size();
+            boolean duplicateConfig = true; //hard coded to always duplicate layers
             layerSpaces.add(new LayerConf(layerSpace, layerName, null, numLayersDistribution, duplicateConfig, null));
             return this;
         }