Various fixes (#43)

* #8172 Enable DL4J MKLDNN batch norm backward pass Signed-off-by: AlexDBlack <blacka101@gmail.com> * #8382 INDArray.toString() rank 1 brackets / ambiguity fix Signed-off-by: AlexDBlack <blacka101@gmail.com> * #8308 Fix handful of broken links (inc. some in errors) Signed-off-by: AlexDBlack <blacka101@gmail.com> * Unused dependencies, round 1 Signed-off-by: AlexDBlack <blacka101@gmail.com> * Unused dependencies, round 2 Signed-off-by: AlexDBlack <blacka101@gmail.com> * Unused dependencies, round 3 Signed-off-by: AlexDBlack <blacka101@gmail.com> * Small fix Signed-off-by: AlexDBlack <blacka101@gmail.com> * Uniform distribution TF import fix Signed-off-by: AlexDBlack <blacka101@gmail.com>
deeplearning4j · Nov 14, 2019 · 47d1990 · 47d1990
1 parent 48df1ac
commit 47d1990
Show file tree

Hide file tree

Showing 87 changed files with 261 additions and 1,013 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -9,7 +9,7 @@ Deeplearning4j's [open issues are here](https://github.com/eclipse/deeplearning4
 
 Note that you will need to [build dl4j from source](https://deeplearning4j.org/docs/latest/deeplearning4j-build-from-source)
 
-For some tips on contributing to open source, this [post is helpful](http://blog.smartbear.com/programming/14-ways-to-contribute-to-open-source-without-being-a-programming-genius-or-a-rock-star/).
+For some tips on contributing to open source, this [post is helpful](https://smartbear.com/blog/test-and-monitor/14-ways-to-contribute-to-open-source-without-being/).
 
 ## Contributions
 

diff --git a/arbiter/arbiter-core/src/assembly/bin.xml b/arbiter/arbiter-core/src/assembly/bin.xml
@@ -61,7 +61,7 @@
 			<outputDirectory>examples</outputDirectory>
 			<!--
 				<lineEnding>unix</lineEnding>
-				http://stackoverflow.com/questions/2958282/stranges-files-in-my-assembly-since-switching-to-lineendingunix-lineending
+				https://stackoverflow.com/questions/2958282/stranges-files-in-my-assembly-since-switching-to-lineendingunix-lineending
 			-->
 		</fileSet>
 

diff --git a/datavec/datavec-api/pom.xml b/datavec/datavec-api/pom.xml
@@ -52,11 +52,6 @@
             <artifactId>joda-time</artifactId>
             <version>${jodatime.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-            <version>${snakeyaml.version}</version>
-        </dependency>
         <!-- ND4J Shaded Jackson Dependencies -->
         <dependency>
             <groupId>org.nd4j</groupId>

diff --git a/datavec/datavec-arrow/pom.xml b/datavec/datavec-arrow/pom.xml
@@ -29,21 +29,11 @@
     <name>datavec-arrow</name>
 
     <dependencies>
-        <dependency>
-            <groupId>org.nd4j</groupId>
-            <artifactId>nd4j-arrow</artifactId>
-            <version>${project.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.datavec</groupId>
             <artifactId>datavec-api</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.carrotsearch</groupId>
-            <artifactId>hppc</artifactId>
-            <version>${hppc.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-vector</artifactId>

diff --git a/datavec/datavec-data/datavec-data-nlp/pom.xml b/datavec/datavec-data/datavec-data-nlp/pom.xml
@@ -44,26 +44,6 @@
             <artifactId>datavec-api</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>commons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-            <version>${commons-logging.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-core</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-context</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-beans</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.cleartk</groupId>
             <artifactId>cleartk-snowball</artifactId>

diff --git a/datavec/datavec-geo/pom.xml b/datavec/datavec-geo/pom.xml
@@ -31,36 +31,6 @@
             <artifactId>datavec-api</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>${geo.jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${geo.jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>${geo.jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-yaml</artifactId>
-            <version>${geo.jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-xml</artifactId>
-            <version>${geo.jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-joda</artifactId>
-            <version>${geo.jackson.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.maxmind.geoip2</groupId>
             <artifactId>geoip2</artifactId>

diff --git a/datavec/datavec-hadoop/pom.xml b/datavec/datavec-hadoop/pom.xml
@@ -35,41 +35,11 @@
             <version>${project.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>com.sun.xml.bind</groupId>
-            <artifactId>jaxb-core</artifactId>
-            <version>${jaxb.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.sun.xml.bind</groupId>
-            <artifactId>jaxb-impl</artifactId>
-            <version>${jaxb.version}</version>
-        </dependency>
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty</artifactId>
             <version>${netty.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-compress</artifactId>
-            <version>${commons-compress.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.zookeeper</groupId>
-            <artifactId>zookeeper</artifactId>
-            <version>${zookeeper.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>

diff --git a/datavec/datavec-local/pom.xml b/datavec/datavec-local/pom.xml
@@ -73,42 +73,7 @@
         </dependency>
 
 
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>${geo.jackson.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${geo.jackson.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>${geo.jackson.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-yaml</artifactId>
-            <version>${geo.jackson.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-xml</artifactId>
-            <version>${geo.jackson.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-joda</artifactId>
-            <version>${geo.jackson.version}</version>
-            <scope>test</scope>
-        </dependency>
+
         <dependency>
             <groupId>org.datavec</groupId>
             <artifactId>datavec-python</artifactId>

diff --git a/datavec/datavec-perf/pom.xml b/datavec/datavec-perf/pom.xml
@@ -41,11 +41,6 @@
             <artifactId>slf4j-api</artifactId>
             <version>${slf4j.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.github.oshi</groupId>
-            <artifactId>oshi-core</artifactId>
-            <version>${oshi.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.datavec</groupId>
             <artifactId>datavec-data-image</artifactId>

diff --git a/datavec/datavec-spark-inference-parent/datavec-spark-inference-client/pom.xml b/datavec/datavec-spark-inference-parent/datavec-spark-inference-client/pom.xml
@@ -41,26 +41,6 @@
             <version>1.0.0-SNAPSHOT</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-            <version>${commons-codec.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpclient</artifactId>
-            <version>${httpclient.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpcore</artifactId>
-            <version>${httpcore.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpmime</artifactId>
-            <version>${httpmime.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.mashape.unirest</groupId>
             <artifactId>unirest-java</artifactId>

diff --git a/datavec/datavec-spark-inference-parent/datavec-spark-inference-server/pom.xml b/datavec/datavec-spark-inference-parent/datavec-spark-inference-server/pom.xml
@@ -94,12 +94,6 @@
             <version>${scala.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-            <version>${snakeyaml.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>com.typesafe.play</groupId>
             <artifactId>play-java_2.11</artifactId>

diff --git a/datavec/datavec-spark/pom.xml b/datavec/datavec-spark/pom.xml
@@ -39,11 +39,6 @@
             <artifactId>scala-library</artifactId>
             <version>${scala.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.scala-lang</groupId>
-            <artifactId>scala-reflect</artifactId>
-            <version>${scala.version}</version>
-        </dependency>
 
         <dependency>
             <groupId>org.apache.spark</groupId>

diff --git a/...eeplearning4j-common/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java b/...eeplearning4j-common/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java
@@ -64,7 +64,7 @@ public class DL4JResources {
     /**
      * Set the base download URL for (most) DL4J datasets and models.<br>
      * This usually doesn't need to be set manually unless there is some issue with the default location
-     * @param baseDownloadURL Base download URL to set. For example, http://blob.deeplearning4j.org/
+     * @param baseDownloadURL Base download URL to set. For example, https://dl4jdata.blob.core.windows.net/
      */
     public static void setBaseDownloadURL(@NonNull String baseDownloadURL){
         baseURL = baseDownloadURL;
@@ -79,8 +79,8 @@ public static String getBaseDownloadURL(){
 
     /**
      * Get the URL relative to the base URL.<br>
-     * For example, if baseURL is "http://blob.deeplearning4j.org/", and relativeToBase is "/datasets/iris.dat"
-     * this simply returns "http://blob.deeplearning4j.org/datasets/iris.dat"
+     * For example, if baseURL is "https://dl4jdata.blob.core.windows.net/", and relativeToBase is "/datasets/iris.dat"
+     * this simply returns "https://dl4jdata.blob.core.windows.net/datasets/iris.dat"
      *
      * @param relativeToBase Relative URL
      * @return URL
@@ -92,8 +92,8 @@ public static URL getURL(String relativeToBase) throws MalformedURLException {
 
     /**
      * Get the URL relative to the base URL as a String.<br>
-     * For example, if baseURL is "http://blob.deeplearning4j.org/", and relativeToBase is "/datasets/iris.dat"
-     * this simply returns "http://blob.deeplearning4j.org/datasets/iris.dat"
+     * For example, if baseURL is "https://dl4jdata.blob.core.windows.net/", and relativeToBase is "/datasets/iris.dat"
+     * this simply returns "https://dl4jdata.blob.core.windows.net/datasets/iris.dat"
      *
      * @param relativeToBase Relative URL
      * @return URL

diff --git a/...ning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java b/...ning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java
@@ -138,52 +138,55 @@ public void validateBatchNorm() {
         ConvolutionMode cm = ConvolutionMode.Truncate;
 
         for (int minibatch : new int[]{1, 3}) {
+            for (boolean b : new boolean[]{true, false}) {
 
-            inputSize[0] = minibatch;
-            INDArray f = Nd4j.rand(Nd4j.defaultFloatingPointType(), inputSize);
-            INDArray l = TestUtils.randomOneHot(minibatch, 10);
-
-            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
-                    .updater(new Adam(0.01))
-                    .convolutionMode(cm)
-                    .seed(12345)
-                    .list()
-                    .layer(new ConvolutionLayer.Builder().activation(Activation.TANH)
-                            .kernelSize(kernel)
-                            .stride(stride)
-                            .padding(0, 0)
-                            .nOut(3)
-                            .build())
-                    .layer(new BatchNormalization.Builder().helperAllowFallback(false)/*.eps(0)*/.build())
-                    .layer(new ConvolutionLayer.Builder().activation(Activation.TANH)
-                            .kernelSize(kernel)
-                            .stride(stride)
-                            .padding(0, 0)
-                            .nOut(3)
-                            .build())
-                    .layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
-                    .setInputType(InputType.convolutional(inputSize[2], inputSize[3], inputSize[1]))
-                    .build();
-
-            MultiLayerNetwork netWith = new MultiLayerNetwork(conf.clone());
-            netWith.init();
-
-            MultiLayerNetwork netWithout = new MultiLayerNetwork(conf.clone());
-            netWithout.init();
-
-            LayerHelperValidationUtil.TestCase tc = LayerHelperValidationUtil.TestCase.builder()
-                    .allowHelpersForClasses(Collections.<Class<?>>singletonList(org.deeplearning4j.nn.layers.normalization.BatchNormalization.class))
-                    .testForward(true)
-                    .testScore(true)
-                    .testBackward(true)
-                    .testTraining(true)
-                    .features(f)
-                    .labels(l)
-                    .data(new SingletonDataSetIterator(new DataSet(f, l)))
-                    .maxRelError(1e-4)
-                    .build();
-
-            LayerHelperValidationUtil.validateMLN(netWith, tc);
+                inputSize[0] = minibatch;
+                INDArray f = Nd4j.rand(Nd4j.defaultFloatingPointType(), inputSize);
+                INDArray l = TestUtils.randomOneHot(minibatch, 10);
+
+                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
+                        .dataType(DataType.FLOAT)
+                        .updater(new Adam(0.01))
+                        .convolutionMode(cm)
+                        .seed(12345)
+                        .list()
+                        .layer(new ConvolutionLayer.Builder().activation(Activation.TANH)
+                                .kernelSize(kernel)
+                                .stride(stride)
+                                .padding(0, 0)
+                                .nOut(3)
+                                .build())
+                        .layer(new BatchNormalization.Builder().useLogStd(b).helperAllowFallback(false)/*.eps(0)*/.build())
+                        .layer(new ConvolutionLayer.Builder().activation(Activation.TANH)
+                                .kernelSize(kernel)
+                                .stride(stride)
+                                .padding(0, 0)
+                                .nOut(3)
+                                .build())
+                        .layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
+                        .setInputType(InputType.convolutional(inputSize[2], inputSize[3], inputSize[1]))
+                        .build();
+
+                MultiLayerNetwork netWith = new MultiLayerNetwork(conf.clone());
+                netWith.init();
+
+                MultiLayerNetwork netWithout = new MultiLayerNetwork(conf.clone());
+                netWithout.init();
+
+                LayerHelperValidationUtil.TestCase tc = LayerHelperValidationUtil.TestCase.builder()
+                        .allowHelpersForClasses(Collections.<Class<?>>singletonList(org.deeplearning4j.nn.layers.normalization.BatchNormalization.class))
+                        .testForward(true)
+                        .testScore(true)
+                        .testBackward(true)
+                        .testTraining(true)
+                        .features(f)
+                        .labels(l)
+                        .data(new SingletonDataSetIterator(new DataSet(f, l)))
+                        .maxRelError(1e-4)
+                        .build();
+
+                LayerHelperValidationUtil.validateMLN(netWith, tc);
+            }
         }
     }