Support MKL2017 DNN API With New Branch

.gitignore

@@ -20,3 +20,4 @@ project/plugins/project/
 
 # other
 *.txt
+*.swp # vim swap file

dl/src/main/scala/com/intel/analytics/sparkdl/example/ImageNetLocal.scala

@@ -21,7 +21,7 @@ import java.awt.color.ColorSpace
 import java.util
 
 import com.intel.analytics.sparkdl.nn.ClassNLLCriterion
-import com.intel.analytics.sparkdl.optim.SGD
+import com.intel.analytics.sparkdl.optim.{EvaluateMethods, SGD}
 import com.intel.analytics.sparkdl.tensor.Tensor
 import com.intel.analytics.sparkdl.utils.{File, T}
 
@@ -49,160 +49,9 @@ object ImageNetLocal {
     println(s"[${(System.nanoTime() - startTime) / 1e9}s] $msg")
   }
 
-  def runDouble(donkey: Donkey, dataSet: DataSets, netType: String, classNum: Int,
+  def run(donkey: Donkey, dataSet: DataSets, netType: String, classNum: Int,
     labelsMap: Map[String, Double], testInterval: Int, donkeyVal: Donkey,
-    dataSetVal: DataSets, batchSize: Int): Unit = {
-    // Compute Mean on amount of samples
-    val samples = 10000
-    log(s"Start to calculate Mean on $samples samples")
-    var (meanR, meanG, meanB) = Array.tabulate(samples)(n => {
-      print(".")
-      val data = donkey.pull
-      dataSet.post(data._2)
-      ImageNetUtils.computeMean(data._1, data._2.dataOffset)
-    }).reduce((a, b) => (a._1 + b._1, a._2 + b._2, a._3 + b._3))
-    meanR /= samples
-    meanG /= samples
-    meanB /= samples
-    println()
-
-    // Compute std on amount of samples
-    log(s"Start to calculate std on $samples samples")
-    var (varR, varG, varB) = Array.tabulate(samples)(n => {
-      print(".")
-      val data = donkey.pull
-      dataSet.post(data._2)
-      ImageNetUtils.computeVar(data._1, meanR, meanG, meanB, data._2.dataOffset)
-    }).reduce((a, b) => (a._1 + b._1, a._2 + b._2, a._3 + b._3))
-    varR /= samples
-    varG /= samples
-    varB /= samples
-
-    val model = netType match {
-      case "alexnet" => AlexNet.getModel[Double](classNum)
-      case "googlenet" => GoogleNet.getModel[Double](classNum)
-      case "googlenet-bn" => GoogleNet.getModel[Double](classNum, "googlenet-bn")
-      case "googlenet-cf" => GoogleNet.getModelCaffe[Double](classNum)
-      case _ => throw new IllegalArgumentException
-    }
-    val (weights, grad) = model.getParameters()
-    println(s"modelsize ${weights.nElement()}")
-    println(model)
-    val criterion = new ClassNLLCriterion[Double]()
-    val epochNum = 90
-    val featureShape = Array(3, 224, 224)
-    val targetShape = Array(1)
-    val sgd = new SGD[Double]
-    val state = T("momentum" -> 0.9, "dampening" -> 0.0)
-    val stageImgs = new util.ArrayDeque[Image](batchSize)
-    val input = Tensor[Double](batchSize, 3, 224, 224)
-    val target = Tensor[Double](batchSize)
-    val iter = ImageNetUtils.toTensorDouble(
-      donkey.map(d => {
-        stageImgs.push(d._2)
-        (labelsMap(d._2.label), d._1)
-      }),
-      featureShape,
-      targetShape,
-      batchSize,
-      (meanR, meanG, meanB),
-      (varR, varG, varB),
-      input,
-      target
-    )
-
-    val stageImgsVal = new util.ArrayDeque[Image](batchSize)
-    val iterVal = ImageNetUtils.toTensorDouble(
-      donkeyVal.map(d => {
-        stageImgsVal.push(d._2)
-        (labelsMap(d._2.label), d._1)
-      }),
-      featureShape,
-      targetShape,
-      batchSize,
-      (meanR, meanG, meanB),
-      (varR, varG, varB),
-      input,
-      target
-    )
-
-    log(s"meanR is $meanR meanG is $meanG meanB is $meanB")
-    log(s"varR is $varR varG is $varG varB is $varB")
-    log("Start to train...")
-
-    var wallClockTime = 0L
-    for (i <- 1 to epochNum) {
-      println(s"Epoch[$i] Train")
-
-      for (regime <- regimes(netType)) {
-        if (i >= regime._1 && i <= regime._2) {
-          state("learningRate") = regime._3
-          state("weightDecay") = regime._4
-        }
-      }
-
-      var j = 0
-      var c = 0
-      model.training()
-      while (j < dataSet.getTotal) {
-        val start = System.nanoTime()
-        val (input, target) = iter.next()
-        val readImgTime = System.nanoTime()
-        model.zeroGradParameters()
-        val output = model.forward(input)
-        val loss = criterion.forward(output, target)
-        val gradOutput = criterion.backward(output, target)
-        model.backward(input, gradOutput)
-        sgd.optimize(_ => (loss, grad), weights, state, state)
-        val end = System.nanoTime()
-        wallClockTime += end - start
-        log(s"Epoch[$i][Iteration $c $j/${dataSet.getTotal}][Wall Clock ${wallClockTime / 1e9}s]" +
-          s" loss is $loss time ${(end - start) / 1e9}s read " +
-          s"time ${(readImgTime - start) / 1e9}s train time ${(end - readImgTime) / 1e9}s." +
-          s" Throughput is ${input.size(1).toDouble / (end - start) * 1e9} img / second")
-        while (!stageImgs.isEmpty) {
-          dataSet.post(stageImgs.poll())
-        }
-        j += input.size(1)
-        c += 1
-      }
-
-      if (i % testInterval == 0) {
-        model.evaluate()
-        var correct = 0
-        var k = 0
-        while (k < dataSetVal.getTotal) {
-          val (input, target) = iterVal.next()
-          val output = model.forward(input)
-          output.max(2)._2.squeeze().map(target, (a, b) => {
-            if (a == b) {
-              correct += 1
-            }
-            a
-          })
-          while (!stageImgsVal.isEmpty) {
-            dataSetVal.post(stageImgsVal.poll())
-          }
-          k += input.size(1)
-        }
-
-        val accuracy = correct.toDouble / dataSetVal.getTotal
-        println(s"[Wall Clock ${wallClockTime / 1e9}s] Accuracy is $accuracy")
-
-        // Save model to a file each epoch
-        File.save(model, s"${netType}${accuracy}.model${i}", true)
-        File.save(state, s"${netType}${accuracy}.state${i}", true)
-      }
-
-      log("shuffle")
-      dataSet.shuffle
-      log("shuffle end")
-    }
-  }
-
-  def runFloat(donkey: Donkey, dataSet: DataSets, netType: String, classNum: Int,
-    labelsMap: Map[String, Double], testInterval: Int, donkeyVal: Donkey,
-    dataSetVal: DataSets, batchSize: Int): Unit = {
+    dataSetVal: DataSets, batchSize: Int, modelPath : String): Unit = {
     // Compute Mean on amount of samples
     val samples = 10000
     log(s"Start to calculate Mean on $samples samples")
@@ -327,25 +176,27 @@ object ImageNetLocal {
 
       if (i % testInterval == 0) {
         model.evaluate()
-        var correct = 0
+        var top1Correct = 0
+        var top5Correct = 0
         var k = 0
         while (k < dataSetVal.getTotal) {
           val (input, target) = iterVal.next()
           val output = model.forward(input)
-          output.max(2)._2.squeeze().map(target, (a, b) => {
-            if (a == b) {
-              correct += 1
-            }
-            a
-          })
+          top1Correct += EvaluateMethods.calcAccuracy(output, target)._1
+          top5Correct += EvaluateMethods.calcTop5Accuracy(output, target)._1
           while (!stageImgsVal.isEmpty) {
             dataSetVal.post(stageImgsVal.poll())
           }
           k += input.size(1)
         }
 
-        val accuracy = correct.toDouble / dataSetVal.getTotal
-        println(s"[Wall Clock ${wallClockTime / 1e9}s] Accuracy is $accuracy")
+        val top1Accuracy = top1Correct.toDouble / dataSetVal.getTotal
+        val top5Accuracy = top5Correct.toDouble / dataSetVal.getTotal
+        println(s"[Wall Clock ${wallClockTime / 1e9}s] Top-1 Accuracy is $top1Accuracy")
+        println(s"[Wall Clock ${wallClockTime / 1e9}s] Top-5 Accuracy is $top5Accuracy")
+        println(s"Save model and state to $modelPath-$i")
+        File.save(model, modelPath + s"-$i.model")
+        File.save(state, modelPath + s"-$i.state")
       }
 
       log("shuffle")
@@ -371,8 +222,8 @@ object ImageNetLocal {
     val testInterval = args(4).toInt
     val netType = args(5)
     val classNum = args(6).toInt
-    val dataType = args(7)
-    val batchSize = args(8).toInt
+    val batchSize = args(7).toInt
+    val modelPath = args(8)
 
     val dataSet = new DataSets(path, classNum, labelsMap)
     val donkey = new Donkey(parallelism, dataSet)
@@ -383,12 +234,7 @@ object ImageNetLocal {
     dataSet.shuffle
     log("shuffle end")
 
-    dataType match {
-      case "double" => runDouble(donkey, dataSet, netType, classNum, labelsMap, testInterval,
-        donkeyVal, dataSetVal, batchSize)
-      case "float" => runFloat(donkey, dataSet, netType, classNum, labelsMap, testInterval,
-        donkeyVal, dataSetVal, batchSize)
-      case _ => throw new IllegalArgumentException
-    }
+    run(donkey, dataSet, netType, classNum, labelsMap, testInterval,
+      donkeyVal, dataSetVal, batchSize, modelPath)
   }
 }

dl/src/main/scala/com/intel/analytics/sparkdl/models/GoogleNet.scala

@@ -232,7 +232,7 @@ object GoogleNet_v2 {
 
     val conv3 = new Sequential[D]
     conv3.add(new SpatialConvolution[D](inputSize, config[Table](2)(1), 1, 1, 1, 1)
-      .setName(namePrefix + "3x3_s2"))
+      .setName(namePrefix + "3x3_reduce"))
     conv3.add(new SpatialBatchNormalization(config[Table](2)(1), 1e-3)
       .setName(namePrefix + "3x3_reduce/bn"))
     conv3.add(new ReLU[D](true). setName(namePrefix + "3x3_reduce/bn/sc/relu"))

dl/src/main/scala/com/intel/analytics/sparkdl/models/Perf.scala

@@ -79,7 +79,7 @@ object Perf {
 
   def performance[T: ClassTag](param: Params)(implicit tn: TensorNumeric[T]): Unit = {
     val (model, input) = param.module match {
-      case "alexnet" => (AlexNet(1000), Tensor[T](param.batchSize, 3, 224, 224))
+      case "alexnet" => (AlexNet(1000), Tensor[T](param.batchSize, 3, 227, 227))
       case "alexnetowt" => (AlexNet_OWT(1000), Tensor[T](param.batchSize, 3, 224, 224))
       case "googlenet_v1" => (GoogleNet_v1(1000), Tensor[T](param.batchSize, 3, 224, 224))
       case "googlenet_v2" => (GoogleNet_v2(1000), Tensor[T](param.batchSize, 3, 224, 224))
@@ -139,8 +139,6 @@ object Perf {
   }
 }
 
-case class TestCase[T](input: Tensor[T], target: Tensor[T], model: Module[T])
-
 case class Params(
   batchSize: Int = 128,
   iteration: Int = 10,

dl/src/main/scala/com/intel/analytics/sparkdl/nn/Container.scala

@@ -19,6 +19,7 @@ package com.intel.analytics.sparkdl.nn
 
 import com.intel.analytics.sparkdl.tensor.Tensor
 import com.intel.analytics.sparkdl.tensor.TensorNumericMath.TensorNumeric
+import com.intel.analytics.sparkdl.mkl.MKL
 
 import scala.collection.mutable.ArrayBuffer
 import scala.reflect.ClassTag
@@ -93,4 +94,25 @@ private[nn] abstract class Container[@specialized(Float, Double) T: ClassTag](
     })
     (result, offset, newIndexes)
   }
+
+  override def initMkl() : Unit = {
+    def containMkl(module : Module[T]) : Boolean = {
+      return if (module.toString.startsWith("mkl.")) true else false
+    }
+
+    for (i <- 0 until modules.length) {
+      if (containMkl(modules(i))) {
+        if (i >= 1 && containMkl(modules(i - 1))) {
+          ev.getType() match {
+            case "Float" => MKL.SetPrevFloat(modules(i - 1).getClassPtr(),
+                                             modules(i).getClassPtr())
+            case "Double" => MKL.SetPrevDouble(modules(i - 1).getClassPtr(),
+                                               modules(i).getClassPtr())
+          }
+        }
+      } else {
+        modules(i).initMkl()
+      }
+    }
+  }
 }

dl/src/main/scala/com/intel/analytics/sparkdl/nn/Module.scala

@@ -43,6 +43,17 @@ abstract class Module[T: ClassTag](implicit ev: TensorNumeric[T]) extends Serial
     if (this.name == null) this.toString else this.name
   }
 
+  private var needComputeBack = true
+
+  def setNeedComputeBack(need: Boolean): this.type = {
+    needComputeBack = need
+    this
+  }
+
+  def isNeedComputeBack(): Boolean = {
+    needComputeBack
+  }
+
   // list of sub modules
   val modules: ArrayBuffer[Module[T]] = ArrayBuffer[Module[T]]()
 
@@ -199,6 +210,10 @@ abstract class Module[T: ClassTag](implicit ev: TensorNumeric[T]) extends Serial
   def cloneModule(): Module[T] = {
     SerializationUtils.clone(this)
   }
+
+  // Support for mkl init.
+  def getClassPtr() : Long = {0L}
+  def initMkl() : Unit = {}
 }
 
 object Module {