SparseLinear SparseJoinTable DenseToSparse (#1652)

* SparseLinear SparseJoinTable DenseToSparse

* Python api

* add DenseToSparseSpec

* update to upstream

* add some method

* meet code review

* fix python unit test

* fix python unit test

pyspark/bigdl/nn/layer.py

@@ -744,6 +744,74 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None):
                                    weight_init_method, bias_init_method)
         return self
 
+class SparseLinear(Layer):
+
+    '''
+    SparseLinear is the sparse version of module Linear. SparseLinear has two different from Linear:
+    firstly, SparseLinear's input Tensor is a SparseTensor. Secondly, SparseLinear doesn't backward
+    gradient to next layer in the backpropagation by default, as the gradInput of SparseLinear is
+    useless and very big in most cases.
+
+    But, considering model like Wide&Deep, we provide backwardStart and backwardLength to backward
+    part of the gradient to next layer.
+
+    :param input_size the size the each input sample
+    :param output_size the size of the module output of each sample
+    :param backwardStart backwardStart index, counting from 1
+    :param backwardLength backward length
+    :param withBias if has bias
+    :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices.
+    :param bRegularizer: instance of [[Regularizer]]applied to the bias.
+    :param init_weight: the optional initial value for the weight
+    :param init_bias: the optional initial value for the bias
+    :param init_grad_weight: the optional initial value for the grad_weight
+    :param init_grad_bias: the optional initial value for the grad_bias
+
+
+    >>> sparselinear = SparseLinear(100, 10, True, wRegularizer=L1Regularizer(0.5), bRegularizer=L1Regularizer(0.5))
+    creating: createL1Regularizer
+    creating: createL1Regularizer
+    creating: createSparseLinear
+    >>> import numpy as np
+    >>> init_weight = np.random.randn(10, 100)
+    >>> init_bias = np.random.randn(10)
+    >>> init_grad_weight = np.zeros([10, 100])
+    >>> init_grad_bias = np.zeros([10])
+    >>> sparselinear = SparseLinear(100, 10, True, 1, 5, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias)
+    creating: createL1Regularizer
+    creating: createL1Regularizer
+    creating: createSparseLinear
+    '''
+
+    def __init__(self, input_size, output_size, with_bias=True, backwardStart=-1, backwardLength=-1,
+                 wRegularizer=None, bRegularizer=None, init_weight=None, init_bias=None,
+                 init_grad_weight=None, init_grad_bias=None, bigdl_type="float"):
+        super(SparseLinear, self).__init__(None, bigdl_type, input_size, output_size,
+                                     with_bias, backwardStart, backwardLength,
+                                     wRegularizer, bRegularizer,
+                                     JTensor.from_ndarray(init_weight),
+                                     JTensor.from_ndarray(init_bias),
+                                     JTensor.from_ndarray(init_grad_weight),
+                                     JTensor.from_ndarray(init_grad_bias))
+
+    def set_init_method(self, weight_init_method = None, bias_init_method = None):
+        callBigDlFunc(self.bigdl_type, "setInitMethod", self.value,
+                      weight_init_method, bias_init_method)
+        return self
+
+class DenseToSparse(Layer):
+
+    '''
+    Convert DenseTensor to SparseTensor.
+
+
+    >>> DenseToSparse = DenseToSparse()
+    creating: createDenseToSparse
+    '''
+
+    def __init__(self,
+                 bigdl_type="float"):
+        super(DenseToSparse, self).__init__(None, bigdl_type)
 
 class ReLU(Layer):
 
@@ -2294,6 +2362,28 @@ def __init__(self,
                                         dimension,
                                         n_input_dims)
 
+class SparseJoinTable(Layer):
+
+    '''
+    :: Experimental ::
+
+    Sparse version of JoinTable. Backward just pass the origin gradOutput back to
+    the next layers without split. So this layer may just works in Wide&Deep like models.
+
+
+    :param dimension: to be join in this dimension
+
+
+    >>> joinTable = SparseJoinTable(1)
+    creating: createSparseJoinTable
+    '''
+
+    def __init__(self,
+                 dimension,
+                 bigdl_type="float"):
+        super(SparseJoinTable, self).__init__(None, bigdl_type,
+                                        dimension)
+
 
 class L1Penalty(Layer):
 

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/DenseToSparse.scala

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2016 The BigDL Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.analytics.bigdl.nn
+
+import com.intel.analytics.bigdl.nn.abstractnn.{TensorCriterion, TensorModule}
+import com.intel.analytics.bigdl.tensor.{DenseType, SparseType, Tensor}
+import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
+
+import scala.reflect.ClassTag
+
+/**
+ * Convert DenseTensor to SparseTensor.
+ * @param ev$1
+ * @param ev
+ * @tparam T The numeric type in the criterion, usually which are [[Float]] or [[Double]]
+ */
+class DenseToSparse[T: ClassTag](implicit ev: TensorNumeric[T]) extends TensorModule[T] {
+  override def updateOutput(input: Tensor[T]): Tensor[T] = {
+    require(input.getTensorType == DenseType, "DenseToSparse: input should be a DenseTensor," +
+      s"but got ${input.getTensorType}")
+    output = Tensor.sparse(input)
+    output
+  }
+  override def updateGradInput(input: Tensor[T], gradOutput: Tensor[T]): Tensor[T] = {
+    this.gradInput.resizeAs(input)
+    Tensor.dense(gradOutput, gradInput)
+    this.gradInput
+  }
+
+  override def toString(): String = s"DenseToSparse()"
+}
+
+object DenseToSparse {
+  def apply[@specialized(Float, Double) T: ClassTag]()
+        (implicit ev: TensorNumeric[T]) : DenseToSparse[T] = {
+    new DenseToSparse()
+  }
+}

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SparseJoinTable.scala

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2016 The BigDL Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.analytics.bigdl.nn
+
+import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule
+import com.intel.analytics.bigdl.tensor.{DenseTensor, SparseTensor, Tensor}
+import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
+import com.intel.analytics.bigdl.utils.{Engine, Table}
+
+import scala.concurrent.Future
+import scala.reflect.ClassTag
+
+/**
+ * :: Experimental ::
+ *
+ * Sparse version of JoinTable. Backward just pass the origin gradOutput back to
+ * the next layers without split. So this layer may just works in Wide&Deep like models.
+ *
+ * @param dimension the dimension to join.
+ * @tparam T Numeric type of parameter(e.g. weight, bias). Only support float/double now
+ */
+class SparseJoinTable[T: ClassTag] (
+    val dimension: Int)(implicit ev: TensorNumeric[T])
+    extends AbstractModule[Table, Tensor[T], T] {
+
+  private var results: Array[Future[Unit]] = null
+  output = Tensor.sparse(Array(1, 1), 1)
+
+  var size: Array[Int] = null
+
+  override def updateOutput(input: Table): Tensor[T] = {
+    var nElements = 0
+
+    var i = 1
+    while (i <= input.length()) {
+      val currentOutput: Tensor[T] = input(i)
+      if (i == 1) {
+        size = currentOutput.size()
+      } else {
+        size(dimension - 1) += currentOutput.size(dimension)
+      }
+      nElements += currentOutput.nElement()
+      i += 1
+    }
+    output.resize(size, nElements)
+
+    Tensor.sparseConcat(2, input, output)
+
+    output
+  }
+
+  override def updateGradInput(input: Table, gradOutput: Tensor[T]): Table = {
+    var i = 1
+    while (i <= input.length()) {
+      gradInput(i) = gradOutput
+      i += 1
+    }
+    gradInput
+  }
+
+  override def clearState(): this.type = {
+    super.clearState()
+    size = null
+    results = null
+    this
+  }
+
+  override def toString: String = s"nn.SparseJoinTable($dimension)"
+
+
+  override def canEqual(other: Any): Boolean = other.isInstanceOf[SparseJoinTable[T]]
+
+  override def equals(other: Any): Boolean = other match {
+    case that: SparseJoinTable[T] =>
+      super.equals(that) &&
+        (that canEqual this) &&
+        dimension == that.dimension
+    case _ => false
+  }
+
+  override def hashCode(): Int = {
+    val state = Seq(super.hashCode(), dimension)
+    state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b)
+  }
+}
+
+object SparseJoinTable {
+  def apply[@specialized(Float, Double) T: ClassTag](
+        dimension: Int)(implicit ev: TensorNumeric[T]) : SparseJoinTable[T] = {
+    new SparseJoinTable[T](dimension)
+  }
+}