From bff76f5926af1139384576d280ad4b3f0b9acb8d Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 1 Jun 2017 19:30:24 +0800 Subject: [PATCH 01/23] nn refactor --- .../intel/analytics/bigdl/nn/Padding.scala | 178 ++++++++++++------ .../analytics/bigdl/nn/PaddingSpec.scala | 83 ++++++++ 2 files changed, 206 insertions(+), 55 deletions(-) create mode 100644 spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala index 7004f68de58..5477d79a287 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala @@ -17,81 +17,88 @@ package com.intel.analytics.bigdl.nn import com.intel.analytics.bigdl.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.tensor.{Storage, Tensor} +import com.intel.analytics.bigdl.tensor.Tensor +import com.intel.analytics.bigdl.nn.Padding._ import scala.reflect.ClassTag /** - * This module adds pad units of padding to dimension dim of the input. If pad is negative, - * padding is added to the left, otherwise, it is added to the right of the dimension. + * This module adds paddings to the input tensor. * * The input to this layer is expected to be a tensor, or a batch of tensors; * when using mini-batch, a batch of sample tensors will be passed to the layer and * the user need to specify the number of dimensions of each sample tensor in the * batch using nInputDims. * - * @param dim the dimension to be applied padding operation - * @param pad num of the pad units - * @param nInputDim specify the number of dimensions that this module will receive - * If it is more than the dimension of input tensors, the first dimension + * @param paddings how to pad the input + * @param nInputDim specify the number of dimensions that this module will work on + * If it is not equal to the dimension number of input tensors, the first dimension * would be considered as batch size * @param value padding value */ @SerialVersionUID(- 3401298839313169602L) class Padding[T: ClassTag]( - val dim: Int, - val pad: Int, + val paddings: Array[PaddingInfo], val nInputDim: Int, - val value: Double = 0.0, - val nIndex: Int = 1)(implicit ev: TensorNumeric[T]) extends TensorModule[T] { - - var outputSize = Storage[Int]() + val value: Double = 0.0)(implicit ev: TensorNumeric[T]) extends TensorModule[T] { override def updateOutput(input: Tensor[T]): Tensor[T] = { - outputSize.resize(input.dim()).copy(Storage(input.size())) - val dim = if (input.dim() != nInputDim) this.dim + 1 else this.dim - - outputSize(dim - 1) += math.abs(this.pad) - output.resize(outputSize.array()).fill(ev.fromType(value)) - - val index = if (this.pad > 0) input.size(dim) - nIndex + 2 else nIndex - val pad = if (this.pad > 0) this.pad else -this.pad - - if (index == 1) { - output.narrow(dim, 1 + pad, input.size(dim)).copy(input) - } else if (index == (input.size(dim) + 1)) { - output.narrow(dim, 1, input.size(dim)).copy(input) - } else { - output.narrow(dim, 1, index - 1).copy(input.narrow(dim, 1, index - 1)) - output.narrow(dim, index + pad, input.size(dim) - (index - 1)). - copy(input.narrow(dim, index, input.size(dim) - (index - 1))) + // Expand the output tensor + val batch = (nInputDim != input.nDimension()) + inputs(0) = input + var i = 0 + while(i < paddings.length) { + inputs(i + 1) = paddingOneDimForward(inputs(i), paddings(i), batch) + i += 1 } + output = inputs(i) + output } - override def updateGradInput(input: Tensor[T], gradOutput: Tensor[T]): Tensor[T] = { - gradInput.resizeAs(input) + val batch = (nInputDim != input.nDimension()) + // Copy the tensor content + var gradOutputDim = gradOutput + var i = 0 + while(i < paddings.length) { + gradOutputDim = paddingOneDimBackward(inputs(paddings.length - i - 1), + gradOutputDim, paddings(paddings.length - i - 1), batch) + i += 1 + } - val dim = if (input.dim() != nInputDim) this.dim + 1 else this.dim - val index = if (this.pad > 0) input.size(dim) - nIndex + 2 else nIndex - val pad = if (this.pad > 0) this.pad else -this.pad + gradInput = gradOutputDim + gradInput + } - if (index == 1) { - gradInput.copy(gradOutput.narrow(dim, 1 + pad, input.size(dim))) - } else if (index == input.size(dim) + 1) { - gradInput.copy(gradOutput.narrow(dim, 1, input.size(dim))) + private def paddingOneDimBackward(input: Tensor[T], gradOutput: Tensor[T], + padInfo: PaddingInfo, batch: Boolean): Tensor[T] = { + val gradInput = Tensor[T]().resizeAs(input) + val dim = if (batch) padInfo.dim + 1 else padInfo.dim + if(padInfo.leftStartIndex == 1 && padInfo.rightStartIndex == 1) { + gradInput.narrow(dim, 1, input.size(dim)) + .copy(gradOutput.narrow(dim, padInfo.leftPadding + 1, input.size(dim))) } else { - gradInput.narrow(dim, 1, index - 1). - copy(gradOutput.narrow(dim, 1, index - 1)) - gradInput.narrow(dim, index, input.size(dim) - (index - 1)).copy( - gradOutput.narrow(dim, index + pad, input.size(dim) - (index - 1))) + val central = input.size(dim) - (padInfo.leftStartIndex - 1) - (padInfo.rightStartIndex - 1) + if(padInfo.leftStartIndex != 1) { + gradInput.narrow(dim, 1, padInfo.leftStartIndex - 1) + .copy(gradOutput.narrow(dim, 1, padInfo.leftStartIndex - 1)) + } + if (central != 0) { + gradInput.narrow(dim, padInfo.leftStartIndex, central) + .copy(gradOutput.narrow(dim, padInfo.leftStartIndex + padInfo.leftPadding, central)) + } + if(padInfo.rightStartIndex != 1) { + gradInput.narrow(dim, input.size(dim) - padInfo.rightStartIndex + 2, + padInfo.rightStartIndex - 1).copy(gradOutput.narrow(dim, + padInfo.leftStartIndex + central + padInfo.rightPadding, padInfo.rightStartIndex - 1)) + } } gradInput } override def toString(): String = { - s"${getPrintName}($dim, $pad, $nInputDim, $value, $nIndex)" + s"${getPrintName}((${paddings.mkString(",")}), $nInputDim, $value)" } override def canEqual(other: Any): Boolean = other.isInstanceOf[Padding[T]] @@ -100,27 +107,88 @@ class Padding[T: ClassTag]( case that: Padding[T] => super.equals(that) && (that canEqual this) && - dim == that.dim && - pad == that.pad && + paddings == that.paddings && nInputDim == that.nInputDim && - value == that.value && - nIndex == that.nIndex + value == that.value case _ => false } override def hashCode(): Int = { def getHashCode(a: Any): Int = if (a == null) 0 else a.hashCode() - val state = Seq(super.hashCode(), dim, pad, nInputDim, value, nIndex) + val state = Seq(super.hashCode(), paddings, nInputDim, value) state.map(getHashCode).foldLeft(0)((a, b) => 31 * a + b) } + + private val inputs = new Array[Tensor[T]](paddings.length + 1) + inputs.foreach(_ => Tensor[T]()) + + private def paddingOneDimForward(input: Tensor[T], padInfo: PaddingInfo, + batch: Boolean): Tensor[T] = { + // Expand the output tensor + val outputSize = input.size() + val dim = if (batch) padInfo.dim + 1 else padInfo.dim + outputSize(dim - 1) += padInfo.leftPadding + padInfo.rightPadding + val output = Tensor(outputSize).fill(ev.fromType(value)) + + // Copy the tensor content + if(padInfo.leftStartIndex == 1 && padInfo.rightStartIndex == 1) { + output.narrow(dim, padInfo.leftPadding + 1, input.size(dim)) + .copy(input.narrow(dim, 1, input.size(dim))) + } else { + val central = input.size(dim) - (padInfo.leftStartIndex - 1) - (padInfo.rightStartIndex - 1) + if (padInfo.leftStartIndex != 1) { + output.narrow(dim, 1, padInfo.leftStartIndex - 1) + .copy(input.narrow(dim, 1, padInfo.leftStartIndex - 1)) + } + if (central != 0) { + output.narrow(dim, padInfo.leftStartIndex + padInfo.leftPadding, central) + .copy(input.narrow(dim, padInfo.leftStartIndex, central)) + } + if (padInfo.rightStartIndex != 1) { + output.narrow(dim, + outputSize(dim - 1) - padInfo.rightStartIndex + 2, + padInfo.rightStartIndex - 1).copy(input.narrow(dim, + input.size(dim) - padInfo.rightStartIndex + 2, padInfo.rightStartIndex - 1)) + } + } + output + } } -object Padding{ - def apply[@specialized(Float, Double) T: ClassTag]( +object Padding { + /** + * Padding information. It define which dimensions to add padding + * @param dim the index of the dimension to add padding, start from 1 + * @param leftPadding left padding length + * @param leftStartIndex left padding start index + * @param rightPadding right padding length + * @param rightStartIndex right padding start index + */ + case class PaddingInfo( dim: Int, - pad: Int, + leftPadding: Int, + leftStartIndex: Int, + rightPadding: Int, + rightStartIndex: Int + ) + + def apply[T: ClassTag]( + dim: Int, + pad: Int, + nInputDim: Int, + value: Double = 0.0, + nIndex: Int = 1)(implicit ev: TensorNumeric[T]): Padding[T] = { + if (pad > 0) { + new Padding[T](Array(PaddingInfo(dim, 0, 1, pad, nIndex)), nInputDim, value) + } else { + new Padding[T](Array(PaddingInfo(dim, -pad, nIndex, 0, 1)), nInputDim, value) + } + } + + def apply[T: ClassTag]( + paddings: Array[PaddingInfo], nInputDim: Int, - value: Double = 0.0, - nIndex: Int = 1)(implicit ev: TensorNumeric[T]) : Padding[T] = - new Padding[T](dim, pad, nInputDim, value, nIndex) + value: Double)(implicit ev: TensorNumeric[T]) : Padding[T] = { + new Padding[T](paddings, nInputDim, value) + } } diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala new file mode 100644 index 00000000000..11b4eb93eb0 --- /dev/null +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.nn + +import com.intel.analytics.bigdl.nn.Padding.PaddingInfo +import com.intel.analytics.bigdl.tensor.Tensor +import org.scalatest.{FlatSpec, Matchers} + +class PaddingSpec extends FlatSpec with Matchers { + "A Padding(Array(Array(2, 3), Array(-1, 1, -2, 2))" should + "generate correct output and grad for multi dimension" in { + val layer = Padding[Float]( + Array(PaddingInfo(2, 1, 1, 1, 1), PaddingInfo(3, 1, 1, 1, 1)), 3, 0.0) + val input = Tensor[Float](1, 2, 2) + input(Array(1, 1, 1)) = 0.01f + input(Array(1, 1, 2)) = 0.02f + input(Array(1, 2, 1)) = 0.03f + input(Array(1, 2, 2)) = 0.04f + val expectedOutput = Tensor[Float](1, 4, 4) + expectedOutput(Array(1, 1, 1)) = 0.00f + expectedOutput(Array(1, 1, 2)) = 0.00f + expectedOutput(Array(1, 1, 3)) = 0.00f + expectedOutput(Array(1, 1, 4)) = 0.00f + expectedOutput(Array(1, 2, 1)) = 0.00f + expectedOutput(Array(1, 2, 2)) = 0.01f + expectedOutput(Array(1, 2, 3)) = 0.02f + expectedOutput(Array(1, 2, 4)) = 0.00f + expectedOutput(Array(1, 3, 1)) = 0.00f + expectedOutput(Array(1, 3, 2)) = 0.03f + expectedOutput(Array(1, 3, 3)) = 0.04f + expectedOutput(Array(1, 3, 4)) = 0.00f + expectedOutput(Array(1, 4, 1)) = 0.00f + expectedOutput(Array(1, 4, 2)) = 0.00f + expectedOutput(Array(1, 4, 3)) = 0.00f + expectedOutput(Array(1, 4, 4)) = 0.00f + + val gradOutput = Tensor[Float](1, 4, 4) + gradOutput(Array(1, 1, 1)) = 0.01f + gradOutput(Array(1, 1, 2)) = 0.02f + gradOutput(Array(1, 1, 3)) = 0.03f + gradOutput(Array(1, 1, 4)) = 0.04f + gradOutput(Array(1, 2, 1)) = 0.05f + gradOutput(Array(1, 2, 2)) = 0.06f + gradOutput(Array(1, 2, 3)) = 0.07f + gradOutput(Array(1, 2, 4)) = 0.08f + gradOutput(Array(1, 3, 1)) = 0.09f + gradOutput(Array(1, 3, 2)) = 0.10f + gradOutput(Array(1, 3, 3)) = 0.11f + gradOutput(Array(1, 3, 4)) = 0.12f + gradOutput(Array(1, 4, 1)) = 0.13f + gradOutput(Array(1, 4, 2)) = 0.14f + gradOutput(Array(1, 4, 3)) = 0.15f + gradOutput(Array(1, 4, 4)) = 0.16f + val expectedGrad = Tensor[Float](1, 2, 2) + expectedGrad(Array(1, 1, 1)) = 0.06f + expectedGrad(Array(1, 1, 2)) = 0.07f + expectedGrad(Array(1, 2, 1)) = 0.10f + expectedGrad(Array(1, 2, 2)) = 0.11f + + val inputOrg = input.clone() + val gradOutputOrg = gradOutput.clone() + val output = layer.forward(input) + val gradInput = layer.backward(input, gradOutput) + + expectedOutput should be (output) + expectedGrad should be (gradInput) + input should be (inputOrg) + gradOutput should be (gradOutputOrg) + } +} \ No newline at end of file From 269681c9462314f52a0f39ec89a6fb7109b69fbf Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Mon, 5 Jun 2017 14:45:44 +0800 Subject: [PATCH 02/23] fix code style issue --- .../com/intel/analytics/bigdl/nn/Padding.scala | 14 +++++++------- .../com/intel/analytics/bigdl/nn/PaddingSpec.scala | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala index 5477d79a287..106936a97e4 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala @@ -157,13 +157,13 @@ class Padding[T: ClassTag]( object Padding { /** - * Padding information. It define which dimensions to add padding - * @param dim the index of the dimension to add padding, start from 1 - * @param leftPadding left padding length - * @param leftStartIndex left padding start index - * @param rightPadding right padding length - * @param rightStartIndex right padding start index - */ + * Padding information. It define which dimensions to add padding + * @param dim the index of the dimension to add padding, start from 1 + * @param leftPadding left padding length + * @param leftStartIndex left padding start index + * @param rightPadding right padding length + * @param rightStartIndex right padding start index + */ case class PaddingInfo( dim: Int, leftPadding: Int, diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala index 11b4eb93eb0..554f51addf5 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala @@ -80,4 +80,4 @@ class PaddingSpec extends FlatSpec with Matchers { input should be (inputOrg) gradOutput should be (gradOutputOrg) } -} \ No newline at end of file +} From 27362e802ee348ea8a4aa45929f12c1f7de9c7ab Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Wed, 7 Jun 2017 14:34:40 +0800 Subject: [PATCH 03/23] change back the layers --- .../intel/analytics/bigdl/nn/Padding.scala | 178 ++++++------------ .../analytics/bigdl/nn/PaddingSpec.scala | 83 -------- 2 files changed, 55 insertions(+), 206 deletions(-) delete mode 100644 spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala index 106936a97e4..7004f68de58 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala @@ -17,88 +17,81 @@ package com.intel.analytics.bigdl.nn import com.intel.analytics.bigdl.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.tensor.Tensor -import com.intel.analytics.bigdl.nn.Padding._ +import com.intel.analytics.bigdl.tensor.{Storage, Tensor} import scala.reflect.ClassTag /** - * This module adds paddings to the input tensor. + * This module adds pad units of padding to dimension dim of the input. If pad is negative, + * padding is added to the left, otherwise, it is added to the right of the dimension. * * The input to this layer is expected to be a tensor, or a batch of tensors; * when using mini-batch, a batch of sample tensors will be passed to the layer and * the user need to specify the number of dimensions of each sample tensor in the * batch using nInputDims. * - * @param paddings how to pad the input - * @param nInputDim specify the number of dimensions that this module will work on - * If it is not equal to the dimension number of input tensors, the first dimension + * @param dim the dimension to be applied padding operation + * @param pad num of the pad units + * @param nInputDim specify the number of dimensions that this module will receive + * If it is more than the dimension of input tensors, the first dimension * would be considered as batch size * @param value padding value */ @SerialVersionUID(- 3401298839313169602L) class Padding[T: ClassTag]( - val paddings: Array[PaddingInfo], + val dim: Int, + val pad: Int, val nInputDim: Int, - val value: Double = 0.0)(implicit ev: TensorNumeric[T]) extends TensorModule[T] { + val value: Double = 0.0, + val nIndex: Int = 1)(implicit ev: TensorNumeric[T]) extends TensorModule[T] { + + var outputSize = Storage[Int]() override def updateOutput(input: Tensor[T]): Tensor[T] = { - // Expand the output tensor - val batch = (nInputDim != input.nDimension()) - inputs(0) = input - var i = 0 - while(i < paddings.length) { - inputs(i + 1) = paddingOneDimForward(inputs(i), paddings(i), batch) - i += 1 - } - output = inputs(i) + outputSize.resize(input.dim()).copy(Storage(input.size())) + val dim = if (input.dim() != nInputDim) this.dim + 1 else this.dim + + outputSize(dim - 1) += math.abs(this.pad) + output.resize(outputSize.array()).fill(ev.fromType(value)) + val index = if (this.pad > 0) input.size(dim) - nIndex + 2 else nIndex + val pad = if (this.pad > 0) this.pad else -this.pad + + if (index == 1) { + output.narrow(dim, 1 + pad, input.size(dim)).copy(input) + } else if (index == (input.size(dim) + 1)) { + output.narrow(dim, 1, input.size(dim)).copy(input) + } else { + output.narrow(dim, 1, index - 1).copy(input.narrow(dim, 1, index - 1)) + output.narrow(dim, index + pad, input.size(dim) - (index - 1)). + copy(input.narrow(dim, index, input.size(dim) - (index - 1))) + } output } + override def updateGradInput(input: Tensor[T], gradOutput: Tensor[T]): Tensor[T] = { - val batch = (nInputDim != input.nDimension()) - // Copy the tensor content - var gradOutputDim = gradOutput - var i = 0 - while(i < paddings.length) { - gradOutputDim = paddingOneDimBackward(inputs(paddings.length - i - 1), - gradOutputDim, paddings(paddings.length - i - 1), batch) - i += 1 - } + gradInput.resizeAs(input) - gradInput = gradOutputDim - gradInput - } + val dim = if (input.dim() != nInputDim) this.dim + 1 else this.dim + val index = if (this.pad > 0) input.size(dim) - nIndex + 2 else nIndex + val pad = if (this.pad > 0) this.pad else -this.pad - private def paddingOneDimBackward(input: Tensor[T], gradOutput: Tensor[T], - padInfo: PaddingInfo, batch: Boolean): Tensor[T] = { - val gradInput = Tensor[T]().resizeAs(input) - val dim = if (batch) padInfo.dim + 1 else padInfo.dim - if(padInfo.leftStartIndex == 1 && padInfo.rightStartIndex == 1) { - gradInput.narrow(dim, 1, input.size(dim)) - .copy(gradOutput.narrow(dim, padInfo.leftPadding + 1, input.size(dim))) + if (index == 1) { + gradInput.copy(gradOutput.narrow(dim, 1 + pad, input.size(dim))) + } else if (index == input.size(dim) + 1) { + gradInput.copy(gradOutput.narrow(dim, 1, input.size(dim))) } else { - val central = input.size(dim) - (padInfo.leftStartIndex - 1) - (padInfo.rightStartIndex - 1) - if(padInfo.leftStartIndex != 1) { - gradInput.narrow(dim, 1, padInfo.leftStartIndex - 1) - .copy(gradOutput.narrow(dim, 1, padInfo.leftStartIndex - 1)) - } - if (central != 0) { - gradInput.narrow(dim, padInfo.leftStartIndex, central) - .copy(gradOutput.narrow(dim, padInfo.leftStartIndex + padInfo.leftPadding, central)) - } - if(padInfo.rightStartIndex != 1) { - gradInput.narrow(dim, input.size(dim) - padInfo.rightStartIndex + 2, - padInfo.rightStartIndex - 1).copy(gradOutput.narrow(dim, - padInfo.leftStartIndex + central + padInfo.rightPadding, padInfo.rightStartIndex - 1)) - } + gradInput.narrow(dim, 1, index - 1). + copy(gradOutput.narrow(dim, 1, index - 1)) + gradInput.narrow(dim, index, input.size(dim) - (index - 1)).copy( + gradOutput.narrow(dim, index + pad, input.size(dim) - (index - 1))) } gradInput } override def toString(): String = { - s"${getPrintName}((${paddings.mkString(",")}), $nInputDim, $value)" + s"${getPrintName}($dim, $pad, $nInputDim, $value, $nIndex)" } override def canEqual(other: Any): Boolean = other.isInstanceOf[Padding[T]] @@ -107,88 +100,27 @@ class Padding[T: ClassTag]( case that: Padding[T] => super.equals(that) && (that canEqual this) && - paddings == that.paddings && + dim == that.dim && + pad == that.pad && nInputDim == that.nInputDim && - value == that.value + value == that.value && + nIndex == that.nIndex case _ => false } override def hashCode(): Int = { def getHashCode(a: Any): Int = if (a == null) 0 else a.hashCode() - val state = Seq(super.hashCode(), paddings, nInputDim, value) + val state = Seq(super.hashCode(), dim, pad, nInputDim, value, nIndex) state.map(getHashCode).foldLeft(0)((a, b) => 31 * a + b) } - - private val inputs = new Array[Tensor[T]](paddings.length + 1) - inputs.foreach(_ => Tensor[T]()) - - private def paddingOneDimForward(input: Tensor[T], padInfo: PaddingInfo, - batch: Boolean): Tensor[T] = { - // Expand the output tensor - val outputSize = input.size() - val dim = if (batch) padInfo.dim + 1 else padInfo.dim - outputSize(dim - 1) += padInfo.leftPadding + padInfo.rightPadding - val output = Tensor(outputSize).fill(ev.fromType(value)) - - // Copy the tensor content - if(padInfo.leftStartIndex == 1 && padInfo.rightStartIndex == 1) { - output.narrow(dim, padInfo.leftPadding + 1, input.size(dim)) - .copy(input.narrow(dim, 1, input.size(dim))) - } else { - val central = input.size(dim) - (padInfo.leftStartIndex - 1) - (padInfo.rightStartIndex - 1) - if (padInfo.leftStartIndex != 1) { - output.narrow(dim, 1, padInfo.leftStartIndex - 1) - .copy(input.narrow(dim, 1, padInfo.leftStartIndex - 1)) - } - if (central != 0) { - output.narrow(dim, padInfo.leftStartIndex + padInfo.leftPadding, central) - .copy(input.narrow(dim, padInfo.leftStartIndex, central)) - } - if (padInfo.rightStartIndex != 1) { - output.narrow(dim, - outputSize(dim - 1) - padInfo.rightStartIndex + 2, - padInfo.rightStartIndex - 1).copy(input.narrow(dim, - input.size(dim) - padInfo.rightStartIndex + 2, padInfo.rightStartIndex - 1)) - } - } - output - } } -object Padding { - /** - * Padding information. It define which dimensions to add padding - * @param dim the index of the dimension to add padding, start from 1 - * @param leftPadding left padding length - * @param leftStartIndex left padding start index - * @param rightPadding right padding length - * @param rightStartIndex right padding start index - */ - case class PaddingInfo( +object Padding{ + def apply[@specialized(Float, Double) T: ClassTag]( dim: Int, - leftPadding: Int, - leftStartIndex: Int, - rightPadding: Int, - rightStartIndex: Int - ) - - def apply[T: ClassTag]( - dim: Int, - pad: Int, - nInputDim: Int, - value: Double = 0.0, - nIndex: Int = 1)(implicit ev: TensorNumeric[T]): Padding[T] = { - if (pad > 0) { - new Padding[T](Array(PaddingInfo(dim, 0, 1, pad, nIndex)), nInputDim, value) - } else { - new Padding[T](Array(PaddingInfo(dim, -pad, nIndex, 0, 1)), nInputDim, value) - } - } - - def apply[T: ClassTag]( - paddings: Array[PaddingInfo], + pad: Int, nInputDim: Int, - value: Double)(implicit ev: TensorNumeric[T]) : Padding[T] = { - new Padding[T](paddings, nInputDim, value) - } + value: Double = 0.0, + nIndex: Int = 1)(implicit ev: TensorNumeric[T]) : Padding[T] = + new Padding[T](dim, pad, nInputDim, value, nIndex) } diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala deleted file mode 100644 index 554f51addf5..00000000000 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/PaddingSpec.scala +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2016 The BigDL Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intel.analytics.bigdl.nn - -import com.intel.analytics.bigdl.nn.Padding.PaddingInfo -import com.intel.analytics.bigdl.tensor.Tensor -import org.scalatest.{FlatSpec, Matchers} - -class PaddingSpec extends FlatSpec with Matchers { - "A Padding(Array(Array(2, 3), Array(-1, 1, -2, 2))" should - "generate correct output and grad for multi dimension" in { - val layer = Padding[Float]( - Array(PaddingInfo(2, 1, 1, 1, 1), PaddingInfo(3, 1, 1, 1, 1)), 3, 0.0) - val input = Tensor[Float](1, 2, 2) - input(Array(1, 1, 1)) = 0.01f - input(Array(1, 1, 2)) = 0.02f - input(Array(1, 2, 1)) = 0.03f - input(Array(1, 2, 2)) = 0.04f - val expectedOutput = Tensor[Float](1, 4, 4) - expectedOutput(Array(1, 1, 1)) = 0.00f - expectedOutput(Array(1, 1, 2)) = 0.00f - expectedOutput(Array(1, 1, 3)) = 0.00f - expectedOutput(Array(1, 1, 4)) = 0.00f - expectedOutput(Array(1, 2, 1)) = 0.00f - expectedOutput(Array(1, 2, 2)) = 0.01f - expectedOutput(Array(1, 2, 3)) = 0.02f - expectedOutput(Array(1, 2, 4)) = 0.00f - expectedOutput(Array(1, 3, 1)) = 0.00f - expectedOutput(Array(1, 3, 2)) = 0.03f - expectedOutput(Array(1, 3, 3)) = 0.04f - expectedOutput(Array(1, 3, 4)) = 0.00f - expectedOutput(Array(1, 4, 1)) = 0.00f - expectedOutput(Array(1, 4, 2)) = 0.00f - expectedOutput(Array(1, 4, 3)) = 0.00f - expectedOutput(Array(1, 4, 4)) = 0.00f - - val gradOutput = Tensor[Float](1, 4, 4) - gradOutput(Array(1, 1, 1)) = 0.01f - gradOutput(Array(1, 1, 2)) = 0.02f - gradOutput(Array(1, 1, 3)) = 0.03f - gradOutput(Array(1, 1, 4)) = 0.04f - gradOutput(Array(1, 2, 1)) = 0.05f - gradOutput(Array(1, 2, 2)) = 0.06f - gradOutput(Array(1, 2, 3)) = 0.07f - gradOutput(Array(1, 2, 4)) = 0.08f - gradOutput(Array(1, 3, 1)) = 0.09f - gradOutput(Array(1, 3, 2)) = 0.10f - gradOutput(Array(1, 3, 3)) = 0.11f - gradOutput(Array(1, 3, 4)) = 0.12f - gradOutput(Array(1, 4, 1)) = 0.13f - gradOutput(Array(1, 4, 2)) = 0.14f - gradOutput(Array(1, 4, 3)) = 0.15f - gradOutput(Array(1, 4, 4)) = 0.16f - val expectedGrad = Tensor[Float](1, 2, 2) - expectedGrad(Array(1, 1, 1)) = 0.06f - expectedGrad(Array(1, 1, 2)) = 0.07f - expectedGrad(Array(1, 2, 1)) = 0.10f - expectedGrad(Array(1, 2, 2)) = 0.11f - - val inputOrg = input.clone() - val gradOutputOrg = gradOutput.clone() - val output = layer.forward(input) - val gradInput = layer.backward(input, gradOutput) - - expectedOutput should be (output) - expectedGrad should be (gradInput) - input should be (inputOrg) - gradOutput should be (gradOutputOrg) - } -} From fd0b347128ec4aa108224743ae5f47cabe5cdcb8 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 1 Jun 2017 19:30:24 +0800 Subject: [PATCH 04/23] nn refactor --- .../com/intel/analytics/bigdl/nn/Module.scala | 14 +- .../analytics/bigdl/utils/TFToBigDL.scala | 909 ++++++++++++++++++ .../bigdl/utils/TensorFlowSaver.scala | 319 ++++++ .../bigdl/utils/TensorflowLoader.scala | 230 +++++ .../bigdl/utils/TensorflowUtils.scala | 481 +++++++++ .../intel/analytics/bigdl/utils/package.scala | 27 + spark/dl/src/test/resources/tf/.gitignore | 5 + spark/dl/src/test/resources/tf/alexnet.py | 48 + .../test/resources/tf/inception_resnet_v2.py | 52 + .../dl/src/test/resources/tf/inception_v3.py | 51 + spark/dl/src/test/resources/tf/lenet.py | 44 + spark/dl/src/test/resources/tf/load.py | 37 + spark/dl/src/test/resources/tf/overfeat.py | 52 + spark/dl/src/test/resources/tf/resnet_v1.py | 50 + spark/dl/src/test/resources/tf/rnn.py | 53 + .../test/resources/tf/rnn_cell_zero_state.py | 51 + spark/dl/src/test/resources/tf/rnn_lstm.py | 53 + spark/dl/src/test/resources/tf/save.py | 30 + .../resources/tf/saveTest/LinearSaveTest.py | 39 + .../resources/tf/saveTest/ReLUSaveTest.py | 39 + .../dl/src/test/resources/tf/share_weight.py | 47 + spark/dl/src/test/resources/tf/test.pb | Bin 0 -> 1100 bytes spark/dl/src/test/resources/tf/test.py | 45 + spark/dl/src/test/resources/tf/vgg16.py | 48 + spark/dl/src/test/resources/tf/vgg19.py | 48 + spark/dl/src/test/resources/tf/vgga.py | 48 + .../bigdl/utils/TensorflowLoaderSpec.scala | 543 +++++++++++ .../bigdl/utils/TensorflowSaverSpec.scala | 60 ++ 28 files changed, 3422 insertions(+), 1 deletion(-) create mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala create mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorFlowSaver.scala create mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowLoader.scala create mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowUtils.scala create mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala create mode 100644 spark/dl/src/test/resources/tf/.gitignore create mode 100644 spark/dl/src/test/resources/tf/alexnet.py create mode 100644 spark/dl/src/test/resources/tf/inception_resnet_v2.py create mode 100644 spark/dl/src/test/resources/tf/inception_v3.py create mode 100644 spark/dl/src/test/resources/tf/lenet.py create mode 100644 spark/dl/src/test/resources/tf/load.py create mode 100644 spark/dl/src/test/resources/tf/overfeat.py create mode 100644 spark/dl/src/test/resources/tf/resnet_v1.py create mode 100644 spark/dl/src/test/resources/tf/rnn.py create mode 100644 spark/dl/src/test/resources/tf/rnn_cell_zero_state.py create mode 100644 spark/dl/src/test/resources/tf/rnn_lstm.py create mode 100644 spark/dl/src/test/resources/tf/save.py create mode 100644 spark/dl/src/test/resources/tf/saveTest/LinearSaveTest.py create mode 100644 spark/dl/src/test/resources/tf/saveTest/ReLUSaveTest.py create mode 100644 spark/dl/src/test/resources/tf/share_weight.py create mode 100644 spark/dl/src/test/resources/tf/test.pb create mode 100644 spark/dl/src/test/resources/tf/test.py create mode 100644 spark/dl/src/test/resources/tf/vgg16.py create mode 100644 spark/dl/src/test/resources/tf/vgg19.py create mode 100644 spark/dl/src/test/resources/tf/vgga.py create mode 100644 spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowLoaderSpec.scala create mode 100644 spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowSaverSpec.scala diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala index 34a7b7b88b3..0412b120e1a 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala @@ -15,11 +15,12 @@ */ package com.intel.analytics.bigdl.nn +import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.nn.abstractnn.Activity import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.{CaffeLoader, File} +import com.intel.analytics.bigdl.utils.{CaffeLoader, File, TensorflowLoader} import scala.reflect.ClassTag @@ -38,6 +39,17 @@ object Module { CaffeLoader.load[T](model, defPath, modelPath, matchAll) } + /** + * Load tensorflow model from its saved protobuf file. + * @param file where is the protobuf model file + * @param inputs input node names + * @param outputs output node names, the output tensor order is same with the node order + * @return BigDL model + */ + def loadTF(file: String, inputs: Seq[String], outputs: Seq[String]): Module[Float] = { + TensorflowLoader.load(file, inputs, outputs) + } + def flatten[@specialized(Float, Double) T: ClassTag](parameters: Array[Tensor[T]])( implicit ev: TensorNumeric[T]): Tensor[T] = { val compactedTensor = isCompact(parameters) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala new file mode 100644 index 00000000000..287230cb26f --- /dev/null +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala @@ -0,0 +1,909 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils + +import java.nio.charset.Charset +import java.nio.{ByteBuffer, ByteOrder} + +import collection.JavaConverters._ +import com.intel.analytics.bigdl.nn._ +import com.intel.analytics.bigdl.tensor.{Storage, Tensor} +import org.tensorflow.framework.{DataType, NodeDef, TensorProto} +import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} + +import scala.collection.mutable.ArrayBuffer + +trait TFToBigDL { + def topology: DirectedGraph[String] + + def layer(tfGraph: DirectedGraph[NodeDef], + context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) + + + /** + * @param tfGraph + * @return the nodes of this subgraph that accept input data + */ + def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = null + + /** + * + * @param tfGraph + * @return the nodes of this subgraph that emit output data + */ + def getOutputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { + Seq(tfGraph.source) + } + + protected def getOrSetTensor(node: NodeDef, context: Context) + (f: Tensor[Float] => Tensor[Float]) + : (Tensor[Float], Tensor[Float]) = { + if (context.contains(node)) { + context(node) + } else { + val weight = f(TFToBigDL.toTensor(node.getAttrMap.get("value").getTensor)).contiguous() + val gradient = Tensor[Float](weight.size()) + context.put(node, (weight, gradient)) + (weight, gradient) + } + } +} + +object FullConnectionTF extends TFToBigDL{ + private val graph = { + val add = Node("BiasAdd") + val mul = Node("MatMul") + Node("*") -> mul + Node("Const") -> Node("Identity") -> mul -> add + Node("Const") -> Node("Identity") -> add + add.graph(reverse = true) + } + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + + val biasNode = tfGraph.source.prevNodes(1).prevNodes.head.element + val weightNode = tfGraph.source.prevNodes.head.prevNodes(1).prevNodes.head.element + val (bias, gradBias) = getOrSetTensor(biasNode, context)(t => t) + val (weight, gradWeight) = getOrSetTensor(weightNode, context) { t => + t.transpose(1, 2) + } + + val linearLayer = Linear[Float](inputSize = weight.size(2), outputSize = weight.size(1), + initWeight = weight, initGradWeight = gradWeight, initBias = bias, initGradBias = gradBias) + linearLayer.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } + + override def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { + Seq(tfGraph.source.prevNodes.head) + } +} + +object SqueezeTF extends TFToBigDL { + private val graph = (Node("*") -> Node("Squeeze")).graph(reverse = true) + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val dataFormatMatch = Map("N" -> 0, "H" -> 2, "W" -> 3, "C" -> 1) + val dims = tfGraph.source.element.getAttrOrThrow("squeeze_dims").getList().getIList() + .asScala.map(_.toInt).toArray + .map(i => dataFormatMatch(TFToBigDL.dataFormat.charAt(i).toString)) + Squeeze[Float](dims, batchMode = true) + .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object Conv2D extends TFToBigDL{ + private val graph = { + val add = Node("BiasAdd") + val conv = Node("Conv2D") + + Node("*") -> conv + Node("Const") -> Node("Identity") -> conv -> add + Node("Const") -> Node("Identity") -> add + add.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { + Seq(tfGraph.source.prevNodes.head) + } + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val attributes = tfGraph.source.prevNodes(0).element.getAttrMap + require(attributes.get("strides").getList.getI(0).toInt == 1, s"not support strides on batch") + + val (strideH, strideW) = if (attributes.get("data_format").getS + .toString(Charset.defaultCharset()) == "NHWC") { + require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") + (attributes.get("strides").getList.getI(1).toInt, + attributes.get("strides").getList.getI(2).toInt) + } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { + require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") + (attributes.get("strides").getList.getI(2).toInt, + attributes.get("strides").getList.getI(3).toInt) + } else { + throw new IllegalArgumentException("no supported data format") + } + val biasNode = tfGraph.source.prevNodes(1).prevNodes.head.element + val (bias, gradBias) = getOrSetTensor(biasNode, context)(t => t) + + val weightNode = tfGraph.source.prevNodes.head.prevNodes(1).prevNodes.head.element + val (weights, gradWeights) = getOrSetTensor(weightNode, context) { t => + t.transpose(1, 4).transpose(2, 3).transpose(3, 4) + } + + val nOuputPlane = weights.size(1) + val nInputPlane = weights.size(2) + val kernelH = weights.size(3) + val kernelW = weights.size(4) + + val (pW, pH) = + if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { + require((kernelW - strideW) % 2 == 0) + require((kernelH - strideH) % 2 == 0) + ((kernelW - strideW) / 2, (kernelH - strideH) / 2) + } else { + (0, 0) + } + + val convLayer = SpatialConvolution[Float]( + nInputPlane = nInputPlane, nOutputPlane = nOuputPlane, + kernelW = kernelW, kernelH = kernelH, + strideW = strideW, strideH = strideH, + padW = pW, padH = pH, + initWeight = weights, + initBias = bias, + initGradWeight = gradWeights, + initGradBias = gradBias + ) + convLayer.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object ReluTF extends TFToBigDL { + private val graph = { + (Node("*") -> Node("Relu")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + ReLU[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object TanhTF extends TFToBigDL{ + private val graph = { + (Node("*") -> Node("Tanh")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + Tanh[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object SigmoidTF extends TFToBigDL{ + private val graph = { + (Node("*") -> Node("Sigmoid")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + Sigmoid[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object ReshapeTF extends TFToBigDL { + private val graph = { + val nodeReshape = Node("Reshape") + Node("*") -> nodeReshape + Node("Const") -> nodeReshape + nodeReshape.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val sizes = TFToBigDL.toTensor( + tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor) + + val batchMode = sizes.valueAt(1) == -1 + val arraySize = new Array[Int](if (batchMode) sizes.nElement() - 1 else sizes.nElement()) + var i = if (batchMode) 2 else 1 + var k = 0 + while(i <= sizes.nElement()) { + arraySize(k) = sizes.valueAt(i).toInt + k += 1 + i += 1 + } + Reshape[Float](size = arraySize, Some(batchMode)) + .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object MaxPoolingTF extends TFToBigDL { + private val graph = { + (Node("*") -> Node("MaxPool")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val attributes = tfGraph.source.element.getAttrMap + + val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS + .toString(Charset.defaultCharset()) == "NHWC") { + require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(1).toInt, + attributes.get("strides").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(1).toInt, + attributes.get("ksize").getList.getI(2).toInt + ) + } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { + require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(2).toInt, + attributes.get("strides").getList.getI(3).toInt, + attributes.get("ksize").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(3).toInt + ) + } else { + throw new IllegalArgumentException("no supported data format") + } + + val (pW, pH) = + if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { + require((ksizeW - strideW) % 2 == 0) + require((ksizeH - strideH) % 2 == 0) + ((ksizeW - strideW) / 2, (ksizeH - strideH) / 2) + } else { + (0, 0) + } + + val maxpool = SpatialMaxPooling[Float](ksizeW, ksizeH, strideW, strideH, pW, pH) + maxpool.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object AvgPoolingTF extends TFToBigDL{ + private val graph = { + (Node("*") -> Node("AvgPool")).graph(reverse = true) + } + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val attributes = tfGraph.source.element.getAttrMap + + val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS + .toString(Charset.defaultCharset()) == "NHWC") { + require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(1).toInt, + attributes.get("strides").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(1).toInt, + attributes.get("ksize").getList.getI(2).toInt + ) + } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { + require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(2).toInt, + attributes.get("strides").getList.getI(3).toInt, + attributes.get("ksize").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(3).toInt + ) + } else { + throw new IllegalArgumentException("no supported data format") + } + + val (pW, pH) = + if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { + require((ksizeW - strideW) % 2 == 0) + require((ksizeH - strideH) % 2 == 0) + ((ksizeW - strideW) / 2, (ksizeH - strideH) / 2) + } else { + (0, 0) + } + + SpatialAveragePooling[Float](ksizeW, ksizeH, strideW, strideH, pW, pH, countIncludePad = false) + .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object DropoutTF extends TFToBigDL{ + private val graph = { + val nodediv = Node("RealDiv") + val nodeP = Node("Const") + val nodeadd = Node("Add") + val noderandom = Node("Add") + val nodemin = Node("Const") + val nodesub = Node("Sub") + val nodemul = Node("Mul") + val nodedrop = Node("Mul") + Node("*") -> nodediv -> nodedrop + nodeP -> nodediv + nodeP -> nodeadd -> Node("Floor") -> nodedrop + Node("*") -> Node("Shape") -> Node("RandomUniform") -> nodemul -> noderandom -> nodeadd + Node("Const") -> nodesub -> nodemul + nodemin -> nodesub + nodemin -> noderandom + nodedrop.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val keepProp = tfGraph.source.prevNodes(0).prevNodes(1).element + .getAttrMap.get("value").getTensor.getFloatVal(0) + + Dropout[Float](keepProp).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object Placeholder extends TFToBigDL { + private val graph = Node("Placeholder").graph(reverse = true) + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + new Input[Float].asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + + +object ConstTF extends TFToBigDL { + private val graph = Node("Const").graph(reverse = true) + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + val value = TFToBigDL.toTensor(tfGraph.source.element.getAttrMap.get("value").getTensor) + Const(value).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object ShapeTF extends TFToBigDL { + private val graph = { + val node = Node("Shape") + Node("*") -> node + node.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + + new Shape[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object InputTF extends TFToBigDL { + private val graph = (Node("Const") -> Node("Identity")).graph(reverse = true) + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + new Input[Float].asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object IdentityTF extends TFToBigDL { + private val graph = (Node("*") -> Node("Identity")).graph(reverse = true) + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + new Input[Float].asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object BatchNormTF extends TFToBigDL{ + private val graph = { + val nodeInput = Node("*") + val nodeMean1 = Node("Mean") + val nodeStopGrad = Node("StopGradient") + val nodeSub1 = Node("Sub") + val nodeSquare = Node("SquaredDifference") + val nodeMeanss = Node("Sum") + val nodeVarss = Node("Sum") + val nodeShape = Node("Reshape") + val nodeDivisor = Node("Reciprocal") + val nodeShiftedMean = Node("Mul") + val nodeMean2 = Node("Add") + val nodeMul1 = Node("Mul") + val nodeVariance = Node("Sub") + val nodeAdd1 = Node("Add") + val nodeMul2 = Node("Mul") + val nodeMul3 = Node("Mul") + val nodeMul4 = Node("Mul") + val nodeSub2 = Node("Sub") + val nodeAdd2 = Node("Add") + + nodeInput -> nodeMul3 -> nodeAdd2 + Node("Const") -> Node("Identity") -> nodeSub2 + nodeInput -> nodeMean1 -> nodeStopGrad -> nodeShape + Node("Const") -> nodeMean1 + nodeInput -> nodeSub1 -> nodeMeanss -> nodeShiftedMean -> nodeMean2 -> nodeMul4 + nodeStopGrad -> nodeSub1 + nodeInput -> nodeSquare -> nodeVarss -> nodeMul1 -> nodeVariance + nodeStopGrad -> nodeSquare + Node("Const") -> nodeDivisor -> nodeShiftedMean -> Node("Square") -> nodeVariance -> nodeAdd1 + Node("Const") -> nodeMeanss -> nodeDivisor -> nodeMul1 + Node("Const") -> nodeVarss -> nodeDivisor + Node("Const") -> nodeAdd1 -> Node("Rsqrt") -> nodeMul2 -> nodeMul3 + Node("Const") -> Node("Identity") -> nodeMul2 -> nodeMul4 -> nodeSub2 -> nodeAdd2 + Node("Const") -> nodeShape -> nodeMean2 + nodeAdd2.graph(reverse = true) + } + + override def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { + Seq(tfGraph.source.prevNodes.head.prevNodes.head) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val nOutput = tfGraph.source.prevNodes(1).prevNodes(1).prevNodes(1) + .prevNodes(1).prevNodes(0).element.getAttrMap.get("value").getTensor.getIntVal(0) + + val weightNode = tfGraph.source.prevNodes(1).prevNodes.head.prevNodes.head.element + val biasNode = tfGraph.source.prevNodes(1).prevNodes(1).prevNodes(1) + .prevNodes.head.prevNodes.head.element + val (weights, gradWeights) = getOrSetTensor(weightNode, context)(t => t) + val (bias, gradBias) = getOrSetTensor(weightNode, context)(t => t) + + val spatialBatchNorm = SpatialBatchNormalization[Float]( + nOutput = nOutput, + initWeight = weights, + initBias = bias, + initGradWeight = weights, + initGradBias = gradBias + ) + spatialBatchNorm.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object FillTF extends TFToBigDL{ + private val graph = { + val nodeFill = Node("Fill") + Node("*") -> nodeFill + Node("Const") -> nodeFill + nodeFill.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val constNode = tfGraph.source.prevNodes(1) + val const = constNode.element.getAttrMap.get("value").getTensor.getFloatVal(0) + + Fill[Float](const).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object PackTF extends TFToBigDL{ + private val graph = { + val nodePack = Node("Pack") + Node("...") -> nodePack + nodePack.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val dim = tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1 + + Pack[Float](dim).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object UnpackTF extends TFToBigDL{ + private val graph = { + val nodePack = Node("Unpack") + Node("*") -> nodePack + nodePack.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val dim = tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1 + val index = tfGraph.source.element.getName.split(":").toList match { + case _::Nil => 1 + case _::i::Nil => i.toInt + 1 + } + Select[Float](dim, index).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object StrideSliceTF extends TFToBigDL { + private val graph = { + val nodeSlice = Node("StridedSlice") + Node("*") -> nodeSlice + Node("Const") -> nodeSlice + Node("Const") -> nodeSlice + Node("Const") -> nodeSlice + nodeSlice.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + val startNode = tfGraph.source.prevNodes(1) + val endNode = tfGraph.source.prevNodes(2) + val strideNode = tfGraph.source.prevNodes(3) + + def getIntArray(node: Node[NodeDef]) = { + node.element.getAttrMap.get("value").getTensor.getIntValList.asScala.map(_.toInt) + } + + val start = getIntArray(startNode) + val end = getIntArray(endNode) + val stride = getIntArray(strideNode) + + val specs = (start zip end zip stride).zipWithIndex + .map(elem => (elem._2 + 1, elem._1._1._1 + 1, elem._1._1._2 + 1, elem._1._2)).toArray + + + StrideSlice[Float](specs).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + + +object ConcatTF extends TFToBigDL{ + private val graph = { + val nodeConcat = Node("ConcatV2") + Node("...") -> nodeConcat + (Node("Const") -> nodeConcat).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val inputNumber = tfGraph.source.element.getAttrMap.get("N").getI.toInt + val nodeaxis = tfGraph.source.prevNodes(inputNumber) + val axis = nodeaxis.element.getAttrMap.get("value").getTensor.getIntVal(0) + val nInputDims = 4 + + new JoinTable[Float](dimension = axis + 1, nInputDims = -1) + .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object AddConstTF1 extends TFToBigDL{ + private val graph = { + val nodeAdd = Node("Add") + Node("Const") -> nodeAdd + (Node("*") -> nodeAdd).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val value = tfGraph.source.prevNodes.head.element + .getAttrMap.get("value").getTensor.getFloatVal(0) + AddConstant[Float](value).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object AddConstTF2 extends TFToBigDL{ + private val graph = { + val nodeAdd = Node("Add") + Node("*") -> nodeAdd + (Node("Const") -> nodeAdd).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val value = tfGraph.source.prevNodes(1).element + .getAttrMap.get("value").getTensor.getFloatVal(0) + AddConstant[Float](value).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object AddTF extends TFToBigDL{ + private val graph = { + val nodeAdd = Node("Add") + Node("*") -> nodeAdd + (Node("*") -> nodeAdd).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + CAddTable[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object SoftMaxTF extends TFToBigDL{ + private val graph = { + (Node("*") -> Node("Softmax")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + SoftMax[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + + +object MulTF extends TFToBigDL{ + private val graph = { + val nodeMul = Node("Mul") + Node("Const") -> nodeMul + (Node("*") -> nodeMul).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val mul = Mul[Float]() + val scale = TFToBigDL.toTensor( + tfGraph.source.prevNodes(0).element.getAttrMap.get("value").getTensor) + require(scale.dim() == 1 && scale.size(1) == 1, s"scale must be one number") + mul.weight.copy(scale) + mul.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object ElementWiseMulTF extends TFToBigDL{ + private val graph = { + val nodeMul = Node("Mul") + Node("*") -> nodeMul + (Node("*") -> nodeMul).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val mul = CMulTable[Float]() + mul.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object SplitTF extends TFToBigDL { + + private val graph = { + val nodeSplit = Node("Split") + Node("Const") -> nodeSplit + (Node("*") -> nodeSplit).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : (AbstractModule[Activity, Tensor[Float], Float]) = { + val numSplit = tfGraph.source.element.getAttrMap.get("num_split").getI.toInt + val dim = tfGraph.source.prevNodes.head.element + .getAttrMap.get("value").getTensor.getIntVal(0) + 1 + val index = tfGraph.source.element.getName.split(":").toList match { + case _::Nil => 1 + case _::i::Nil => i.toInt + 1 + } + SplitAndSelect[Float](dim, index, numSplit) + .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } + +} + + +object PaddingTF extends TFToBigDL{ + private val graph = { + val nodePad = Node("Pad") + Node("*") -> nodePad + (Node("Const") -> nodePad).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + val paddings = TFToBigDL.toTensor( + tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor) + val pad = ArrayBuffer[Int]() + val dataFormatMatch = Map("N" -> 0, "H" -> 2, "W" -> 3, "C" -> 1) + val padding = Sequential[Float]() + + for(i <- 1 to paddings.size(1)) { + if (paddings(Array(i, 1)) != 0 || paddings(Array(i, 2)) != 0 ) { + val dim = dataFormatMatch(TFToBigDL.dataFormat.charAt(i-1).toString) + 1 + if (paddings(Array(i, 1)) != 0) { + padding.add(Padding[Float](dim, -paddings(Array(i, 1)).toInt, 4)) + } + if (paddings(Array(i, 2)) != 0) { + padding.add(Padding[Float](dim, paddings(Array(i, 1)).toInt, 4)) + } + } + } + + padding.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object MeanTF extends TFToBigDL{ + private val graph = { + val nodeMean = Node("Mean") + Node("*") -> nodeMean + (Node("Const") -> nodeMean).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) + : AbstractModule[Activity, Tensor[Float], Float] = { + val dims = TFToBigDL.toTensor( + tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor) + val dim = ArrayBuffer[Int]() + val dataFormatMatch = Map("N" -> 0, "H" -> 2, "W" -> 3, "C" -> 1) + val mean = Sequential[Float]() + for (i <- 1 to dims.size(1)) { + dim += dataFormatMatch(TFToBigDL + .dataFormat.charAt(dims.valueAt(i).toInt).toString) + 1 + } + dim.foreach(i => mean.add(Mean[Float](i, squeeze = false))) + mean.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] + } +} + +object TFToBigDL { + + /** + * Get the pattern list. + * @return + */ + def patterns : Array[TFToBigDL] = { + patternList.toArray + } + + /** + * Switch endianess to big endian. You should do this when you save the model in a big endian + * environment. The default endianess is little endian. + */ + def bigEndian : Unit = endian = ByteOrder.BIG_ENDIAN + + /** + * Switch endianess to little endian. You should do this when you save the model in a little + * endian environment. This is the default endianess. + */ + def littleEndian : Unit = endian = ByteOrder.LITTLE_ENDIAN + + /** + * Register a new mapping from tensor flow operations to BigDL layer. The mapping is defined as + * a subclass of TFToBigDL, which defines an operation topology(reversed graph) and how to get + * constructor parameters from the topology. + * @param pattern + */ + def registerPattern(pattern : TFToBigDL): Unit = { + require(pattern.topology.reverse == true, "the topology should be a reversed graph") + patternList.append(pattern) + sortPattern() + } + + private var endian = ByteOrder.LITTLE_ENDIAN + + var dataFormat : String = "NHWC" + + def dataNCHW : Unit = dataFormat = "NCHW" + + /** + * Convert a tensorflow tensor proto to BigDL tensor + * @param tfTensor + * @return + */ + private[utils] def toTensor(tfTensor: TensorProto): Tensor[Float] = { + require(tfTensor.getDtype == DataType.DT_FLOAT || tfTensor.getDtype == DataType.DT_INT32, + s"Data type ${tfTensor.getDtype} is not supported now") + val shape = tfTensor.getTensorShape.getDimList.asScala.map(_.getSize.toInt).toArray + + if (shape.product == 1) { + if (tfTensor.getDtype == DataType.DT_FLOAT) { + return Tensor[Float](T(tfTensor.getFloatVal(0))) + } else { + return Tensor[Float](T(tfTensor.getIntVal(0).toFloat)) + } + } + + val buffer = ByteBuffer.wrap(tfTensor.getTensorContent.toByteArray) + buffer.order(endian) + + if (tfTensor.getDtype == DataType.DT_FLOAT) { + val params = buffer.asFloatBuffer + val tmp = new Array[Float](params.capacity()) + var j = 0 + while(j < params.capacity()) { + tmp(j) = params.get(j) + j += 1 + } + Tensor(Storage(tmp), 1, shape) + } else { + val params = buffer.asIntBuffer + val tmp = new Array[Float](params.capacity()) + var j = 0 + while(j < params.capacity()) { + tmp(j) = params.get(j) + j += 1 + } + Tensor(Storage(tmp), 1, shape) + } + } + + private var patternList : ArrayBuffer[TFToBigDL] = { + val res = new ArrayBuffer[TFToBigDL]() + // ElementWiseMulTF must be after MulTF + res.append( + FullConnectionTF, DropoutTF, AvgPoolingTF, MaxPoolingTF, ReshapeTF, InputTF, + TanhTF, ReluTF, SigmoidTF, Conv2D, Placeholder, SqueezeTF, IdentityTF, ConcatTF, + BatchNormTF, AddConstTF1, AddConstTF2, AddTF, SoftMaxTF, MulTF, ElementWiseMulTF, + SplitTF, PaddingTF, MeanTF, UnpackTF, StrideSliceTF, ShapeTF, FillTF, PackTF, ConstTF + ) + res + } + + sortPattern() + + /** + * Sort the pattern list to make sure the graph match first should not be a sub-graph of the graph + * match later + */ + private def sortPattern() : Unit = { + // do not calculate size and edges of a graph every time + val topToNNodes = patternList.map(g => g -> g.topology.size).toMap + val topToNEdges = patternList.map(g => g -> g.topology.edges).toMap + patternList = patternList.sortWith((l, r) => { + if (topToNNodes(l) != topToNNodes(r)) { + // graph with more nodes comes first + topToNNodes(l) > topToNNodes(r) + } else { + // same node number, graph with more edges come first + topToNEdges(l) > topToNEdges(r) + } + }) + } +} diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorFlowSaver.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorFlowSaver.scala new file mode 100644 index 00000000000..d4292df76b9 --- /dev/null +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorFlowSaver.scala @@ -0,0 +1,319 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils + +import java.io.FileOutputStream + +import com.google.protobuf.CodedOutputStream +import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} +import com.intel.analytics.bigdl.nn._ +import com.intel.analytics.bigdl.tensor.Tensor +import org.apache.log4j.Logger +import org.tensorflow.framework._ + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +import TensorflowUtils._ + +object TensorFlowSaver { + /** + * Save a graph model to protobuf files so that it can be used in tensorflow inference. + * + * When save the model, placeholders will be added to the tf model as input nodes. So you need to + * pass in the names and shape for the placeholders. BigDL model doesn't have such information. + * The order of the placeholde information should be same as the inputs of the graph model + * + * @param model graph model instance + * @param inputs placeholder information + * @param path where to save + * @tparam T + */ + def saveGraph[T]( + model : Graph[T], + inputs : Seq[(String, Seq[Int])], + path: String): Unit = { + val inputNodeDefs = inputs.map(input => + placeholder(model.getNumericType(), input._2, input._1) + ) + val inputNodeCache = + new mutable.HashMap[AbstractModule[Activity, Tensor[T], T], ArrayBuffer[NodeDef]]() + model.inputs.zip(inputNodeDefs).foreach(n => { + inputNodeCache(n._1.element) = ArrayBuffer(n._2) + }) + + val graphBuilder = GraphDef.newBuilder() + inputNodeDefs.foreach(graphBuilder.addNode(_)) + + model.executions.foreach(n => { + val nodeDefs = maps(n.element.getClass.getName).toTFDef(n.element, inputNodeCache(n.element)) + nodeDefs.foreach(nDef => { + graphBuilder.addNode(nDef) + }) + n.nextNodes.foreach(n => { + val list = inputNodeCache.getOrElse(n.element, ArrayBuffer()) + list.append(nodeDefs(0)) + }) + }) + + // Save to file + val os = new FileOutputStream(path) + val output = CodedOutputStream.newInstance(os) + val graph = graphBuilder.build() + logger.debug("Graph definition is:") + logger.debug(graph.toString) + graph.writeTo(output) + output.flush() + os.close() + logger.info(s"Save as tensorflow model file to $path") + } + + private val logger = Logger.getLogger(getClass) + + private val maps = mutable.Map[String, BigDLToTF]( + getNameFromObj(ReLU.getClass.getName) -> ReLUToTF, + getNameFromObj(Linear.getClass.getName) -> LinearToTF + ) + + private def getNameFromObj(name: String) : String = name.substring(0, name.length - 1) +} + +/** + * Wrapper of logic to convert module to tensorflow node definition + */ +trait BigDLToTF { + + /** + * Convert the module to a tensorflow nodedef + * @return Mapped nodedef list, the first is the output node + */ + def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] +} + +object ReLUToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Relu only accept one input") + + Seq(relu(inputs(0), module.getName())) + } +} + +object LinearToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Linear only accept one input") + val linear = module.asInstanceOf[Linear[_]] + val weight = const(linear.weight, linear.getName() + "/weight") + val weightReader = identity(weight, linear.getName() + "/weightReader") + val mm = matmul(inputs(0), weightReader, linear.getName() + "matmul") + val bias = const(linear.bias, linear.getName() + "/bias") + val biasReader = identity(bias, linear.getName() + "/biasReader") + val add = biasAdd(mm, biasReader, NCHW, linear.getName() + "/biasAdd") + Seq(add, biasReader, bias, mm, weightReader, weight) + } +} + +object SpatialConvolutionToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "SpatialConvolution only accept one input") + val spatialConv = module.asInstanceOf[SpatialConvolution[_]] + val filter = const(spatialConv.weight, spatialConv.getName() + "/filter") + val filterReader = identity(filter, spatialConv.getName() + "/filterReader") + val conv = conv2D(inputs(0), filterReader, spatialConv.strideH, spatialConv.strideW, + spatialConv.kernelW, spatialConv.kernelH, spatialConv.strideW, spatialConv.strideH, NCHW, + spatialConv.getName() + "/conv2D") + val bias = const(spatialConv.bias, spatialConv.getName() + "/bias") + val biasReader = identity(bias, spatialConv.getName() + "/biasReader") + val add = biasAdd(conv, biasReader, NCHW, spatialConv.getName() + "/biasAdd") + Seq(add, biasReader, bias, conv, filterReader, filter) + } +} + +object SqueezeToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Squeeze only accept one input") + val sq = module.asInstanceOf[Squeeze[_]] + Seq(squeeze(inputs(0), sq.dims.map(_ - 1), sq.getName())) + } +} + +object TanhToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Tanh only accept one input") + Seq(tanh(inputs(0), module.getName())) + } +} + +object ReshapeToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Reshape only accept one input") + val rh = module.asInstanceOf[Reshape[_]] + val size = Tensor[Float](rh.size.length) + var i = 0 + while(i < rh.size.length) { + size.setValue(i + 1, rh.size(i)) + i += 1 + } + val shape = const(size, rh.getName() + "/shape", DataType.DT_INT32) + val reshapeNode = reshape(inputs(0), shape, rh.getName()) + Seq(reshapeNode, shape) + } +} + +object ViewToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Reshape only accept one input") + val viewLayer = module.asInstanceOf[View[_]] + val size = Tensor[Float](viewLayer.sizes.length) + var i = 0 + while(i < viewLayer.sizes.length) { + size.setValue(i + 1, viewLayer.sizes(i)) + i += 1 + } + val shape = const(size, viewLayer.getName() + "/shape", DataType.DT_INT32) + val reshapeNode = reshape(inputs(0), shape, viewLayer.getName()) + Seq(reshapeNode, shape) + } +} + +object MaxpoolToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Maxpool only accept one input") + val layer = module.asInstanceOf[SpatialMaxPooling[_]] + Seq(maxPool(inputs(0), layer.kW, layer.kH, layer.padW, layer.padH, + layer.dW, layer.dH, NCHW, layer.getName())) + } +} + +object PaddingToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Padding only accept one input") + val layer = module.asInstanceOf[Padding[_]] + val padding = Tensor[Float](1, 2) + if (layer.pad < 0) { + padding.setValue(1, 1, -layer.pad) + padding.setValue(1, 2, 0) + } + else { + padding.setValue(1, 1, 0) + padding.setValue(1, 2, layer.pad) + } + val paddingsNode = const(padding, layer.getName() + "/padding", DataType.DT_INT32) + val padNode = pad(inputs(0), paddingsNode, layer.getName() + "/output") + Seq(padNode, paddingsNode) + } +} + +object AvgpoolToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Avgpool only accept one input") + val layer = module.asInstanceOf[SpatialAveragePooling[_]] + Seq(avgPool(inputs(0), layer.kW, layer.kH, layer.padW, layer.padH, + layer.dW, layer.dH, NCHW, layer.getName())) + } +} + +object SigmoidToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Sigmoid only accept one input") + Seq(sigmoid(inputs(0), module.getName())) + } +} + +object DropoutToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Dropout only accept one input") + val layer = module.asInstanceOf[Dropout[_]] + val shapeNode = shape(inputs(0), layer.getName() + "/shape") + val rand = randomUniform(shapeNode, layer.getName() + "/random") + val maxNode = const(Tensor[Float](T(1.0f)), layer.getName() + "/max") + val minNode = const(Tensor[Float](T(0.0f)), layer.getName() + "/max") + val sub = subtract(maxNode, minNode, layer.getName() + "/sub") + val mul = multiply(rand, sub, layer.getName() + "/mul") + val randOutput = add(minNode, mul, layer.getName() + "/rand_output") + val keepProb = const(Tensor[Float](T(0.5f)), layer.getName() + "/keep_prob") + val div1 = realdiv(keepProb, inputs(0), layer.getName() + "/div1") + val div2 = realdiv(keepProb, randOutput, layer.getName() + "/div2") + val floorNode = floor(div2, layer.getName() + "/floor") + val output = multiply(div1, floorNode, layer.getName() + "/output") + Seq(output, floorNode, div2, div1, keepProb, randOutput, mul, sub, minNode, maxNode, + rand, shapeNode) + } +} + +object CAddTableToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + Seq(addN(inputs, module.getName())) + } +} + +object CMultTableToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 2, "Tensorflow only support two tensor multiply together") + + Seq(multiply(inputs(0), inputs(1), module.getName())) + } +} + +object JoinTableToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + val layer = module.asInstanceOf[JoinTable[_]] + Seq(concat(inputs, layer.dimension - 1, layer.getName())) + } +} + +object MeanToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Mean only accept one input") + val layer = module.asInstanceOf[Mean[_]] + val dimsTensor = Tensor[Float](layer.dimension) + dimsTensor.setValue(1, layer.dimension) + + val dims = const(dimsTensor, layer.getName() + "/dims") + val mean = reduceMean(inputs(0), dims, false, layer.getName() + "/output") + Seq(mean, dims) + } +} + +object SoftMaxToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "Softmax only accept one input") + Seq(softmax(inputs(0), module.getName())) + } +} + +object LogSoftMaxToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "LogSoftmax only accept one input") + Seq(logSoftmax(inputs(0), module.getName())) + } +} + +object BatchNormToTF extends BigDLToTF { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { + require(inputs.length == 1, "BatchNorm only accept one input") + val layer = module.asInstanceOf[SpatialBatchNormalization[_]] + val stdVar = const(layer.saveStd, layer.getName() + "/std") + val mean = const(layer.saveMean, layer.getName() + "/mean") + val scale = const(layer.weight, layer.getName() + "/scale") + val offset = const(layer.bias, layer.getName() + "/offset") + val div = realdiv(scale, stdVar, layer.getName() + "/div") + val mul1 = multiply(inputs(0), div, layer.getName() + "/mul1") + val mul2 = multiply(scale, div, layer.getName() + "/mul2") + val sub = multiply(offset, scale, layer.getName() + "/sub") + val output = add(mul1, sub, layer.getName() + "/output") + Seq(output, sub, mul2, mul1, div, offset, scale, mean, stdVar) + } +} \ No newline at end of file diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowLoader.scala new file mode 100644 index 00000000000..c2c96eab92b --- /dev/null +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowLoader.scala @@ -0,0 +1,230 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils + +import java.io.{DataInputStream, FileInputStream} +import java.util + +import org.tensorflow.framework.{GraphDef, NodeDef} +import com.google.protobuf.CodedInputStream +import java.util.{Collections, List} + +import com.intel.analytics.bigdl.Module +import com.intel.analytics.bigdl.nn.Graph +import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} +import com.intel.analytics.bigdl.tensor.Tensor + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +case class TFInputOutputNodes(input: Seq[Node[NodeDef]], output: Seq[Node[NodeDef]]) + +object TensorflowLoader{ + /** + * Load tensor flow module from protobuf file + * @param graphPrototxt where is the tf protobuf file + * @return + */ + def load(graphPrototxt: String, inputs: Seq[String], outputs: Seq[String]): Module[Float] = { + val nodeList = parse(graphPrototxt) + val tfGraph = buildTFGraph(nodeList) + buildBigDLModel(tfGraph, inputs, outputs) + } + + private val inputPlaceholder : String = "*" + + private val nInputPlaceholder : String = "..." + + /** + * Extract one module and the corresponding node list from the given graph + * @param graph + * @return + */ + private[bigdl] def extract(graph: DirectedGraph[NodeDef], + context: Context) + : (Option[AbstractModule[Activity, Tensor[Float], Float]], + List[Node[NodeDef]], + TFInputOutputNodes) = { + var i = 0 + while(i < TFToBigDL.patterns.length) { + val result = matchGraph(graph, TFToBigDL.patterns(i).topology) + if (result.size != 0) { + // get model + val input = TFToBigDL.patterns(i).getInputNodes(graph) + val output = TFToBigDL.patterns(i).getOutputNodes(graph) + val inputOutput = TFInputOutputNodes(input, output) + return (Some(TFToBigDL.patterns(i).layer(graph, context)), result, inputOutput) + } + i += 1 + } + (None, Collections.emptyList(), null) + } + + private def matchGraph(graph: DirectedGraph[NodeDef], pattern: DirectedGraph[String]) + : List[Node[NodeDef]] = { + require(graph.reverse && pattern.reverse, "Must pass in reversed graph") + val patternToGraph = new mutable.HashMap[Node[String], Node[NodeDef]]() + patternToGraph(pattern.source) = graph.source + + pattern.BFS.foreach(patternNode => { + if (patternNode.element != nInputPlaceholder && patternNode.element != inputPlaceholder) { + // Normal operation node + if (patternToGraph.get(patternNode).isEmpty) return util.Collections.emptyList() + + val graphNode = patternToGraph.get(patternNode).get + // Operation type should match + if (patternNode.element != graphNode.element.getOp) return util.Collections.emptyList() + + // Prev nodes number should be same except for the Ninput case + if (patternNode.prevNodes.length != graphNode.prevNodes.length && + patternNode.prevNodes.filter(_.element == nInputPlaceholder).length == 0) { + return util.Collections.emptyList() + } + + var i = 0 + var direction = 0 + var j = 0 + while (i < patternNode.prevNodes.length) { + if (patternNode.prevNodes(i).element == nInputPlaceholder) { + require(patternNode.prevNodes.count(_.element == nInputPlaceholder) == 1, + s"only support one $nInputPlaceholder ") + direction = 1 + // skip the left input nodes of graphNode, + // once we find the placeholder, we start from another side + } else if (patternNode.prevNodes(i).element == inputPlaceholder) { + // skip input placeholder + } else { + val posPattern = { if (direction == 0) i else patternNode.prevNodes.length - 1 - j} + val posGraph = { if (direction == 0) i else graphNode.prevNodes.length - 1 - j} + val pn = patternNode.prevNodes(posPattern) + val gn = graphNode.prevNodes(posGraph) + if (patternToGraph.keySet.contains(pn)) { + if (!patternToGraph(pn).eq(gn)) return util.Collections.emptyList() + } else { + patternToGraph(pn) = gn + } + if (direction == 1) j += 1 + } + i += 1 + } + } + }) + import scala.collection.JavaConverters._ + return patternToGraph.valuesIterator.toList.asJava + } + + private[bigdl] def buildBigDLModel( + tfGraph: DirectedGraph[NodeDef], + inputs: Seq[String], + outputs: Seq[String] + ): Module[Float] = { + import scala.collection.JavaConverters._ + + val convertedNode = new mutable.HashMap[Node[NodeDef], + Node[AbstractModule[Activity, Tensor[Float], Float]]]() + val nameToNode = + new mutable.HashMap[String, Node[AbstractModule[Activity, Tensor[Float], Float]]]() + + val context = new Context + + tfGraph.BFS.foreach(n => { + if (n.element == null) { + // Dummy node, skip + } else if (convertedNode.get(n).isDefined) { + // converted node, skip + } else { + val (module, nodes, inputOutput) = extract(n.graph(reverse = true), context) + require(module.isDefined, s"Can not find matched graph ${n}") + val node = new Node(module.get) + nodes.asScala.foreach(m => { + convertedNode(m) = node + nameToNode(m.element.getName) = node + }) + + val outputNodes = if (inputOutput.output == null) nodes.asScala else inputOutput.output + val nextNodes = outputNodes.flatMap(_.nextNodes) + .filter(n => n.element != null && convertedNode.contains(n) + && !context.contains(n.element)) + .map(convertedNode(_)).filter(_ != node).toSet + nextNodes.foreach(node -> _) + + val inputNodes = if (inputOutput.input == null) nodes.asScala else inputOutput.input + val preNodes = inputNodes.flatMap(_.prevNodes) + .filter(n => n.element != null && convertedNode.contains(n) + && !context.contains(n.element)) + .map(convertedNode(_)).filter(_ != node).toSet + preNodes.foreach(_ -> node) + } + }) + + val inputNodes = inputs + .map(n => nameToNode.getOrElse(n, throw new IllegalArgumentException(s"Can't find node $n"))) + val outputNodes = outputs + .map(n => nameToNode.getOrElse(n, throw new IllegalArgumentException(s"Can't find node $n"))) + + + val weights = ArrayBuffer[Tensor[Float]]() + val gradients = ArrayBuffer[Tensor[Float]]() + for ((weight, grad) <- context.values) { + weights += weight + gradients += grad + } + + Graph(inputNodes.toArray, outputNodes.toArray, Some((weights.toArray, gradients.toArray))) + } + + /** + * Build tf ops graph from a given node list + * @param nodes + * @return + */ + private[bigdl] def buildTFGraph(nodes : List[NodeDef]): DirectedGraph[NodeDef] = { + import scala.collection.JavaConverters._ + var name2Node = nodes.asScala.map(n => n.getName -> new Node(n)).toMap + nodes.asScala + .flatMap(_.getInputList.asScala) + .filter(_.split(":").length > 1) + .foreach { nameWithChannel => + val name = nameWithChannel.split(":").head + val tfNode = NodeDef.newBuilder(name2Node(name).element) + .setName(nameWithChannel).build() + name2Node += nameWithChannel -> new Node(tfNode) + } + // Connect nodes + name2Node.valuesIterator.foreach(n => { + n.element.getInputList.asScala.foreach(name2Node(_) -> n) + }) + val outputNodes = name2Node.valuesIterator.filter(_.nextNodes.length == 0) + val dummyOutput = new Node[NodeDef](null) + outputNodes.foreach(_ -> dummyOutput) + dummyOutput.graph(reverse = true) + } + + /** + * Parse a tensor flow model protobuf file, read a list of op nodes from it + * @param graphPrototxt where is the tf protobuf file + * @return + */ + private[bigdl] def parse(graphPrototxt: String) : List[NodeDef] = { + val f = new java.io.File(graphPrototxt) + val reader = CodedInputStream.newInstance(new DataInputStream(new FileInputStream(f))) + reader.setSizeLimit(0x7fffffff) + require(f.exists(), graphPrototxt + " does not exists") + + val graph = GraphDef.parseFrom(reader) + graph.getNodeList + } +} \ No newline at end of file diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowUtils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowUtils.scala new file mode 100644 index 00000000000..204d3369779 --- /dev/null +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowUtils.scala @@ -0,0 +1,481 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils + +import java.nio.charset.Charset + +import com.google.protobuf.ByteString +import com.intel.analytics.bigdl.tensor.{DoubleType, FloatType, Tensor, TensorDataType} +import org.tensorflow.framework.AttrValue.ListValue +import org.tensorflow.framework._ +import org.tensorflow.framework.TensorShapeProto.Dim + +/** + * Tensorflow data format. It is mostly applied in processing image type data + */ +sealed trait TensorflowDataFormat { + def value : AttrValue +} + +/** + * Store the image data in tensor as batch, height, width, channel + */ +object NHWC extends TensorflowDataFormat { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("NHWC", Charset.defaultCharset())).build() + + override def value: AttrValue = v +} + +sealed trait PaddingType { + def value : AttrValue +} + +object PADDING_SAME extends PaddingType { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("SAME", Charset.defaultCharset())).build() + + override def value: AttrValue = v +} + +object PADDING_VALID extends PaddingType { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("VALID", Charset.defaultCharset())).build() + + override def value: AttrValue = v +} + +/** + * Store the image data in tensor as batch, channel, height, width + */ +object NCHW extends TensorflowDataFormat { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("NCHW", Charset.defaultCharset())).build() + + override def value: AttrValue = v +} + +object TensorflowUtils { + /** + * Generate a placeholder tensorflow protobuf node + * @param dtype numeric type + * @param shape shape + * @param name node name + * @return + */ + def placeholder(dtype: TensorDataType, shape: Seq[Int], name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Placeholder") + .putAttr("dtype", typeAttr(dtype)) + .putAttr("shape", shapeAttr(shape)) + .build() + } + + /** + * Generate a const tensorflow protobuf node + * @param value + * @param name + * @return + */ + def const(value : Tensor[_], name : String, dtype: DataType = DataType.DT_FLOAT): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Const") + .putAttr("dtype", AttrValue.newBuilder().setType(dtype).build()) + .putAttr("value", tensorAttr(value, dtype)) + .build() + } + + /** + * Generate an identity tensorflow protobuf node + * @param input + * @param name + * @return + */ + def identity(input : NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Identity") + .addInput(input.getName) + .putAttr("T", getDataType(input)) + .build() + } + + /** + * Generate a matmul tensorflow protobuf node + * @param a + * @param b + * @param name + * @param transposeA + * @param transposeB + * @return + */ + def matmul(a: NodeDef, b: NodeDef, name: String, + transposeA: Boolean = false, transposeB: Boolean = false): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("MatMul") + .addInput(a.getName) + .addInput(b.getName) + .putAttr("T", getDataType(a)) + .putAttr("transpose_a", booleanAttr(transposeA)) + .putAttr("transpose_b", booleanAttr(transposeB)) + .build() + } + + /** + * Generate a biasAdd tensorflow protobuf node + * @param value + * @param bias + * @param dataFormat + * @param name + * @return + */ + def biasAdd(value: NodeDef, bias: NodeDef, dataFormat: TensorflowDataFormat, + name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("BiasAdd") + .addInput(value.getName) + .addInput(bias.getName) + .putAttr("T", getDataType(value)) + .putAttr("data_format", dataFormat.value) + .build() + } + + /** + * Generate a relu tensorflow protobuf node + * @param features + * @param name + * @return + */ + def relu(features: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Relu") + .addInput(features.getName) + .putAttr("T", getDataType(features)) + .build() + } + + def conv2D(input: NodeDef, filter: NodeDef, sW: Int, sH: Int, kW: Int, kH: Int, pW: Int, pH: Int, + dataFormat: TensorflowDataFormat, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Conv2D") + .addInput(filter.getName) + .addInput(input.getName) + .putAttr("T", getDataType(input)) + .putAttr("data_format", dataFormat.value) + .putAttr("padding", getPaddingType(pW, pH, kW, kH, sW, sH).value) + .putAttr("strides", listIntAttr(Seq(sH, sW))) + .build() + } + + def squeeze(input: NodeDef, axis: Seq[Int], name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Squeeze") + .addInput(input.getName) + .putAttr("T", getDataType(input)) + .putAttr("squeeze_dims", listIntAttr(axis)) + .build() + } + + def tanh(input: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Tanh") + .addInput(input.getName) + .putAttr("T", getDataType(input)) + .build() + } + + def reshape(tensor: NodeDef, shape: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Reshape") + .addInput(tensor.getName) + .addInput(shape.getName) + .putAttr("T", getDataType(tensor)) + .putAttr("Tshape", getDataType(shape)) + .build() + + } + + def maxPool(value: NodeDef, kW: Int, kH: Int, pW: Int, pH: Int, sW: Int, sH: Int, + dataFormat: TensorflowDataFormat, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("MaxPool") + .addInput(value.getName) + .putAttr("T", getDataType(value)) + .putAttr("data_format", dataFormat.value) + .putAttr("ksize", listIntAttr(Seq(kH, kW))) + .putAttr("padding", getPaddingType(pW, pH, kW, kH, sW, sH).value) + .putAttr("strides", listIntAttr(Seq(sH, sW))) + .build() + } + + def avgPool(value: NodeDef, kW: Int, kH: Int, pW: Int, pH: Int, sW: Int, sH: Int, + dataFormat: TensorflowDataFormat, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("AvgPool") + .putAttr("T", getDataType(value)) + .addInput(value.getName) + .putAttr("data_format", dataFormat.value) + .putAttr("ksize", listIntAttr(Seq(kH, kW))) + .putAttr("padding", getPaddingType(pW, pH, kW, kH, sW, sH).value) + .putAttr("strides", listIntAttr(Seq(sH, sW))) + .build() + } + + def sigmoid(x: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Sigmoid") + .putAttr("T", getDataType(x)) + .addInput(x.getName) + .build() + } + + def multiply(x: NodeDef, y: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Mul") + .putAttr("T", getDataType(x)) + .addInput(x.getName) + .addInput(y.getName) + .build() + } + + def floor(x: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Floor") + .putAttr("T", getDataType(x)) + .addInput(x.getName) + .build() + } + + def add(x: NodeDef, y: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Add") + .putAttr("T", getDataType(x)) + .addInput(x.getName) + .addInput(y.getName) + .build() + } + + def realdiv(x: NodeDef, y: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("RealDiv") + .putAttr("T", getDataType(x)) + .addInput(x.getName) + .addInput(y.getName) + .build() + } + + def subtract(x: NodeDef, y: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Sub") + .putAttr("T", getDataType(x)) + .addInput(x.getName) + .addInput(y.getName) + .build() + } + + def shape(input: NodeDef, name: String, outType: DataType = DataType.DT_INT32): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Shape") + .putAttr("T", getDataType(input)) + .putAttr("out_type", AttrValue.newBuilder().setType(outType).build()) + .build() + } + + def randomUniform(shape: NodeDef, name: String, dtype: DataType = DataType.DT_FLOAT, + seed: Int = 0): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("RandomUniform") + .putAttr("T", getDataType(shape)) + .putAttr("dtype", AttrValue.newBuilder().setType(dtype).build()) + .putAttr("seed", intAttr(seed)) + .putAttr("seed2", intAttr(seed)) + .build() + } + + def addN(inputs: Seq[NodeDef], name: String): NodeDef = { + require(inputs.length >= 2, "at least two inputs for addN") + val node = NodeDef.newBuilder() + .setName(name) + .putAttr("N", intAttr(inputs.length)) + .putAttr("T", getDataType(inputs(0))) + .setOp("AddN") + inputs.foreach(i => node.addInput(i.getName)) + node.build() + } + + def concat(inputs: Seq[NodeDef], axis: Int, name: String): NodeDef = { + require(inputs.length >= 1, "at least one inputs for addN") + + val node = NodeDef.newBuilder() + .setName(name) + .setOp("ConcatV2") + .putAttr("N", intAttr(axis)) + .putAttr("T", getDataType(inputs(0))) + .putAttr("Tidx", AttrValue.newBuilder().setType(DataType.DT_INT32).build()) + + inputs.foreach(i => node.addInput(i.getName)) + + node.build() + } + + def pad(tensor: NodeDef, paddings: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Pad") + .putAttr("T", getDataType(tensor)) + .putAttr("Tpaddings", getDataType(paddings)) + .addInput(tensor.getName) + .build() + } + + def reduceMean(inputTensor: NodeDef, axis: NodeDef, keepDim: Boolean, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Mean") + .putAttr("T", getDataType(inputTensor)) + .putAttr("Tidx", getDataType(axis)) + .putAttr("keep_dims", booleanAttr(keepDim)) + .addInput(inputTensor.getName) + .addInput(axis.getName) + .build() + } + + def softmax(logits: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Softmax") + .putAttr("T", getDataType(logits)) + .addInput(logits.getName) + .build() + } + + def logSoftmax(logits: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("LogSoftmax") + .putAttr("T", getDataType(logits)) + .addInput(logits.getName) + .build() + } + + def rsqrt(x: NodeDef, name: String): NodeDef = { + NodeDef.newBuilder() + .setName(name) + .setOp("Rsqrt") + .putAttr("T", getDataType(x)) + .addInput(x.getName) + .build() + } + + private def booleanAttr(value: Boolean): AttrValue = { + AttrValue.newBuilder().setB(value).build() + } + + private def intAttr(value: Int): AttrValue = { + AttrValue.newBuilder().setI(value).build() + } + + private def listIntAttr(value: Seq[Int]): AttrValue = { + val list = ListValue.newBuilder() + value.foreach(list.addI(_)) + AttrValue.newBuilder().setList(list).build() + } + + private def tensorAttr(value: Tensor[_], dtype: DataType): AttrValue = { + val shape = TensorShapeProto.newBuilder() + value.size().foreach(dim => { + shape.addDim(Dim.newBuilder().setSize(dim)) + }) + + AttrValue.newBuilder().setTensor( + TensorProto.newBuilder().setTensorShape(shape).setDtype(dtype) + ).build() + } + + private def tensorAttr(value: Seq[Int]): AttrValue = { + val shape = TensorShapeProto.newBuilder() + shape.addDim(Dim.newBuilder().setSize(value.length)) + val dtype = DataType.DT_INT32 + AttrValue.newBuilder().setTensor( + TensorProto.newBuilder().setTensorShape(shape).setDtype(dtype) + ).build() + } + + private def typeAttr(dtype : TensorDataType): AttrValue = { + if (dtype == FloatType) { + AttrValue.newBuilder().setType(DataType.DT_FLOAT).build() + } else if (dtype == DoubleType) { + AttrValue.newBuilder().setType(DataType.DT_DOUBLE).build() + } else { + throw new NotImplementedError(s"type $dtype is not supported") + } + } + + private def shapeAttr(shape: Seq[Int]): AttrValue = { + val attr = TensorShapeProto.newBuilder() + shape.foreach(dim => { + attr.addDim(Dim.newBuilder().setSize(dim)) + }) + AttrValue.newBuilder().setShape(attr).build() + } + + private def getDataType(node: NodeDef): AttrValue = { + var attr = node.getAttrOrDefault("T", null) + if (attr != null) { + return attr + } + + attr = node.getAttrOrDefault("dtype", null) + if (attr != null) { + return attr + } + + throw new IllegalArgumentException("TensorflowSaver: Can not find data type") + } + + private def getPaddingType(padW: Int, padH: Int, kW: Int, kH: Int, sW: Int, sH: Int) + : PaddingType = { + if (padW == 0 && padH == 0) { + return PADDING_VALID + } else if (2 * padW == (kW - sW) && 2 * padH == (kH - sH)) { + return PADDING_SAME + } else { + throw new IllegalArgumentException( + s"Can not get padding type from given parameter " + + s"(padW: $padW, padH: $padH, kW: $kW, kH: $kH, sW: $sW, sH: $sH )") + } + } + +} diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala new file mode 100644 index 00000000000..bdf6cff58fd --- /dev/null +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl + +import com.intel.analytics.bigdl.tensor.Tensor +import org.tensorflow.framework.NodeDef + +import scala.collection.mutable + +package object utils { + + type Context = mutable.HashMap[NodeDef, (Tensor[Float], Tensor[Float])] + +} diff --git a/spark/dl/src/test/resources/tf/.gitignore b/spark/dl/src/test/resources/tf/.gitignore new file mode 100644 index 00000000000..9ffaa437c01 --- /dev/null +++ b/spark/dl/src/test/resources/tf/.gitignore @@ -0,0 +1,5 @@ +model/ +freeze_graph.py +log/ +*.pb +!test.pb diff --git a/spark/dl/src/test/resources/tf/alexnet.py b/spark/dl/src/test/resources/tf/alexnet.py new file mode 100644 index 00000000000..46359b4ecca --- /dev/null +++ b/spark/dl/src/test/resources/tf/alexnet.py @@ -0,0 +1,48 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.alexnet as alexnet + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 1. mkdir model + 2. python alexnet.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/alexnet.pbtxt --input_checkpoint model/alexnet.chkp --output_node_names="alexnet_v2/fc8/squeezed,output" --output_graph alexnet_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 224, 224 + #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = alexnet.alexnet_v2(inputs, is_training=False) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + print(sess.run(result)) + checkpointpath = saver.save(sess, dir + '/model/alexnet.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'alexnet.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/inception_resnet_v2.py b/spark/dl/src/test/resources/tf/inception_resnet_v2.py new file mode 100644 index 00000000000..32e817253fc --- /dev/null +++ b/spark/dl/src/test/resources/tf/inception_resnet_v2.py @@ -0,0 +1,52 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.inception_resnet_v2 as inception_resnet_v2 + + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder, eg. /home/models/ + 1. mkdir model + 2. python inception_resnet_v2.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/inception_resnet_v2.pbtxt --input_checkpoint model/inception_resnet_v2.chkp --output_node_names="InceptionResnetV2/AuxLogits/Logits/BiasAdd,InceptionResnetV2/Logits/Logits/BiasAdd,output1,output2" --output_graph inception_resnet_v2_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 299, 299 + num_classes = 1001 + # inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') + net, end_points = inception_resnet_v2.inception_resnet_v2(inputs,is_training = False) + output1 = tf.Variable(tf.random_uniform(tf.shape(net)),name='output1') + result1 = tf.assign(output1,net) + output2 = tf.Variable(tf.random_uniform(tf.shape(end_points['AuxLogits'])),name='output2') + result2 = tf.assign(output2,end_points['AuxLogits']) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run([result1,result2]) + checkpointpath = saver.save(sess, dir + '/model/inception_resnet_v2.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'inception_resnet_v2.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/inception_v3.py b/spark/dl/src/test/resources/tf/inception_v3.py new file mode 100644 index 00000000000..edcaa5337bb --- /dev/null +++ b/spark/dl/src/test/resources/tf/inception_v3.py @@ -0,0 +1,51 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from nets import inception + +slim = tf.contrib.slim + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder/models/slim, eg. /home/tensorflow/models/slim/ + 1. mkdir model + 2. python inception_v3.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/inception_v3.pbtxt --input_checkpoint model/inception_v3.chkp --output_node_names="InceptionV3/Logits/SpatialSqueeze,output" --output_graph inception_v3_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 299, 299 + num_classes = 1000 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = inception.inception_v3(inputs, num_classes,is_training=False) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + result = sess.run(output) + sess.run(init) + checkpointpath = saver.save(sess, dir + '/model/inception_v3.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'inception_v3.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/lenet.py b/spark/dl/src/test/resources/tf/lenet.py new file mode 100644 index 00000000000..63a9cfd48a4 --- /dev/null +++ b/spark/dl/src/test/resources/tf/lenet.py @@ -0,0 +1,44 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.lenet as lenet + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 1. mkdir model + 2. python lenet.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/lenet.pbtxt --input_checkpoint model/lenet.chkp --output_node_names="LeNet/fc4/BiasAdd" --output_graph lenet.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 32, 32 + inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + net, end_points = lenet.lenet(inputs) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + checkpointpath = saver.save(sess, dir + '/model/lenet.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'lenet.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/load.py b/spark/dl/src/test/resources/tf/load.py new file mode 100644 index 00000000000..92340c50015 --- /dev/null +++ b/spark/dl/src/test/resources/tf/load.py @@ -0,0 +1,37 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from tensorflow.python.platform import gfile + +def main(): + with gfile.FastGFile("/tmp/tensorflow9012247417719342743saver",'rb') as f: + graph_def = tf.GraphDef() + content = f.read() + graph_def.ParseFromString(content) + with tf.Graph().as_default() as graph: + tf.import_graph_def(graph_def, name='') + sess = tf.Session() + for op in graph.get_operations(): + print(op.name) + prediction = graph.get_tensor_by_name('relu:0') + ix = graph.get_tensor_by_name('input:0') + rand_array = np.random.rand(2, 4) + print(sess.run(prediction, feed_dict={ix: rand_array})) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/overfeat.py b/spark/dl/src/test/resources/tf/overfeat.py new file mode 100644 index 00000000000..a792a761b46 --- /dev/null +++ b/spark/dl/src/test/resources/tf/overfeat.py @@ -0,0 +1,52 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.overfeat as overfeat + +slim = tf.contrib.slim + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder, eg. /home/models/ + 1. mkdir model + 2. python overfeat.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/overfeat.pbtxt --input_checkpoint model/overfeat.chkp --output_node_names="overfeat/fc8/squeezed,output" --output_graph overfeat_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 231, 231 + num_classes = 1000 + #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + with slim.arg_scope(overfeat.overfeat_arg_scope()): + net, end_points = overfeat.overfeat(inputs, is_training = False) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run(result) + checkpointpath = saver.save(sess, dir + '/model/overfeat.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'overfeat.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/resnet_v1.py b/spark/dl/src/test/resources/tf/resnet_v1.py new file mode 100644 index 00000000000..0ea03ee385f --- /dev/null +++ b/spark/dl/src/test/resources/tf/resnet_v1.py @@ -0,0 +1,50 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from nets import resnet_utils +from nets import resnet_v1 + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder/models/slim, eg. /home/tensorflow/models/slim/ + 1. mkdir model + 2. python resnet_v1.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/resnet_v1.pbtxt --input_checkpoint model/resnet_v1.chkp --output_node_names="resnet_v1_101/SpatialSqueeze,output" --output_graph resnet_v1_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 224, 224 + num_classes = 1000 + #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') + net, end_points = resnet_v1.resnet_v1_101(inputs, 1000, is_training=True) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run(result) + checkpointpath = saver.save(sess, dir + '/model/resnet_v1.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'resnet_v1.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/rnn.py b/spark/dl/src/test/resources/tf/rnn.py new file mode 100644 index 00000000000..e5ec1c0c3d8 --- /dev/null +++ b/spark/dl/src/test/resources/tf/rnn.py @@ -0,0 +1,53 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from tensorflow.contrib import rnn + +def main(): + """ + Run this command to generate the pb file + 1. mkdir model + 2. python rnn.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/rnn.pbtxt --input_checkpoint model/rnn.chkp --output_node_names=output --output_graph "rnn.pb" + """ + dir = os.path.dirname(os.path.realpath(__file__)) + n_steps = 5 + n_input = 10 + n_hidden = 20 + n_output = 5 + xs = tf.placeholder(tf.float32, [None, n_steps, n_input]) + weight = tf.Variable(tf.random_normal([n_hidden, n_output]), name="weight") + bias = tf.Variable(tf.random_normal([n_output]), name="bias") + + x = tf.unstack(xs, n_steps, 1) + + cell = rnn.BasicRNNCell(n_hidden) + + output, states = rnn.static_rnn(cell, x, dtype=tf.float32) + + final = tf.nn.bias_add(tf.matmul(output[-1], weight), bias, name='output') + saver = tf.train.Saver() + with tf.Session() as sess: + file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) + init = tf.global_variables_initializer() + sess.run(init) + checkpointpath = saver.save(sess, dir + '/model/rnn.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'rnn.pbtxt') +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/rnn_cell_zero_state.py b/spark/dl/src/test/resources/tf/rnn_cell_zero_state.py new file mode 100644 index 00000000000..efe5321bf5d --- /dev/null +++ b/spark/dl/src/test/resources/tf/rnn_cell_zero_state.py @@ -0,0 +1,51 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from tensorflow.contrib import rnn +from tensorflow.python.ops import array_ops + +def main(): + """ + Run this command to generate the pb file + 1. mkdir model + 2. python rnn_cell_zero_state.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/rnn_cell.pbtxt --input_checkpoint model/rnn_cell.chkp --output_node_names=output --output_graph "rnn_cell.pb" + """ + dir = os.path.dirname(os.path.realpath(__file__)) + n_input = 10 + n_hidden = 20 + + xs = tf.placeholder(tf.float32, [None, n_input]) + W = tf.Variable(tf.constant(1.0, shape=[n_hidden, 5], dtype=tf.float32)) + b = tf.Variable(tf.constant(2.0, shape=[5], dtype=tf.float32)) + + cell = rnn.BasicRNNCell(n_hidden) + batch_size = array_ops.shape(xs)[0] + state = cell.zero_state(batch_size, tf.float32) + tf.nn.bias_add(tf.matmul(state, W), b, name="output") + + saver = tf.train.Saver() + with tf.Session() as sess: + file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) + init = tf.global_variables_initializer() + sess.run(init) + checkpointpath = saver.save(sess, dir + '/model/rnn_cell.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'rnn_cell.pbtxt') +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/rnn_lstm.py b/spark/dl/src/test/resources/tf/rnn_lstm.py new file mode 100644 index 00000000000..72a58e12946 --- /dev/null +++ b/spark/dl/src/test/resources/tf/rnn_lstm.py @@ -0,0 +1,53 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from tensorflow.contrib import rnn + +def main(): + """ + Run this command to generate the pb file + 1. mkdir model + 2. python rnn.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/lstm.pbtxt --input_checkpoint model/lstm.chkp --output_node_names=output --output_graph "lstm.pb" + """ + dir = os.path.dirname(os.path.realpath(__file__)) + n_steps = 5 + n_input = 10 + n_hidden = 20 + n_output = 5 + xs = tf.placeholder(tf.float32, [None, n_steps, n_input]) + weight = tf.Variable(tf.random_normal([n_hidden, n_output]), name="weight") + bias = tf.Variable(tf.random_normal([n_output]), name="bias") + + x = tf.unstack(xs, n_steps, 1) + + cell = rnn.BasicLSTMCell(n_hidden) + + output, states = rnn.static_rnn(cell, x, dtype=tf.float32) + + final = tf.nn.bias_add(tf.matmul(output[-1], weight), bias, name='output') + saver = tf.train.Saver() + with tf.Session() as sess: + file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) + init = tf.global_variables_initializer() + sess.run(init) + checkpointpath = saver.save(sess, dir + '/model/lstm.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'lstm.pbtxt') +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/save.py b/spark/dl/src/test/resources/tf/save.py new file mode 100644 index 00000000000..1ab44e17be1 --- /dev/null +++ b/spark/dl/src/test/resources/tf/save.py @@ -0,0 +1,30 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +from tensorflow.python.platform import gfile +import os + +def main(): + dir = os.path.dirname(os.path.realpath(__file__)) + xs = tf.placeholder(tf.float32, [4, 3, 4]) + tf.nn.relu(xs) + + with tf.Session() as sess: + tf.train.write_graph(sess.graph, dir, 'model.pb') + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/saveTest/LinearSaveTest.py b/spark/dl/src/test/resources/tf/saveTest/LinearSaveTest.py new file mode 100644 index 00000000000..788f44fb1d9 --- /dev/null +++ b/spark/dl/src/test/resources/tf/saveTest/LinearSaveTest.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from tensorflow.python.platform import gfile +from sys import argv + +def main(): + with gfile.FastGFile(argv[1],'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + with tf.Graph().as_default() as graph: + tf.import_graph_def(graph_def, name='') + sess = tf.Session() + for op in graph.get_operations(): + print(op.name) + prediction = graph.get_tensor_by_name('linear/biasAdd:0') + ix = graph.get_tensor_by_name('input:0') + test = np.matrix([[1, 2, 5], [-3, -4, -7]]) + result = sess.run(prediction, {ix: test}) + + #np.testing.assert_equal(result, np.array([[1, 2, 5, 6], [0, 0, 0, 0]])) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/saveTest/ReLUSaveTest.py b/spark/dl/src/test/resources/tf/saveTest/ReLUSaveTest.py new file mode 100644 index 00000000000..f990de085e2 --- /dev/null +++ b/spark/dl/src/test/resources/tf/saveTest/ReLUSaveTest.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +from tensorflow.python.platform import gfile +from sys import argv + +def main(): + with gfile.FastGFile(argv[1],'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + with tf.Graph().as_default() as graph: + tf.import_graph_def(graph_def, name='') + sess = tf.Session() + for op in graph.get_operations(): + print(op.name) + prediction = graph.get_tensor_by_name('relu:0') + ix = graph.get_tensor_by_name('input:0') + test = np.matrix([[1, 2, 5, 6], [-3, -4, -7, -8]]) + result = sess.run(prediction, {ix: test}) + + np.testing.assert_equal(result, np.array([[1, 2, 5, 6], [0, 0, 0, 0]])) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/share_weight.py b/spark/dl/src/test/resources/tf/share_weight.py new file mode 100644 index 00000000000..6c958b64f6e --- /dev/null +++ b/spark/dl/src/test/resources/tf/share_weight.py @@ -0,0 +1,47 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os + +def main(): + """ + Run this command to generate the pb file + 1. mkdir model + 2. python test.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/share_weight.pbtxt --input_checkpoint model/share_weight.chkp --output_node_names=output --output_graph "share_weight.pb" + """ + dir = os.path.dirname(os.path.realpath(__file__)) + xs = tf.placeholder(tf.float32, [None, 10]) + W1 = tf.Variable(tf.random_normal([10,10])) + b1 = tf.Variable(tf.random_normal([10])) + Wx_plus_b1 = tf.nn.bias_add(tf.matmul(xs,W1), b1) + output= tf.nn.tanh(Wx_plus_b1) + + Wx_plus_b2 = tf.nn.bias_add(tf.matmul(output,W1), b1) + W2 = tf.Variable(tf.random_normal([10, 1])) + b2 = tf.Variable(tf.random_normal([1])) + final = tf.nn.bias_add(tf.matmul(Wx_plus_b2, W2), b2, name='output') + saver = tf.train.Saver() + with tf.Session() as sess: + file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) + init = tf.global_variables_initializer() + sess.run(init) + checkpointpath = saver.save(sess, dir + '/model/share_weight.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'share_weight.pbtxt') +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/test.pb b/spark/dl/src/test/resources/tf/test.pb new file mode 100644 index 0000000000000000000000000000000000000000..839ceff1b14da3d3805c78b1c740a3fa774551a9 GIT binary patch literal 1100 zcmbV~!B4_46vo*y#_|?KYCJfj2Tx=dWe{U97#SCdpdN^tcv#9-1IgUT)|v1>!v7>= zwA%=v@vyE<)AxPvdp~G``r=}njD6Q30m>e96^ag>O$idOg!mASaeQbM0E6<1g9#qF z1Qol!7gD_g#c%9J1l^d|ihvX(E>}^3i`rt|zx&IV(}NzAnZ#v4u!EGILp(a6GmUHO zB@l)ycc&m*wu{3MRiT2)u5Y*Q*>kV}*xsr$zisBXX&OtX67xE5FXq30d9Tk?197461`Ewm$vf3sWQ`>FMFE3+EsWu{TDmRb%z|Gq5w+8!Y=R ztU%1me_6@qZ|u(B*q^_Vy1vc0ZZOv;jF?07U`>pmOEE3GX=vl!S&Q sy=+iP-s!lKw)5joP{`4-B7h4+A7nqG(}<=da49VDyWO0n`sf*8l(j literal 0 HcmV?d00001 diff --git a/spark/dl/src/test/resources/tf/test.py b/spark/dl/src/test/resources/tf/test.py new file mode 100644 index 00000000000..c0f882304d0 --- /dev/null +++ b/spark/dl/src/test/resources/tf/test.py @@ -0,0 +1,45 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os + +def main(): + """ + Run this command to generate the pb file + 1. mkdir model + 2. python test.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/test.pbtxt --input_checkpoint model/test.chkp --output_node_names=output --output_graph "test.pb" + """ + dir = os.path.dirname(os.path.realpath(__file__)) + xs = tf.placeholder(tf.float32, [None, 1]) + W1 = tf.Variable(tf.zeros([1,10])+0.2) + b1 = tf.Variable(tf.zeros([10])+0.1) + Wx_plus_b1 = tf.nn.bias_add(tf.matmul(xs,W1), b1) + output= tf.nn.tanh(Wx_plus_b1) + + W2 = tf.Variable(tf.zeros([10,1])+0.2) + b2 = tf.Variable(tf.zeros([1])+0.1) + Wx_plus_b2 = tf.nn.bias_add(tf.matmul(output,W2), b2, name='output') + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + checkpointpath = saver.save(sess, dir + '/model/test.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'test.pbtxt') +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/vgg16.py b/spark/dl/src/test/resources/tf/vgg16.py new file mode 100644 index 00000000000..ea61a43713c --- /dev/null +++ b/spark/dl/src/test/resources/tf/vgg16.py @@ -0,0 +1,48 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.vgg as vgg + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 1. mkdir model + 2. python vgg16.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/vgg16.pbtxt --input_checkpoint model/vgg16.chkp --output_node_names="vgg_16/fc8/squeezed,output" --output_graph vgg16_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 224, 224 + #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = vgg.vgg_16(inputs, is_training = False) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run(result) + checkpointpath = saver.save(sess, dir + '/model/vgg16.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'vgg16.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/vgg19.py b/spark/dl/src/test/resources/tf/vgg19.py new file mode 100644 index 00000000000..5e0de05a5ca --- /dev/null +++ b/spark/dl/src/test/resources/tf/vgg19.py @@ -0,0 +1,48 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.vgg as vgg + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 1. mkdir model + 2. python vgg19.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/vgg19.pbtxt --input_checkpoint model/vgg19.chkp --output_node_names="vgg_19/fc8/squeezed,output" --output_graph vgg19_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 224, 224 + #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = vgg.vgg_19(inputs, is_training = False) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run(result) + checkpointpath = saver.save(sess, dir + '/model/vgg19.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'vgg19.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/vgga.py b/spark/dl/src/test/resources/tf/vgga.py new file mode 100644 index 00000000000..cc53f1a30a1 --- /dev/null +++ b/spark/dl/src/test/resources/tf/vgga.py @@ -0,0 +1,48 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.vgg as vgg + +def main(): + """ + Run this command to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 1. mkdir model + 2. python vgga.py + 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 4. python freeze_graph.py --input_graph model/vgga.pbtxt --input_checkpoint model/vgga.chkp --output_node_names="vgg_a/fc8/squeezed,output" --output_graph vgga_save.pb + """ + dir = os.path.dirname(os.path.realpath(__file__)) + batch_size = 5 + height, width = 224, 224 + #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = vgg.vgg_a(inputs, is_training = False) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run(result) + checkpointpath = saver.save(sess, dir + '/model/vgga.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'vgga.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowLoaderSpec.scala new file mode 100644 index 00000000000..c71425e1b61 --- /dev/null +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowLoaderSpec.scala @@ -0,0 +1,543 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils + +import java.io.{File => JFile} + +import com.intel.analytics.bigdl.dataset.{DistributedDataSet, MiniBatch} +import com.intel.analytics.bigdl.nn._ +import com.intel.analytics.bigdl.optim.{DistriOptimizer, Trigger} +import com.intel.analytics.bigdl.tensor.{Storage, Tensor} +import org.apache.log4j.{Level, Logger} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD +import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} +import com.intel.analytics.bigdl.utils.TestUtils.processPath +import scala.math._ + +object TensorflowLoaderSpec { + private val data1 = Array(0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.1f) + private val data2 = Array(0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.1f) + private val input1: Tensor[Float] = Tensor[Float](Storage[Float](data1)) + private val input2: Tensor[Float] = Tensor[Float](Storage[Float](data2)) + private val nodeNumber = 4 + private val coreNumber = 4 + + Engine.init(nodeNumber, coreNumber, true) + + private val batchSize = 2 * coreNumber + + private val prepareData: Int => (MiniBatch[Float]) = index => { + val input = Tensor[Float]().resize(batchSize, 10) + val target = Tensor[Float]().resize(batchSize) + var i = 0 + while (i < batchSize) { + if (i % 2 == 0) { + target.setValue(i + 1, 0.0f) + input.select(1, i + 1).copy(input1) + } else { + target.setValue(i + 1, 0.1f) + input.select(1, i + 1).copy(input2) + } + i += 1 + } + MiniBatch(input, target) + } +} + +@com.intel.analytics.bigdl.tags.Parallel +class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { + + Logger.getLogger("org").setLevel(Level.WARN) + Logger.getLogger("akka").setLevel(Level.WARN) + + import TensorflowLoaderSpec._ + + var sc: SparkContext = null + + var dataSet: DistributedDataSet[MiniBatch[Float]] = null + + before { + sc = new SparkContext("local[1]", "RDDOptimizerSpec") + + val rdd = sc.parallelize(1 to (256 * 4), 4).map(prepareData) + + dataSet = new DistributedDataSet[MiniBatch[Float]] { + override def originRDD(): RDD[_] = rdd + + override def data(train : Boolean): RDD[MiniBatch[Float]] = rdd + + override def size(): Long = 256 * nodeNumber + + override def shuffle(): Unit = {} + } + + Engine.model.setPoolSize(1) + } + + after { + if (sc != null) { + sc.stop() + } + } + + "TensorFlow loader" should "read a list of nodes from pb file" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "test.pb" + val results = TensorflowLoader.parse(path) + results.size() should be(14) + } + + "TensorFlow loader" should "be able to build a TF graph" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "test.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + tfGraph.size should be(15) // there's a dummy output + val topSort = tfGraph.topologySort// It can do topology sort + topSort.length should be(15) + topSort(0).element should be(null) + topSort(1).element.getName should be("output") + topSort(2).element.getName should be("MatMul_1") + topSort(3).element.getName should be("Variable_3/read") + topSort(4).element.getName should be("Variable_3") + topSort(5).element.getName should be("Tanh") + topSort(6).element.getName should be("Variable_2/read") + topSort(7).element.getName should be("Variable_2") + topSort(8).element.getName should be("BiasAdd") + topSort(9).element.getName should be("MatMul") + topSort(10).element.getName should be("Variable_1/read") + topSort(11).element.getName should be("Variable_1") + topSort(12).element.getName should be("Placeholder") + topSort(13).element.getName should be("Variable/read") + topSort(14).element.getName should be("Variable") + } + + "TensorFlow loader" should "be able to build a BigDL graph" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "test.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output")) + val container = model.asInstanceOf[Graph[Float]] + container.modules.length should be(4) + RandomGenerator.RNG.setSeed(100) + val input = Tensor[Float](4, 1).rand() + val output1 = container.forward(input) + + val model2 = Sequential[Float]() + val fc1 = Linear[Float](1, 10) + fc1.parameters()._1(0).fill(0.2f) + fc1.parameters()._1(1).fill(0.1f) + model2.add(fc1).add(Tanh()) + + val fc2 = Linear[Float](10, 1) + fc2.parameters()._1(0).fill(0.2f) + fc2.parameters()._1(1).fill(0.1f) + model2.add(fc2) + + val output2 = model2.forward(input) + output1 should be(output2) + } + + "Shared weights" should "be the same instance" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "share_weight.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output")) + val container = model.asInstanceOf[Graph[Float]] + container.modules.length should be(4) + val l1 = container.modules(1).asInstanceOf[Linear[Float]] + val l2 = container.modules(3).asInstanceOf[Linear[Float]] + assert(l1.weight eq l2.weight) + assert(l1.bias eq l2.bias) + } + + "Shared weights" should "be the same after running optimizer" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "share_weight.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output")) + val container = model.asInstanceOf[Graph[Float]] + + val optimizer = new DistriOptimizer[Float](container, dataSet, new MSECriterion[Float]()) + .setState(T("learningRate" -> 20.0)) + .setEndWhen(Trigger.maxEpoch(5)) + optimizer.optimize() + + val l1 = container.modules(1).asInstanceOf[Linear[Float]] + val l2 = container.modules(3).asInstanceOf[Linear[Float]] + assert(l1.weight == l2.weight) + assert(l1.bias == l2.bias) + } + + "TensorFlow loader" should "be able to load rnn_cell with zero state" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "rnn_cell.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("output")) + val input = Tensor[Float](4, 10).rand() + val gradient = Tensor[Float](4, 5).rand() + val result: Tensor[Float] = model.forward(input).asInstanceOf[Tensor[Float]] + val expectedResult = Tensor[Float](4, 5).fill(2.0f) + val expectedGrad = Tensor[Float](4, 10) + result should be(expectedResult) + val grad = model.backward(input, gradient) + grad should be(expectedGrad) + } + + "TensorFlow loader" should "be able to load static simple rnn model" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "rnn.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("output")) + + val input = Tensor[Float](4, 5, 10).rand() + val gradient = Tensor[Float](4, 5).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load static lstm rnn model" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "lstm.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("output")) + + val input = Tensor[Float](4, 5, 10).rand() + val gradient = Tensor[Float](4, 5).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim alexnetv2" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "alexnet.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("alexnet_v2/fc8/squeezed")) + val input = Tensor[Float](4, 3, 224, 224).rand() + val gradient = Tensor[Float](4, 1000).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim vgga" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "vgga.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("vgg_a/fc8/squeezed")) + val input = Tensor[Float](4, 3, 224, 224).rand() + val gradient = Tensor[Float](4, 1000).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim vgg16" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "vgg16.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("vgg_16/fc8/squeezed")) + val input = Tensor[Float](4, 3, 224, 224).rand() + val gradient = Tensor[Float](4, 1000).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim vgg19" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "vgg19.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("vgg_19/fc8/squeezed")) + val input = Tensor[Float](2, 3, 224, 224).rand() + val gradient = Tensor[Float](2, 1000).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim lenet" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "lenet.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("LeNet/fc4/BiasAdd")) + val input = Tensor[Float](4, 3, 32, 32).rand() + val gradient = Tensor[Float](4, 10).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim inception_v3" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "inception_v3.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("InceptionV3/Logits/SpatialSqueeze")) + val input = Tensor[Float](2, 3, 299, 299).rand() + val gradient = Tensor[Float](2, 1000).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim resnet_v1" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "resnet_v1.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("resnet_v1_101/SpatialSqueeze")) + val input = Tensor[Float](2, 3, 224, 224).rand() + val gradient = Tensor[Float](2, 1000).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim overfeat" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "overfeat.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("overfeat/fc8/squeezed")) + val input = Tensor[Float](5, 3, 231, 231).rand() + val gradient = Tensor[Float](5, 1000).rand() + model.forward(input) + model.backward(input, gradient) + } + + "TensorFlow loader" should "be able to load slim inception_resnet_v2" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "inception_resnet_v2.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), + Seq("InceptionResnetV2/Logits/Predictions", "InceptionResnetV2/AuxLogits/Logits/BiasAdd")) + val input = Tensor[Float](5, 3, 299, 299).rand() + val gradient1 = Tensor[Float](5, 1001).rand() + val gradient2 = Tensor[Float](5, 1001).rand() + model.forward(input) + model.backward(input, T(gradient1, gradient2)) + } + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim alexnet" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "alexnet_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("alexnet_v2/fc8/squeezed")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient = Tensor[Float](1, 1000).rand() + val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult.map( BigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, gradient) + } + + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim vgga" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "vgga_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("vgg_a/fc8/squeezed")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient = Tensor[Float](1, 1000).rand() + val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult.map( BigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, gradient) + } + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim vgg_16" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "vgg16_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("vgg_16/fc8/squeezed")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient = Tensor[Float](1, 1000).rand() + val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult.map( BigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, gradient) + } + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim vgg_19" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "vgg19_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("vgg_19/fc8/squeezed")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient = Tensor[Float](1, 1000).rand() + val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult.map( BigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, gradient) + } + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim overfeat" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "overfeat_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("overfeat/fc8/squeezed")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient = Tensor[Float](1, 1000).rand() + val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult.map( BigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, gradient) + } + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim inception_v3" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "inception_v3_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("InceptionV3/Logits/SpatialSqueeze")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient = Tensor[Float](1, 1000).rand() + val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult.map( BigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, gradient) + } + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim resnet_v1" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "resnet_v1_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("resnet_v1_101/SpatialSqueeze")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient = Tensor[Float](2, 1000).rand() + val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult.map( BigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) < 2e-7); + v2 + }) + model.backward(input, gradient) + } + + "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "after loading slim inception_resnet_v2" in { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + "inception_resnet_v2_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-2)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd")) + val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient1 = Tensor[Float](2, 1001).rand() + val gradient2 = Tensor[Float](2, 1001).rand() + val tfResult1 = TFToBigDL.toTensor(results.get(results.size()-2) + .getAttrMap.get("value").getTensor) + val tfResult2 = TFToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor) + val BigDLResult = model.forward(input) + + tfResult1.map( BigDLResult.toTable(1), (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + tfResult2.map( BigDLResult.toTable(2), (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, T(gradient1, gradient2)) + } + + private def processPath(path: String): String = { + if (path.contains(":")) { + path.substring(1) + } else { + path + } + } +} diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowSaverSpec.scala new file mode 100644 index 00000000000..c99654af7b3 --- /dev/null +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowSaverSpec.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils + +import com.intel.analytics.bigdl.nn.{Graph, Linear, ReLU} +import org.scalatest.{FlatSpec, Matchers} +import com.intel.analytics.bigdl.numeric.NumericFloat + +import scala.sys.process._ +import java.io.{File => JFile} + +import com.intel.analytics.bigdl.utils.TestUtils.processPath + +class TensorflowSaverSpec extends FlatSpec with Matchers { + "ReLU layer" should "be correctly saved" in { + val relu = ReLU().setName("relu").apply() + val graph = Graph(relu, relu) + + val tmpFile = java.io.File.createTempFile("tensorflowSaverTest", "ReLU") + TensorFlowSaver.saveGraph(graph, Seq(("input", Seq(2, 4))), tmpFile.getPath) + runPython(testScriptsPath("ReLUSaveTest.py ") + tmpFile) should be(true) + } + + "Linear layer" should "be correctly saved" in { + val linear = Linear(3, 4).setName("linear").apply() + val graph = Graph(linear, linear) + val tmpFile = java.io.File.createTempFile("tensorflowSaverTest", "Linear") + TensorFlowSaver.saveGraph(graph, Seq(("input", Seq(2, 3))), tmpFile.getPath) + println(tmpFile.getPath) + // runPython(testScriptsPath("LinearSaveTest.py ") + tmpFile) should be(true) + } + + private def testScriptsPath(script: String) : String = { + val resource = getClass().getClassLoader().getResource("tf") + processPath(resource.getPath()) + JFile.separator + "saveTest" + + JFile.separator + script + } + + private def runPython(cmd: String): Boolean = { + try { + (("python " + cmd) !!) + return true + } catch { + case _: Throwable => false + } + } +} \ No newline at end of file From 1f5edcf1ce6883e1c7b85de5da4c326ccf49e617 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 8 Jun 2017 18:58:28 +0800 Subject: [PATCH 05/23] code refactor --- .../com/intel/analytics/bigdl/nn/Module.scala | 16 +- .../analytics/bigdl/utils/TFToBigDL.scala | 909 -------------- .../intel/analytics/bigdl/utils/package.scala | 27 - .../BigDLToTensorflow.scala} | 208 ++-- .../Tensorflow.scala} | 66 +- .../utils/{ => tf}/TensorflowLoader.scala | 290 +++-- .../bigdl/utils/tf/TensorflowSaver.scala | 98 ++ .../bigdl/utils/tf/TensorflowToBigDL.scala | 1091 +++++++++++++++++ .../tf/loadTest/inception_resnet_v2.py | 63 + .../resources/tf/loadTest/merge_checkpoint.py | 58 + .../tf/{saveTest => save}/LinearSaveTest.py | 0 .../tf/{saveTest => save}/ReLUSaveTest.py | 0 .../utils/{ => tf}/TensorflowLoaderSpec.scala | 166 ++- .../utils/{ => tf}/TensorflowSaverSpec.scala | 14 +- .../bigdl/utils/tf/TensorflowSpecHelper.scala | 36 + 15 files changed, 1764 insertions(+), 1278 deletions(-) delete mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala delete mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala rename spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/{TensorFlowSaver.scala => tf/BigDLToTensorflow.scala} (66%) rename spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/{TensorflowUtils.scala => tf/Tensorflow.scala} (91%) rename spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/{ => tf}/TensorflowLoader.scala (56%) create mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala create mode 100644 spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala create mode 100644 spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py create mode 100644 spark/dl/src/test/resources/tf/loadTest/merge_checkpoint.py rename spark/dl/src/test/resources/tf/{saveTest => save}/LinearSaveTest.py (100%) rename spark/dl/src/test/resources/tf/{saveTest => save}/ReLUSaveTest.py (100%) rename spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/{ => tf}/TensorflowLoaderSpec.scala (75%) rename spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/{ => tf}/TensorflowSaverSpec.scala (92%) create mode 100644 spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala index 0412b120e1a..47754056dc5 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala @@ -15,12 +15,15 @@ */ package com.intel.analytics.bigdl.nn +import java.nio.ByteOrder + import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.nn.abstractnn.Activity import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.{CaffeLoader, File, TensorflowLoader} +import com.intel.analytics.bigdl.utils.{CaffeLoader, File} +import com.intel.analytics.bigdl.utils.tf.{TensorflowDataFormat, TensorflowLoader} import scala.reflect.ClassTag @@ -44,10 +47,17 @@ object Module { * @param file where is the protobuf model file * @param inputs input node names * @param outputs output node names, the output tensor order is same with the node order + * @param byteOrder byte order in the tensorflow file. The default value is little endian + * @param dataFormat the model dataFormat. Note that BigDL only support NCHW, so for NHWC + * tensorflow model, the model define will be converted to NCHW * @return BigDL model */ - def loadTF(file: String, inputs: Seq[String], outputs: Seq[String]): Module[Float] = { - TensorflowLoader.load(file, inputs, outputs) + def loadTF[T: ClassTag](file: String, inputs: Seq[String], outputs: Seq[String], + byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN, + dataFormat: TensorflowDataFormat = TensorflowDataFormat.NCHW)( + implicit ev: TensorNumeric[T]): Module[T] = { + + TensorflowLoader.load(file, inputs, outputs, byteOrder, dataFormat) } def flatten[@specialized(Float, Double) T: ClassTag](parameters: Array[Tensor[T]])( diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala deleted file mode 100644 index 287230cb26f..00000000000 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TFToBigDL.scala +++ /dev/null @@ -1,909 +0,0 @@ -/* - * Copyright 2016 The BigDL Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intel.analytics.bigdl.utils - -import java.nio.charset.Charset -import java.nio.{ByteBuffer, ByteOrder} - -import collection.JavaConverters._ -import com.intel.analytics.bigdl.nn._ -import com.intel.analytics.bigdl.tensor.{Storage, Tensor} -import org.tensorflow.framework.{DataType, NodeDef, TensorProto} -import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} - -import scala.collection.mutable.ArrayBuffer - -trait TFToBigDL { - def topology: DirectedGraph[String] - - def layer(tfGraph: DirectedGraph[NodeDef], - context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) - - - /** - * @param tfGraph - * @return the nodes of this subgraph that accept input data - */ - def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = null - - /** - * - * @param tfGraph - * @return the nodes of this subgraph that emit output data - */ - def getOutputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { - Seq(tfGraph.source) - } - - protected def getOrSetTensor(node: NodeDef, context: Context) - (f: Tensor[Float] => Tensor[Float]) - : (Tensor[Float], Tensor[Float]) = { - if (context.contains(node)) { - context(node) - } else { - val weight = f(TFToBigDL.toTensor(node.getAttrMap.get("value").getTensor)).contiguous() - val gradient = Tensor[Float](weight.size()) - context.put(node, (weight, gradient)) - (weight, gradient) - } - } -} - -object FullConnectionTF extends TFToBigDL{ - private val graph = { - val add = Node("BiasAdd") - val mul = Node("MatMul") - Node("*") -> mul - Node("Const") -> Node("Identity") -> mul -> add - Node("Const") -> Node("Identity") -> add - add.graph(reverse = true) - } - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - - val biasNode = tfGraph.source.prevNodes(1).prevNodes.head.element - val weightNode = tfGraph.source.prevNodes.head.prevNodes(1).prevNodes.head.element - val (bias, gradBias) = getOrSetTensor(biasNode, context)(t => t) - val (weight, gradWeight) = getOrSetTensor(weightNode, context) { t => - t.transpose(1, 2) - } - - val linearLayer = Linear[Float](inputSize = weight.size(2), outputSize = weight.size(1), - initWeight = weight, initGradWeight = gradWeight, initBias = bias, initGradBias = gradBias) - linearLayer.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } - - override def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { - Seq(tfGraph.source.prevNodes.head) - } -} - -object SqueezeTF extends TFToBigDL { - private val graph = (Node("*") -> Node("Squeeze")).graph(reverse = true) - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val dataFormatMatch = Map("N" -> 0, "H" -> 2, "W" -> 3, "C" -> 1) - val dims = tfGraph.source.element.getAttrOrThrow("squeeze_dims").getList().getIList() - .asScala.map(_.toInt).toArray - .map(i => dataFormatMatch(TFToBigDL.dataFormat.charAt(i).toString)) - Squeeze[Float](dims, batchMode = true) - .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object Conv2D extends TFToBigDL{ - private val graph = { - val add = Node("BiasAdd") - val conv = Node("Conv2D") - - Node("*") -> conv - Node("Const") -> Node("Identity") -> conv -> add - Node("Const") -> Node("Identity") -> add - add.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { - Seq(tfGraph.source.prevNodes.head) - } - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val attributes = tfGraph.source.prevNodes(0).element.getAttrMap - require(attributes.get("strides").getList.getI(0).toInt == 1, s"not support strides on batch") - - val (strideH, strideW) = if (attributes.get("data_format").getS - .toString(Charset.defaultCharset()) == "NHWC") { - require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") - (attributes.get("strides").getList.getI(1).toInt, - attributes.get("strides").getList.getI(2).toInt) - } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { - require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") - (attributes.get("strides").getList.getI(2).toInt, - attributes.get("strides").getList.getI(3).toInt) - } else { - throw new IllegalArgumentException("no supported data format") - } - val biasNode = tfGraph.source.prevNodes(1).prevNodes.head.element - val (bias, gradBias) = getOrSetTensor(biasNode, context)(t => t) - - val weightNode = tfGraph.source.prevNodes.head.prevNodes(1).prevNodes.head.element - val (weights, gradWeights) = getOrSetTensor(weightNode, context) { t => - t.transpose(1, 4).transpose(2, 3).transpose(3, 4) - } - - val nOuputPlane = weights.size(1) - val nInputPlane = weights.size(2) - val kernelH = weights.size(3) - val kernelW = weights.size(4) - - val (pW, pH) = - if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { - require((kernelW - strideW) % 2 == 0) - require((kernelH - strideH) % 2 == 0) - ((kernelW - strideW) / 2, (kernelH - strideH) / 2) - } else { - (0, 0) - } - - val convLayer = SpatialConvolution[Float]( - nInputPlane = nInputPlane, nOutputPlane = nOuputPlane, - kernelW = kernelW, kernelH = kernelH, - strideW = strideW, strideH = strideH, - padW = pW, padH = pH, - initWeight = weights, - initBias = bias, - initGradWeight = gradWeights, - initGradBias = gradBias - ) - convLayer.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object ReluTF extends TFToBigDL { - private val graph = { - (Node("*") -> Node("Relu")).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - ReLU[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object TanhTF extends TFToBigDL{ - private val graph = { - (Node("*") -> Node("Tanh")).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - Tanh[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object SigmoidTF extends TFToBigDL{ - private val graph = { - (Node("*") -> Node("Sigmoid")).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - Sigmoid[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object ReshapeTF extends TFToBigDL { - private val graph = { - val nodeReshape = Node("Reshape") - Node("*") -> nodeReshape - Node("Const") -> nodeReshape - nodeReshape.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val sizes = TFToBigDL.toTensor( - tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor) - - val batchMode = sizes.valueAt(1) == -1 - val arraySize = new Array[Int](if (batchMode) sizes.nElement() - 1 else sizes.nElement()) - var i = if (batchMode) 2 else 1 - var k = 0 - while(i <= sizes.nElement()) { - arraySize(k) = sizes.valueAt(i).toInt - k += 1 - i += 1 - } - Reshape[Float](size = arraySize, Some(batchMode)) - .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object MaxPoolingTF extends TFToBigDL { - private val graph = { - (Node("*") -> Node("MaxPool")).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val attributes = tfGraph.source.element.getAttrMap - - val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS - .toString(Charset.defaultCharset()) == "NHWC") { - require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") - ( - attributes.get("strides").getList.getI(1).toInt, - attributes.get("strides").getList.getI(2).toInt, - attributes.get("ksize").getList.getI(1).toInt, - attributes.get("ksize").getList.getI(2).toInt - ) - } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { - require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") - ( - attributes.get("strides").getList.getI(2).toInt, - attributes.get("strides").getList.getI(3).toInt, - attributes.get("ksize").getList.getI(2).toInt, - attributes.get("ksize").getList.getI(3).toInt - ) - } else { - throw new IllegalArgumentException("no supported data format") - } - - val (pW, pH) = - if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { - require((ksizeW - strideW) % 2 == 0) - require((ksizeH - strideH) % 2 == 0) - ((ksizeW - strideW) / 2, (ksizeH - strideH) / 2) - } else { - (0, 0) - } - - val maxpool = SpatialMaxPooling[Float](ksizeW, ksizeH, strideW, strideH, pW, pH) - maxpool.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object AvgPoolingTF extends TFToBigDL{ - private val graph = { - (Node("*") -> Node("AvgPool")).graph(reverse = true) - } - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val attributes = tfGraph.source.element.getAttrMap - - val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS - .toString(Charset.defaultCharset()) == "NHWC") { - require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") - ( - attributes.get("strides").getList.getI(1).toInt, - attributes.get("strides").getList.getI(2).toInt, - attributes.get("ksize").getList.getI(1).toInt, - attributes.get("ksize").getList.getI(2).toInt - ) - } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { - require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") - ( - attributes.get("strides").getList.getI(2).toInt, - attributes.get("strides").getList.getI(3).toInt, - attributes.get("ksize").getList.getI(2).toInt, - attributes.get("ksize").getList.getI(3).toInt - ) - } else { - throw new IllegalArgumentException("no supported data format") - } - - val (pW, pH) = - if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { - require((ksizeW - strideW) % 2 == 0) - require((ksizeH - strideH) % 2 == 0) - ((ksizeW - strideW) / 2, (ksizeH - strideH) / 2) - } else { - (0, 0) - } - - SpatialAveragePooling[Float](ksizeW, ksizeH, strideW, strideH, pW, pH, countIncludePad = false) - .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object DropoutTF extends TFToBigDL{ - private val graph = { - val nodediv = Node("RealDiv") - val nodeP = Node("Const") - val nodeadd = Node("Add") - val noderandom = Node("Add") - val nodemin = Node("Const") - val nodesub = Node("Sub") - val nodemul = Node("Mul") - val nodedrop = Node("Mul") - Node("*") -> nodediv -> nodedrop - nodeP -> nodediv - nodeP -> nodeadd -> Node("Floor") -> nodedrop - Node("*") -> Node("Shape") -> Node("RandomUniform") -> nodemul -> noderandom -> nodeadd - Node("Const") -> nodesub -> nodemul - nodemin -> nodesub - nodemin -> noderandom - nodedrop.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val keepProp = tfGraph.source.prevNodes(0).prevNodes(1).element - .getAttrMap.get("value").getTensor.getFloatVal(0) - - Dropout[Float](keepProp).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object Placeholder extends TFToBigDL { - private val graph = Node("Placeholder").graph(reverse = true) - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - new Input[Float].asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - - -object ConstTF extends TFToBigDL { - private val graph = Node("Const").graph(reverse = true) - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - val value = TFToBigDL.toTensor(tfGraph.source.element.getAttrMap.get("value").getTensor) - Const(value).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object ShapeTF extends TFToBigDL { - private val graph = { - val node = Node("Shape") - Node("*") -> node - node.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - - new Shape[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object InputTF extends TFToBigDL { - private val graph = (Node("Const") -> Node("Identity")).graph(reverse = true) - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - new Input[Float].asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object IdentityTF extends TFToBigDL { - private val graph = (Node("*") -> Node("Identity")).graph(reverse = true) - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - new Input[Float].asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object BatchNormTF extends TFToBigDL{ - private val graph = { - val nodeInput = Node("*") - val nodeMean1 = Node("Mean") - val nodeStopGrad = Node("StopGradient") - val nodeSub1 = Node("Sub") - val nodeSquare = Node("SquaredDifference") - val nodeMeanss = Node("Sum") - val nodeVarss = Node("Sum") - val nodeShape = Node("Reshape") - val nodeDivisor = Node("Reciprocal") - val nodeShiftedMean = Node("Mul") - val nodeMean2 = Node("Add") - val nodeMul1 = Node("Mul") - val nodeVariance = Node("Sub") - val nodeAdd1 = Node("Add") - val nodeMul2 = Node("Mul") - val nodeMul3 = Node("Mul") - val nodeMul4 = Node("Mul") - val nodeSub2 = Node("Sub") - val nodeAdd2 = Node("Add") - - nodeInput -> nodeMul3 -> nodeAdd2 - Node("Const") -> Node("Identity") -> nodeSub2 - nodeInput -> nodeMean1 -> nodeStopGrad -> nodeShape - Node("Const") -> nodeMean1 - nodeInput -> nodeSub1 -> nodeMeanss -> nodeShiftedMean -> nodeMean2 -> nodeMul4 - nodeStopGrad -> nodeSub1 - nodeInput -> nodeSquare -> nodeVarss -> nodeMul1 -> nodeVariance - nodeStopGrad -> nodeSquare - Node("Const") -> nodeDivisor -> nodeShiftedMean -> Node("Square") -> nodeVariance -> nodeAdd1 - Node("Const") -> nodeMeanss -> nodeDivisor -> nodeMul1 - Node("Const") -> nodeVarss -> nodeDivisor - Node("Const") -> nodeAdd1 -> Node("Rsqrt") -> nodeMul2 -> nodeMul3 - Node("Const") -> Node("Identity") -> nodeMul2 -> nodeMul4 -> nodeSub2 -> nodeAdd2 - Node("Const") -> nodeShape -> nodeMean2 - nodeAdd2.graph(reverse = true) - } - - override def getInputNodes(tfGraph: DirectedGraph[NodeDef]): Seq[Node[NodeDef]] = { - Seq(tfGraph.source.prevNodes.head.prevNodes.head) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val nOutput = tfGraph.source.prevNodes(1).prevNodes(1).prevNodes(1) - .prevNodes(1).prevNodes(0).element.getAttrMap.get("value").getTensor.getIntVal(0) - - val weightNode = tfGraph.source.prevNodes(1).prevNodes.head.prevNodes.head.element - val biasNode = tfGraph.source.prevNodes(1).prevNodes(1).prevNodes(1) - .prevNodes.head.prevNodes.head.element - val (weights, gradWeights) = getOrSetTensor(weightNode, context)(t => t) - val (bias, gradBias) = getOrSetTensor(weightNode, context)(t => t) - - val spatialBatchNorm = SpatialBatchNormalization[Float]( - nOutput = nOutput, - initWeight = weights, - initBias = bias, - initGradWeight = weights, - initGradBias = gradBias - ) - spatialBatchNorm.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object FillTF extends TFToBigDL{ - private val graph = { - val nodeFill = Node("Fill") - Node("*") -> nodeFill - Node("Const") -> nodeFill - nodeFill.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val constNode = tfGraph.source.prevNodes(1) - val const = constNode.element.getAttrMap.get("value").getTensor.getFloatVal(0) - - Fill[Float](const).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object PackTF extends TFToBigDL{ - private val graph = { - val nodePack = Node("Pack") - Node("...") -> nodePack - nodePack.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val dim = tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1 - - Pack[Float](dim).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object UnpackTF extends TFToBigDL{ - private val graph = { - val nodePack = Node("Unpack") - Node("*") -> nodePack - nodePack.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val dim = tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1 - val index = tfGraph.source.element.getName.split(":").toList match { - case _::Nil => 1 - case _::i::Nil => i.toInt + 1 - } - Select[Float](dim, index).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object StrideSliceTF extends TFToBigDL { - private val graph = { - val nodeSlice = Node("StridedSlice") - Node("*") -> nodeSlice - Node("Const") -> nodeSlice - Node("Const") -> nodeSlice - Node("Const") -> nodeSlice - nodeSlice.graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - val startNode = tfGraph.source.prevNodes(1) - val endNode = tfGraph.source.prevNodes(2) - val strideNode = tfGraph.source.prevNodes(3) - - def getIntArray(node: Node[NodeDef]) = { - node.element.getAttrMap.get("value").getTensor.getIntValList.asScala.map(_.toInt) - } - - val start = getIntArray(startNode) - val end = getIntArray(endNode) - val stride = getIntArray(strideNode) - - val specs = (start zip end zip stride).zipWithIndex - .map(elem => (elem._2 + 1, elem._1._1._1 + 1, elem._1._1._2 + 1, elem._1._2)).toArray - - - StrideSlice[Float](specs).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - - -object ConcatTF extends TFToBigDL{ - private val graph = { - val nodeConcat = Node("ConcatV2") - Node("...") -> nodeConcat - (Node("Const") -> nodeConcat).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val inputNumber = tfGraph.source.element.getAttrMap.get("N").getI.toInt - val nodeaxis = tfGraph.source.prevNodes(inputNumber) - val axis = nodeaxis.element.getAttrMap.get("value").getTensor.getIntVal(0) - val nInputDims = 4 - - new JoinTable[Float](dimension = axis + 1, nInputDims = -1) - .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object AddConstTF1 extends TFToBigDL{ - private val graph = { - val nodeAdd = Node("Add") - Node("Const") -> nodeAdd - (Node("*") -> nodeAdd).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val value = tfGraph.source.prevNodes.head.element - .getAttrMap.get("value").getTensor.getFloatVal(0) - AddConstant[Float](value).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object AddConstTF2 extends TFToBigDL{ - private val graph = { - val nodeAdd = Node("Add") - Node("*") -> nodeAdd - (Node("Const") -> nodeAdd).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val value = tfGraph.source.prevNodes(1).element - .getAttrMap.get("value").getTensor.getFloatVal(0) - AddConstant[Float](value).asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object AddTF extends TFToBigDL{ - private val graph = { - val nodeAdd = Node("Add") - Node("*") -> nodeAdd - (Node("*") -> nodeAdd).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - CAddTable[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object SoftMaxTF extends TFToBigDL{ - private val graph = { - (Node("*") -> Node("Softmax")).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - SoftMax[Float]().asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - - -object MulTF extends TFToBigDL{ - private val graph = { - val nodeMul = Node("Mul") - Node("Const") -> nodeMul - (Node("*") -> nodeMul).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val mul = Mul[Float]() - val scale = TFToBigDL.toTensor( - tfGraph.source.prevNodes(0).element.getAttrMap.get("value").getTensor) - require(scale.dim() == 1 && scale.size(1) == 1, s"scale must be one number") - mul.weight.copy(scale) - mul.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object ElementWiseMulTF extends TFToBigDL{ - private val graph = { - val nodeMul = Node("Mul") - Node("*") -> nodeMul - (Node("*") -> nodeMul).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val mul = CMulTable[Float]() - mul.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object SplitTF extends TFToBigDL { - - private val graph = { - val nodeSplit = Node("Split") - Node("Const") -> nodeSplit - (Node("*") -> nodeSplit).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : (AbstractModule[Activity, Tensor[Float], Float]) = { - val numSplit = tfGraph.source.element.getAttrMap.get("num_split").getI.toInt - val dim = tfGraph.source.prevNodes.head.element - .getAttrMap.get("value").getTensor.getIntVal(0) + 1 - val index = tfGraph.source.element.getName.split(":").toList match { - case _::Nil => 1 - case _::i::Nil => i.toInt + 1 - } - SplitAndSelect[Float](dim, index, numSplit) - .asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } - -} - - -object PaddingTF extends TFToBigDL{ - private val graph = { - val nodePad = Node("Pad") - Node("*") -> nodePad - (Node("Const") -> nodePad).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - val paddings = TFToBigDL.toTensor( - tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor) - val pad = ArrayBuffer[Int]() - val dataFormatMatch = Map("N" -> 0, "H" -> 2, "W" -> 3, "C" -> 1) - val padding = Sequential[Float]() - - for(i <- 1 to paddings.size(1)) { - if (paddings(Array(i, 1)) != 0 || paddings(Array(i, 2)) != 0 ) { - val dim = dataFormatMatch(TFToBigDL.dataFormat.charAt(i-1).toString) + 1 - if (paddings(Array(i, 1)) != 0) { - padding.add(Padding[Float](dim, -paddings(Array(i, 1)).toInt, 4)) - } - if (paddings(Array(i, 2)) != 0) { - padding.add(Padding[Float](dim, paddings(Array(i, 1)).toInt, 4)) - } - } - } - - padding.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object MeanTF extends TFToBigDL{ - private val graph = { - val nodeMean = Node("Mean") - Node("*") -> nodeMean - (Node("Const") -> nodeMean).graph(reverse = true) - } - - override def topology: DirectedGraph[String] = graph - - override def layer(tfGraph: DirectedGraph[NodeDef], context: Context) - : AbstractModule[Activity, Tensor[Float], Float] = { - val dims = TFToBigDL.toTensor( - tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor) - val dim = ArrayBuffer[Int]() - val dataFormatMatch = Map("N" -> 0, "H" -> 2, "W" -> 3, "C" -> 1) - val mean = Sequential[Float]() - for (i <- 1 to dims.size(1)) { - dim += dataFormatMatch(TFToBigDL - .dataFormat.charAt(dims.valueAt(i).toInt).toString) + 1 - } - dim.foreach(i => mean.add(Mean[Float](i, squeeze = false))) - mean.asInstanceOf[AbstractModule[Activity, Tensor[Float], Float]] - } -} - -object TFToBigDL { - - /** - * Get the pattern list. - * @return - */ - def patterns : Array[TFToBigDL] = { - patternList.toArray - } - - /** - * Switch endianess to big endian. You should do this when you save the model in a big endian - * environment. The default endianess is little endian. - */ - def bigEndian : Unit = endian = ByteOrder.BIG_ENDIAN - - /** - * Switch endianess to little endian. You should do this when you save the model in a little - * endian environment. This is the default endianess. - */ - def littleEndian : Unit = endian = ByteOrder.LITTLE_ENDIAN - - /** - * Register a new mapping from tensor flow operations to BigDL layer. The mapping is defined as - * a subclass of TFToBigDL, which defines an operation topology(reversed graph) and how to get - * constructor parameters from the topology. - * @param pattern - */ - def registerPattern(pattern : TFToBigDL): Unit = { - require(pattern.topology.reverse == true, "the topology should be a reversed graph") - patternList.append(pattern) - sortPattern() - } - - private var endian = ByteOrder.LITTLE_ENDIAN - - var dataFormat : String = "NHWC" - - def dataNCHW : Unit = dataFormat = "NCHW" - - /** - * Convert a tensorflow tensor proto to BigDL tensor - * @param tfTensor - * @return - */ - private[utils] def toTensor(tfTensor: TensorProto): Tensor[Float] = { - require(tfTensor.getDtype == DataType.DT_FLOAT || tfTensor.getDtype == DataType.DT_INT32, - s"Data type ${tfTensor.getDtype} is not supported now") - val shape = tfTensor.getTensorShape.getDimList.asScala.map(_.getSize.toInt).toArray - - if (shape.product == 1) { - if (tfTensor.getDtype == DataType.DT_FLOAT) { - return Tensor[Float](T(tfTensor.getFloatVal(0))) - } else { - return Tensor[Float](T(tfTensor.getIntVal(0).toFloat)) - } - } - - val buffer = ByteBuffer.wrap(tfTensor.getTensorContent.toByteArray) - buffer.order(endian) - - if (tfTensor.getDtype == DataType.DT_FLOAT) { - val params = buffer.asFloatBuffer - val tmp = new Array[Float](params.capacity()) - var j = 0 - while(j < params.capacity()) { - tmp(j) = params.get(j) - j += 1 - } - Tensor(Storage(tmp), 1, shape) - } else { - val params = buffer.asIntBuffer - val tmp = new Array[Float](params.capacity()) - var j = 0 - while(j < params.capacity()) { - tmp(j) = params.get(j) - j += 1 - } - Tensor(Storage(tmp), 1, shape) - } - } - - private var patternList : ArrayBuffer[TFToBigDL] = { - val res = new ArrayBuffer[TFToBigDL]() - // ElementWiseMulTF must be after MulTF - res.append( - FullConnectionTF, DropoutTF, AvgPoolingTF, MaxPoolingTF, ReshapeTF, InputTF, - TanhTF, ReluTF, SigmoidTF, Conv2D, Placeholder, SqueezeTF, IdentityTF, ConcatTF, - BatchNormTF, AddConstTF1, AddConstTF2, AddTF, SoftMaxTF, MulTF, ElementWiseMulTF, - SplitTF, PaddingTF, MeanTF, UnpackTF, StrideSliceTF, ShapeTF, FillTF, PackTF, ConstTF - ) - res - } - - sortPattern() - - /** - * Sort the pattern list to make sure the graph match first should not be a sub-graph of the graph - * match later - */ - private def sortPattern() : Unit = { - // do not calculate size and edges of a graph every time - val topToNNodes = patternList.map(g => g -> g.topology.size).toMap - val topToNEdges = patternList.map(g => g -> g.topology.edges).toMap - patternList = patternList.sortWith((l, r) => { - if (topToNNodes(l) != topToNNodes(r)) { - // graph with more nodes comes first - topToNNodes(l) > topToNNodes(r) - } else { - // same node number, graph with more edges come first - topToNEdges(l) > topToNEdges(r) - } - }) - } -} diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala deleted file mode 100644 index bdf6cff58fd..00000000000 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/package.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2016 The BigDL Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intel.analytics.bigdl - -import com.intel.analytics.bigdl.tensor.Tensor -import org.tensorflow.framework.NodeDef - -import scala.collection.mutable - -package object utils { - - type Context = mutable.HashMap[NodeDef, (Tensor[Float], Tensor[Float])] - -} diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorFlowSaver.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala similarity index 66% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorFlowSaver.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala index d4292df76b9..d2cfa66ad42 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorFlowSaver.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala @@ -13,106 +13,56 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.utils.tf -import java.io.FileOutputStream +import java.nio.ByteOrder -import com.google.protobuf.CodedOutputStream -import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.nn._ +import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.tensor.Tensor -import org.apache.log4j.Logger -import org.tensorflow.framework._ - -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer - -import TensorflowUtils._ - -object TensorFlowSaver { - /** - * Save a graph model to protobuf files so that it can be used in tensorflow inference. - * - * When save the model, placeholders will be added to the tf model as input nodes. So you need to - * pass in the names and shape for the placeholders. BigDL model doesn't have such information. - * The order of the placeholde information should be same as the inputs of the graph model - * - * @param model graph model instance - * @param inputs placeholder information - * @param path where to save - * @tparam T - */ - def saveGraph[T]( - model : Graph[T], - inputs : Seq[(String, Seq[Int])], - path: String): Unit = { - val inputNodeDefs = inputs.map(input => - placeholder(model.getNumericType(), input._2, input._1) - ) - val inputNodeCache = - new mutable.HashMap[AbstractModule[Activity, Tensor[T], T], ArrayBuffer[NodeDef]]() - model.inputs.zip(inputNodeDefs).foreach(n => { - inputNodeCache(n._1.element) = ArrayBuffer(n._2) - }) - - val graphBuilder = GraphDef.newBuilder() - inputNodeDefs.foreach(graphBuilder.addNode(_)) - - model.executions.foreach(n => { - val nodeDefs = maps(n.element.getClass.getName).toTFDef(n.element, inputNodeCache(n.element)) - nodeDefs.foreach(nDef => { - graphBuilder.addNode(nDef) - }) - n.nextNodes.foreach(n => { - val list = inputNodeCache.getOrElse(n.element, ArrayBuffer()) - list.append(nodeDefs(0)) - }) - }) - - // Save to file - val os = new FileOutputStream(path) - val output = CodedOutputStream.newInstance(os) - val graph = graphBuilder.build() - logger.debug("Graph definition is:") - logger.debug(graph.toString) - graph.writeTo(output) - output.flush() - os.close() - logger.info(s"Save as tensorflow model file to $path") - } - - private val logger = Logger.getLogger(getClass) - - private val maps = mutable.Map[String, BigDLToTF]( - getNameFromObj(ReLU.getClass.getName) -> ReLUToTF, - getNameFromObj(Linear.getClass.getName) -> LinearToTF - ) - - private def getNameFromObj(name: String) : String = name.substring(0, name.length - 1) -} +import com.intel.analytics.bigdl.utils.T +import Tensorflow._ +import BigDLToTensorflow._ +import org.tensorflow.framework.{DataType, NodeDef} /** * Wrapper of logic to convert module to tensorflow node definition */ -trait BigDLToTF { +trait BigDLToTensorflow { /** * Convert the module to a tensorflow nodedef * @return Mapped nodedef list, the first is the output node */ - def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] + def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] +} + +object BigDLToTensorflow { + private[tf] def processSaveDim(dim: Int, dataFormat: TensorflowDataFormat): Int = { + if (dataFormat == TensorflowDataFormat.NHWC) { + if (dim == 2) return 4 + if (dim == 3) return 2 + if (dim == 4) return 3 + dim + } else { + dim + } + } } -object ReLUToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object ReLUToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Relu only accept one input") Seq(relu(inputs(0), module.getName())) } } -object LinearToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object LinearToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Linear only accept one input") val linear = module.asInstanceOf[Linear[_]] val weight = const(linear.weight, linear.getName() + "/weight") @@ -120,44 +70,49 @@ object LinearToTF extends BigDLToTF { val mm = matmul(inputs(0), weightReader, linear.getName() + "matmul") val bias = const(linear.bias, linear.getName() + "/bias") val biasReader = identity(bias, linear.getName() + "/biasReader") - val add = biasAdd(mm, biasReader, NCHW, linear.getName() + "/biasAdd") + val add = biasAdd(mm, biasReader, dataFormat, linear.getName() + "/biasAdd") Seq(add, biasReader, bias, mm, weightReader, weight) } } -object SpatialConvolutionToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object SpatialConvolutionToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "SpatialConvolution only accept one input") val spatialConv = module.asInstanceOf[SpatialConvolution[_]] val filter = const(spatialConv.weight, spatialConv.getName() + "/filter") val filterReader = identity(filter, spatialConv.getName() + "/filterReader") val conv = conv2D(inputs(0), filterReader, spatialConv.strideH, spatialConv.strideW, - spatialConv.kernelW, spatialConv.kernelH, spatialConv.strideW, spatialConv.strideH, NCHW, - spatialConv.getName() + "/conv2D") + spatialConv.kernelW, spatialConv.kernelH, spatialConv.strideW, spatialConv.strideH, + dataFormat, spatialConv.getName() + "/conv2D") val bias = const(spatialConv.bias, spatialConv.getName() + "/bias") val biasReader = identity(bias, spatialConv.getName() + "/biasReader") - val add = biasAdd(conv, biasReader, NCHW, spatialConv.getName() + "/biasAdd") + val add = biasAdd(conv, biasReader, dataFormat, + spatialConv.getName() + "/biasAdd") Seq(add, biasReader, bias, conv, filterReader, filter) } } -object SqueezeToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object SqueezeToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Squeeze only accept one input") val sq = module.asInstanceOf[Squeeze[_]] - Seq(squeeze(inputs(0), sq.dims.map(_ - 1), sq.getName())) + Seq(squeeze(inputs(0), sq.dims.map(processSaveDim(_, dataFormat) - 1), sq.getName())) } } -object TanhToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object TanhToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Tanh only accept one input") Seq(tanh(inputs(0), module.getName())) } } -object ReshapeToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object ReshapeToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Reshape only accept one input") val rh = module.asInstanceOf[Reshape[_]] val size = Tensor[Float](rh.size.length) @@ -172,8 +127,9 @@ object ReshapeToTF extends BigDLToTF { } } -object ViewToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object ViewToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Reshape only accept one input") val viewLayer = module.asInstanceOf[View[_]] val size = Tensor[Float](viewLayer.sizes.length) @@ -188,17 +144,19 @@ object ViewToTF extends BigDLToTF { } } -object MaxpoolToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object MaxpoolToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Maxpool only accept one input") val layer = module.asInstanceOf[SpatialMaxPooling[_]] Seq(maxPool(inputs(0), layer.kW, layer.kH, layer.padW, layer.padH, - layer.dW, layer.dH, NCHW, layer.getName())) + layer.dW, layer.dH, dataFormat, layer.getName())) } } -object PaddingToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object PaddingToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Padding only accept one input") val layer = module.asInstanceOf[Padding[_]] val padding = Tensor[Float](1, 2) @@ -216,24 +174,27 @@ object PaddingToTF extends BigDLToTF { } } -object AvgpoolToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object AvgpoolToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Avgpool only accept one input") val layer = module.asInstanceOf[SpatialAveragePooling[_]] Seq(avgPool(inputs(0), layer.kW, layer.kH, layer.padW, layer.padH, - layer.dW, layer.dH, NCHW, layer.getName())) + layer.dW, layer.dH, dataFormat, layer.getName())) } } -object SigmoidToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object SigmoidToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Sigmoid only accept one input") Seq(sigmoid(inputs(0), module.getName())) } } -object DropoutToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object DropoutToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Dropout only accept one input") val layer = module.asInstanceOf[Dropout[_]] val shapeNode = shape(inputs(0), layer.getName() + "/shape") @@ -253,29 +214,33 @@ object DropoutToTF extends BigDLToTF { } } -object CAddTableToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object CAddTableToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { Seq(addN(inputs, module.getName())) } } -object CMultTableToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object CMultTableToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 2, "Tensorflow only support two tensor multiply together") Seq(multiply(inputs(0), inputs(1), module.getName())) } } -object JoinTableToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object JoinTableToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { val layer = module.asInstanceOf[JoinTable[_]] Seq(concat(inputs, layer.dimension - 1, layer.getName())) } } -object MeanToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object MeanToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Mean only accept one input") val layer = module.asInstanceOf[Mean[_]] val dimsTensor = Tensor[Float](layer.dimension) @@ -287,22 +252,25 @@ object MeanToTF extends BigDLToTF { } } -object SoftMaxToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object SoftMaxToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Softmax only accept one input") Seq(softmax(inputs(0), module.getName())) } } -object LogSoftMaxToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object LogSoftMaxToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "LogSoftmax only accept one input") Seq(logSoftmax(inputs(0), module.getName())) } } -object BatchNormToTF extends BigDLToTF { - override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef]): Seq[NodeDef] = { +object BatchNormToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "BatchNorm only accept one input") val layer = module.asInstanceOf[SpatialBatchNormalization[_]] val stdVar = const(layer.saveStd, layer.getName() + "/std") diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowUtils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala similarity index 91% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowUtils.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala index 204d3369779..6a6622160ad 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowUtils.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.utils.tf import java.nio.charset.Charset @@ -30,45 +30,53 @@ sealed trait TensorflowDataFormat { def value : AttrValue } -/** - * Store the image data in tensor as batch, height, width, channel - */ -object NHWC extends TensorflowDataFormat { - private val v = AttrValue.newBuilder().setS(ByteString - .copyFrom("NHWC", Charset.defaultCharset())).build() +object TensorflowDataFormat { + /** + * Store the image data in tensor as batch, height, width, channel + */ + object NHWC extends TensorflowDataFormat { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("NHWC", Charset.defaultCharset())).build() + + override def value: AttrValue = v + } - override def value: AttrValue = v + /** + * Store the image data in tensor as batch, channel, height, width + */ + object NCHW extends TensorflowDataFormat { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("NCHW", Charset.defaultCharset())).build() + + override def value: AttrValue = v + } } +/** + * Tensorflow padding type + */ sealed trait PaddingType { def value : AttrValue } -object PADDING_SAME extends PaddingType { - private val v = AttrValue.newBuilder().setS(ByteString - .copyFrom("SAME", Charset.defaultCharset())).build() +object PaddingType { - override def value: AttrValue = v -} + object PADDING_SAME extends PaddingType { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("SAME", Charset.defaultCharset())).build() -object PADDING_VALID extends PaddingType { - private val v = AttrValue.newBuilder().setS(ByteString - .copyFrom("VALID", Charset.defaultCharset())).build() - - override def value: AttrValue = v -} + override def value: AttrValue = v + } -/** - * Store the image data in tensor as batch, channel, height, width - */ -object NCHW extends TensorflowDataFormat { - private val v = AttrValue.newBuilder().setS(ByteString - .copyFrom("NCHW", Charset.defaultCharset())).build() + object PADDING_VALID extends PaddingType { + private val v = AttrValue.newBuilder().setS(ByteString + .copyFrom("VALID", Charset.defaultCharset())).build() - override def value: AttrValue = v + override def value: AttrValue = v + } } -object TensorflowUtils { +object Tensorflow { /** * Generate a placeholder tensorflow protobuf node * @param dtype numeric type @@ -468,9 +476,9 @@ object TensorflowUtils { private def getPaddingType(padW: Int, padH: Int, kW: Int, kH: Int, sW: Int, sH: Int) : PaddingType = { if (padW == 0 && padH == 0) { - return PADDING_VALID + return PaddingType.PADDING_VALID } else if (2 * padW == (kW - sW) && 2 * padH == (kH - sH)) { - return PADDING_SAME + return PaddingType.PADDING_SAME } else { throw new IllegalArgumentException( s"Can not get padding type from given parameter " + diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala similarity index 56% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowLoader.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala index c2c96eab92b..92dd816bc3c 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/TensorflowLoader.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala @@ -13,155 +13,141 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.utils.tf import java.io.{DataInputStream, FileInputStream} +import java.nio.ByteOrder import java.util import org.tensorflow.framework.{GraphDef, NodeDef} import com.google.protobuf.CodedInputStream -import java.util.{Collections, List} +import java.util.List import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.nn.Graph import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.tensor.Tensor +import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.utils.{DirectedGraph, Node} +import com.intel.analytics.bigdl.utils.tf.TensorflowToBigDL._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer - -case class TFInputOutputNodes(input: Seq[Node[NodeDef]], output: Seq[Node[NodeDef]]) +import scala.reflect.ClassTag object TensorflowLoader{ + + type Context[T] = mutable.HashMap[NodeDef, (Tensor[T], Tensor[T])] + /** - * Load tensor flow module from protobuf file - * @param graphPrototxt where is the tf protobuf file + * Load tensorflow model from a prototxt file + * @param graphPrototxt where is the tensorflow protobuf file + * @param inputs input node names + * @param outputs output node names + * @param byteOrder file byteOrder + * @param dataFormat dataformat * @return */ - def load(graphPrototxt: String, inputs: Seq[String], outputs: Seq[String]): Module[Float] = { + def load[T: ClassTag](graphPrototxt: String, inputs: Seq[String], outputs: Seq[String], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): Module[T] = { + // Get node list val nodeList = parse(graphPrototxt) - val tfGraph = buildTFGraph(nodeList) - buildBigDLModel(tfGraph, inputs, outputs) - } - private val inputPlaceholder : String = "*" + // Construct tf node graph + val tfGraph = buildTFGraph(nodeList) - private val nInputPlaceholder : String = "..." + // Build BigDL model from the tf node graph + buildBigDLModel(tfGraph, inputs, outputs, byteOrder, dataFormat) + } /** - * Extract one module and the corresponding node list from the given graph - * @param graph + * Parse a tensorflow model protobuf file, read a list of op nodes from it + * @param graphProtoTxt where is the tf protobuf file * @return */ - private[bigdl] def extract(graph: DirectedGraph[NodeDef], - context: Context) - : (Option[AbstractModule[Activity, Tensor[Float], Float]], - List[Node[NodeDef]], - TFInputOutputNodes) = { - var i = 0 - while(i < TFToBigDL.patterns.length) { - val result = matchGraph(graph, TFToBigDL.patterns(i).topology) - if (result.size != 0) { - // get model - val input = TFToBigDL.patterns(i).getInputNodes(graph) - val output = TFToBigDL.patterns(i).getOutputNodes(graph) - val inputOutput = TFInputOutputNodes(input, output) - return (Some(TFToBigDL.patterns(i).layer(graph, context)), result, inputOutput) - } - i += 1 - } - (None, Collections.emptyList(), null) - } - - private def matchGraph(graph: DirectedGraph[NodeDef], pattern: DirectedGraph[String]) - : List[Node[NodeDef]] = { - require(graph.reverse && pattern.reverse, "Must pass in reversed graph") - val patternToGraph = new mutable.HashMap[Node[String], Node[NodeDef]]() - patternToGraph(pattern.source) = graph.source + private[bigdl] def parse(graphProtoTxt: String) : List[NodeDef] = { + val f = new java.io.File(graphProtoTxt) + require(f.exists(), graphProtoTxt + " does not exists") - pattern.BFS.foreach(patternNode => { - if (patternNode.element != nInputPlaceholder && patternNode.element != inputPlaceholder) { - // Normal operation node - if (patternToGraph.get(patternNode).isEmpty) return util.Collections.emptyList() + val reader = CodedInputStream.newInstance(new DataInputStream(new FileInputStream(f))) + reader.setSizeLimit(0x7fffffff) - val graphNode = patternToGraph.get(patternNode).get - // Operation type should match - if (patternNode.element != graphNode.element.getOp) return util.Collections.emptyList() + val graph = GraphDef.parseFrom(reader) + graph.getNodeList + } - // Prev nodes number should be same except for the Ninput case - if (patternNode.prevNodes.length != graphNode.prevNodes.length && - patternNode.prevNodes.filter(_.element == nInputPlaceholder).length == 0) { - return util.Collections.emptyList() - } + /** + * Build tf ops graph from a given node list + * @param nodes + * @return + */ + private[bigdl] def buildTFGraph(nodes : List[NodeDef]): DirectedGraph[NodeDef] = { + import scala.collection.JavaConverters._ + var name2Node = nodes.asScala.map(n => n.getName -> new Node(n)).toMap - var i = 0 - var direction = 0 - var j = 0 - while (i < patternNode.prevNodes.length) { - if (patternNode.prevNodes(i).element == nInputPlaceholder) { - require(patternNode.prevNodes.count(_.element == nInputPlaceholder) == 1, - s"only support one $nInputPlaceholder ") - direction = 1 - // skip the left input nodes of graphNode, - // once we find the placeholder, we start from another side - } else if (patternNode.prevNodes(i).element == inputPlaceholder) { - // skip input placeholder - } else { - val posPattern = { if (direction == 0) i else patternNode.prevNodes.length - 1 - j} - val posGraph = { if (direction == 0) i else graphNode.prevNodes.length - 1 - j} - val pn = patternNode.prevNodes(posPattern) - val gn = graphNode.prevNodes(posGraph) - if (patternToGraph.keySet.contains(pn)) { - if (!patternToGraph(pn).eq(gn)) return util.Collections.emptyList() - } else { - patternToGraph(pn) = gn - } - if (direction == 1) j += 1 - } - i += 1 - } + // Process node with multiple tensor output, each tensor is regarded as a node + nodes.asScala + .flatMap(_.getInputList.asScala) + .filter(_.split(TENSOR_SEPARATOR).length > 1) + .foreach { nameWithChannel => + val name = nameWithChannel.split(TENSOR_SEPARATOR).head + val tfNode = NodeDef.newBuilder(name2Node(name).element) + .setName(nameWithChannel).build() + name2Node += nameWithChannel -> new Node(tfNode) } + + // Connect nodes + name2Node.valuesIterator.foreach(n => { + n.element.getInputList.asScala.foreach(name2Node(_) -> n) }) - import scala.collection.JavaConverters._ - return patternToGraph.valuesIterator.toList.asJava + + // Build graph + val outputNodes = name2Node.valuesIterator.filter(_.nextNodes.length == 0) + val dummyOutput = new Node[NodeDef](null) + outputNodes.foreach(_ -> dummyOutput) + dummyOutput.graph(reverse = true) } - private[bigdl] def buildBigDLModel( + private[bigdl] def buildBigDLModel[T: ClassTag]( tfGraph: DirectedGraph[NodeDef], inputs: Seq[String], - outputs: Seq[String] - ): Module[Float] = { + outputs: Seq[String], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat + )(implicit ev: TensorNumeric[T]): Module[T] = { import scala.collection.JavaConverters._ + // Map from tensorflow node to the converted BigDL node val convertedNode = new mutable.HashMap[Node[NodeDef], - Node[AbstractModule[Activity, Tensor[Float], Float]]]() + Node[AbstractModule[Activity, Tensor[T], T]]]() val nameToNode = - new mutable.HashMap[String, Node[AbstractModule[Activity, Tensor[Float], Float]]]() - - val context = new Context + new mutable.HashMap[String, Node[AbstractModule[Activity, Tensor[T], T]]]() + val context = new mutable.HashMap[NodeDef, (Tensor[T], Tensor[T])] + // BFS to keep the input order same tfGraph.BFS.foreach(n => { if (n.element == null) { // Dummy node, skip } else if (convertedNode.get(n).isDefined) { // converted node, skip } else { - val (module, nodes, inputOutput) = extract(n.graph(reverse = true), context) - require(module.isDefined, s"Can not find matched graph ${n}") - val node = new Node(module.get) + val (module, nodes, inputNodes) = + extract[T](n.graph(reverse = true), context, byteOrder, dataFormat).getOrElse( + throw new UnsupportedOperationException(s"Can not find matched graph ${n}")) + + val node = new Node(module) nodes.asScala.foreach(m => { convertedNode(m) = node nameToNode(m.element.getName) = node }) - val outputNodes = if (inputOutput.output == null) nodes.asScala else inputOutput.output - val nextNodes = outputNodes.flatMap(_.nextNodes) - .filter(n => n.element != null && convertedNode.contains(n) - && !context.contains(n.element)) - .map(convertedNode(_)).filter(_ != node).toSet + // These two pieces of code are all necessary + val nextNodes = n.nextNodes.filter( + n => n.element != null && convertedNode.contains(n) && !context.contains(n.element) + ).map(convertedNode(_)).filter(_ != node).toSet nextNodes.foreach(node -> _) - val inputNodes = if (inputOutput.input == null) nodes.asScala else inputOutput.input val preNodes = inputNodes.flatMap(_.prevNodes) .filter(n => n.element != null && convertedNode.contains(n) && !context.contains(n.element)) @@ -176,8 +162,8 @@ object TensorflowLoader{ .map(n => nameToNode.getOrElse(n, throw new IllegalArgumentException(s"Can't find node $n"))) - val weights = ArrayBuffer[Tensor[Float]]() - val gradients = ArrayBuffer[Tensor[Float]]() + val weights = ArrayBuffer[Tensor[T]]() + val gradients = ArrayBuffer[Tensor[T]]() for ((weight, grad) <- context.values) { weights += weight gradients += grad @@ -187,44 +173,88 @@ object TensorflowLoader{ } /** - * Build tf ops graph from a given node list - * @param nodes + * Extract one module and the corresponding node list from the given graph + * @param graph * @return */ - private[bigdl] def buildTFGraph(nodes : List[NodeDef]): DirectedGraph[NodeDef] = { - import scala.collection.JavaConverters._ - var name2Node = nodes.asScala.map(n => n.getName -> new Node(n)).toMap - nodes.asScala - .flatMap(_.getInputList.asScala) - .filter(_.split(":").length > 1) - .foreach { nameWithChannel => - val name = nameWithChannel.split(":").head - val tfNode = NodeDef.newBuilder(name2Node(name).element) - .setName(nameWithChannel).build() - name2Node += nameWithChannel -> new Node(tfNode) + private[bigdl] def extract[T: ClassTag](graph: DirectedGraph[NodeDef], + context: Context[T], byteOrder: ByteOrder, dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): Option[( + AbstractModule[Activity, Tensor[T], T], + List[Node[NodeDef]], + Seq[Node[NodeDef]] + )] = { + + var i = 0 + while(i < patterns.length) { + val (result, inputs) = matchGraph(graph, patterns(i).topology) + if (result.size != 0) { + // get model + return Some(patterns(i).layer(graph, context, byteOrder, dataFormat), result, inputs) } - // Connect nodes - name2Node.valuesIterator.foreach(n => { - n.element.getInputList.asScala.foreach(name2Node(_) -> n) - }) - val outputNodes = name2Node.valuesIterator.filter(_.nextNodes.length == 0) - val dummyOutput = new Node[NodeDef](null) - outputNodes.foreach(_ -> dummyOutput) - dummyOutput.graph(reverse = true) + i += 1 + } + None } - /** - * Parse a tensor flow model protobuf file, read a list of op nodes from it - * @param graphPrototxt where is the tf protobuf file - * @return - */ - private[bigdl] def parse(graphPrototxt: String) : List[NodeDef] = { - val f = new java.io.File(graphPrototxt) - val reader = CodedInputStream.newInstance(new DataInputStream(new FileInputStream(f))) - reader.setSizeLimit(0x7fffffff) - require(f.exists(), graphPrototxt + " does not exists") + private def matchGraph(graph: DirectedGraph[NodeDef], pattern: DirectedGraph[String]) + : (List[Node[NodeDef]], Seq[Node[NodeDef]]) = { + require(graph.reverse && pattern.reverse, "Must pass in reversed graph") + val patternToGraph = new mutable.HashMap[Node[String], Node[NodeDef]]() + val inputs = new ArrayBuffer[Node[NodeDef]]() + patternToGraph(pattern.source) = graph.source - val graph = GraphDef.parseFrom(reader) - graph.getNodeList + pattern.BFS.foreach(patternNode => { + if (patternNode.element != N_INPUT_PLACEHOLDER && patternNode.element != INPUT_PLACEHOLDER) { + // Normal operation node + if (patternToGraph.get(patternNode).isEmpty) return (util.Collections.emptyList(), Seq()) + + val graphNode = patternToGraph.get(patternNode).get + // Operation type should match + if (patternNode.element != graphNode.element.getOp) return ( + util.Collections.emptyList(), Seq()) + + // Prev nodes number should be same except for the Ninput case + if (patternNode.prevNodes.length != graphNode.prevNodes.length && + patternNode.prevNodes.filter(_.element == N_INPUT_PLACEHOLDER).length == 0) { + return (util.Collections.emptyList(), Seq()) + } + + var i = 0 + var direction = 0 + var j = 0 + while (i < patternNode.prevNodes.length) { + if (patternNode.prevNodes(i).element == N_INPUT_PLACEHOLDER) { + require(patternNode.prevNodes.count(_.element == N_INPUT_PLACEHOLDER) == 1, + s"only support one $N_INPUT_PLACEHOLDER ") + direction = 1 + // skip the left input nodes of graphNode, + // once we find the placeholder, we start from another side + if (!inputs.contains(graphNode)) { + inputs.append(graphNode) + } + } else if (patternNode.prevNodes(i).element == INPUT_PLACEHOLDER) { + // skip input placeholder + if (!inputs.contains(graphNode)) { + inputs.append(graphNode) + } + } else { + val posPattern = { if (direction == 0) i else patternNode.prevNodes.length - 1 - j} + val posGraph = { if (direction == 0) i else graphNode.prevNodes.length - 1 - j} + val pn = patternNode.prevNodes(posPattern) + val gn = graphNode.prevNodes(posGraph) + if (patternToGraph.keySet.contains(pn)) { + if (!patternToGraph(pn).eq(gn)) return (util.Collections.emptyList(), Seq()) + } else { + patternToGraph(pn) = gn + } + if (direction == 1) j += 1 + } + i += 1 + } + } + }) + import scala.collection.JavaConverters._ + return (patternToGraph.valuesIterator.toList.asJava, inputs) } } \ No newline at end of file diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala new file mode 100644 index 00000000000..98bbc2a35e3 --- /dev/null +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala @@ -0,0 +1,98 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils.tf + +import java.io.FileOutputStream +import java.nio.ByteOrder + +import com.google.protobuf.CodedOutputStream +import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} +import com.intel.analytics.bigdl.nn._ +import com.intel.analytics.bigdl.tensor.Tensor +import org.apache.log4j.Logger +import org.tensorflow.framework._ + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer +import com.intel.analytics.bigdl.utils.tf.Tensorflow._ + +object TensorflowSaver { + /** + * Save a graph model to protobuf files so that it can be used in tensorflow inference. + * + * When save the model, placeholders will be added to the tf model as input nodes. So you need to + * pass in the names and shape for the placeholders. BigDL model doesn't have such information. + * The order of the placeholde information should be same as the inputs of the graph model + * + * @param model graph model instance + * @param inputs placeholder information + * @param path where to save + * @param byteOrder model byte order + * @param dataFormat model data format + * @tparam T + */ + def saveGraph[T]( + model : Graph[T], + inputs : Seq[(String, Seq[Int])], + path: String, + byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN, + dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC): Unit = { + val inputNodeDefs = inputs.map(input => + placeholder(model.getNumericType(), input._2, input._1) + ) + val inputNodeCache = + new mutable.HashMap[AbstractModule[Activity, Tensor[T], T], ArrayBuffer[NodeDef]]() + model.inputs.zip(inputNodeDefs).foreach(n => { + inputNodeCache(n._1.element) = ArrayBuffer(n._2) + }) + + val graphBuilder = GraphDef.newBuilder() + inputNodeDefs.foreach(graphBuilder.addNode(_)) + + model.executions.foreach(n => { + val nodeDefs = maps(n.element.getClass.getName).toTFDef(n.element, inputNodeCache(n.element), + byteOrder, dataFormat) + nodeDefs.foreach(nDef => { + graphBuilder.addNode(nDef) + }) + n.nextNodes.foreach(n => { + val list = inputNodeCache.getOrElse(n.element, ArrayBuffer()) + list.append(nodeDefs(0)) + }) + }) + + // Save to file + val os = new FileOutputStream(path) + val output = CodedOutputStream.newInstance(os) + val graph = graphBuilder.build() + logger.debug("Graph definition is:") + logger.debug(graph.toString) + graph.writeTo(output) + output.flush() + os.close() + logger.info(s"Save as tensorflow model file to $path") + } + + private val logger = Logger.getLogger(getClass) + + private val maps = mutable.Map[String, BigDLToTensorflow]( + getNameFromObj(ReLU.getClass.getName) -> ReLUToTF, + getNameFromObj(Linear.getClass.getName) -> LinearToTF + ) + + private def getNameFromObj(name: String) : String = name.substring(0, name.length - 1) +} + diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala new file mode 100644 index 00000000000..47e8ffc8c65 --- /dev/null +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -0,0 +1,1091 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils.tf + +import java.nio.charset.Charset +import java.nio.{ByteBuffer, ByteOrder} + +import collection.JavaConverters._ +import com.intel.analytics.bigdl.nn._ +import com.intel.analytics.bigdl.tensor.{Storage, Tensor} +import org.tensorflow.framework.{DataType, NodeDef, TensorProto} +import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} +import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.utils.{DirectedGraph, Node, T} +import com.intel.analytics.bigdl.utils.tf.TensorflowLoader.Context +import com.intel.analytics.bigdl.utils.tf.TensorflowToBigDL._ + +import scala.collection.mutable.ArrayBuffer +import scala.reflect.{ClassTag, classTag} + +/** + * Represent a mapping from tensorflow operations graph to BigDL Module + */ +trait TensorflowToBigDL { + + /** + * The topology of the tensorflow operation graph + * @return + */ + def topology: DirectedGraph[String] + + /** + * Get the BigDL model + * @param tfGraph operation graph + * @param context variables + * @return (module, input nodes, output nodes) + */ + def layer[T: ClassTag]( + tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat + )(implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] + + protected def getOrSetTensor[T: ClassTag]( + node: NodeDef, context: Context[T], byteOrder: ByteOrder)(f: Tensor[T] => Tensor[T])( + implicit ev: TensorNumeric[T]): (Tensor[T], Tensor[T]) = { + + if (context.contains(node)) { + context(node) + } else { + val weight = f(toTensor[T](node.getAttrMap.get("value").getTensor, byteOrder)).contiguous() + val gradient = Tensor[T](weight.size()) + context.put(node, (weight, gradient)) + (weight, gradient) + } + } +} + +object TensorflowToBigDL { + + /** + * Represent one input + */ + val INPUT_PLACEHOLDER: String = "*" + + /** + * Represent one or many inputs. Note this can only be the first or the last of the input names + */ + val N_INPUT_PLACEHOLDER: String = "..." + + /** + * Separate operation name and its output tensor. In tensorflow, if one operation output multiple + * tensors, the tensor will be referred as Op:n, which n is a integer. + */ + val TENSOR_SEPARATOR: String = ":" + + /** + * Get the pattern list. + * @return + */ + def patterns: Array[TensorflowToBigDL] = { + patternList.toArray + } + + /** + * Register a new mapping from tensor flow operations to BigDL layer. The mapping is defined as + * a subclass of TFToBigDL, which defines an operation topology(reversed graph) and how to get + * constructor parameters from the topology. + * @param pattern + */ + def registerPattern(pattern: TensorflowToBigDL): Unit = { + require(pattern.topology.reverse == true, "the topology should be a reversed graph") + patternList.append(pattern) + sortPattern() + } + + var dataFormat: String = "NHWC" + + def dataNCHW: Unit = dataFormat = "NCHW" + + /** + * Convert a tensorflow tensor proto to BigDL tensor + * @param tfTensor + * @return + */ + private[utils] def toTensor[T: ClassTag](tfTensor: TensorProto, endian: ByteOrder)( + implicit ev: TensorNumeric[T]): Tensor[T] = { + + require( + tfTensor.getDtype == DataType.DT_FLOAT || + tfTensor.getDtype == DataType.DT_FLOAT || + tfTensor.getDtype == DataType.DT_INT32, + s"Data type ${tfTensor.getDtype} is not supported now") + + val shape = tfTensor.getTensorShape.getDimList.asScala.map(_.getSize.toInt).toArray + + if (shape.product == 1) { + if (classTag[T] == classTag[Float]) { + if (tfTensor.getDtype == DataType.DT_FLOAT) { + return Tensor[Float](T(tfTensor.getFloatVal(0))).asInstanceOf[Tensor[T]] + } + + if (tfTensor.getDtype == DataType.DT_INT32) { + return Tensor[Float](T(tfTensor.getIntVal(0).toFloat)).asInstanceOf[Tensor[T]] + } + + throw new IllegalArgumentException("Can not convert double to float") + } else if (classTag[T] == classTag[Double]) { + if (tfTensor.getDtype == DataType.DT_DOUBLE) { + return Tensor[Float](T(tfTensor.getDoubleVal(0))).asInstanceOf[Tensor[T]] + } + + if (tfTensor.getDtype == DataType.DT_FLOAT) { + return Tensor[Float](T(tfTensor.getFloatVal(0).toDouble)).asInstanceOf[Tensor[T]] + } + + if (tfTensor.getDtype == DataType.DT_INT32) { + return Tensor[Float](T(tfTensor.getIntVal(0).toDouble)).asInstanceOf[Tensor[T]] + } + } + } + + val buffer = ByteBuffer.wrap(tfTensor.getTensorContent.toByteArray) + buffer.order(endian) + + if (classTag[T] == classTag[Float]) { + if (tfTensor.getDtype == DataType.DT_FLOAT) { + val params = buffer.asFloatBuffer + val tmp = new Array[Float](params.capacity()) + var j = 0 + while (j < params.capacity()) { + tmp(j) = params.get(j) + j += 1 + } + Tensor(Storage(tmp), 1, shape).asInstanceOf[Tensor[T]] + } else if (tfTensor.getDtype == DataType.DT_INT32) { + val params = buffer.asIntBuffer + val tmp = new Array[Float](params.capacity()) + var j = 0 + while (j < params.capacity()) { + tmp(j) = params.get(j) + j += 1 + } + Tensor(Storage(tmp), 1, shape).asInstanceOf[Tensor[T]] + } else { + throw new IllegalArgumentException("Can not convert double to float") + } + } else if (classTag[T] == classTag[Double]) { + if (tfTensor.getDtype == DataType.DT_FLOAT) { + val params = buffer.asFloatBuffer + val tmp = new Array[Double](params.capacity()) + var j = 0 + while (j < params.capacity()) { + tmp(j) = params.get(j) + j += 1 + } + Tensor(Storage(tmp), 1, shape).asInstanceOf[Tensor[T]] + } else if (tfTensor.getDtype == DataType.DT_INT32) { + val params = buffer.asIntBuffer + val tmp = new Array[Double](params.capacity()) + var j = 0 + while (j < params.capacity()) { + tmp(j) = params.get(j) + j += 1 + } + Tensor(Storage(tmp), 1, shape).asInstanceOf[Tensor[T]] + } else if (tfTensor.getDtype == DataType.DT_DOUBLE) { + val params = buffer.asDoubleBuffer() + val tmp = new Array[Double](params.capacity()) + var j = 0 + while (j < params.capacity()) { + tmp(j) = params.get(j) + j += 1 + } + Tensor(Storage(tmp), 1, shape).asInstanceOf[Tensor[T]] + } else { + throw new IllegalArgumentException("Data type ${tfTensor.getDtype} is not supported now") + } + } else { + throw new IllegalArgumentException("Only support Float/Double") + } + } + + private var patternList : ArrayBuffer[TensorflowToBigDL] = { + val res = new ArrayBuffer[TensorflowToBigDL]() + // ElementWiseMulTF must be after MulTF + res.append( + FullConnectionTF, DropoutTF, AvgPoolingTF, MaxPoolingTF, ReshapeTF, InputTF, + TanhTF, ReluTF, SigmoidTF, Conv2D, Placeholder, SqueezeTF, IdentityTF, ConcatTF, + BatchNormTF, AddConstTF1, AddConstTF2, AddTF, SoftMaxTF, MulTF, ElementWiseMulTF, + SplitTF, PaddingTF, MeanTF, UnpackTF, StrideSliceTF, ShapeTF, FillTF, PackTF, ConstTF + ) + res + } + + sortPattern() + + /** + * Sort the pattern list to make sure the graph match first should not be a sub-graph of the graph + * match later + */ + private def sortPattern() : Unit = { + // do not calculate size and edges of a graph every time + val topToNNodes = patternList.map(g => g -> g.topology.size).toMap + val topToNEdges = patternList.map(g => g -> g.topology.edges).toMap + patternList = patternList.sortWith((l, r) => { + if (topToNNodes(l) != topToNNodes(r)) { + // graph with more nodes comes first + topToNNodes(l) > topToNNodes(r) + } else { + // same node number, graph with more edges come first + topToNEdges(l) > topToNEdges(r) + } + }) + } + + private[utils] def processDims(dim: Int, dataFormat: TensorflowDataFormat): Int = { + if (dataFormat == TensorflowDataFormat.NHWC) { + // exchange the dims as BigDL only support NCHW now + if (dim == 1) return 2 + if (dim == 2) return 3 + if (dim == 3) return 1 + dim + } else { + dim + } + } +} + +object FullConnectionTF extends TensorflowToBigDL{ + private val graph = { + val add = Node("BiasAdd") + val mul = Node("MatMul") + Node("*") -> mul + Node("Const") -> Node("Identity") -> mul -> add + Node("Const") -> Node("Identity") -> add + add.graph(reverse = true) + } + override def topology: DirectedGraph[String] = graph + + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + + val biasNode = tfGraph.source.prevNodes(1).prevNodes.head.element + val weightNode = tfGraph.source.prevNodes.head.prevNodes(1).prevNodes.head.element + val (bias, gradBias) = getOrSetTensor(biasNode, context, byteOrder)(t => t) + val (weight, gradWeight) = getOrSetTensor(weightNode, context, byteOrder) { t => + t.transpose(1, 2) + } + + Linear[T](inputSize = weight.size(2), outputSize = weight.size(1), + initWeight = weight, initGradWeight = gradWeight, initBias = bias, initGradBias = gradBias) + .asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object SqueezeTF extends TensorflowToBigDL { + private val graph = (Node("*") -> Node("Squeeze")).graph(reverse = true) + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val dims = tfGraph.source.element.getAttrOrThrow("squeeze_dims").getList().getIList() + .asScala.map(_.toInt).toArray.map(processDims(_, dataFormat)) + + Squeeze[T](dims, batchMode = true).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object Conv2D extends TensorflowToBigDL{ + private val graph = { + val add = Node("BiasAdd") + val conv = Node("Conv2D") + + Node("*") -> conv + Node("Const") -> Node("Identity") -> conv -> add + Node("Const") -> Node("Identity") -> add + add.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + + val attributes = tfGraph.source.prevNodes(0).element.getAttrMap + require(attributes.get("strides").getList.getI(0).toInt == 1, s"not support strides on batch") + + val (strideH, strideW) = if (attributes.get("data_format").getS + .toString(Charset.defaultCharset()) == "NHWC") { + require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") + (attributes.get("strides").getList.getI(1).toInt, + attributes.get("strides").getList.getI(2).toInt) + } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { + require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") + (attributes.get("strides").getList.getI(2).toInt, + attributes.get("strides").getList.getI(3).toInt) + } else { + throw new IllegalArgumentException("no supported data format") + } + val biasNode = tfGraph.source.prevNodes(1).prevNodes.head.element + val (bias, gradBias) = getOrSetTensor(biasNode, context, byteOrder)(t => t) + + val weightNode = tfGraph.source.prevNodes.head.prevNodes(1).prevNodes.head.element + val (weights, gradWeights) = getOrSetTensor(weightNode, context, byteOrder) { t => + t.transpose(1, 4).transpose(2, 3).transpose(3, 4) + } + + val nOuputPlane = weights.size(1) + val nInputPlane = weights.size(2) + val kernelH = weights.size(3) + val kernelW = weights.size(4) + + val (pW, pH) = + if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { + require((kernelW - strideW) % 2 == 0) + require((kernelH - strideH) % 2 == 0) + ((kernelW - strideW) / 2, (kernelH - strideH) / 2) + } else { + (0, 0) + } + + SpatialConvolution[T]( + nInputPlane = nInputPlane, nOutputPlane = nOuputPlane, + kernelW = kernelW, kernelH = kernelH, + strideW = strideW, strideH = strideH, + padW = pW, padH = pH, + initWeight = weights, + initBias = bias, + initGradWeight = gradWeights, + initGradBias = gradBias).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object ReluTF extends TensorflowToBigDL { + private val graph = { + (Node("*") -> Node("Relu")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + ReLU[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object TanhTF extends TensorflowToBigDL{ + private val graph = { + (Node("*") -> Node("Tanh")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + + Tanh[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object SigmoidTF extends TensorflowToBigDL{ + private val graph = { + (Node("*") -> Node("Sigmoid")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + Sigmoid[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object ReshapeTF extends TensorflowToBigDL { + private val graph = { + val nodeReshape = Node("Reshape") + Node("*") -> nodeReshape + Node("Const") -> nodeReshape + nodeReshape.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val sizes = TensorflowToBigDL.toTensor( + tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor, byteOrder) + + val batchMode = sizes.valueAt(1) == -1 + val arraySize = new Array[Int](if (batchMode) sizes.nElement() - 1 else sizes.nElement()) + var i = if (batchMode) 2 else 1 + var k = 0 + while(i <= sizes.nElement()) { + arraySize(k) = ev.toType[Int](sizes.valueAt(i)) + k += 1 + i += 1 + } + Reshape[T](size = arraySize, Some(batchMode)) + .asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object MaxPoolingTF extends TensorflowToBigDL { + private val graph = { + (Node("*") -> Node("MaxPool")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val attributes = tfGraph.source.element.getAttrMap + + val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS + .toString(Charset.defaultCharset()) == "NHWC") { + require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(1).toInt, + attributes.get("strides").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(1).toInt, + attributes.get("ksize").getList.getI(2).toInt + ) + } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { + require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(2).toInt, + attributes.get("strides").getList.getI(3).toInt, + attributes.get("ksize").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(3).toInt + ) + } else { + throw new IllegalArgumentException("no supported data format") + } + + val (pW, pH) = + if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { + require((ksizeW - strideW) % 2 == 0) + require((ksizeH - strideH) % 2 == 0) + ((ksizeW - strideW) / 2, (ksizeH - strideH) / 2) + } else { + (0, 0) + } + + SpatialMaxPooling[T](ksizeW, ksizeH, strideW, strideH, pW, pH) + .asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object AvgPoolingTF extends TensorflowToBigDL{ + private val graph = { + (Node("*") -> Node("AvgPool")).graph(reverse = true) + } + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val attributes = tfGraph.source.element.getAttrMap + + val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS + .toString(Charset.defaultCharset()) == "NHWC") { + require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(1).toInt, + attributes.get("strides").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(1).toInt, + attributes.get("ksize").getList.getI(2).toInt + ) + } else if (attributes.get("data_format").getS.toString(Charset.defaultCharset()) == "NCHW") { + require(attributes.get("strides").getList.getI(2).toInt == 1, s"not support strides on depth") + ( + attributes.get("strides").getList.getI(2).toInt, + attributes.get("strides").getList.getI(3).toInt, + attributes.get("ksize").getList.getI(2).toInt, + attributes.get("ksize").getList.getI(3).toInt + ) + } else { + throw new IllegalArgumentException("no supported data format") + } + + val (pW, pH) = + if (attributes.get("padding").getS.toString(Charset.defaultCharset()) == "SAME") { + require((ksizeW - strideW) % 2 == 0) + require((ksizeH - strideH) % 2 == 0) + ((ksizeW - strideW) / 2, (ksizeH - strideH) / 2) + } else { + (0, 0) + } + + SpatialAveragePooling[T](ksizeW, ksizeH, strideW, strideH, pW, pH, countIncludePad = false) + .asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object DropoutTF extends TensorflowToBigDL{ + private val graph = { + val nodediv = Node("RealDiv") + val nodeP = Node("Const") + val nodeadd = Node("Add") + val noderandom = Node("Add") + val nodemin = Node("Const") + val nodesub = Node("Sub") + val nodemul = Node("Mul") + val nodedrop = Node("Mul") + Node("*") -> nodediv -> nodedrop + nodeP -> nodediv + nodeP -> nodeadd -> Node("Floor") -> nodedrop + Node("*") -> Node("Shape") -> Node("RandomUniform") -> nodemul -> noderandom -> nodeadd + Node("Const") -> nodesub -> nodemul + nodemin -> nodesub + nodemin -> noderandom + nodedrop.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val keepProp = tfGraph.source.prevNodes(0).prevNodes(1).element + .getAttrMap.get("value").getTensor.getFloatVal(0) + + Dropout[T](keepProp).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object Placeholder extends TensorflowToBigDL { + private val graph = Node("Placeholder").graph(reverse = true) + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + Input[T].element.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + + +object ConstTF extends TensorflowToBigDL { + private val graph = Node("Const").graph(reverse = true) + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val value = TensorflowToBigDL + .toTensor(tfGraph.source.element.getAttrMap.get("value").getTensor, byteOrder) + Const(value).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object ShapeTF extends TensorflowToBigDL { + private val graph = { + val node = Node("Shape") + Node("*") -> node + node.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + + Shape[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object InputTF extends TensorflowToBigDL { + private val graph = (Node("Const") -> Node("Identity")).graph(reverse = true) + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + Input[T].element.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object IdentityTF extends TensorflowToBigDL { + private val graph = (Node("*") -> Node("Identity")).graph(reverse = true) + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + Input[T].element.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object BatchNormTF extends TensorflowToBigDL{ + private val graph = { + val nodeInput = Node("*") + val nodeMean1 = Node("Mean") + val nodeStopGrad = Node("StopGradient") + val nodeSub1 = Node("Sub") + val nodeSquare = Node("SquaredDifference") + val nodeMeanss = Node("Sum") + val nodeVarss = Node("Sum") + val nodeShape = Node("Reshape") + val nodeDivisor = Node("Reciprocal") + val nodeShiftedMean = Node("Mul") + val nodeMean2 = Node("Add") + val nodeMul1 = Node("Mul") + val nodeVariance = Node("Sub") + val nodeAdd1 = Node("Add") + val nodeMul2 = Node("Mul") + val nodeMul3 = Node("Mul") + val nodeMul4 = Node("Mul") + val nodeSub2 = Node("Sub") + val nodeAdd2 = Node("Add") + + nodeInput -> nodeMul3 -> nodeAdd2 + Node("Const") -> Node("Identity") -> nodeSub2 + nodeInput -> nodeMean1 -> nodeStopGrad -> nodeShape + Node("Const") -> nodeMean1 + nodeInput -> nodeSub1 -> nodeMeanss -> nodeShiftedMean -> nodeMean2 -> nodeMul4 + nodeStopGrad -> nodeSub1 + nodeInput -> nodeSquare -> nodeVarss -> nodeMul1 -> nodeVariance + nodeStopGrad -> nodeSquare + Node("Const") -> nodeDivisor -> nodeShiftedMean -> Node("Square") -> nodeVariance -> nodeAdd1 + Node("Const") -> nodeMeanss -> nodeDivisor -> nodeMul1 + Node("Const") -> nodeVarss -> nodeDivisor + Node("Const") -> nodeAdd1 -> Node("Rsqrt") -> nodeMul2 -> nodeMul3 + Node("Const") -> Node("Identity") -> nodeMul2 -> nodeMul4 -> nodeSub2 -> nodeAdd2 + Node("Const") -> nodeShape -> nodeMean2 + nodeAdd2.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val nOutput = tfGraph.source.prevNodes(1).prevNodes(1).prevNodes(1) + .prevNodes(1).prevNodes(0).element.getAttrMap.get("value").getTensor.getIntVal(0) + + val weightNode = tfGraph.source.prevNodes(1).prevNodes.head.prevNodes.head.element + val biasNode = tfGraph.source.prevNodes(1).prevNodes(1).prevNodes(1) + .prevNodes.head.prevNodes.head.element + val (weights, gradWeights) = getOrSetTensor[T](weightNode, context, byteOrder)(t => t) + val (bias, gradBias) = getOrSetTensor[T](weightNode, context, byteOrder)(t => t) + + SpatialBatchNormalization[T]( + nOutput = nOutput, + initWeight = weights, + initBias = bias, + initGradWeight = gradWeights, + initGradBias = gradBias + ).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object FillTF extends TensorflowToBigDL{ + private val graph = { + val nodeFill = Node("Fill") + Node("*") -> nodeFill + Node("Const") -> nodeFill + nodeFill.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val constNode = tfGraph.source.prevNodes(1) + val const = constNode.element.getAttrMap.get("value").getTensor.getFloatVal(0) + + Fill[T](const).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object PackTF extends TensorflowToBigDL{ + private val graph = { + val nodePack = Node("Pack") + Node("...") -> nodePack + nodePack.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + val dim = processDims(tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1, dataFormat) + + Pack[T](dim).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object UnpackTF extends TensorflowToBigDL{ + private val graph = { + val nodePack = Node("Unpack") + Node("*") -> nodePack + nodePack.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val dim = processDims(tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1, dataFormat) + val index = tfGraph.source.element.getName.split(":").toList match { + case _::Nil => 1 + case _::i::Nil => i.toInt + 1 + } + Select[T](dim, index).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object StrideSliceTF extends TensorflowToBigDL { + private val graph = { + val nodeSlice = Node("StridedSlice") + Node("*") -> nodeSlice + Node("Const") -> nodeSlice + Node("Const") -> nodeSlice + Node("Const") -> nodeSlice + nodeSlice.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val startNode = tfGraph.source.prevNodes(1) + val endNode = tfGraph.source.prevNodes(2) + val strideNode = tfGraph.source.prevNodes(3) + + def getIntArray(node: Node[NodeDef]) = { + node.element.getAttrMap.get("value").getTensor.getIntValList.asScala.map(_.toInt) + } + + val start = getIntArray(startNode) + val end = getIntArray(endNode) + val stride = getIntArray(strideNode) + + val specs = (start zip end zip stride).zipWithIndex + .map(elem => (elem._2 + 1, elem._1._1._1 + 1, elem._1._1._2 + 1, elem._1._2)).toArray + + + StrideSlice[T](specs).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + + +object ConcatTF extends TensorflowToBigDL{ + private val graph = { + val nodeConcat = Node("ConcatV2") + Node("...") -> nodeConcat + (Node("Const") -> nodeConcat).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val inputNumber = tfGraph.source.element.getAttrMap.get("N").getI.toInt + val nodeaxis = tfGraph.source.prevNodes(inputNumber) + val axis = processDims( + nodeaxis.element.getAttrMap.get("value").getTensor.getIntVal(0), dataFormat) + val nInputDims = 4 + + JoinTable[T](dimension = axis + 1, nInputDims = -1) + .asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object AddConstTF1 extends TensorflowToBigDL{ + private val graph = { + val nodeAdd = Node("Add") + Node("Const") -> nodeAdd + (Node("*") -> nodeAdd).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + val value = tfGraph.source.prevNodes.head.element + .getAttrMap.get("value").getTensor.getFloatVal(0) + AddConstant[T](value).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object AddConstTF2 extends TensorflowToBigDL{ + private val graph = { + val nodeAdd = Node("Add") + Node("*") -> nodeAdd + (Node("Const") -> nodeAdd).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val value = tfGraph.source.prevNodes(1).element + .getAttrMap.get("value").getTensor.getFloatVal(0) + AddConstant[T](value).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object AddTF extends TensorflowToBigDL{ + private val graph = { + val nodeAdd = Node("Add") + Node("*") -> nodeAdd + (Node("*") -> nodeAdd).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + CAddTable[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object SoftMaxTF extends TensorflowToBigDL{ + private val graph = { + (Node("*") -> Node("Softmax")).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + SoftMax[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + + +object MulTF extends TensorflowToBigDL{ + private val graph = { + val nodeMul = Node("Mul") + Node("Const") -> nodeMul + (Node("*") -> nodeMul).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val mul = Mul[T]() + val scale = TensorflowToBigDL.toTensor( + tfGraph.source.prevNodes(0).element.getAttrMap.get("value").getTensor, byteOrder) + require(scale.dim() == 1 && scale.size(1) == 1, s"scale must be one number") + mul.weight.copy(scale) + mul.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object ElementWiseMulTF extends TensorflowToBigDL{ + private val graph = { + val nodeMul = Node("Mul") + Node("*") -> nodeMul + (Node("*") -> nodeMul).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + CMulTable[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object SplitTF extends TensorflowToBigDL { + + private val graph = { + val nodeSplit = Node("Split") + Node("Const") -> nodeSplit + (Node("*") -> nodeSplit).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val numSplit = tfGraph.source.element.getAttrMap.get("num_split").getI.toInt + val dim = tfGraph.source.prevNodes.head.element + .getAttrMap.get("value").getTensor.getIntVal(0) + 1 + val index = tfGraph.source.element.getName.split(":").toList match { + case _::Nil => 1 + case _::i::Nil => i.toInt + 1 + } + SplitAndSelect[T](dim, index, numSplit) + .asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } + +} + + +object PaddingTF extends TensorflowToBigDL{ + private val graph = { + val nodePad = Node("Pad") + Node("*") -> nodePad + (Node("Const") -> nodePad).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val paddings = TensorflowToBigDL.toTensor( + tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor, byteOrder) + val pad = ArrayBuffer[Int]() + val padding = Sequential[T]() + + for(i <- 1 to paddings.size(1)) { + if (paddings.valueAt(i, 1) != 0 || paddings.valueAt(i, 2) != 0 ) { + val dim = processDims(i, dataFormat) + if (paddings(Array(i, 1)) != 0) { + padding.add(Padding[T](dim, -ev.toType[Int](paddings.valueAt(i, 1)), 4)) + } + if (paddings(Array(i, 2)) != 0) { + padding.add(Padding[T](dim, ev.toType[Int](paddings.valueAt(i, 1)), 4)) + } + } + } + + padding.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + +object MeanTF extends TensorflowToBigDL{ + private val graph = { + val nodeMean = Node("Mean") + Node("*") -> nodeMean + (Node("Const") -> nodeMean).graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder, + dataFormat: TensorflowDataFormat)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + + val dims = TensorflowToBigDL.toTensor( + tfGraph.source.prevNodes(1).element.getAttrMap.get("value").getTensor, byteOrder) + val dim = ArrayBuffer[Int]() + val mean = Sequential[T]() + for (i <- 1 to dims.size(1)) { + dim += processDims(ev.toType[Int](dims.valueAt(i)) + 1, dataFormat) + } + dim.foreach(i => mean.add(Mean[T](i, squeeze = false))) + mean.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} diff --git a/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py b/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py new file mode 100644 index 00000000000..7aa7875a180 --- /dev/null +++ b/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py @@ -0,0 +1,63 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +import numpy as np +import os +import slim.nets.inception_resnet_v2 as inception_resnet_v2 + +import merge_checkpoint as merge + + + +def main(): + """ + You can run these commands to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder, eg. /home/models/ + 3. python inception_resnet_v2.py + """ + dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') + batch_size = 5 + height, width = 299, 299 + num_classes = 1001 + # inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') + net, end_points = inception_resnet_v2.inception_resnet_v2(inputs,is_training = False) + output1 = tf.Variable(tf.random_uniform(tf.shape(net)),name='output1') + result1 = tf.assign(output1,net) + output2 = tf.Variable(tf.random_uniform(tf.shape(end_points['AuxLogits'])),name='output2') + result2 = tf.assign(output2,end_points['AuxLogits']) + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run([result1,result2]) + checkpointpath = saver.save(sess, dir + '/model/inception_resnet_v2.chkp') + tf.train.write_graph(sess.graph, dir + '/model', 'inception_resnet_v2.pbtxt') + tf.summary.FileWriter(dir + '/log', sess.graph) + + + input_graph = dir + "/model/inception_resnet_v2.pbtxt" + input_checkpoint = dir + "/model/inception_resnet_v2.chkp" + output_node_names= "InceptionResnetV2/AuxLogits/Logits/BiasAdd,InceptionResnetV2/Logits/Logits/BiasAdd,output1,output2" + output_graph = dir + "/inception_resnet_v2_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/loadTest/merge_checkpoint.py b/spark/dl/src/test/resources/tf/loadTest/merge_checkpoint.py new file mode 100644 index 00000000000..04b450ce840 --- /dev/null +++ b/spark/dl/src/test/resources/tf/loadTest/merge_checkpoint.py @@ -0,0 +1,58 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from google.protobuf import text_format + +from tensorflow.core.framework import graph_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import graph_util +from tensorflow.python.framework import importer +from tensorflow.python.platform import gfile + +def merge_checkpoint(input_graph, + input_checkpoint, + output_node_names, + output_graph): + """ + merge the checkpoint file with the non-binary graph file to + generate one GraphDef file with the variable values + Args: + input_graph: the GraphDef file, not in the binary form + input_checkpoint: the checkpoint file + output_node_names: A string of name of the output names, + use comma to seperate multi outputs + output_graph: String of the location and the name of the + output graph + """ + restore_op_name = "save/restore_all" + filename_tensor_name = "save/Const:0" + + input_graph_def = graph_pb2.GraphDef() + mode = "r" + with gfile.FastGFile(input_graph, mode) as f: + text_format.Merge(f.read().decode("utf-8"), input_graph_def) + for node in input_graph_def.node: + node.device = "" + _ = importer.import_graph_def(input_graph_def, name="") + with session.Session() as sess: + sess.run([restore_op_name], {filename_tensor_name: input_checkpoint}) + output_graph_def = graph_util.convert_variables_to_constants( + sess, + input_graph_def, + output_node_names.split(","), + variable_names_blacklist="") + with gfile.GFile(output_graph, "wb") as f: + f.write(output_graph_def.SerializeToString()) + diff --git a/spark/dl/src/test/resources/tf/saveTest/LinearSaveTest.py b/spark/dl/src/test/resources/tf/save/LinearSaveTest.py similarity index 100% rename from spark/dl/src/test/resources/tf/saveTest/LinearSaveTest.py rename to spark/dl/src/test/resources/tf/save/LinearSaveTest.py diff --git a/spark/dl/src/test/resources/tf/saveTest/ReLUSaveTest.py b/spark/dl/src/test/resources/tf/save/ReLUSaveTest.py similarity index 100% rename from spark/dl/src/test/resources/tf/saveTest/ReLUSaveTest.py rename to spark/dl/src/test/resources/tf/save/ReLUSaveTest.py diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala similarity index 75% rename from spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowLoaderSpec.scala rename to spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index c71425e1b61..4cedb50ff76 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -13,19 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.utils.tf import java.io.{File => JFile} +import java.nio.ByteOrder import com.intel.analytics.bigdl.dataset.{DistributedDataSet, MiniBatch} import com.intel.analytics.bigdl.nn._ import com.intel.analytics.bigdl.optim.{DistriOptimizer, Trigger} import com.intel.analytics.bigdl.tensor.{Storage, Tensor} +import com.intel.analytics.bigdl.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.TestUtils.processPath +import com.intel.analytics.bigdl.numeric.NumericFloat +import scala.sys.process._ + import scala.math._ object TensorflowLoaderSpec { @@ -59,7 +63,7 @@ object TensorflowLoaderSpec { } @com.intel.analytics.bigdl.tags.Parallel -class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { +class TensorflowLoaderSpec extends TensorflowSpecHelper{ Logger.getLogger("org").setLevel(Level.WARN) Logger.getLogger("akka").setLevel(Level.WARN) @@ -131,7 +135,8 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val path = processPath(resource.getPath()) + JFile.separator + "test.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output")) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] container.modules.length should be(4) RandomGenerator.RNG.setSeed(100) @@ -154,11 +159,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { } "Shared weights" should "be the same instance" in { + tfCheck() val resource = getClass().getClassLoader().getResource("tf") val path = processPath(resource.getPath()) + JFile.separator + "share_weight.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output")) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] container.modules.length should be(4) val l1 = container.modules(1).asInstanceOf[Linear[Float]] @@ -172,7 +179,8 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val path = processPath(resource.getPath()) + JFile.separator + "share_weight.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output")) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] val optimizer = new DistriOptimizer[Float](container, dataSet, new MSECriterion[Float]()) @@ -192,7 +200,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("output")) + Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](4, 10).rand() val gradient = Tensor[Float](4, 5).rand() val result: Tensor[Float] = model.forward(input).asInstanceOf[Tensor[Float]] @@ -209,7 +217,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("output")) + Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](4, 5, 10).rand() val gradient = Tensor[Float](4, 5).rand() @@ -223,7 +231,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("output")) + Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](4, 5, 10).rand() val gradient = Tensor[Float](4, 5).rand() @@ -237,7 +245,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("alexnet_v2/fc8/squeezed")) + Seq("alexnet_v2/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](4, 3, 224, 224).rand() val gradient = Tensor[Float](4, 1000).rand() model.forward(input) @@ -250,7 +258,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("vgg_a/fc8/squeezed")) + Seq("vgg_a/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](4, 3, 224, 224).rand() val gradient = Tensor[Float](4, 1000).rand() model.forward(input) @@ -263,7 +271,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("vgg_16/fc8/squeezed")) + Seq("vgg_16/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](4, 3, 224, 224).rand() val gradient = Tensor[Float](4, 1000).rand() model.forward(input) @@ -276,7 +284,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("vgg_19/fc8/squeezed")) + Seq("vgg_19/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](2, 3, 224, 224).rand() val gradient = Tensor[Float](2, 1000).rand() model.forward(input) @@ -289,7 +297,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("LeNet/fc4/BiasAdd")) + Seq("LeNet/fc4/BiasAdd"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](4, 3, 32, 32).rand() val gradient = Tensor[Float](4, 10).rand() model.forward(input) @@ -302,7 +310,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("InceptionV3/Logits/SpatialSqueeze")) + Seq("InceptionV3/Logits/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](2, 3, 299, 299).rand() val gradient = Tensor[Float](2, 1000).rand() model.forward(input) @@ -315,7 +323,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("resnet_v1_101/SpatialSqueeze")) + Seq("resnet_v1_101/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](2, 3, 224, 224).rand() val gradient = Tensor[Float](2, 1000).rand() model.forward(input) @@ -328,7 +336,7 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("overfeat/fc8/squeezed")) + Seq("overfeat/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](5, 3, 231, 231).rand() val gradient = Tensor[Float](5, 1000).rand() model.forward(input) @@ -341,7 +349,8 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("InceptionResnetV2/Logits/Predictions", "InceptionResnetV2/AuxLogits/Logits/BiasAdd")) + Seq("InceptionResnetV2/Logits/Predictions", "InceptionResnetV2/AuxLogits/Logits/BiasAdd"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val input = Tensor[Float](5, 3, 299, 299).rand() val gradient1 = Tensor[Float](5, 1001).rand() val gradient2 = Tensor[Float](5, 1001).rand() @@ -356,12 +365,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("alexnet_v2/fc8/squeezed")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) - .transpose(2, 4).transpose(3, 4).contiguous() + Seq("alexnet_v2/fc8/squeezed"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN).transpose(2, 4).transpose(3, 4).contiguous() val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { @@ -379,12 +389,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("vgg_a/fc8/squeezed")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + Seq("vgg_a/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) .transpose(2, 4).transpose(3, 4).contiguous() val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { @@ -401,12 +412,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("vgg_16/fc8/squeezed")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + Seq("vgg_16/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) .transpose(2, 4).transpose(3, 4).contiguous() val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { @@ -423,12 +435,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("vgg_19/fc8/squeezed")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + Seq("vgg_19/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) .transpose(2, 4).transpose(3, 4).contiguous() val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { @@ -445,12 +458,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("overfeat/fc8/squeezed")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + Seq("overfeat/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) .transpose(2, 4).transpose(3, 4).contiguous() val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { @@ -467,12 +481,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("InceptionV3/Logits/SpatialSqueeze")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + Seq("InceptionV3/Logits/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) .transpose(2, 4).transpose(3, 4).contiguous() val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { @@ -489,12 +504,13 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("resnet_v1_101/SpatialSqueeze")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + Seq("resnet_v1_101/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) .transpose(2, 4).transpose(3, 4).contiguous() val gradient = Tensor[Float](2, 1000).rand() - val tfResult = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { @@ -504,6 +520,42 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { model.backward(input, gradient) } + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + + "after loading slim inception_resnet_v2" in { + // check python path for tensorflow models + tfCheck() + (("python " + testScriptsPath("inception_resnet_v2.py")) !!) + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "inception_resnet_v2_save.pb" + val results = TensorflowLoader.parse(path) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-2)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd") + , ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) + .transpose(2, 4).transpose(3, 4).contiguous() + val gradient1 = Tensor[Float](2, 1001).rand() + val gradient2 = Tensor[Float](2, 1001).rand() + val tfResult1 = TensorflowToBigDL.toTensor(results.get(results.size()-2) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) + val tfResult2 = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) + val BigDLResult = model.forward(input) + + tfResult1.map( BigDLResult.toTable(1), (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + tfResult2.map( BigDLResult.toTable(2), (v1, v2) => { + assert(abs(v1 - v2) < 1e-7); + v2 + }) + model.backward(input, T(gradient1, gradient2)) + } + "TensorFlow loader" should "have the same inferrence result with tensorflow " + "after loading slim inception_resnet_v2" in { val resource = getClass().getClassLoader().getResource("tf") @@ -511,15 +563,17 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-2)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd")) - val input = TFToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor) + Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN) .transpose(2, 4).transpose(3, 4).contiguous() val gradient1 = Tensor[Float](2, 1001).rand() val gradient2 = Tensor[Float](2, 1001).rand() - val tfResult1 = TFToBigDL.toTensor(results.get(results.size()-2) - .getAttrMap.get("value").getTensor) - val tfResult2 = TFToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor) + val tfResult1 = TensorflowToBigDL.toTensor(results.get(results.size()-2) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) + val tfResult2 = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val BigDLResult = model.forward(input) tfResult1.map( BigDLResult.toTable(1), (v1, v2) => { @@ -540,4 +594,10 @@ class TensorflowLoaderSpec extends FlatSpec with Matchers with BeforeAndAfter { path } } + + private def testScriptsPath(script: String) : String = { + val resource = getClass().getClassLoader().getResource("tf") + processPath(resource.getPath()) + JFile.separator + "loadTest" + + JFile.separator + script + } } diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala similarity index 92% rename from spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowSaverSpec.scala rename to spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala index c99654af7b3..f83dca7d29a 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/TensorflowSaverSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.utils.tf + +import java.io.{File => JFile} import com.intel.analytics.bigdl.nn.{Graph, Linear, ReLU} -import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat +import com.intel.analytics.bigdl.utils.TestUtils.processPath +import org.scalatest.{FlatSpec, Matchers} import scala.sys.process._ -import java.io.{File => JFile} - -import com.intel.analytics.bigdl.utils.TestUtils.processPath class TensorflowSaverSpec extends FlatSpec with Matchers { "ReLU layer" should "be correctly saved" in { @@ -30,7 +30,7 @@ class TensorflowSaverSpec extends FlatSpec with Matchers { val graph = Graph(relu, relu) val tmpFile = java.io.File.createTempFile("tensorflowSaverTest", "ReLU") - TensorFlowSaver.saveGraph(graph, Seq(("input", Seq(2, 4))), tmpFile.getPath) + TensorflowSaver.saveGraph(graph, Seq(("input", Seq(2, 4))), tmpFile.getPath) runPython(testScriptsPath("ReLUSaveTest.py ") + tmpFile) should be(true) } @@ -38,7 +38,7 @@ class TensorflowSaverSpec extends FlatSpec with Matchers { val linear = Linear(3, 4).setName("linear").apply() val graph = Graph(linear, linear) val tmpFile = java.io.File.createTempFile("tensorflowSaverTest", "Linear") - TensorFlowSaver.saveGraph(graph, Seq(("input", Seq(2, 3))), tmpFile.getPath) + TensorflowSaver.saveGraph(graph, Seq(("input", Seq(2, 3))), tmpFile.getPath) println(tmpFile.getPath) // runPython(testScriptsPath("LinearSaveTest.py ") + tmpFile) should be(true) } diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala new file mode 100644 index 00000000000..a00c43e5992 --- /dev/null +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.analytics.bigdl.utils.tf + +import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} + +import scala.sys.process._ + +class TensorflowSpecHelper extends FlatSpec with Matchers with BeforeAndAfter { + protected def tfCheck(): Unit = { + var exitValue : String = "" + try { + exitValue = ((Seq("python", "-c", "import sys; print ','.join(sys.path)"))!!) + ((Seq("python", "-c", "import tensorflow"))!!) + } catch { + case _: Throwable => cancel("python or tensorflow is not installed") + } + + if (!exitValue.contains("models")) { + cancel("Tensorflow models path is not exported") + } + } +} From 7df9a62c1b926b23c94806d0d02aa58eb3a7c220 Mon Sep 17 00:00:00 2001 From: SujieZhu Date: Fri, 9 Jun 2017 15:03:21 +0800 Subject: [PATCH 06/23] change tests to automatic test --- .../bigdl/utils/tf/TensorflowToBigDL.scala | 6 +- .../test/resources/tf/inception_resnet_v2.py | 52 ----------- .../resources/tf/{ => loadTest}/alexnet.py | 19 +++- .../tf/{ => loadTest}/inception_v3.py | 23 +++-- .../resources/tf/{ => loadTest}/overfeat.py | 18 +++- .../resources/tf/{ => loadTest}/resnet_v1.py | 19 +++- .../test/resources/tf/{ => loadTest}/vgg16.py | 18 ++-- .../test/resources/tf/{ => loadTest}/vgg19.py | 19 +++- .../test/resources/tf/{ => loadTest}/vgga.py | 20 ++-- .../bigdl/utils/tf/TensorflowLoaderSpec.scala | 92 +++++++++---------- 10 files changed, 144 insertions(+), 142 deletions(-) delete mode 100644 spark/dl/src/test/resources/tf/inception_resnet_v2.py rename spark/dl/src/test/resources/tf/{ => loadTest}/alexnet.py (76%) rename spark/dl/src/test/resources/tf/{ => loadTest}/inception_v3.py (74%) rename spark/dl/src/test/resources/tf/{ => loadTest}/overfeat.py (76%) rename spark/dl/src/test/resources/tf/{ => loadTest}/resnet_v1.py (75%) rename spark/dl/src/test/resources/tf/{ => loadTest}/vgg16.py (76%) rename spark/dl/src/test/resources/tf/{ => loadTest}/vgg19.py (74%) rename spark/dl/src/test/resources/tf/{ => loadTest}/vgga.py (74%) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala index 47e8ffc8c65..42127895830 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -1049,12 +1049,12 @@ object PaddingTF extends TensorflowToBigDL{ for(i <- 1 to paddings.size(1)) { if (paddings.valueAt(i, 1) != 0 || paddings.valueAt(i, 2) != 0 ) { - val dim = processDims(i, dataFormat) + val dim = processDims(i - 1, dataFormat) + 1 if (paddings(Array(i, 1)) != 0) { padding.add(Padding[T](dim, -ev.toType[Int](paddings.valueAt(i, 1)), 4)) } if (paddings(Array(i, 2)) != 0) { - padding.add(Padding[T](dim, ev.toType[Int](paddings.valueAt(i, 1)), 4)) + padding.add(Padding[T](dim, ev.toType[Int](paddings.valueAt(i, 2)), 4)) } } } @@ -1083,7 +1083,7 @@ object MeanTF extends TensorflowToBigDL{ val dim = ArrayBuffer[Int]() val mean = Sequential[T]() for (i <- 1 to dims.size(1)) { - dim += processDims(ev.toType[Int](dims.valueAt(i)) + 1, dataFormat) + dim += processDims(ev.toType[Int](dims.valueAt(i)), dataFormat) + 1 } dim.foreach(i => mean.add(Mean[T](i, squeeze = false))) mean.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] diff --git a/spark/dl/src/test/resources/tf/inception_resnet_v2.py b/spark/dl/src/test/resources/tf/inception_resnet_v2.py deleted file mode 100644 index 32e817253fc..00000000000 --- a/spark/dl/src/test/resources/tf/inception_resnet_v2.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -import slim.nets.inception_resnet_v2 as inception_resnet_v2 - - -def main(): - """ - Run this command to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder, eg. /home/models/ - 1. mkdir model - 2. python inception_resnet_v2.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/inception_resnet_v2.pbtxt --input_checkpoint model/inception_resnet_v2.chkp --output_node_names="InceptionResnetV2/AuxLogits/Logits/BiasAdd,InceptionResnetV2/Logits/Logits/BiasAdd,output1,output2" --output_graph inception_resnet_v2_save.pb - """ - dir = os.path.dirname(os.path.realpath(__file__)) - batch_size = 5 - height, width = 299, 299 - num_classes = 1001 - # inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') - net, end_points = inception_resnet_v2.inception_resnet_v2(inputs,is_training = False) - output1 = tf.Variable(tf.random_uniform(tf.shape(net)),name='output1') - result1 = tf.assign(output1,net) - output2 = tf.Variable(tf.random_uniform(tf.shape(end_points['AuxLogits'])),name='output2') - result2 = tf.assign(output2,end_points['AuxLogits']) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run([result1,result2]) - checkpointpath = saver.save(sess, dir + '/model/inception_resnet_v2.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'inception_resnet_v2.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/alexnet.py b/spark/dl/src/test/resources/tf/loadTest/alexnet.py similarity index 76% rename from spark/dl/src/test/resources/tf/alexnet.py rename to spark/dl/src/test/resources/tf/loadTest/alexnet.py index 46359b4ecca..3ec91d2e785 100644 --- a/spark/dl/src/test/resources/tf/alexnet.py +++ b/spark/dl/src/test/resources/tf/loadTest/alexnet.py @@ -18,17 +18,18 @@ import os import slim.nets.alexnet as alexnet +import merge_checkpoint as merge + def main(): """ - Run this command to generate the pb file + You can also run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder - 1. mkdir model - 2. python alexnet.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/alexnet.pbtxt --input_checkpoint model/alexnet.chkp --output_node_names="alexnet_v2/fc8/squeezed,output" --output_graph alexnet_save.pb + 3. python alexnet.py """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 224, 224 #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) @@ -44,5 +45,13 @@ def main(): checkpointpath = saver.save(sess, dir + '/model/alexnet.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'alexnet.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + input_graph = dir + "/model/alexnet.pbtxt" + input_checkpoint = dir + "/model/alexnet.chkp" + output_node_names= "alexnet_v2/fc8/squeezed,output" + output_graph = dir + "/alexnet_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/inception_v3.py b/spark/dl/src/test/resources/tf/loadTest/inception_v3.py similarity index 74% rename from spark/dl/src/test/resources/tf/inception_v3.py rename to spark/dl/src/test/resources/tf/loadTest/inception_v3.py index edcaa5337bb..b2ee6d7ffa9 100644 --- a/spark/dl/src/test/resources/tf/inception_v3.py +++ b/spark/dl/src/test/resources/tf/loadTest/inception_v3.py @@ -18,19 +18,20 @@ import os from nets import inception +import merge_checkpoint as merge + slim = tf.contrib.slim def main(): """ - Run this command to generate the pb file + You can run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder/models/slim, eg. /home/tensorflow/models/slim/ - 1. mkdir model - 2. python inception_v3.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/inception_v3.pbtxt --input_checkpoint model/inception_v3.chkp --output_node_names="InceptionV3/Logits/SpatialSqueeze,output" --output_graph inception_v3_save.pb + 3. python inception_v3.py """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 299, 299 num_classes = 1000 @@ -42,10 +43,18 @@ def main(): with tf.Session() as sess: init = tf.global_variables_initializer() sess.run(init) - result = sess.run(output) - sess.run(init) + sess.run(result) checkpointpath = saver.save(sess, dir + '/model/inception_v3.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'inception_v3.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + + input_graph = dir + "/model/inception_v3.pbtxt" + input_checkpoint = dir + "/model/inception_v3.chkp" + output_node_names= "InceptionV3/Logits/SpatialSqueeze,output" + output_graph = dir + "/inception_v3_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + if __name__ == "__main__": main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/overfeat.py b/spark/dl/src/test/resources/tf/loadTest/overfeat.py similarity index 76% rename from spark/dl/src/test/resources/tf/overfeat.py rename to spark/dl/src/test/resources/tf/loadTest/overfeat.py index a792a761b46..a95106dc249 100644 --- a/spark/dl/src/test/resources/tf/overfeat.py +++ b/spark/dl/src/test/resources/tf/loadTest/overfeat.py @@ -17,20 +17,20 @@ import numpy as np import os import slim.nets.overfeat as overfeat +import merge_checkpoint as merge slim = tf.contrib.slim def main(): """ - Run this command to generate the pb file + You can also run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder, eg. /home/models/ - 1. mkdir model - 2. python overfeat.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/overfeat.pbtxt --input_checkpoint model/overfeat.chkp --output_node_names="overfeat/fc8/squeezed,output" --output_graph overfeat_save.pb + 3. python overfeat.py """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 231, 231 num_classes = 1000 @@ -48,5 +48,13 @@ def main(): checkpointpath = saver.save(sess, dir + '/model/overfeat.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'overfeat.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + input_graph = dir + "/model/overfeat.pbtxt" + input_checkpoint = dir + "/model/overfeat.chkp" + output_node_names= "overfeat/fc8/squeezed,output" + output_graph = dir + "/overfeat_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + if __name__ == "__main__": main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/resnet_v1.py b/spark/dl/src/test/resources/tf/loadTest/resnet_v1.py similarity index 75% rename from spark/dl/src/test/resources/tf/resnet_v1.py rename to spark/dl/src/test/resources/tf/loadTest/resnet_v1.py index 0ea03ee385f..925e68ff3f8 100644 --- a/spark/dl/src/test/resources/tf/resnet_v1.py +++ b/spark/dl/src/test/resources/tf/loadTest/resnet_v1.py @@ -19,17 +19,18 @@ from nets import resnet_utils from nets import resnet_v1 +import merge_checkpoint as merge + def main(): """ - Run this command to generate the pb file + You can run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder/models/slim, eg. /home/tensorflow/models/slim/ - 1. mkdir model - 2. python resnet_v1.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/resnet_v1.pbtxt --input_checkpoint model/resnet_v1.chkp --output_node_names="resnet_v1_101/SpatialSqueeze,output" --output_graph resnet_v1_save.pb + 3. python resnet_v1.py """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 224, 224 num_classes = 1000 @@ -46,5 +47,13 @@ def main(): checkpointpath = saver.save(sess, dir + '/model/resnet_v1.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'resnet_v1.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + input_graph = dir + "/model/resnet_v1.pbtxt" + input_checkpoint = dir + "/model/resnet_v1.chkp" + output_node_names= "resnet_v1_101/SpatialSqueeze,output" + output_graph = dir + "/resnet_v1_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + if __name__ == "__main__": main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/vgg16.py b/spark/dl/src/test/resources/tf/loadTest/vgg16.py similarity index 76% rename from spark/dl/src/test/resources/tf/vgg16.py rename to spark/dl/src/test/resources/tf/loadTest/vgg16.py index ea61a43713c..049898e2028 100644 --- a/spark/dl/src/test/resources/tf/vgg16.py +++ b/spark/dl/src/test/resources/tf/loadTest/vgg16.py @@ -17,18 +17,17 @@ import numpy as np import os import slim.nets.vgg as vgg - +import merge_checkpoint as merge def main(): """ - Run this command to generate the pb file + You can also run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder - 1. mkdir model - 2. python vgg16.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/vgg16.pbtxt --input_checkpoint model/vgg16.chkp --output_node_names="vgg_16/fc8/squeezed,output" --output_graph vgg16_save.pb + 3. python vgg16.py """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 224, 224 #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) @@ -44,5 +43,12 @@ def main(): checkpointpath = saver.save(sess, dir + '/model/vgg16.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'vgg16.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + input_graph = dir + "/model/vgg16.pbtxt" + input_checkpoint = dir + "/model/vgg16.chkp" + output_node_names= "vgg_16/fc8/squeezed,output" + output_graph = dir + "/vgg16_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/vgg19.py b/spark/dl/src/test/resources/tf/loadTest/vgg19.py similarity index 74% rename from spark/dl/src/test/resources/tf/vgg19.py rename to spark/dl/src/test/resources/tf/loadTest/vgg19.py index 5e0de05a5ca..a2cae99098d 100644 --- a/spark/dl/src/test/resources/tf/vgg19.py +++ b/spark/dl/src/test/resources/tf/loadTest/vgg19.py @@ -17,18 +17,20 @@ import numpy as np import os import slim.nets.vgg as vgg +import merge_checkpoint as merge def main(): """ - Run this command to generate the pb file + You can also run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder - 1. mkdir model - 2. python vgg19.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/vgg19.pbtxt --input_checkpoint model/vgg19.chkp --output_node_names="vgg_19/fc8/squeezed,output" --output_graph vgg19_save.pb + 3. python vgg19.py + 4. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 5. python freeze_graph.py --input_graph model/vgg19.pbtxt --input_checkpoint model/vgg19.chkp --output_node_names="vgg_19/fc8/squeezed,output" --output_graph vgg19_save.pb """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 224, 224 #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) @@ -44,5 +46,12 @@ def main(): checkpointpath = saver.save(sess, dir + '/model/vgg19.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'vgg19.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + input_graph = dir + "/model/vgg19.pbtxt" + input_checkpoint = dir + "/model/vgg19.chkp" + output_node_names= "vgg_19/fc8/squeezed,output" + output_graph = dir + "/vgg19_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/vgga.py b/spark/dl/src/test/resources/tf/loadTest/vgga.py similarity index 74% rename from spark/dl/src/test/resources/tf/vgga.py rename to spark/dl/src/test/resources/tf/loadTest/vgga.py index cc53f1a30a1..b869adabc17 100644 --- a/spark/dl/src/test/resources/tf/vgga.py +++ b/spark/dl/src/test/resources/tf/loadTest/vgga.py @@ -17,18 +17,19 @@ import numpy as np import os import slim.nets.vgg as vgg - +import merge_checkpoint as merge def main(): """ - Run this command to generate the pb file + You can also run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder - 1. mkdir model - 2. python vgga.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/vgga.pbtxt --input_checkpoint model/vgga.chkp --output_node_names="vgg_a/fc8/squeezed,output" --output_graph vgga_save.pb + 3. python vgga.py + 4. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py + 5. python freeze_graph.py --input_graph model/vgga.pbtxt --input_checkpoint model/vgga.chkp --output_node_names="vgg_a/fc8/squeezed,output" --output_graph vgga_save.pb """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 224, 224 #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) @@ -44,5 +45,12 @@ def main(): checkpointpath = saver.save(sess, dir + '/model/vgga.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'vgga.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + input_graph = dir + "/model/vgga.pbtxt" + input_checkpoint = dir + "/model/vgga.chkp" + output_node_names= "vgg_a/fc8/squeezed,output" + output_graph = dir + "/vgga_save.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index 4cedb50ff76..84d6442b786 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -358,10 +358,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, T(gradient1, gradient2)) } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + "after loading slim alexnet" in { + tfCheck() + (("python " + testScriptsPath("alexnet.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "alexnet_save.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "alexnet_save.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), @@ -382,10 +386,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + - "after loading slim vgga" in { + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + + "after loading slim vgg_a" in { + tfCheck() + (("python " + testScriptsPath("vgga.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "vgga_save.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "vgga_save.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), @@ -405,10 +413,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, gradient) } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + "after loading slim vgg_16" in { + tfCheck() + (("python " + testScriptsPath("vgg16.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "vgg16_save.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "vgg16_save.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), @@ -428,10 +440,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, gradient) } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + "after loading slim vgg_19" in { + tfCheck() + (("python " + testScriptsPath("vgg19.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "vgg19_save.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "vgg19_save.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), @@ -451,10 +467,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, gradient) } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + "after loading slim overfeat" in { + tfCheck() + (("python " + testScriptsPath("overfeat.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "overfeat_save.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "overfeat_save.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), @@ -474,10 +494,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, gradient) } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + "after loading slim inception_v3" in { + tfCheck() + (("python " + testScriptsPath("inception_v3.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "inception_v3_save.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "inception_v3_save.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), @@ -497,10 +521,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, gradient) } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + + "TensorFlow loader" should "run the python to save the modle and " + + "have the same inferrence result with tensorflow " + "after loading slim resnet_v1" in { + tfCheck() + (("python " + testScriptsPath("resnet_v1.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "resnet_v1_save.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "resnet_v1_save.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), @@ -514,7 +542,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val BigDLResult = model.forward(input) tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 2e-7); + assert(abs(v1 - v2) < 1e-6); v2 }) model.backward(input, gradient) @@ -523,7 +551,6 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "TensorFlow loader" should "run the python to save the modle and " + "have the same inferrence result with tensorflow " + "after loading slim inception_resnet_v2" in { - // check python path for tensorflow models tfCheck() (("python " + testScriptsPath("inception_resnet_v2.py")) !!) val resource = getClass().getClassLoader().getResource("tf") @@ -556,37 +583,6 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, T(gradient1, gradient2)) } - "TensorFlow loader" should "have the same inferrence result with tensorflow " + - "after loading slim inception_resnet_v2" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "inception_resnet_v2_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-2)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient1 = Tensor[Float](2, 1001).rand() - val gradient2 = Tensor[Float](2, 1001).rand() - val tfResult1 = TensorflowToBigDL.toTensor(results.get(results.size()-2) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val tfResult2 = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult1.map( BigDLResult.toTable(1), (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - tfResult2.map( BigDLResult.toTable(2), (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, T(gradient1, gradient2)) - } - private def processPath(path: String): String = { if (path.contains(":")) { path.substring(1) From 0ab6677d053240d2b1786fa87a66f8980b165cef Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Fri, 9 Jun 2017 16:31:41 +0800 Subject: [PATCH 07/23] add more test for model save --- .../bigdl/nn/SpatialAveragePooling.scala | 2 +- .../bigdl/utils/tf/BigDLToTensorflow.scala | 30 +++--- .../analytics/bigdl/utils/tf/Tensorflow.scala | 85 ++++++++++++++++- .../bigdl/utils/tf/TensorflowSaver.scala | 72 +++++++++++++-- .../test/resources/tf/save/ReLUSaveTest.py | 39 -------- .../{save/LinearSaveTest.py => save_test.py} | 16 ++-- .../bigdl/utils/tf/TensorflowSaverSpec.scala | 92 +++++++++++++------ .../bigdl/utils/tf/TensorflowSpecHelper.scala | 27 ++++++ 8 files changed, 260 insertions(+), 103 deletions(-) delete mode 100644 spark/dl/src/test/resources/tf/save/ReLUSaveTest.py rename spark/dl/src/test/resources/tf/{save/LinearSaveTest.py => save_test.py} (70%) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialAveragePooling.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialAveragePooling.scala index c61a3bd6e73..9957e0a10a4 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialAveragePooling.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialAveragePooling.scala @@ -473,7 +473,7 @@ class SpatialAveragePooling[@specialized(Float, Double) T: ClassTag]( } object SpatialAveragePooling { - def apply[@specialized(Float, Double) T: ClassTag]( + def apply[T: ClassTag]( kW: Int, kH: Int, dW: Int = 1, diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala index d2cfa66ad42..fa6e0bcc8d2 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala @@ -65,10 +65,10 @@ object LinearToTF extends BigDLToTensorflow { byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Linear only accept one input") val linear = module.asInstanceOf[Linear[_]] - val weight = const(linear.weight, linear.getName() + "/weight") + val weight = const(linear.weight.t().contiguous(), linear.getName() + "/weight", byteOrder) val weightReader = identity(weight, linear.getName() + "/weightReader") val mm = matmul(inputs(0), weightReader, linear.getName() + "matmul") - val bias = const(linear.bias, linear.getName() + "/bias") + val bias = const(linear.bias, linear.getName() + "/bias", byteOrder) val biasReader = identity(bias, linear.getName() + "/biasReader") val add = biasAdd(mm, biasReader, dataFormat, linear.getName() + "/biasAdd") Seq(add, biasReader, bias, mm, weightReader, weight) @@ -80,12 +80,12 @@ object SpatialConvolutionToTF extends BigDLToTensorflow { byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "SpatialConvolution only accept one input") val spatialConv = module.asInstanceOf[SpatialConvolution[_]] - val filter = const(spatialConv.weight, spatialConv.getName() + "/filter") + val filter = const(spatialConv.weight, spatialConv.getName() + "/filter", byteOrder) val filterReader = identity(filter, spatialConv.getName() + "/filterReader") val conv = conv2D(inputs(0), filterReader, spatialConv.strideH, spatialConv.strideW, spatialConv.kernelW, spatialConv.kernelH, spatialConv.strideW, spatialConv.strideH, dataFormat, spatialConv.getName() + "/conv2D") - val bias = const(spatialConv.bias, spatialConv.getName() + "/bias") + val bias = const(spatialConv.bias, spatialConv.getName() + "/bias", byteOrder) val biasReader = identity(bias, spatialConv.getName() + "/biasReader") val add = biasAdd(conv, biasReader, dataFormat, spatialConv.getName() + "/biasAdd") @@ -121,7 +121,7 @@ object ReshapeToTF extends BigDLToTensorflow { size.setValue(i + 1, rh.size(i)) i += 1 } - val shape = const(size, rh.getName() + "/shape", DataType.DT_INT32) + val shape = const(size, rh.getName() + "/shape", byteOrder, DataType.DT_INT32) val reshapeNode = reshape(inputs(0), shape, rh.getName()) Seq(reshapeNode, shape) } @@ -138,7 +138,7 @@ object ViewToTF extends BigDLToTensorflow { size.setValue(i + 1, viewLayer.sizes(i)) i += 1 } - val shape = const(size, viewLayer.getName() + "/shape", DataType.DT_INT32) + val shape = const(size, viewLayer.getName() + "/shape", byteOrder, DataType.DT_INT32) val reshapeNode = reshape(inputs(0), shape, viewLayer.getName()) Seq(reshapeNode, shape) } @@ -168,7 +168,7 @@ object PaddingToTF extends BigDLToTensorflow { padding.setValue(1, 1, 0) padding.setValue(1, 2, layer.pad) } - val paddingsNode = const(padding, layer.getName() + "/padding", DataType.DT_INT32) + val paddingsNode = const(padding, layer.getName() + "/padding", byteOrder, DataType.DT_INT32) val padNode = pad(inputs(0), paddingsNode, layer.getName() + "/output") Seq(padNode, paddingsNode) } @@ -199,12 +199,12 @@ object DropoutToTF extends BigDLToTensorflow { val layer = module.asInstanceOf[Dropout[_]] val shapeNode = shape(inputs(0), layer.getName() + "/shape") val rand = randomUniform(shapeNode, layer.getName() + "/random") - val maxNode = const(Tensor[Float](T(1.0f)), layer.getName() + "/max") - val minNode = const(Tensor[Float](T(0.0f)), layer.getName() + "/max") + val maxNode = const(Tensor[Float](T(1.0f)), layer.getName() + "/max", byteOrder) + val minNode = const(Tensor[Float](T(0.0f)), layer.getName() + "/max", byteOrder) val sub = subtract(maxNode, minNode, layer.getName() + "/sub") val mul = multiply(rand, sub, layer.getName() + "/mul") val randOutput = add(minNode, mul, layer.getName() + "/rand_output") - val keepProb = const(Tensor[Float](T(0.5f)), layer.getName() + "/keep_prob") + val keepProb = const(Tensor[Float](T(0.5f)), layer.getName() + "/keep_prob", byteOrder) val div1 = realdiv(keepProb, inputs(0), layer.getName() + "/div1") val div2 = realdiv(keepProb, randOutput, layer.getName() + "/div2") val floorNode = floor(div2, layer.getName() + "/floor") @@ -246,7 +246,7 @@ object MeanToTF extends BigDLToTensorflow { val dimsTensor = Tensor[Float](layer.dimension) dimsTensor.setValue(1, layer.dimension) - val dims = const(dimsTensor, layer.getName() + "/dims") + val dims = const(dimsTensor, layer.getName() + "/dims", byteOrder) val mean = reduceMean(inputs(0), dims, false, layer.getName() + "/output") Seq(mean, dims) } @@ -273,10 +273,10 @@ object BatchNormToTF extends BigDLToTensorflow { byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "BatchNorm only accept one input") val layer = module.asInstanceOf[SpatialBatchNormalization[_]] - val stdVar = const(layer.saveStd, layer.getName() + "/std") - val mean = const(layer.saveMean, layer.getName() + "/mean") - val scale = const(layer.weight, layer.getName() + "/scale") - val offset = const(layer.bias, layer.getName() + "/offset") + val stdVar = const(layer.saveStd, layer.getName() + "/std", byteOrder) + val mean = const(layer.saveMean, layer.getName() + "/mean", byteOrder) + val scale = const(layer.weight, layer.getName() + "/scale", byteOrder) + val offset = const(layer.bias, layer.getName() + "/offset", byteOrder) val div = realdiv(scale, stdVar, layer.getName() + "/div") val mul1 = multiply(inputs(0), div, layer.getName() + "/mul1") val mul2 = multiply(scale, div, layer.getName() + "/mul2") diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala index 6a6622160ad..d4427ee6d4b 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala @@ -15,6 +15,7 @@ */ package com.intel.analytics.bigdl.utils.tf +import java.nio.{ByteBuffer, ByteOrder} import java.nio.charset.Charset import com.google.protobuf.ByteString @@ -23,6 +24,8 @@ import org.tensorflow.framework.AttrValue.ListValue import org.tensorflow.framework._ import org.tensorflow.framework.TensorShapeProto.Dim +import scala.reflect.{ClassTag, classTag} + /** * Tensorflow data format. It is mostly applied in processing image type data */ @@ -99,12 +102,23 @@ object Tensorflow { * @param name * @return */ - def const(value : Tensor[_], name : String, dtype: DataType = DataType.DT_FLOAT): NodeDef = { + def const[T: ClassTag](value : Tensor[T], name : String, byteOrder: ByteOrder, + dataType: DataType = null): NodeDef = { + val dtype = if (dataType == null) { + if (value.getType() == DoubleType) { + DataType.DT_DOUBLE + } else { + DataType.DT_FLOAT + } + } else { + dataType + } + NodeDef.newBuilder() .setName(name) .setOp("Const") .putAttr("dtype", AttrValue.newBuilder().setType(dtype).build()) - .putAttr("value", tensorAttr(value, dtype)) + .putAttr("value", tensorAttr(value, dtype, byteOrder)) .build() } @@ -421,14 +435,79 @@ object Tensorflow { AttrValue.newBuilder().setList(list).build() } - private def tensorAttr(value: Tensor[_], dtype: DataType): AttrValue = { + private def tensorAttr[T: ClassTag](value: Tensor[T], dtype: DataType, + byteOrder: ByteOrder): AttrValue = { val shape = TensorShapeProto.newBuilder() value.size().foreach(dim => { shape.addDim(Dim.newBuilder().setSize(dim)) }) + require(value.isContiguous(), "only support save a contiguous tensor") + + val content = if (value.getType() == DoubleType) { + val array = value.asInstanceOf[Tensor[Double]].storage().array() + val offset = value.storageOffset() - 1 + if (dtype == DataType.DT_INT32) { + val buffer = ByteBuffer.allocate(array.length * 4) + buffer.order(byteOrder) + var i = 0 + while (i < value.nElement()) { + buffer.putInt(array(i + offset).toInt) + i += 1 + } + buffer + } else if (dtype == DataType.DT_FLOAT) { + val buffer = ByteBuffer.allocate(array.length * 4) + buffer.order(byteOrder) + var i = 0 + while (i < value.nElement()) { + buffer.putFloat(array(i + offset).toFloat) + i += 1 + } + buffer + } else if (dtype == DataType.DT_DOUBLE) { + val buffer = ByteBuffer.allocate(array.length * 8) + buffer.order(byteOrder) + var i = 0 + while (i < value.nElement()) { + buffer.putDouble(array(i + offset)) + i += 1 + } + buffer + } else { + throw new UnsupportedOperationException(s"data type ${dtype} is not supported currently") + } + } else { + val array = value.asInstanceOf[Tensor[Float]].storage().array() + val offset = value.storageOffset() - 1 + if (dtype == DataType.DT_INT32) { + val buffer = ByteBuffer.allocate(array.length * 4) + buffer.order(byteOrder) + var i = 0 + while (i < value.nElement()) { + buffer.putInt(array(i + offset).toInt) + i += 1 + } + buffer + } else if (dtype == DataType.DT_FLOAT) { + val buffer = ByteBuffer.allocate(array.length * 4) + buffer.order(byteOrder) + var i = 0 + while (i < value.nElement()) { + buffer.putFloat(array(i + offset)) + i += 1 + } + buffer + } else if (dtype == DataType.DT_DOUBLE) { + throw new IllegalArgumentException(s"can not convert a float tensor to double tensor") + } else { + throw new UnsupportedOperationException(s"data type ${dtype} is not supported currently") + } + } + AttrValue.newBuilder().setTensor( TensorProto.newBuilder().setTensorShape(shape).setDtype(dtype) + .setTensorContent(ByteString.copyFrom(content.array())) ).build() } diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala index 98bbc2a35e3..d4350a83a45 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala @@ -38,29 +38,27 @@ object TensorflowSaver { * The order of the placeholde information should be same as the inputs of the graph model * * @param model graph model instance - * @param inputs placeholder information + * @param inputs input node defs * @param path where to save * @param byteOrder model byte order * @param dataFormat model data format * @tparam T */ - def saveGraph[T]( + def saveGraphWitNodeDef[T]( model : Graph[T], - inputs : Seq[(String, Seq[Int])], + inputs : Seq[NodeDef], path: String, byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN, - dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC): Unit = { - val inputNodeDefs = inputs.map(input => - placeholder(model.getNumericType(), input._2, input._1) - ) + dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC, + extraNodes: Set[NodeDef] = Set()): Unit = { val inputNodeCache = new mutable.HashMap[AbstractModule[Activity, Tensor[T], T], ArrayBuffer[NodeDef]]() - model.inputs.zip(inputNodeDefs).foreach(n => { + model.inputs.zip(inputs).foreach(n => { inputNodeCache(n._1.element) = ArrayBuffer(n._2) }) val graphBuilder = GraphDef.newBuilder() - inputNodeDefs.foreach(graphBuilder.addNode(_)) + inputs.foreach(graphBuilder.addNode(_)) model.executions.foreach(n => { val nodeDefs = maps(n.element.getClass.getName).toTFDef(n.element, inputNodeCache(n.element), @@ -74,6 +72,8 @@ object TensorflowSaver { }) }) + extraNodes.foreach(graphBuilder.addNode(_)) + // Save to file val os = new FileOutputStream(path) val output = CodedOutputStream.newInstance(os) @@ -86,11 +86,63 @@ object TensorflowSaver { logger.info(s"Save as tensorflow model file to $path") } + /** + * Save a graph model to protobuf files so that it can be used in tensorflow inference. + * + * When save the model, placeholders will be added to the tf model as input nodes. So you need to + * pass in the names and shape for the placeholders. BigDL model doesn't have such information. + * The order of the placeholde information should be same as the inputs of the graph model + * + * @param model graph model instance + * @param inputs placeholder information + * @param path where to save + * @param byteOrder model byte order + * @param dataFormat model data format + * @tparam T + */ + def saveGraph[T]( + model : Graph[T], + inputs : Seq[(String, Seq[Int])], + path: String, + byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN, + dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC): Unit = { + val inputNodeDefs = inputs.map(input => + placeholder(model.getNumericType(), input._2, input._1) + ) + saveGraphWitNodeDef(model, inputNodeDefs, path, byteOrder, dataFormat) + } + + /** + * Register a customized BigDL module saver. + * @param className class name of the BigDL module + * @param saver customized saver + */ + def register(className : String, saver: BigDLToTensorflow): Unit = { + maps(className) = saver + } + private val logger = Logger.getLogger(getClass) private val maps = mutable.Map[String, BigDLToTensorflow]( getNameFromObj(ReLU.getClass.getName) -> ReLUToTF, - getNameFromObj(Linear.getClass.getName) -> LinearToTF + getNameFromObj(Linear.getClass.getName) -> LinearToTF, + getNameFromObj(SpatialConvolution.getClass.getName) -> SpatialConvolutionToTF, + getNameFromObj(Squeeze.getClass.getName) -> SqueezeToTF, + getNameFromObj(Tanh.getClass.getName) -> TanhToTF, + getNameFromObj(Reshape.getClass.getName) -> ReshapeToTF, + getNameFromObj(View.getClass.getName) -> ViewToTF, + getNameFromObj(SpatialMaxPooling.getClass.getName) -> MaxpoolToTF, + getNameFromObj(Padding.getClass.getName) -> PaddingToTF, + getNameFromObj(SpatialAveragePooling.getClass.getName) -> AvgpoolToTF, + getNameFromObj(Sigmoid.getClass.getName) -> SigmoidToTF, + getNameFromObj(Dropout.getClass.getName) -> DropoutToTF, + getNameFromObj(CAddTable.getClass.getName) -> CAddTableToTF, + getNameFromObj(CMulTable.getClass.getName) -> CMultTableToTF, + getNameFromObj(JoinTable.getClass.getName) -> JoinTableToTF, + getNameFromObj(Mean.getClass.getName) -> MeanToTF, + getNameFromObj(SoftMax.getClass.getName) -> SoftMaxToTF, + getNameFromObj(LogSoftMax.getClass.getName) -> LogSoftMaxToTF, + getNameFromObj(SpatialBatchNormalization.getClass.getName) -> BatchNormToTF ) private def getNameFromObj(name: String) : String = name.substring(0, name.length - 1) diff --git a/spark/dl/src/test/resources/tf/save/ReLUSaveTest.py b/spark/dl/src/test/resources/tf/save/ReLUSaveTest.py deleted file mode 100644 index f990de085e2..00000000000 --- a/spark/dl/src/test/resources/tf/save/ReLUSaveTest.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from tensorflow.python.platform import gfile -from sys import argv - -def main(): - with gfile.FastGFile(argv[1],'rb') as f: - graph_def = tf.GraphDef() - graph_def.ParseFromString(f.read()) - with tf.Graph().as_default() as graph: - tf.import_graph_def(graph_def, name='') - sess = tf.Session() - for op in graph.get_operations(): - print(op.name) - prediction = graph.get_tensor_by_name('relu:0') - ix = graph.get_tensor_by_name('input:0') - test = np.matrix([[1, 2, 5, 6], [-3, -4, -7, -8]]) - result = sess.run(prediction, {ix: test}) - - np.testing.assert_equal(result, np.array([[1, 2, 5, 6], [0, 0, 0, 0]])) - -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/save/LinearSaveTest.py b/spark/dl/src/test/resources/tf/save_test.py similarity index 70% rename from spark/dl/src/test/resources/tf/save/LinearSaveTest.py rename to spark/dl/src/test/resources/tf/save_test.py index 788f44fb1d9..1a4afc096cb 100644 --- a/spark/dl/src/test/resources/tf/save/LinearSaveTest.py +++ b/spark/dl/src/test/resources/tf/save_test.py @@ -28,12 +28,16 @@ def main(): sess = tf.Session() for op in graph.get_operations(): print(op.name) - prediction = graph.get_tensor_by_name('linear/biasAdd:0') - ix = graph.get_tensor_by_name('input:0') - test = np.matrix([[1, 2, 5], [-3, -4, -7]]) - result = sess.run(prediction, {ix: test}) - - #np.testing.assert_equal(result, np.array([[1, 2, 5, 6], [0, 0, 0, 0]])) + output_suffix = '' + if len(argv) == 3: + output_suffix = argv[2] + output = graph.get_tensor_by_name('output' + output_suffix + ':0') + target = graph.get_tensor_by_name('target:0') + tf_output = sess.run(output) + bigdl_output = sess.run(target) + print(tf_output) + print(bigdl_output) + np.testing.assert_equal(tf_output, bigdl_output) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala index f83dca7d29a..0e43ce37cec 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala @@ -15,46 +15,80 @@ */ package com.intel.analytics.bigdl.utils.tf -import java.io.{File => JFile} -import com.intel.analytics.bigdl.nn.{Graph, Linear, ReLU} +import java.nio.ByteOrder +import java.util.UUID + +import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule +import com.intel.analytics.bigdl.nn.{Graph, Linear, ReLU, SpatialAveragePooling} import com.intel.analytics.bigdl.numeric.NumericFloat -import com.intel.analytics.bigdl.utils.TestUtils.processPath -import org.scalatest.{FlatSpec, Matchers} +import com.intel.analytics.bigdl.tensor.Tensor +import com.intel.analytics.bigdl.utils.T +import org.apache.log4j.Logger -import scala.sys.process._ +class TensorflowSaverSpec extends TensorflowSpecHelper { -class TensorflowSaverSpec extends FlatSpec with Matchers { - "ReLU layer" should "be correctly saved" in { - val relu = ReLU().setName("relu").apply() - val graph = Graph(relu, relu) + private val logger = Logger.getLogger(getClass) - val tmpFile = java.io.File.createTempFile("tensorflowSaverTest", "ReLU") - TensorflowSaver.saveGraph(graph, Seq(("input", Seq(2, 4))), tmpFile.getPath) - runPython(testScriptsPath("ReLUSaveTest.py ") + tmpFile) should be(true) + "ReLU layer" should "be correctly saved" in { + val inputTensor = Tensor[Float](T( + T(1.0f, 2.0f, 5.0f, 6.0f), + T(-3.0f, -4.0f, -7.0f, -8.0f) + )) + test(ReLU[Float](), inputTensor) should be(true) } "Linear layer" should "be correctly saved" in { - val linear = Linear(3, 4).setName("linear").apply() - val graph = Graph(linear, linear) - val tmpFile = java.io.File.createTempFile("tensorflowSaverTest", "Linear") - TensorflowSaver.saveGraph(graph, Seq(("input", Seq(2, 3))), tmpFile.getPath) - println(tmpFile.getPath) - // runPython(testScriptsPath("LinearSaveTest.py ") + tmpFile) should be(true) + val layer = Linear[Float](3, 4, + initWeight = Tensor(T( + T(1.0f, 2.0f, 3.0f), + T(4.0f, 5.0f, 6.0f), + T(7.0f, 8.0f, 9.0f), + T(10.0f, 11.0f, 12.0f) + )), + initBias = Tensor(T(1.0f, 2.0f, 3.0f, 4.0f)) + ) + val input = Tensor[Float](T( + T(1.0f, 2.0f, 5.0f), + T(-3.0f, -4.0f, -7.0f) + )) + test(layer, input, "/biasAdd") should be(true) } - private def testScriptsPath(script: String) : String = { - val resource = getClass().getClassLoader().getResource("tf") - processPath(resource.getPath()) + JFile.separator + "saveTest" + - JFile.separator + script + "AvgPooling" should "be correctly saved" in { + val layer = SpatialAveragePooling(2, 2) + val input = Tensor[Float](T( + T( + T(1.0f, 2.0f, 5.0f), + T(-3.0f, -4.0f, -7.0f), + T(-4.0f, -2.0f, -1.0f) + ), + T( + T(-1.0f, -2.0f, -5.0f), + T(3.0f, 4.0f, 7.0f), + T(4.0f, 2.0f, 1.0f) + ) + )) + test(layer, input) should be(true) } - private def runPython(cmd: String): Boolean = { - try { - (("python " + cmd) !!) - return true - } catch { - case _: Throwable => false - } + private def test(layer: AbstractModule[Tensor[Float], Tensor[Float], Float], + inputTensor: Tensor[Float], outputSuffix: String = "") : Boolean = { + tfCheck() + val layerNode = layer.setName("output").apply() + val graph = Graph(layerNode, layerNode) + val outputTensor = layer.forward(inputTensor) + + val tmpFile = java.io.File.createTempFile("tensorflowSaverTest" + UUID.randomUUID(), "Layer") + logger.info(s"Save model to ${tmpFile}") + TensorflowSaver.saveGraphWitNodeDef( + graph, + Seq(Tensorflow.const(inputTensor, "input", ByteOrder.LITTLE_ENDIAN)), + tmpFile.getPath, + ByteOrder.LITTLE_ENDIAN, + TensorflowDataFormat.NHWC, + Set(Tensorflow.const(outputTensor, "target", ByteOrder.LITTLE_ENDIAN)) + ) + runPythonSaveTest(tmpFile.getPath, outputSuffix) } } \ No newline at end of file diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala index a00c43e5992..9a1aa1936da 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala @@ -15,11 +15,18 @@ */ package com.intel.analytics.bigdl.utils.tf +import com.intel.analytics.bigdl.utils.TestUtils.processPath import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} +import java.io.{File => JFile} + +import org.apache.log4j.Logger import scala.sys.process._ class TensorflowSpecHelper extends FlatSpec with Matchers with BeforeAndAfter { + + private val logger = Logger.getLogger(getClass) + protected def tfCheck(): Unit = { var exitValue : String = "" try { @@ -33,4 +40,24 @@ class TensorflowSpecHelper extends FlatSpec with Matchers with BeforeAndAfter { cancel("Tensorflow models path is not exported") } } + + protected def runPython(cmd: String): Boolean = { + var result = "" + try { + logger.info("run command\n" + cmd) + result = (("python " + cmd) !!) + return true + } catch { + case _: Throwable => false + } finally { + logger.info("stdout is \n" + result) + } + } + + protected def runPythonSaveTest(graphPath: String, outputSuffix: String) : Boolean = { + val resource = getClass().getClassLoader().getResource("tf") + val path = processPath(resource.getPath()) + JFile.separator + + s"save_test.py $graphPath $outputSuffix" + runPython(path) + } } From bf56fe3430bc3fed3b956ddcfe32c8ba31798459 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Fri, 9 Jun 2017 16:43:28 +0800 Subject: [PATCH 08/23] remove some useless unit test --- .../bigdl/utils/tf/TensorflowLoaderSpec.scala | 106 ------------------ 1 file changed, 106 deletions(-) diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index 84d6442b786..541cd1bdc2d 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -239,58 +239,6 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, gradient) } - "TensorFlow loader" should "be able to load slim alexnetv2" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "alexnet.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("alexnet_v2/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](4, 3, 224, 224).rand() - val gradient = Tensor[Float](4, 1000).rand() - model.forward(input) - model.backward(input, gradient) - } - - "TensorFlow loader" should "be able to load slim vgga" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "vgga.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("vgg_a/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](4, 3, 224, 224).rand() - val gradient = Tensor[Float](4, 1000).rand() - model.forward(input) - model.backward(input, gradient) - } - - "TensorFlow loader" should "be able to load slim vgg16" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "vgg16.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("vgg_16/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](4, 3, 224, 224).rand() - val gradient = Tensor[Float](4, 1000).rand() - model.forward(input) - model.backward(input, gradient) - } - - "TensorFlow loader" should "be able to load slim vgg19" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "vgg19.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("vgg_19/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](2, 3, 224, 224).rand() - val gradient = Tensor[Float](2, 1000).rand() - model.forward(input) - model.backward(input, gradient) - } - "TensorFlow loader" should "be able to load slim lenet" in { val resource = getClass().getClassLoader().getResource("tf") val path = processPath(resource.getPath()) + JFile.separator + "lenet.pb" @@ -304,60 +252,6 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, gradient) } - "TensorFlow loader" should "be able to load slim inception_v3" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "inception_v3.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("InceptionV3/Logits/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](2, 3, 299, 299).rand() - val gradient = Tensor[Float](2, 1000).rand() - model.forward(input) - model.backward(input, gradient) - } - - "TensorFlow loader" should "be able to load slim resnet_v1" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "resnet_v1.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("resnet_v1_101/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](2, 3, 224, 224).rand() - val gradient = Tensor[Float](2, 1000).rand() - model.forward(input) - model.backward(input, gradient) - } - - "TensorFlow loader" should "be able to load slim overfeat" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "overfeat.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("overfeat/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](5, 3, 231, 231).rand() - val gradient = Tensor[Float](5, 1000).rand() - model.forward(input) - model.backward(input, gradient) - } - - "TensorFlow loader" should "be able to load slim inception_resnet_v2" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "inception_resnet_v2.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("InceptionResnetV2/Logits/Predictions", "InceptionResnetV2/AuxLogits/Logits/BiasAdd"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](5, 3, 299, 299).rand() - val gradient1 = Tensor[Float](5, 1001).rand() - val gradient2 = Tensor[Float](5, 1001).rand() - model.forward(input) - model.backward(input, T(gradient1, gradient2)) - } - "TensorFlow loader" should "run the python to save the modle and " + "have the same inferrence result with tensorflow " + "after loading slim alexnet" in { From db5f3853789c68a7880ea337424e05b390e5e3f5 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Fri, 9 Jun 2017 17:59:31 +0800 Subject: [PATCH 09/23] add more save test --- .../bigdl/nn/SpatialMaxPooling.scala | 4 +- .../com/intel/analytics/bigdl/nn/Tanh.scala | 2 +- .../analytics/bigdl/utils/tf/Tensorflow.scala | 25 +++++++-- spark/dl/src/test/resources/tf/save_test.py | 6 +- .../bigdl/utils/tf/TensorflowSaverSpec.scala | 55 ++++++++++++++++--- .../bigdl/utils/tf/TensorflowSpecHelper.scala | 7 +-- 6 files changed, 77 insertions(+), 22 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialMaxPooling.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialMaxPooling.scala index 31230ad575a..b7d155a158f 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialMaxPooling.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialMaxPooling.scala @@ -290,8 +290,8 @@ object SpatialMaxPooling { def apply[@specialized(Float, Double) T: ClassTag]( kW: Int, kH: Int, - dW: Int, - dH: Int, + dW: Int = 1, + dH: Int = 1, padW: Int = 0, padH: Int = 0)(implicit ev: TensorNumeric[T]): SpatialMaxPooling[T] = { new SpatialMaxPooling[T](kW, kH, dW, dH, padW, padH) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Tanh.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Tanh.scala index 244a01d1b2c..caba8c4b04b 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Tanh.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Tanh.scala @@ -48,7 +48,7 @@ class Tanh[@specialized(Float, Double) T: ClassTag]( object Tanh { - def apply[@specialized(Float, Double) T: ClassTag]() + def apply[T: ClassTag]() (implicit ev: TensorNumeric[T]) : Tanh[T] = { new Tanh[T]() } diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala index d4427ee6d4b..33c258967a7 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala @@ -247,9 +247,9 @@ object Tensorflow { .addInput(value.getName) .putAttr("T", getDataType(value)) .putAttr("data_format", dataFormat.value) - .putAttr("ksize", listIntAttr(Seq(kH, kW))) + .putAttr("ksize", kernelAttr(kW, kH, dataFormat)) .putAttr("padding", getPaddingType(pW, pH, kW, kH, sW, sH).value) - .putAttr("strides", listIntAttr(Seq(sH, sW))) + .putAttr("strides", strideAttr(sW, sH, dataFormat)) .build() } @@ -261,9 +261,9 @@ object Tensorflow { .putAttr("T", getDataType(value)) .addInput(value.getName) .putAttr("data_format", dataFormat.value) - .putAttr("ksize", listIntAttr(Seq(kH, kW))) + .putAttr("ksize", kernelAttr(kW, kH, dataFormat)) .putAttr("padding", getPaddingType(pW, pH, kW, kH, sW, sH).value) - .putAttr("strides", listIntAttr(Seq(sH, sW))) + .putAttr("strides", strideAttr(sW, sH, dataFormat)) .build() } @@ -565,4 +565,21 @@ object Tensorflow { } } + private def kernelAttr(kW: Int, kH: Int, dataFormat: TensorflowDataFormat): AttrValue = { + val kSize = if (dataFormat == TensorflowDataFormat.NHWC) { + Seq(1, kH, kW, 1) + } else { + Seq(1, 1, kH, kW) + } + listIntAttr(kSize) + } + + private def strideAttr(sW: Int, sH: Int, dataFormat: TensorflowDataFormat): AttrValue = { + val sSize = if (dataFormat == TensorflowDataFormat.NHWC) { + Seq(1, sH, sW, 1) + } else { + Seq(1, 1, sH, sW) + } + listIntAttr(sSize) + } } diff --git a/spark/dl/src/test/resources/tf/save_test.py b/spark/dl/src/test/resources/tf/save_test.py index 1a4afc096cb..a44bb08ba03 100644 --- a/spark/dl/src/test/resources/tf/save_test.py +++ b/spark/dl/src/test/resources/tf/save_test.py @@ -26,8 +26,6 @@ def main(): with tf.Graph().as_default() as graph: tf.import_graph_def(graph_def, name='') sess = tf.Session() - for op in graph.get_operations(): - print(op.name) output_suffix = '' if len(argv) == 3: output_suffix = argv[2] @@ -35,9 +33,11 @@ def main(): target = graph.get_tensor_by_name('target:0') tf_output = sess.run(output) bigdl_output = sess.run(target) + print("Tensorflow output is:") print(tf_output) + print("BigDL output is:") print(bigdl_output) - np.testing.assert_equal(tf_output, bigdl_output) + np.testing.assert_almost_equal(tf_output, bigdl_output) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala index 0e43ce37cec..af263c3ce0e 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala @@ -20,7 +20,7 @@ import java.nio.ByteOrder import java.util.UUID import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule -import com.intel.analytics.bigdl.nn.{Graph, Linear, ReLU, SpatialAveragePooling} +import com.intel.analytics.bigdl.nn._ import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.utils.T @@ -52,12 +52,12 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { T(1.0f, 2.0f, 5.0f), T(-3.0f, -4.0f, -7.0f) )) - test(layer, input, "/biasAdd") should be(true) + test(layer, input, false, "/biasAdd") should be(true) } "AvgPooling" should "be correctly saved" in { val layer = SpatialAveragePooling(2, 2) - val input = Tensor[Float](T( + val input = Tensor[Float](T(T( T( T(1.0f, 2.0f, 5.0f), T(-3.0f, -4.0f, -7.0f), @@ -68,12 +68,43 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { T(3.0f, 4.0f, 7.0f), T(4.0f, 2.0f, 1.0f) ) - )) + ))) + test(layer, input, true) should be(true) + } + + "MaxPooling" should "be correctly saved" in { + val layer = SpatialMaxPooling(2, 2) + val input = Tensor[Float](T(T( + T( + T(1.0f, 2.0f, 5.0f), + T(-3.0f, -4.0f, -7.0f), + T(-4.0f, -2.0f, -1.0f) + ), + T( + T(-1.0f, -2.0f, -5.0f), + T(3.0f, 4.0f, 7.0f), + T(4.0f, 2.0f, 1.0f) + ) + ))) + test(layer, input, true) should be(true) + } + + "Tanh" should "be correctly saved" in { + val layer = Tanh() + val input = Tensor[Float](4).rand() test(layer, input) should be(true) } + "Squeeze" should "be correctly saved" in { + val layer = Squeeze(3) + val input = Tensor[Float](4, 2, 1, 2).rand() + test(layer, input, true) should be(true) + } + private def test(layer: AbstractModule[Tensor[Float], Tensor[Float], Float], - inputTensor: Tensor[Float], outputSuffix: String = "") : Boolean = { + inputTensor: Tensor[Float], + convertNHWC: Boolean = false, + outputSuffix: String = "") : Boolean = { tfCheck() val layerNode = layer.setName("output").apply() val graph = Graph(layerNode, layerNode) @@ -81,13 +112,23 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { val tmpFile = java.io.File.createTempFile("tensorflowSaverTest" + UUID.randomUUID(), "Layer") logger.info(s"Save model to ${tmpFile}") + val tfTensor = if (convertNHWC) { + inputTensor.transpose(2, 3).transpose(3, 4).contiguous() + } else { + inputTensor + } + val outputSave = if (convertNHWC) { + outputTensor.transpose(2, 3).transpose(3, 4).contiguous() + } else { + outputTensor + } TensorflowSaver.saveGraphWitNodeDef( graph, - Seq(Tensorflow.const(inputTensor, "input", ByteOrder.LITTLE_ENDIAN)), + Seq(Tensorflow.const(tfTensor, "input", ByteOrder.LITTLE_ENDIAN)), tmpFile.getPath, ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC, - Set(Tensorflow.const(outputTensor, "target", ByteOrder.LITTLE_ENDIAN)) + Set(Tensorflow.const(outputSave, "target", ByteOrder.LITTLE_ENDIAN)) ) runPythonSaveTest(tmpFile.getPath, outputSuffix) } diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala index 9a1aa1936da..d42fb516eea 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSpecHelper.scala @@ -42,15 +42,12 @@ class TensorflowSpecHelper extends FlatSpec with Matchers with BeforeAndAfter { } protected def runPython(cmd: String): Boolean = { - var result = "" try { logger.info("run command\n" + cmd) - result = (("python " + cmd) !!) - return true + val proc = s"python $cmd".run + return proc.exitValue() == 0 } catch { case _: Throwable => false - } finally { - logger.info("stdout is \n" + result) } } From d481afc2f7196d20b95c0e7066c0d04a3e27bf19 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Mon, 12 Jun 2017 17:57:07 +0800 Subject: [PATCH 10/23] add more writer test --- .../intel/analytics/bigdl/nn/Dropout.scala | 2 +- .../com/intel/analytics/bigdl/nn/Graph.scala | 6 + .../com/intel/analytics/bigdl/nn/Mean.scala | 4 +- .../intel/analytics/bigdl/nn/Padding.scala | 2 +- .../intel/analytics/bigdl/nn/Reshape.scala | 2 +- .../intel/analytics/bigdl/nn/Sigmoid.scala | 2 +- .../analytics/bigdl/tensor/DenseTensor.scala | 2 + .../intel/analytics/bigdl/tensor/Tensor.scala | 6 + .../bigdl/utils/tf/BigDLToTensorflow.scala | 89 +++++---- .../analytics/bigdl/utils/tf/Tensorflow.scala | 32 ++-- .../bigdl/utils/tf/TensorflowSaver.scala | 10 +- spark/dl/src/test/resources/tf/save_test.py | 2 +- .../bigdl/utils/tf/TensorflowSaverSpec.scala | 170 +++++++++++++++++- 13 files changed, 266 insertions(+), 63 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Dropout.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Dropout.scala index 71de6e7ba4d..0d1ffb923e3 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Dropout.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Dropout.scala @@ -207,7 +207,7 @@ class Dropout[T: ClassTag]( } object Dropout { - def apply[@specialized(Float, Double) T: ClassTag]( + def apply[T: ClassTag]( initP: Double = 0.5, inplace: Boolean = false, scale: Boolean = true)(implicit ev: TensorNumeric[T]) : Dropout[T] = { diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala index e8cc86bae31..823091e4ef4 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala @@ -325,6 +325,12 @@ class Input[T: ClassTag]()(implicit ev: TensorNumeric[T]) extends TensorModule[T gradInput = gradOutput gradInput } + override def equals(other: Any): Boolean = { + if (!other.isInstanceOf[Input[_]]) return false + this.eq(other.asInstanceOf[Input[_]]) + } + + override def hashCode(): Int = System.identityHashCode(this) } object Input { diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Mean.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Mean.scala index cbe8c7c9f29..90999b78714 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Mean.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Mean.scala @@ -40,8 +40,8 @@ import scala.reflect.ClassTag @SerialVersionUID(2995626598003841724L) class Mean[T: ClassTag]( val dimension: Int = 1, - nInputDims: Int = -1, - squeeze: Boolean = true) + val nInputDims: Int = -1, + val squeeze: Boolean = true) (implicit ev: TensorNumeric[T]) extends Sum[T](dimension, nInputDims, true, squeeze) { override def toString: String = s"nn.Mean" } diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala index 7004f68de58..b67377d61dc 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Padding.scala @@ -116,7 +116,7 @@ class Padding[T: ClassTag]( } object Padding{ - def apply[@specialized(Float, Double) T: ClassTag]( + def apply[T: ClassTag]( dim: Int, pad: Int, nInputDim: Int, diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Reshape.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Reshape.scala index 2c44ee825e9..48babd206b0 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Reshape.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Reshape.scala @@ -128,7 +128,7 @@ class Reshape[@specialized(Float, Double) T: ClassTag]( } object Reshape { - def apply[@specialized(Float, Double) T: ClassTag]( + def apply[T: ClassTag]( size: Array[Int], batchMode: Option[Boolean] = None)(implicit ev: TensorNumeric[T]) : Reshape[T] = { new Reshape[T](size, batchMode) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Sigmoid.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Sigmoid.scala index 4e6a77b764a..d9f4ba477ec 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Sigmoid.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Sigmoid.scala @@ -47,7 +47,7 @@ class Sigmoid[@specialized(Float, Double) T: ClassTag]( } object Sigmoid { - def apply[@specialized(Float, Double) T: ClassTag]() + def apply[T: ClassTag]() (implicit ev: TensorNumeric[T]) : Sigmoid[T] = { new Sigmoid[T]() } diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/DenseTensor.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/DenseTensor.scala index 2e1f4cc0c3d..aae6d6707e2 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/DenseTensor.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/DenseTensor.scala @@ -1906,6 +1906,8 @@ private[tensor] class DenseTensor[@specialized(Float, Double) T: ClassTag]( "corresponding module, please keep them same.") } } + + override def getTensorNumeric(): TensorNumeric[T] = ev } object DenseTensor { diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala index 1d092b68a6b..2682d59c98d 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala @@ -646,6 +646,12 @@ trait Tensor[T] extends Serializable with TensorMath[T] with Activity { * @return false */ override def isTable: Boolean = false + + /** + * Return tensor numeric + * @return + */ + def getTensorNumeric(): TensorNumeric[T] } /** diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala index fa6e0bcc8d2..530cadf45b4 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala @@ -25,6 +25,8 @@ import Tensorflow._ import BigDLToTensorflow._ import org.tensorflow.framework.{DataType, NodeDef} +import scala.collection.mutable.ArrayBuffer + /** * Wrapper of logic to convert module to tensorflow node definition */ @@ -51,6 +53,15 @@ object BigDLToTensorflow { } } +object InputToTF extends BigDLToTensorflow { + override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], + byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + require(inputs.length == 1, "Input only accept one input") + + Seq(identity(inputs(0), module.getName())) + } +} + object ReLUToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { @@ -67,7 +78,7 @@ object LinearToTF extends BigDLToTensorflow { val linear = module.asInstanceOf[Linear[_]] val weight = const(linear.weight.t().contiguous(), linear.getName() + "/weight", byteOrder) val weightReader = identity(weight, linear.getName() + "/weightReader") - val mm = matmul(inputs(0), weightReader, linear.getName() + "matmul") + val mm = matmul(inputs(0), weightReader, linear.getName() + "/matmul") val bias = const(linear.bias, linear.getName() + "/bias", byteOrder) val biasReader = identity(bias, linear.getName() + "/biasReader") val add = biasAdd(mm, biasReader, dataFormat, linear.getName() + "/biasAdd") @@ -80,10 +91,16 @@ object SpatialConvolutionToTF extends BigDLToTensorflow { byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "SpatialConvolution only accept one input") val spatialConv = module.asInstanceOf[SpatialConvolution[_]] - val filter = const(spatialConv.weight, spatialConv.getName() + "/filter", byteOrder) + // squeeze will modify the weight tensor + // GOIHW -> HWIO + require(spatialConv.weight.size(1) == 1, "convolution group is not supported") + val filterTensor = spatialConv.weight.select(1, 1) + .transpose(2, 3).transpose(3, 4).transpose(1, 2).transpose(2, 3).transpose(3, 4).contiguous() + + val filter = const(filterTensor, spatialConv.getName() + "/filter", byteOrder) val filterReader = identity(filter, spatialConv.getName() + "/filterReader") - val conv = conv2D(inputs(0), filterReader, spatialConv.strideH, spatialConv.strideW, - spatialConv.kernelW, spatialConv.kernelH, spatialConv.strideW, spatialConv.strideH, + val conv = conv2D(inputs(0), filterReader, spatialConv.strideW, spatialConv.strideH, + spatialConv.kernelW, spatialConv.kernelH, spatialConv.padW, spatialConv.padH, dataFormat, spatialConv.getName() + "/conv2D") val bias = const(spatialConv.bias, spatialConv.getName() + "/bias", byteOrder) val biasReader = identity(bias, spatialConv.getName() + "/biasReader") @@ -121,7 +138,7 @@ object ReshapeToTF extends BigDLToTensorflow { size.setValue(i + 1, rh.size(i)) i += 1 } - val shape = const(size, rh.getName() + "/shape", byteOrder, DataType.DT_INT32) + val shape = const(size, rh.getName() + "/shape", byteOrder, false, DataType.DT_INT32) val reshapeNode = reshape(inputs(0), shape, rh.getName()) Seq(reshapeNode, shape) } @@ -138,7 +155,7 @@ object ViewToTF extends BigDLToTensorflow { size.setValue(i + 1, viewLayer.sizes(i)) i += 1 } - val shape = const(size, viewLayer.getName() + "/shape", byteOrder, DataType.DT_INT32) + val shape = const(size, viewLayer.getName() + "/shape", byteOrder, false, DataType.DT_INT32) val reshapeNode = reshape(inputs(0), shape, viewLayer.getName()) Seq(reshapeNode, shape) } @@ -159,16 +176,17 @@ object PaddingToTF extends BigDLToTensorflow { byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Padding only accept one input") val layer = module.asInstanceOf[Padding[_]] - val padding = Tensor[Float](1, 2) + require(layer.nIndex == 1, "only support padding nIndex == 1") + require(layer.nInputDim > 0, "nInputDim must be explicit specified") + val padding = Tensor[Float](layer.nInputDim, 2).zero() if (layer.pad < 0) { - padding.setValue(1, 1, -layer.pad) - padding.setValue(1, 2, 0) + padding.setValue(layer.dim, 1, -layer.pad) } else { - padding.setValue(1, 1, 0) - padding.setValue(1, 2, layer.pad) + padding.setValue(layer.dim, 2, layer.pad) } - val paddingsNode = const(padding, layer.getName() + "/padding", byteOrder, DataType.DT_INT32) + val paddingsNode = const(padding, layer.getName() + "/padding", byteOrder, + false, DataType.DT_INT32) val padNode = pad(inputs(0), paddingsNode, layer.getName() + "/output") Seq(padNode, paddingsNode) } @@ -197,20 +215,9 @@ object DropoutToTF extends BigDLToTensorflow { byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Dropout only accept one input") val layer = module.asInstanceOf[Dropout[_]] - val shapeNode = shape(inputs(0), layer.getName() + "/shape") - val rand = randomUniform(shapeNode, layer.getName() + "/random") - val maxNode = const(Tensor[Float](T(1.0f)), layer.getName() + "/max", byteOrder) - val minNode = const(Tensor[Float](T(0.0f)), layer.getName() + "/max", byteOrder) - val sub = subtract(maxNode, minNode, layer.getName() + "/sub") - val mul = multiply(rand, sub, layer.getName() + "/mul") - val randOutput = add(minNode, mul, layer.getName() + "/rand_output") - val keepProb = const(Tensor[Float](T(0.5f)), layer.getName() + "/keep_prob", byteOrder) - val div1 = realdiv(keepProb, inputs(0), layer.getName() + "/div1") - val div2 = realdiv(keepProb, randOutput, layer.getName() + "/div2") - val floorNode = floor(div2, layer.getName() + "/floor") - val output = multiply(div1, floorNode, layer.getName() + "/output") - Seq(output, floorNode, div2, div1, keepProb, randOutput, mul, sub, minNode, maxNode, - rand, shapeNode) + require(layer.isTraining() == false, "only support evaluating mode dropout") + require(inputs.length == 1, "require only one tensor input") + Seq(identity(inputs(0), layer.getName())) } } @@ -234,7 +241,12 @@ object JoinTableToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { val layer = module.asInstanceOf[JoinTable[_]] - Seq(concat(inputs, layer.dimension - 1, layer.getName())) + val axis = const(Tensor[Float](T((layer.dimension - 1).toFloat)), layer.getName() + "/axis", + byteOrder, true, DataType.DT_INT32) + val updateInputs = new ArrayBuffer[NodeDef]() + updateInputs ++= inputs.reverse + updateInputs.append(axis) + Seq(concat(updateInputs, layer.dimension - 1, layer.getName()), axis) } } @@ -243,10 +255,11 @@ object MeanToTF extends BigDLToTensorflow { byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "Mean only accept one input") val layer = module.asInstanceOf[Mean[_]] + require(layer.squeeze == true, "Mean must squeeze input") val dimsTensor = Tensor[Float](layer.dimension) - dimsTensor.setValue(1, layer.dimension) + dimsTensor.setValue(1, layer.dimension - 1) - val dims = const(dimsTensor, layer.getName() + "/dims", byteOrder) + val dims = const(dimsTensor, layer.getName() + "/dims", byteOrder, false, DataType.DT_INT32) val mean = reduceMean(inputs(0), dims, false, layer.getName() + "/output") Seq(mean, dims) } @@ -268,20 +281,22 @@ object LogSoftMaxToTF extends BigDLToTensorflow { } } -object BatchNormToTF extends BigDLToTensorflow { +object BatchNorm2DToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { require(inputs.length == 1, "BatchNorm only accept one input") val layer = module.asInstanceOf[SpatialBatchNormalization[_]] - val stdVar = const(layer.saveStd, layer.getName() + "/std", byteOrder) - val mean = const(layer.saveMean, layer.getName() + "/mean", byteOrder) + require(!layer.isTraining(), "Only support evaluate mode batch norm") + val varNode = const(layer.runningVar, layer.getName() + "/std", byteOrder) + val mean = const(layer.runningMean, layer.getName() + "/mean", byteOrder) val scale = const(layer.weight, layer.getName() + "/scale", byteOrder) val offset = const(layer.bias, layer.getName() + "/offset", byteOrder) - val div = realdiv(scale, stdVar, layer.getName() + "/div") - val mul1 = multiply(inputs(0), div, layer.getName() + "/mul1") - val mul2 = multiply(scale, div, layer.getName() + "/mul2") - val sub = multiply(offset, scale, layer.getName() + "/sub") + val sqrtVar = rsqrt(varNode, layer.getName() + "/stdvar") + val mul0 = multiply(scale, sqrtVar, layer.getName() + "/mul0") + val mul1 = multiply(inputs(0), mul0, layer.getName() + "/mul1") + val mul2 = multiply(mean, mul0, layer.getName() + "/mul2") + val sub = subtract(offset, mul2, layer.getName() + "/sub") val output = add(mul1, sub, layer.getName() + "/output") - Seq(output, sub, mul2, mul1, div, offset, scale, mean, stdVar) + Seq(output, sub, mul2, mul1, mul0, offset, scale, mean, sqrtVar, varNode) } } \ No newline at end of file diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala index 33c258967a7..a6f0dc5e876 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/Tensorflow.scala @@ -103,7 +103,7 @@ object Tensorflow { * @return */ def const[T: ClassTag](value : Tensor[T], name : String, byteOrder: ByteOrder, - dataType: DataType = null): NodeDef = { + isScalar: Boolean = false, dataType: DataType = null): NodeDef = { val dtype = if (dataType == null) { if (value.getType() == DoubleType) { DataType.DT_DOUBLE @@ -118,7 +118,7 @@ object Tensorflow { .setName(name) .setOp("Const") .putAttr("dtype", AttrValue.newBuilder().setType(dtype).build()) - .putAttr("value", tensorAttr(value, dtype, byteOrder)) + .putAttr("value", tensorAttr(value, dtype, byteOrder, isScalar)) .build() } @@ -199,12 +199,12 @@ object Tensorflow { NodeDef.newBuilder() .setName(name) .setOp("Conv2D") - .addInput(filter.getName) .addInput(input.getName) + .addInput(filter.getName) .putAttr("T", getDataType(input)) .putAttr("data_format", dataFormat.value) .putAttr("padding", getPaddingType(pW, pH, kW, kH, sW, sH).value) - .putAttr("strides", listIntAttr(Seq(sH, sW))) + .putAttr("strides", strideAttr(sW, sH, dataFormat)) .build() } @@ -343,6 +343,7 @@ object Tensorflow { .putAttr("dtype", AttrValue.newBuilder().setType(dtype).build()) .putAttr("seed", intAttr(seed)) .putAttr("seed2", intAttr(seed)) + .addInput(shape.getName) .build() } @@ -363,7 +364,7 @@ object Tensorflow { val node = NodeDef.newBuilder() .setName(name) .setOp("ConcatV2") - .putAttr("N", intAttr(axis)) + .putAttr("N", intAttr(inputs.length - 1)) .putAttr("T", getDataType(inputs(0))) .putAttr("Tidx", AttrValue.newBuilder().setType(DataType.DT_INT32).build()) @@ -379,6 +380,7 @@ object Tensorflow { .putAttr("T", getDataType(tensor)) .putAttr("Tpaddings", getDataType(paddings)) .addInput(tensor.getName) + .addInput(paddings.getName) .build() } @@ -436,12 +438,13 @@ object Tensorflow { } private def tensorAttr[T: ClassTag](value: Tensor[T], dtype: DataType, - byteOrder: ByteOrder): AttrValue = { + byteOrder: ByteOrder, isScalar: Boolean): AttrValue = { val shape = TensorShapeProto.newBuilder() - value.size().foreach(dim => { - shape.addDim(Dim.newBuilder().setSize(dim)) - }) - + if (!isScalar) { + value.size().foreach(dim => { + shape.addDim(Dim.newBuilder().setSize(dim)) + }) + } require(value.isContiguous(), "only support save a contiguous tensor") val content = if (value.getType() == DoubleType) { @@ -539,12 +542,17 @@ object Tensorflow { } private def getDataType(node: NodeDef): AttrValue = { - var attr = node.getAttrOrDefault("T", null) + var attr = node.getAttrOrDefault("dtype", null) + if (attr != null) { + return attr + } + + attr = node.getAttrOrDefault("out_type", null) if (attr != null) { return attr } - attr = node.getAttrOrDefault("dtype", null) + attr = node.getAttrOrDefault("T", null) if (attr != null) { return attr } diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala index d4350a83a45..6963fe6fd0e 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala @@ -55,6 +55,7 @@ object TensorflowSaver { new mutable.HashMap[AbstractModule[Activity, Tensor[T], T], ArrayBuffer[NodeDef]]() model.inputs.zip(inputs).foreach(n => { inputNodeCache(n._1.element) = ArrayBuffer(n._2) + println() }) val graphBuilder = GraphDef.newBuilder() @@ -69,6 +70,7 @@ object TensorflowSaver { n.nextNodes.foreach(n => { val list = inputNodeCache.getOrElse(n.element, ArrayBuffer()) list.append(nodeDefs(0)) + inputNodeCache(n.element) = list }) }) @@ -78,8 +80,8 @@ object TensorflowSaver { val os = new FileOutputStream(path) val output = CodedOutputStream.newInstance(os) val graph = graphBuilder.build() - logger.debug("Graph definition is:") - logger.debug(graph.toString) + logger.info("Graph definition is:") + logger.info(graph.toString) graph.writeTo(output) output.flush() os.close() @@ -142,7 +144,9 @@ object TensorflowSaver { getNameFromObj(Mean.getClass.getName) -> MeanToTF, getNameFromObj(SoftMax.getClass.getName) -> SoftMaxToTF, getNameFromObj(LogSoftMax.getClass.getName) -> LogSoftMaxToTF, - getNameFromObj(SpatialBatchNormalization.getClass.getName) -> BatchNormToTF + getNameFromObj(SpatialBatchNormalization.getClass.getName) -> BatchNorm2DToTF, + getNameFromObj(Input.getClass.getName) -> InputToTF, + getNameFromObj(Sigmoid.getClass.getName) -> SigmoidToTF ) private def getNameFromObj(name: String) : String = name.substring(0, name.length - 1) diff --git a/spark/dl/src/test/resources/tf/save_test.py b/spark/dl/src/test/resources/tf/save_test.py index a44bb08ba03..d42840692f6 100644 --- a/spark/dl/src/test/resources/tf/save_test.py +++ b/spark/dl/src/test/resources/tf/save_test.py @@ -37,7 +37,7 @@ def main(): print(tf_output) print("BigDL output is:") print(bigdl_output) - np.testing.assert_almost_equal(tf_output, bigdl_output) + np.testing.assert_almost_equal(tf_output, bigdl_output, 4) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala index af263c3ce0e..cdfe28d95c5 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.nn._ import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.tensor.Tensor -import com.intel.analytics.bigdl.utils.T +import com.intel.analytics.bigdl.utils.{T, Table} import org.apache.log4j.Logger class TensorflowSaverSpec extends TensorflowSpecHelper { @@ -52,7 +52,7 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { T(1.0f, 2.0f, 5.0f), T(-3.0f, -4.0f, -7.0f) )) - test(layer, input, false, "/biasAdd") should be(true) + test(layer, input, false, TensorflowDataFormat.NHWC, "/biasAdd") should be(true) } "AvgPooling" should "be correctly saved" in { @@ -98,12 +98,133 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { "Squeeze" should "be correctly saved" in { val layer = Squeeze(3) val input = Tensor[Float](4, 2, 1, 2).rand() - test(layer, input, true) should be(true) + test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + } + + "CAddTableToTF" should "be correct" in { + val layer = CAddTable[Float]() + val input1 = Tensor[Float](4, 2, 2).rand() + val input2 = Tensor[Float](4, 2, 2).rand() + testMultiInput(layer, Seq(input1, input2), false, TensorflowDataFormat.NCHW) should be(true) + } + + "CMultToTF" should "be correct" in { + val layer = CMulTable[Float]() + val input1 = Tensor[Float](4, 2, 2).rand() + val input2 = Tensor[Float](4, 2, 2).rand() + testMultiInput(layer, Seq(input1, input2), false, TensorflowDataFormat.NCHW) should be(true) + } + + "JoinTableToTF" should "be correct" in { + val layer = JoinTable[Float](3, -1) + val input1 = Tensor[Float](4, 2, 2).rand() + val input2 = Tensor[Float](4, 2, 2).rand() + testMultiInput(layer, Seq(input1, input2), false, TensorflowDataFormat.NCHW) should be(true) + } + + "LogSoftMax" should "be correctly saved" in { + val layer = LogSoftMax() + val input = Tensor[Float](4, 5).rand() + test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + } + + "SoftMax" should "be correctly saved" in { + val layer = SoftMax() + val input = Tensor[Float](4, 5).rand() + test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + } + + "Sigmoid" should "be correctly saved" in { + val layer = Sigmoid() + val input = Tensor[Float](4, 5).rand() + test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + } + + "SpatialConvolution" should "be correctly saved" in { + val layer = SpatialConvolution(3, 5, 2, 2) + val input = Tensor[Float](4, 3, 5, 5).rand() + test(layer, input, true, TensorflowDataFormat.NHWC, "/biasAdd") should be(true) + } + + "Mean" should "be correctly saved" in { + val layer = Mean(1, -1, true) + val input = Tensor[Float](4, 5).rand() + test(layer, input, false, TensorflowDataFormat.NCHW, "/output") should be(true) + } + + "Padding" should "be correctly saved" in { + val layer = Padding(1, 2, 2) + val input = Tensor[Float](4, 5).rand() + test(layer, input, false, TensorflowDataFormat.NCHW, "/output") should be(true) + } + + "Batch Norm2D" should "be correctly saved" in { + val layer = SpatialBatchNormalization(2) + layer.evaluate() + layer.weight.rand(10.0, 20.0) + layer.bias.rand() + layer.runningVar.rand(0.9, 1.1) + layer.runningMean.rand() + val input = Tensor[Float](3, 2, 4, 5).rand() + test(layer, input, true, TensorflowDataFormat.NHWC, "/output") should be(true) + } + + "Dropout" should "be correctly saved" in { + val layer = Dropout() + layer.evaluate() + val input = Tensor[Float](3, 2).rand() + test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + } + + "View" should "be correctly saved" in { + val layer = View(2, 4) + val input = Tensor[Float](2, 2, 2).rand() + test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + } + + "Reshape" should "be correctly saved" in { + val layer = Reshape(Array(2, 4)) + val input = Tensor[Float](2, 2, 2).rand() + test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + } + + "lenet" should "be correctly saved" in { + val conv1 = SpatialConvolution(1, 6, 5, 5).setName("conv1").apply() + val tanh1 = Tanh().setName("tanh1").apply(conv1) + val pool1 = SpatialMaxPooling(2, 2, 2, 2).setName("pool1").apply(tanh1) + val tanh2 = Tanh().setName("tanh2").apply(pool1) + val conv2 = SpatialConvolution(6, 12, 5, 5).setName("conv2").apply(tanh2) + val pool2 = SpatialMaxPooling(2, 2, 2, 2).setName("pool2").apply(conv2) + val reshape = Reshape(Array(4, 12 * 4 * 4)).setName("reshape2").apply(pool2) + val fc1 = Linear(12 * 4 * 4, 100).setName("fc1").apply(reshape) + val tanh3 = Tanh().setName("tanh3").apply(fc1) + val fc2 = Linear(100, 10).setName("fc2").apply(tanh3) + val output = LogSoftMax().setName("output").apply(fc2) + + val funcModel = Graph(conv1, output) + val inputData = Tensor(4, 1, 28, 28).rand() + val transInput = inputData.transpose(2, 3).transpose(3, 4).contiguous() + val outputData = funcModel.forward(inputData).toTensor + + + val tmpFile = java.io.File.createTempFile("tensorflowSaverTest" + UUID.randomUUID(), "lenet") + TensorflowSaver.saveGraphWitNodeDef( + funcModel, + Seq(Tensorflow.const(transInput, "input", ByteOrder.LITTLE_ENDIAN)), + tmpFile.getPath, + ByteOrder.LITTLE_ENDIAN, + TensorflowDataFormat.NHWC, + Set(Tensorflow.const(outputData, "target", ByteOrder.LITTLE_ENDIAN)) + ) + + runPythonSaveTest(tmpFile.getPath, "") } private def test(layer: AbstractModule[Tensor[Float], Tensor[Float], Float], inputTensor: Tensor[Float], convertNHWC: Boolean = false, + // The default is NHWC so we can test on CPU + dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC, outputSuffix: String = "") : Boolean = { tfCheck() val layerNode = layer.setName("output").apply() @@ -127,7 +248,48 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { Seq(Tensorflow.const(tfTensor, "input", ByteOrder.LITTLE_ENDIAN)), tmpFile.getPath, ByteOrder.LITTLE_ENDIAN, - TensorflowDataFormat.NHWC, + dataFormat, + Set(Tensorflow.const(outputSave, "target", ByteOrder.LITTLE_ENDIAN)) + ) + runPythonSaveTest(tmpFile.getPath, outputSuffix) + } + + private def testMultiInput(layer: AbstractModule[Table, Tensor[Float], Float], + inputTensors: Seq[Tensor[Float]], + convertNHWC: Boolean = false, + // The default is NHWC so we can test on CPU + dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC, + outputSuffix: String = "") : Boolean = { + tfCheck() + val layerNode = layer.setName("output").apply() + val inputNodes = inputTensors.map(_ => Input[Float]()).toArray + inputNodes.foreach(_ -> layerNode) + inputNodes.zipWithIndex.foreach(n => n._1.element.setName("inputNode" + n._2)) + val graph = Graph(inputNodes, layerNode) + val inputTable = T() + inputTensors.foreach(inputTable.insert(_)) + val outputTensor = layer.forward(inputTable) + + val tmpFile = java.io.File.createTempFile("tensorflowSaverTest" + UUID.randomUUID(), "Layer") + logger.info(s"Save model to ${tmpFile}") + val tfTensors = if (convertNHWC) { + inputTensors.map(t => t.transpose(2, 3).transpose(3, 4).contiguous()) + } else { + inputTensors + } + val outputSave = if (convertNHWC) { + outputTensor.transpose(2, 3).transpose(3, 4).contiguous() + } else { + outputTensor + } + + TensorflowSaver.saveGraphWitNodeDef( + graph, + tfTensors.zipWithIndex.map(t => + Tensorflow.const(t._1, "input" + t._2, ByteOrder.LITTLE_ENDIAN)), + tmpFile.getPath, + ByteOrder.LITTLE_ENDIAN, + dataFormat, Set(Tensorflow.const(outputSave, "target", ByteOrder.LITTLE_ENDIAN)) ) runPythonSaveTest(tmpFile.getPath, outputSuffix) From bc4fd870235145cdaa12ee7070b1260a7fd46b37 Mon Sep 17 00:00:00 2001 From: yangw Date: Tue, 13 Jun 2017 09:54:01 +0800 Subject: [PATCH 11/23] rnn test case automation --- .../com/intel/analytics/bigdl/nn/Const.scala | 4 +- .../com/intel/analytics/bigdl/nn/Graph.scala | 3 +- .../test/resources/tf/{ => loadTest}/rnn.py | 22 ++++- .../resources/tf/{ => loadTest}/rnn_lstm.py | 21 ++++- .../tf/{ => loadTest}/share_weight.py | 8 ++ .../test/resources/tf/rnn_cell_zero_state.py | 51 ----------- .../bigdl/utils/tf/TensorflowLoaderSpec.scala | 85 ++++++++++--------- 7 files changed, 95 insertions(+), 99 deletions(-) rename spark/dl/src/test/resources/tf/{ => loadTest}/rnn.py (68%) rename spark/dl/src/test/resources/tf/{ => loadTest}/rnn_lstm.py (68%) rename spark/dl/src/test/resources/tf/{ => loadTest}/share_weight.py (86%) delete mode 100644 spark/dl/src/test/resources/tf/rnn_cell_zero_state.py diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Const.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Const.scala index 8b582b83425..2788d563c3e 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Const.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Const.scala @@ -22,13 +22,15 @@ import com.intel.analytics.bigdl.utils.{T, Table} import scala.reflect.ClassTag +trait WithoutInput + /** * Return a constant tensor defined by value * @param value the constant tensor to be returned in forward */ @SerialVersionUID(-4008935551091949324L) class Const[T: ClassTag](value: Tensor[T])(implicit ev: TensorNumeric[T]) - extends AbstractModule[Activity, Tensor[T], T] { + extends AbstractModule[Activity, Tensor[T], T] with WithoutInput { output = value diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala index 823091e4ef4..4779f8b3b2c 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala @@ -64,7 +64,7 @@ class Graph[T: ClassTag](val inputs : Seq[ModuleNode[T]], var i = 0 while(i < executions.length) { val node = executions(i) - inputsBP(i) = if (node.prevNodes.isEmpty) { + inputsBP(i) = if (node.prevNodes.isEmpty && !node.element.isInstanceOf[WithoutInput]) { inputData(node, input) } else if (node.prevNodes.length == 1) { node.prevNodes.head.element.output.toTensor[T] @@ -198,6 +198,7 @@ class Graph[T: ClassTag](val inputs : Seq[ModuleNode[T]], private def checkRoots : Unit = { val roots = executions.filter(_.prevNodes.size == 0) + .filter(node => !node.element.isInstanceOf[WithoutInput]) require(roots.size == inputs.length, s"There're ${inputs.length} inputs, but graph has ${roots.size} roots") inputs.foreach(n => diff --git a/spark/dl/src/test/resources/tf/rnn.py b/spark/dl/src/test/resources/tf/loadTest/rnn.py similarity index 68% rename from spark/dl/src/test/resources/tf/rnn.py rename to spark/dl/src/test/resources/tf/loadTest/rnn.py index e5ec1c0c3d8..799510a96cb 100644 --- a/spark/dl/src/test/resources/tf/rnn.py +++ b/spark/dl/src/test/resources/tf/loadTest/rnn.py @@ -17,6 +17,7 @@ import numpy as np import os from tensorflow.contrib import rnn +import merge_checkpoint as merge def main(): """ @@ -27,13 +28,14 @@ def main(): 4. python freeze_graph.py --input_graph model/rnn.pbtxt --input_checkpoint model/rnn.chkp --output_node_names=output --output_graph "rnn.pb" """ dir = os.path.dirname(os.path.realpath(__file__)) - n_steps = 5 + n_steps = 2 n_input = 10 n_hidden = 20 n_output = 5 - xs = tf.placeholder(tf.float32, [None, n_steps, n_input]) - weight = tf.Variable(tf.random_normal([n_hidden, n_output]), name="weight") - bias = tf.Variable(tf.random_normal([n_output]), name="bias") + # xs = tf.placeholder(tf.float32, [None, n_steps, n_input]) + xs = tf.Variable(tf.random_uniform([4, n_steps, n_input]) + 10, name='input', dtype=tf.float32) + weight = tf.Variable(tf.random_uniform([n_hidden, n_output]) + 10, name="weight", dtype=tf.float32) + bias = tf.Variable(tf.random_uniform([n_output]) + 10, name="bias", dtype=tf.float32) x = tf.unstack(xs, n_steps, 1) @@ -42,12 +44,24 @@ def main(): output, states = rnn.static_rnn(cell, x, dtype=tf.float32) final = tf.nn.bias_add(tf.matmul(output[-1], weight), bias, name='output') + output = tf.Variable(tf.random_uniform(tf.shape(final)),name='output_result') + result = tf.assign(output, final) + + saver = tf.train.Saver() with tf.Session() as sess: file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) init = tf.global_variables_initializer() sess.run(init) + sess.run(result) checkpointpath = saver.save(sess, dir + '/model/rnn.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'rnn.pbtxt') + + input_graph = dir + "/model/rnn.pbtxt" + input_checkpoint = dir + "/model/rnn.chkp" + output_node_names= "output,output_result" + output_graph = dir + "/rnn.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/rnn_lstm.py b/spark/dl/src/test/resources/tf/loadTest/rnn_lstm.py similarity index 68% rename from spark/dl/src/test/resources/tf/rnn_lstm.py rename to spark/dl/src/test/resources/tf/loadTest/rnn_lstm.py index 72a58e12946..c02686300dd 100644 --- a/spark/dl/src/test/resources/tf/rnn_lstm.py +++ b/spark/dl/src/test/resources/tf/loadTest/rnn_lstm.py @@ -17,6 +17,7 @@ import numpy as np import os from tensorflow.contrib import rnn +import merge_checkpoint as merge def main(): """ @@ -27,13 +28,14 @@ def main(): 4. python freeze_graph.py --input_graph model/lstm.pbtxt --input_checkpoint model/lstm.chkp --output_node_names=output --output_graph "lstm.pb" """ dir = os.path.dirname(os.path.realpath(__file__)) - n_steps = 5 + n_steps = 2 n_input = 10 n_hidden = 20 n_output = 5 - xs = tf.placeholder(tf.float32, [None, n_steps, n_input]) - weight = tf.Variable(tf.random_normal([n_hidden, n_output]), name="weight") - bias = tf.Variable(tf.random_normal([n_output]), name="bias") + # xs = tf.placeholder(tf.float32, [None, n_steps, n_input]) + xs = tf.Variable(tf.random_uniform([4, n_steps, n_input]) + 10, name='input', dtype=tf.float32) + weight = tf.Variable(tf.random_uniform([n_hidden, n_output]) + 10, name="weight", dtype=tf.float32) + bias = tf.Variable(tf.random_uniform([n_output]) + 10, name="bias", dtype=tf.float32) x = tf.unstack(xs, n_steps, 1) @@ -42,12 +44,23 @@ def main(): output, states = rnn.static_rnn(cell, x, dtype=tf.float32) final = tf.nn.bias_add(tf.matmul(output[-1], weight), bias, name='output') + + output = tf.Variable(tf.random_uniform(tf.shape(final)),name='output_result') + result = tf.assign(output, final) saver = tf.train.Saver() with tf.Session() as sess: file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) init = tf.global_variables_initializer() sess.run(init) + sess.run(result) checkpointpath = saver.save(sess, dir + '/model/lstm.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'lstm.pbtxt') + + input_graph = dir + "/model/lstm.pbtxt" + input_checkpoint = dir + "/model/lstm.chkp" + output_node_names= "output,output_result" + output_graph = dir + "/lstm.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/share_weight.py b/spark/dl/src/test/resources/tf/loadTest/share_weight.py similarity index 86% rename from spark/dl/src/test/resources/tf/share_weight.py rename to spark/dl/src/test/resources/tf/loadTest/share_weight.py index 6c958b64f6e..32dd8eb4421 100644 --- a/spark/dl/src/test/resources/tf/share_weight.py +++ b/spark/dl/src/test/resources/tf/loadTest/share_weight.py @@ -16,6 +16,7 @@ import tensorflow as tf import numpy as np import os +import merge_checkpoint as merge def main(): """ @@ -43,5 +44,12 @@ def main(): sess.run(init) checkpointpath = saver.save(sess, dir + '/model/share_weight.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'share_weight.pbtxt') + + input_graph = dir + "/model/share_weight.pbtxt" + input_checkpoint = dir + "/model/share_weight.chkp" + output_node_names= "output" + output_graph = dir + "/share_weight.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/rnn_cell_zero_state.py b/spark/dl/src/test/resources/tf/rnn_cell_zero_state.py deleted file mode 100644 index efe5321bf5d..00000000000 --- a/spark/dl/src/test/resources/tf/rnn_cell_zero_state.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from tensorflow.contrib import rnn -from tensorflow.python.ops import array_ops - -def main(): - """ - Run this command to generate the pb file - 1. mkdir model - 2. python rnn_cell_zero_state.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/rnn_cell.pbtxt --input_checkpoint model/rnn_cell.chkp --output_node_names=output --output_graph "rnn_cell.pb" - """ - dir = os.path.dirname(os.path.realpath(__file__)) - n_input = 10 - n_hidden = 20 - - xs = tf.placeholder(tf.float32, [None, n_input]) - W = tf.Variable(tf.constant(1.0, shape=[n_hidden, 5], dtype=tf.float32)) - b = tf.Variable(tf.constant(2.0, shape=[5], dtype=tf.float32)) - - cell = rnn.BasicRNNCell(n_hidden) - batch_size = array_ops.shape(xs)[0] - state = cell.zero_state(batch_size, tf.float32) - tf.nn.bias_add(tf.matmul(state, W), b, name="output") - - saver = tf.train.Saver() - with tf.Session() as sess: - file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) - init = tf.global_variables_initializer() - sess.run(init) - checkpointpath = saver.save(sess, dir + '/model/rnn_cell.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'rnn_cell.pbtxt') -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index 541cd1bdc2d..2c5b09a4383 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -160,14 +160,15 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "Shared weights" should "be the same instance" in { tfCheck() + (("python " + testScriptsPath("share_weight.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "share_weight.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "share_weight.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] - container.modules.length should be(4) val l1 = container.modules(1).asInstanceOf[Linear[Float]] val l2 = container.modules(3).asInstanceOf[Linear[Float]] assert(l1.weight eq l2.weight) @@ -175,12 +176,15 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ } "Shared weights" should "be the same after running optimizer" in { + tfCheck() + (("python " + testScriptsPath("share_weight.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "share_weight.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "share_weight.pb" val results = TensorflowLoader.parse(path) val tfGraph = TensorflowLoader.buildTFGraph(results) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) val container = model.asInstanceOf[Graph[Float]] val optimizer = new DistriOptimizer[Float](container, dataSet, new MSECriterion[Float]()) @@ -194,49 +198,54 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ assert(l1.bias == l2.bias) } - "TensorFlow loader" should "be able to load rnn_cell with zero state" in { + "static simple rnn " should "have the same inference result as tensorflow" in { + tfCheck() + (("python " + testScriptsPath("rnn.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "rnn_cell.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "rnn.pb" val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](4, 10).rand() + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("output"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN).contiguous() + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) + val bigDLResult = model.forward(input) val gradient = Tensor[Float](4, 5).rand() - val result: Tensor[Float] = model.forward(input).asInstanceOf[Tensor[Float]] - val expectedResult = Tensor[Float](4, 5).fill(2.0f) - val expectedGrad = Tensor[Float](4, 10) - result should be(expectedResult) - val grad = model.backward(input, gradient) - grad should be(expectedGrad) - } - - "TensorFlow loader" should "be able to load static simple rnn model" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "rnn.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](4, 5, 10).rand() - val gradient = Tensor[Float](4, 5).rand() - model.forward(input) - model.backward(input, gradient) + tfResult.map( bigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2) / v1 < 1e-6) + v2 + }) + model.backward(bigDLResult, gradient) } - "TensorFlow loader" should "be able to load static lstm rnn model" in { + "static lstm rnn " should "have the same inference result as tensorflow" in { + tfCheck() + (("python " + testScriptsPath("rnn_lstm.py")) !!) val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "lstm.pb" + val path = processPath(resource.getPath()) + JFile.separator + + "loadTest" + JFile.separator + "lstm.pb" val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - - val input = Tensor[Float](4, 5, 10).rand() + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + Seq("output"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) + val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, + ByteOrder.LITTLE_ENDIAN).contiguous() + val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) + .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) + val bigDLResult = model.forward(input) val gradient = Tensor[Float](4, 5).rand() - model.forward(input) - model.backward(input, gradient) + + tfResult.map( bigDLResult.toTensor, (v1, v2) => { + assert(abs(v1 - v2)/ v1 < 1e-6, s"$v1, $v2") + v2 + }) + model.backward(bigDLResult, gradient) } "TensorFlow loader" should "be able to load slim lenet" in { From 190ecf66257afa9a4e357a8a90cf22f6e310956d Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Wed, 14 Jun 2017 19:27:38 +0800 Subject: [PATCH 12/23] refine save test --- .../intel/analytics/bigdl/tensor/Tensor.scala | 19 ++ .../bigdl/utils/tf/TensorflowLoader.scala | 16 +- .../src/test/resources/tf/loadTest/alexnet.py | 2 +- .../tf/loadTest/inception_resnet_v2.py | 2 +- .../test/resources/tf/{ => loadTest}/lenet.py | 27 +- .../test/resources/tf/loadTest/overfeat.py | 2 +- .../src/test/resources/tf/loadTest/vgg16.py | 2 +- .../src/test/resources/tf/loadTest/vgg19.py | 2 +- .../dl/src/test/resources/tf/loadTest/vgga.py | 2 +- .../src/test/resources/tf/models/alexnet.py | 39 +++ .../test/resources/tf/models/inception_v3.py | 42 +++ .../dl/src/test/resources/tf/models/lenet.py | 39 +++ .../src/test/resources/tf/models/overfeat.py | 42 +++ .../src/test/resources/tf/models/resnet_v1.py | 40 +++ .../merge_checkpoint.py => models/util.py} | 20 +- .../dl/src/test/resources/tf/models/vgg16.py | 39 +++ .../dl/src/test/resources/tf/models/vgg19.py | 39 +++ spark/dl/src/test/resources/tf/models/vgga.py | 39 +++ .../bigdl/utils/tf/TensorflowLoaderSpec.scala | 278 +++++------------- 19 files changed, 473 insertions(+), 218 deletions(-) rename spark/dl/src/test/resources/tf/{ => loadTest}/lenet.py (61%) create mode 100644 spark/dl/src/test/resources/tf/models/alexnet.py create mode 100644 spark/dl/src/test/resources/tf/models/inception_v3.py create mode 100644 spark/dl/src/test/resources/tf/models/lenet.py create mode 100644 spark/dl/src/test/resources/tf/models/overfeat.py create mode 100644 spark/dl/src/test/resources/tf/models/resnet_v1.py rename spark/dl/src/test/resources/tf/{loadTest/merge_checkpoint.py => models/util.py} (75%) create mode 100644 spark/dl/src/test/resources/tf/models/vgg16.py create mode 100644 spark/dl/src/test/resources/tf/models/vgg19.py create mode 100644 spark/dl/src/test/resources/tf/models/vgga.py diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala index 2682d59c98d..ed4d5305f97 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/tensor/Tensor.scala @@ -652,6 +652,25 @@ trait Tensor[T] extends Serializable with TensorMath[T] with Activity { * @return */ def getTensorNumeric(): TensorNumeric[T] + + /** + * Compare with other tensor. The shape of the other tensor must be same with this tensor. + * If element wise difference is less than delta, return true. + * @param other + * @param delta + * @return + */ + def almostEqual(other: Tensor[T], delta : Double): Boolean = { + var result = true + this.map(other, (a, b) => { + val tn = getTensorNumeric() + if (tn.isGreater(tn.abs(tn.minus(a, b)), tn.fromType(delta))) { + result = false + } + a + }) + return result + } } /** diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala index 92dd816bc3c..651547e243e 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala @@ -55,7 +55,7 @@ object TensorflowLoader{ val nodeList = parse(graphPrototxt) // Construct tf node graph - val tfGraph = buildTFGraph(nodeList) + val tfGraph = buildTFGraph(nodeList, outputs) // Build BigDL model from the tf node graph buildBigDLModel(tfGraph, inputs, outputs, byteOrder, dataFormat) @@ -80,9 +80,11 @@ object TensorflowLoader{ /** * Build tf ops graph from a given node list * @param nodes + * @param outputNodeNames * @return */ - private[bigdl] def buildTFGraph(nodes : List[NodeDef]): DirectedGraph[NodeDef] = { + private[bigdl] def buildTFGraph(nodes : List[NodeDef], outputNodeNames: Seq[String]) + : DirectedGraph[NodeDef] = { import scala.collection.JavaConverters._ var name2Node = nodes.asScala.map(n => n.getName -> new Node(n)).toMap @@ -103,7 +105,15 @@ object TensorflowLoader{ }) // Build graph - val outputNodes = name2Node.valuesIterator.filter(_.nextNodes.length == 0) + val outputNodes = if (outputNodeNames == null) { + name2Node.valuesIterator.filter(_.nextNodes.length == 0).toArray + } else { + val results = name2Node.valuesIterator.toArray.filter(n => + outputNodeNames.contains(n.element.getName)) + require(results.length == outputNodeNames.length, "Invalid outputNode names") + results + } + val dummyOutput = new Node[NodeDef](null) outputNodes.foreach(_ -> dummyOutput) dummyOutput.graph(reverse = true) diff --git a/spark/dl/src/test/resources/tf/loadTest/alexnet.py b/spark/dl/src/test/resources/tf/loadTest/alexnet.py index 3ec91d2e785..52478d1d6e1 100644 --- a/spark/dl/src/test/resources/tf/loadTest/alexnet.py +++ b/spark/dl/src/test/resources/tf/loadTest/alexnet.py @@ -16,7 +16,7 @@ import tensorflow as tf import numpy as np import os -import slim.nets.alexnet as alexnet +from nets import alexnet import merge_checkpoint as merge diff --git a/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py b/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py index 7aa7875a180..14eb51f3397 100644 --- a/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py +++ b/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py @@ -16,7 +16,7 @@ import tensorflow as tf import numpy as np import os -import slim.nets.inception_resnet_v2 as inception_resnet_v2 +from nets import inception_resnet_v2 import merge_checkpoint as merge diff --git a/spark/dl/src/test/resources/tf/lenet.py b/spark/dl/src/test/resources/tf/loadTest/lenet.py similarity index 61% rename from spark/dl/src/test/resources/tf/lenet.py rename to spark/dl/src/test/resources/tf/loadTest/lenet.py index 63a9cfd48a4..210e063cc65 100644 --- a/spark/dl/src/test/resources/tf/lenet.py +++ b/spark/dl/src/test/resources/tf/loadTest/lenet.py @@ -16,29 +16,42 @@ import tensorflow as tf import numpy as np import os -import slim.nets.lenet as lenet +from nets import lenet + +import merge_checkpoint as merge def main(): """ - Run this command to generate the pb file + You can also run these commands manually to generate the pb file 1. git clone https://github.com/tensorflow/models.git 2. export PYTHONPATH=Path_to_your_model_folder - 1. mkdir model - 2. python lenet.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/lenet.pbtxt --input_checkpoint model/lenet.chkp --output_node_names="LeNet/fc4/BiasAdd" --output_graph lenet.pb + 3. python alexnet.py """ dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isdir(dir + '/model'): + os.mkdir(dir + '/model') batch_size = 5 height, width = 32, 32 - inputs = tf.placeholder(tf.float32, [None, height, width, 3]) + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + # inputs = tf.placeholder(tf.float32, [None, height, width, 3]) net, end_points = lenet.lenet(inputs) + output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') + result = tf.assign(output,net) saver = tf.train.Saver() with tf.Session() as sess: init = tf.global_variables_initializer() sess.run(init) + print(sess.run(result)) checkpointpath = saver.save(sess, dir + '/model/lenet.chkp') tf.train.write_graph(sess.graph, dir + '/model', 'lenet.pbtxt') tf.summary.FileWriter(dir + '/log', sess.graph) + + input_graph = dir + "/model/lenet.pbtxt" + input_checkpoint = dir + "/model/lenet.chkp" + output_node_names= "LeNet/pool2/MaxPool,output" + output_graph = dir + "/lenet.pb" + + merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/loadTest/overfeat.py b/spark/dl/src/test/resources/tf/loadTest/overfeat.py index a95106dc249..c9cf9fd1387 100644 --- a/spark/dl/src/test/resources/tf/loadTest/overfeat.py +++ b/spark/dl/src/test/resources/tf/loadTest/overfeat.py @@ -16,7 +16,7 @@ import tensorflow as tf import numpy as np import os -import slim.nets.overfeat as overfeat +from nets import overfeat import merge_checkpoint as merge slim = tf.contrib.slim diff --git a/spark/dl/src/test/resources/tf/loadTest/vgg16.py b/spark/dl/src/test/resources/tf/loadTest/vgg16.py index 049898e2028..9d14f3795cf 100644 --- a/spark/dl/src/test/resources/tf/loadTest/vgg16.py +++ b/spark/dl/src/test/resources/tf/loadTest/vgg16.py @@ -16,7 +16,7 @@ import tensorflow as tf import numpy as np import os -import slim.nets.vgg as vgg +from nets import vgg import merge_checkpoint as merge def main(): """ diff --git a/spark/dl/src/test/resources/tf/loadTest/vgg19.py b/spark/dl/src/test/resources/tf/loadTest/vgg19.py index a2cae99098d..79650d62aa0 100644 --- a/spark/dl/src/test/resources/tf/loadTest/vgg19.py +++ b/spark/dl/src/test/resources/tf/loadTest/vgg19.py @@ -16,7 +16,7 @@ import tensorflow as tf import numpy as np import os -import slim.nets.vgg as vgg +from nets import vgg import merge_checkpoint as merge def main(): diff --git a/spark/dl/src/test/resources/tf/loadTest/vgga.py b/spark/dl/src/test/resources/tf/loadTest/vgga.py index b869adabc17..f696ffd0087 100644 --- a/spark/dl/src/test/resources/tf/loadTest/vgga.py +++ b/spark/dl/src/test/resources/tf/loadTest/vgga.py @@ -16,7 +16,7 @@ import tensorflow as tf import numpy as np import os -import slim.nets.vgg as vgg +from nets import vgg import merge_checkpoint as merge def main(): """ diff --git a/spark/dl/src/test/resources/tf/models/alexnet.py b/spark/dl/src/test/resources/tf/models/alexnet.py new file mode 100644 index 00000000000..3e9dcd40280 --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/alexnet.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import alexnet +from sys import argv + +from util import run_model + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 224, 224 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name = 'input') + net, end_points = alexnet.alexnet_v2(inputs, is_training=False) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/models/inception_v3.py b/spark/dl/src/test/resources/tf/models/inception_v3.py new file mode 100644 index 00000000000..cc4b2442696 --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/inception_v3.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import inception +from sys import argv + +from util import run_model + +slim = tf.contrib.slim + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 299, 299 + num_classes = 1000 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = inception.inception_v3(inputs, num_classes,is_training=False) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/models/lenet.py b/spark/dl/src/test/resources/tf/models/lenet.py new file mode 100644 index 00000000000..4725aacd5fd --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/lenet.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import lenet +from sys import argv + +from util import run_model + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 32, 32 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = lenet.lenet(inputs) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/models/overfeat.py b/spark/dl/src/test/resources/tf/models/overfeat.py new file mode 100644 index 00000000000..c82a5b5a67f --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/overfeat.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import overfeat +from sys import argv + +from util import run_model + +slim = tf.contrib.slim + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 231, 231 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + with slim.arg_scope(overfeat.overfeat_arg_scope()): + net, end_points = overfeat.overfeat(inputs, is_training = False) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/models/resnet_v1.py b/spark/dl/src/test/resources/tf/models/resnet_v1.py new file mode 100644 index 00000000000..edb4665873c --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/resnet_v1.py @@ -0,0 +1,40 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import resnet_utils +from nets import resnet_v1 +from sys import argv + +from util import run_model + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 224, 224 + inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') + net, end_points = resnet_v1.resnet_v1_101(inputs, 1000, is_training=True) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/loadTest/merge_checkpoint.py b/spark/dl/src/test/resources/tf/models/util.py similarity index 75% rename from spark/dl/src/test/resources/tf/loadTest/merge_checkpoint.py rename to spark/dl/src/test/resources/tf/models/util.py index 04b450ce840..196361709bc 100644 --- a/spark/dl/src/test/resources/tf/loadTest/merge_checkpoint.py +++ b/spark/dl/src/test/resources/tf/models/util.py @@ -20,6 +20,7 @@ from tensorflow.python.framework import graph_util from tensorflow.python.framework import importer from tensorflow.python.platform import gfile +import tensorflow as tf def merge_checkpoint(input_graph, input_checkpoint, @@ -51,8 +52,25 @@ def merge_checkpoint(input_graph, output_graph_def = graph_util.convert_variables_to_constants( sess, input_graph_def, - output_node_names.split(","), + output_node_names, variable_names_blacklist="") with gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) +def run_model(end_point, output_path): + output=tf.Variable(tf.random_uniform(tf.shape(end_point)), name='output') + result = tf.assign(output, end_point, name = 'assign') + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + sess.run(result) + saver.save(sess, output_path + '/model.chkp') + tf.train.write_graph(sess.graph, output_path, 'model.pbtxt') + + input_graph = output_path + "/model.pbtxt" + input_checkpoint = output_path + "/model.chkp" + output_file = output_path + "/model.pb" + + merge_checkpoint(input_graph, input_checkpoint, ["assign"], output_file) + diff --git a/spark/dl/src/test/resources/tf/models/vgg16.py b/spark/dl/src/test/resources/tf/models/vgg16.py new file mode 100644 index 00000000000..6035cdfbb0b --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/vgg16.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import vgg +from sys import argv + +from util import run_model + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 224, 224 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = vgg.vgg_16(inputs, is_training = False) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/models/vgg19.py b/spark/dl/src/test/resources/tf/models/vgg19.py new file mode 100644 index 00000000000..58dfaf22d5a --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/vgg19.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import vgg +from sys import argv + +from util import run_model + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 224, 224 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = vgg.vgg_19(inputs, is_training = False) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/models/vgga.py b/spark/dl/src/test/resources/tf/models/vgga.py new file mode 100644 index 00000000000..2a70e7b42b7 --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/vgga.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import vgg +from sys import argv + +from util import run_model + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 224, 224 + inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') + net, end_points = vgg.vgg_a(inputs, is_training = False) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) + run_model(net_output, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index 2c5b09a4383..d4d5ec0d0d4 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -17,6 +17,7 @@ package com.intel.analytics.bigdl.utils.tf import java.io.{File => JFile} import java.nio.ByteOrder +import java.util.UUID import com.intel.analytics.bigdl.dataset.{DistributedDataSet, MiniBatch} import com.intel.analytics.bigdl.nn._ @@ -28,8 +29,8 @@ import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat -import scala.sys.process._ +import scala.sys.process._ import scala.math._ object TensorflowLoaderSpec { @@ -109,7 +110,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val resource = getClass().getClassLoader().getResource("tf") val path = processPath(resource.getPath()) + JFile.separator + "test.pb" val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) + val tfGraph = TensorflowLoader.buildTFGraph(results, Seq("output")) tfGraph.size should be(15) // there's a dummy output val topSort = tfGraph.topologySort// It can do topology sort topSort.length should be(15) @@ -133,9 +134,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "TensorFlow loader" should "be able to build a BigDL graph" in { val resource = getClass().getClassLoader().getResource("tf") val path = processPath(resource.getPath()) + JFile.separator + "test.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), + val model = TensorflowLoader.load(path, Seq("Placeholder"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] container.modules.length should be(4) @@ -164,9 +163,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val resource = getClass().getClassLoader().getResource("tf") val path = processPath(resource.getPath()) + JFile.separator + "loadTest" + JFile.separator + "share_weight.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), + val model = TensorflowLoader.load(path, Seq("Placeholder"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] val l1 = container.modules(1).asInstanceOf[Linear[Float]] @@ -181,9 +178,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val resource = getClass().getClassLoader().getResource("tf") val path = processPath(resource.getPath()) + JFile.separator + "loadTest" + JFile.separator + "share_weight.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), Seq("output"), + val model = TensorflowLoader.load(path, Seq("Placeholder"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) val container = model.asInstanceOf[Graph[Float]] @@ -205,7 +200,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val path = processPath(resource.getPath()) + JFile.separator + "loadTest" + JFile.separator + "rnn.pb" val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1), Seq("output")) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) @@ -230,7 +225,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val path = processPath(resource.getPath()) + JFile.separator + "loadTest" + JFile.separator + "lstm.pb" val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1), Seq("output")) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) @@ -248,207 +243,45 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(bigDLResult, gradient) } - "TensorFlow loader" should "be able to load slim lenet" in { - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + "lenet.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("Placeholder"), - Seq("LeNet/fc4/BiasAdd"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = Tensor[Float](4, 3, 32, 32).rand() - val gradient = Tensor[Float](4, 10).rand() - model.forward(input) - model.backward(input, gradient) + "Tensorflow lenet" should "be load correctly" in { + val (tf, bigdl) = testModel("lenet", "LeNet/pool2/MaxPool:0", true) + val transposed = bigdl.transpose(2, 3).transpose(3, 4) + tf.almostEqual(transposed, 1e-6) should be(true) } - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim alexnet" in { - tfCheck() - (("python " + testScriptsPath("alexnet.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "alexnet_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("alexnet_v2/fc8/squeezed"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN).transpose(2, 4).transpose(3, 4).contiguous() - val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, gradient) + "Tensorflow Alexnet" should "be load correctly" in { + val (tf, bigdl) = testModel("alexnet", "alexnet_v2/fc8/squeezed:0", true) + tf.almostEqual(bigdl, 1e-7) should be(true) } - - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim vgg_a" in { - tfCheck() - (("python " + testScriptsPath("vgga.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "vgga_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("vgg_a/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, gradient) + "TensorFlow vgg_a" should "be load correctly" in { + val (tf, bigdl) = testModel("vgga", "vgg_a/fc8/squeezed:0", true) + tf.almostEqual(bigdl, 1e-7) should be(true) } - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim vgg_16" in { - tfCheck() - (("python " + testScriptsPath("vgg16.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "vgg16_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("vgg_16/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, gradient) + "TensorFlow vgg_16" should "be load correctly" in { + val (tf, bigdl) = testModel("vgg16", "vgg_16/fc8/squeezed:0", true) + tf.almostEqual(bigdl, 1e-7) should be(true) } - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim vgg_19" in { - tfCheck() - (("python " + testScriptsPath("vgg19.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "vgg19_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("vgg_19/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, gradient) + "TensorFlow vgg_19" should "be load correctly" in { + val (tf, bigdl) = testModel("vgg19", "vgg_19/fc8/squeezed:0", true) + tf.almostEqual(bigdl, 1e-7) should be(true) } - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim overfeat" in { - tfCheck() - (("python " + testScriptsPath("overfeat.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "overfeat_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("overfeat/fc8/squeezed"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, gradient) + "TensorFlow overfeat" should "be load correctly" in { + val (tf, bigdl) = testModel("overfeat", "overfeat/fc8/squeezed:0", true) + tf.almostEqual(bigdl, 1e-7) should be(true) } - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim inception_v3" in { - tfCheck() - (("python " + testScriptsPath("inception_v3.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "inception_v3_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("InceptionV3/Logits/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient = Tensor[Float](1, 1000).rand() - val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, gradient) + "TensorFlow inception_v3" should "be load correctly" in { + val (tf, bigdl) = testModel("inception_v3", "InceptionV3/Logits/SpatialSqueeze:0", true) + tf.almostEqual(bigdl, 1e-7) should be(true) } - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim resnet_v1" in { - tfCheck() - (("python " + testScriptsPath("resnet_v1.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "resnet_v1_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1)) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("resnet_v1_101/SpatialSqueeze"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient = Tensor[Float](2, 1000).rand() - val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult.map( BigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) < 1e-6); - v2 - }) - model.backward(input, gradient) + "TensorFlow resnet_v1" should "be load correctly" in { + val (tf, bigdl) = testModel("resnet_v1", "resnet_v1_101/SpatialSqueeze:0", true) + tf.almostEqual(bigdl, 1e-6) should be(true) } "TensorFlow loader" should "run the python to save the modle and " + @@ -460,7 +293,8 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val path = processPath(resource.getPath()) + JFile.separator + "loadTest" + JFile.separator + "inception_resnet_v2_save.pb" val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-2)) + val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-2), + Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd")) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd") , ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) @@ -486,6 +320,48 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ model.backward(input, T(gradient1, gradient2)) } + private def testModel(modelName: String, endPoint: String, transInput: Boolean) + : (Tensor[Float], Tensor[Float]) = { + + tfCheck() + // Generate command and prepare the temp folder + val s = JFile.separator + val modelsFolder = processPath(getClass().getClassLoader().getResource("tf").getPath()) + + s + "models" + val modelScript = modelsFolder + s + s"$modelName.py" + val tmpLocation = java.io.File.createTempFile("tensorflowLoaderTest" + UUID.randomUUID(), + modelName) + tmpLocation.delete() + tmpLocation.mkdir() + + require(runPython(s"$modelScript $tmpLocation $endPoint"), "error when run the model script") + + // Load the model and input/output tensors + val modelFile = tmpLocation + s + "model.pb" + val tfNodes = TensorflowLoader.parse(modelFile) + val tfGraph = TensorflowLoader.buildTFGraph(tfNodes, Seq(endPoint.split(":")(0))) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq(endPoint.split(":")(0)), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + + import collection.JavaConverters._ + // Compare the tensor contents + val tfInputTensor = tfNodes.asScala.filter(_.getName == "input")(0) + .getAttrMap.get("value").getTensor + val tfOutputTensor = tfNodes.asScala.filter(_.getName == "output")(0) + .getAttrMap.get("value").getTensor + val input = TensorflowToBigDL.toTensor(tfInputTensor, + ByteOrder.LITTLE_ENDIAN) + + val transposeInput = if (transInput) { + input.transpose(2, 4).transpose(3, 4).contiguous() + } else { + input + } + + (TensorflowToBigDL.toTensor(tfOutputTensor, ByteOrder.LITTLE_ENDIAN), + model.forward(transposeInput).toTensor) + } + private def processPath(path: String): String = { if (path.contains(":")) { path.substring(1) From fab5c4742d7d385151a159dcbf68e6f06563eef5 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 15 Jun 2017 01:12:59 +0800 Subject: [PATCH 13/23] refine save unit test --- spark/dl/src/test/resources/tf/load.py | 37 --- .../src/test/resources/tf/loadTest/alexnet.py | 57 ----- .../tf/loadTest/inception_resnet_v2.py | 63 ----- .../resources/tf/loadTest/inception_v3.py | 60 ----- .../src/test/resources/tf/loadTest/lenet.py | 57 ----- .../test/resources/tf/loadTest/overfeat.py | 60 ----- .../test/resources/tf/loadTest/resnet_v1.py | 59 ----- .../src/test/resources/tf/loadTest/vgg16.py | 54 ---- .../src/test/resources/tf/loadTest/vgg19.py | 57 ----- .../dl/src/test/resources/tf/loadTest/vgga.py | 56 ----- .../src/test/resources/tf/models/alexnet.py | 4 +- .../tf/models/inception_resnet_v2.py | 39 +++ .../test/resources/tf/models/inception_v3.py | 4 +- .../dl/src/test/resources/tf/models/lenet.py | 4 +- .../src/test/resources/tf/models/overfeat.py | 4 +- .../src/test/resources/tf/models/resnet_v1.py | 4 +- .../resources/tf/{loadTest => models}/rnn.py | 30 +-- .../tf/{loadTest => models}/rnn_lstm.py | 27 +- .../tf/{loadTest => models}/share_weight.py | 22 +- spark/dl/src/test/resources/tf/models/util.py | 17 +- .../dl/src/test/resources/tf/models/vgg16.py | 4 +- .../dl/src/test/resources/tf/models/vgg19.py | 4 +- spark/dl/src/test/resources/tf/models/vgga.py | 4 +- spark/dl/src/test/resources/tf/save.py | 30 --- .../bigdl/utils/tf/TensorflowLoaderSpec.scala | 232 ++++++++++-------- .../bigdl/utils/tf/TensorflowSaverSpec.scala | 17 +- 26 files changed, 235 insertions(+), 771 deletions(-) delete mode 100644 spark/dl/src/test/resources/tf/load.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/alexnet.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/inception_v3.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/lenet.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/overfeat.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/resnet_v1.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/vgg16.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/vgg19.py delete mode 100644 spark/dl/src/test/resources/tf/loadTest/vgga.py create mode 100644 spark/dl/src/test/resources/tf/models/inception_resnet_v2.py rename spark/dl/src/test/resources/tf/{loadTest => models}/rnn.py (64%) rename spark/dl/src/test/resources/tf/{loadTest => models}/rnn_lstm.py (66%) rename spark/dl/src/test/resources/tf/{loadTest => models}/share_weight.py (73%) delete mode 100644 spark/dl/src/test/resources/tf/save.py diff --git a/spark/dl/src/test/resources/tf/load.py b/spark/dl/src/test/resources/tf/load.py deleted file mode 100644 index 92340c50015..00000000000 --- a/spark/dl/src/test/resources/tf/load.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from tensorflow.python.platform import gfile - -def main(): - with gfile.FastGFile("/tmp/tensorflow9012247417719342743saver",'rb') as f: - graph_def = tf.GraphDef() - content = f.read() - graph_def.ParseFromString(content) - with tf.Graph().as_default() as graph: - tf.import_graph_def(graph_def, name='') - sess = tf.Session() - for op in graph.get_operations(): - print(op.name) - prediction = graph.get_tensor_by_name('relu:0') - ix = graph.get_tensor_by_name('input:0') - rand_array = np.random.rand(2, 4) - print(sess.run(prediction, feed_dict={ix: rand_array})) - -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/loadTest/alexnet.py b/spark/dl/src/test/resources/tf/loadTest/alexnet.py deleted file mode 100644 index 52478d1d6e1..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/alexnet.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import alexnet - -import merge_checkpoint as merge - -def main(): - """ - You can also run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder - 3. python alexnet.py - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 224, 224 - #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') - net, end_points = alexnet.alexnet_v2(inputs, is_training=False) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - print(sess.run(result)) - checkpointpath = saver.save(sess, dir + '/model/alexnet.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'alexnet.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - input_graph = dir + "/model/alexnet.pbtxt" - input_checkpoint = dir + "/model/alexnet.chkp" - output_node_names= "alexnet_v2/fc8/squeezed,output" - output_graph = dir + "/alexnet_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) - -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py b/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py deleted file mode 100644 index 14eb51f3397..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/inception_resnet_v2.py +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import inception_resnet_v2 - -import merge_checkpoint as merge - - - -def main(): - """ - You can run these commands to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder, eg. /home/models/ - 3. python inception_resnet_v2.py - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 299, 299 - num_classes = 1001 - # inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') - net, end_points = inception_resnet_v2.inception_resnet_v2(inputs,is_training = False) - output1 = tf.Variable(tf.random_uniform(tf.shape(net)),name='output1') - result1 = tf.assign(output1,net) - output2 = tf.Variable(tf.random_uniform(tf.shape(end_points['AuxLogits'])),name='output2') - result2 = tf.assign(output2,end_points['AuxLogits']) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run([result1,result2]) - checkpointpath = saver.save(sess, dir + '/model/inception_resnet_v2.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'inception_resnet_v2.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - - input_graph = dir + "/model/inception_resnet_v2.pbtxt" - input_checkpoint = dir + "/model/inception_resnet_v2.chkp" - output_node_names= "InceptionResnetV2/AuxLogits/Logits/BiasAdd,InceptionResnetV2/Logits/Logits/BiasAdd,output1,output2" - output_graph = dir + "/inception_resnet_v2_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) - -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/loadTest/inception_v3.py b/spark/dl/src/test/resources/tf/loadTest/inception_v3.py deleted file mode 100644 index b2ee6d7ffa9..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/inception_v3.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import inception - -import merge_checkpoint as merge - -slim = tf.contrib.slim - -def main(): - """ - You can run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder/models/slim, eg. /home/tensorflow/models/slim/ - 3. python inception_v3.py - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') - net, end_points = inception.inception_v3(inputs, num_classes,is_training=False) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/inception_v3.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'inception_v3.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - - input_graph = dir + "/model/inception_v3.pbtxt" - input_checkpoint = dir + "/model/inception_v3.chkp" - output_node_names= "InceptionV3/Logits/SpatialSqueeze,output" - output_graph = dir + "/inception_v3_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/loadTest/lenet.py b/spark/dl/src/test/resources/tf/loadTest/lenet.py deleted file mode 100644 index 210e063cc65..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/lenet.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import lenet - -import merge_checkpoint as merge - -def main(): - """ - You can also run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder - 3. python alexnet.py - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 32, 32 - inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') - # inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - net, end_points = lenet.lenet(inputs) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - print(sess.run(result)) - checkpointpath = saver.save(sess, dir + '/model/lenet.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'lenet.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - input_graph = dir + "/model/lenet.pbtxt" - input_checkpoint = dir + "/model/lenet.chkp" - output_node_names= "LeNet/pool2/MaxPool,output" - output_graph = dir + "/lenet.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) - -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/loadTest/overfeat.py b/spark/dl/src/test/resources/tf/loadTest/overfeat.py deleted file mode 100644 index c9cf9fd1387..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/overfeat.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import overfeat -import merge_checkpoint as merge - -slim = tf.contrib.slim - -def main(): - """ - You can also run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder, eg. /home/models/ - 3. python overfeat.py - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 231, 231 - num_classes = 1000 - #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') - with slim.arg_scope(overfeat.overfeat_arg_scope()): - net, end_points = overfeat.overfeat(inputs, is_training = False) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/overfeat.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'overfeat.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - input_graph = dir + "/model/overfeat.pbtxt" - input_checkpoint = dir + "/model/overfeat.chkp" - output_node_names= "overfeat/fc8/squeezed,output" - output_graph = dir + "/overfeat_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/loadTest/resnet_v1.py b/spark/dl/src/test/resources/tf/loadTest/resnet_v1.py deleted file mode 100644 index 925e68ff3f8..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/resnet_v1.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import resnet_utils -from nets import resnet_v1 - -import merge_checkpoint as merge - -def main(): - """ - You can run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder/models/slim, eg. /home/tensorflow/models/slim/ - 3. python resnet_v1.py - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') - net, end_points = resnet_v1.resnet_v1_101(inputs, 1000, is_training=True) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/resnet_v1.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'resnet_v1.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - input_graph = dir + "/model/resnet_v1.pbtxt" - input_checkpoint = dir + "/model/resnet_v1.chkp" - output_node_names= "resnet_v1_101/SpatialSqueeze,output" - output_graph = dir + "/resnet_v1_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/spark/dl/src/test/resources/tf/loadTest/vgg16.py b/spark/dl/src/test/resources/tf/loadTest/vgg16.py deleted file mode 100644 index 9d14f3795cf..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/vgg16.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import vgg -import merge_checkpoint as merge -def main(): - """ - You can also run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder - 3. python vgg16.py - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 224, 224 - #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') - net, end_points = vgg.vgg_16(inputs, is_training = False) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/vgg16.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'vgg16.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - input_graph = dir + "/model/vgg16.pbtxt" - input_checkpoint = dir + "/model/vgg16.chkp" - output_node_names= "vgg_16/fc8/squeezed,output" - output_graph = dir + "/vgg16_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/loadTest/vgg19.py b/spark/dl/src/test/resources/tf/loadTest/vgg19.py deleted file mode 100644 index 79650d62aa0..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/vgg19.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import vgg -import merge_checkpoint as merge - -def main(): - """ - You can also run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder - 3. python vgg19.py - 4. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 5. python freeze_graph.py --input_graph model/vgg19.pbtxt --input_checkpoint model/vgg19.chkp --output_node_names="vgg_19/fc8/squeezed,output" --output_graph vgg19_save.pb - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 224, 224 - #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') - net, end_points = vgg.vgg_19(inputs, is_training = False) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/vgg19.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'vgg19.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - input_graph = dir + "/model/vgg19.pbtxt" - input_checkpoint = dir + "/model/vgg19.chkp" - output_node_names= "vgg_19/fc8/squeezed,output" - output_graph = dir + "/vgg19_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/loadTest/vgga.py b/spark/dl/src/test/resources/tf/loadTest/vgga.py deleted file mode 100644 index f696ffd0087..00000000000 --- a/spark/dl/src/test/resources/tf/loadTest/vgga.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -import os -from nets import vgg -import merge_checkpoint as merge -def main(): - """ - You can also run these commands manually to generate the pb file - 1. git clone https://github.com/tensorflow/models.git - 2. export PYTHONPATH=Path_to_your_model_folder - 3. python vgga.py - 4. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 5. python freeze_graph.py --input_graph model/vgga.pbtxt --input_checkpoint model/vgga.chkp --output_node_names="vgg_a/fc8/squeezed,output" --output_graph vgga_save.pb - """ - dir = os.path.dirname(os.path.realpath(__file__)) - if not os.path.isdir(dir + '/model'): - os.mkdir(dir + '/model') - batch_size = 5 - height, width = 224, 224 - #inputs = tf.placeholder(tf.float32, [None, height, width, 3]) - inputs = tf.Variable(tf.random_uniform((1, height, width, 3)), name='input') - net, end_points = vgg.vgg_a(inputs, is_training = False) - output = tf.Variable(tf.random_uniform(tf.shape(net)),name='output') - result = tf.assign(output,net) - saver = tf.train.Saver() - with tf.Session() as sess: - init = tf.global_variables_initializer() - sess.run(init) - sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/vgga.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'vgga.pbtxt') - tf.summary.FileWriter(dir + '/log', sess.graph) - - input_graph = dir + "/model/vgga.pbtxt" - input_checkpoint = dir + "/model/vgga.chkp" - output_node_names= "vgg_a/fc8/squeezed,output" - output_graph = dir + "/vgga_save.pb" - - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/resources/tf/models/alexnet.py b/spark/dl/src/test/resources/tf/models/alexnet.py index 3e9dcd40280..7759f1f5687 100644 --- a/spark/dl/src/test/resources/tf/models/alexnet.py +++ b/spark/dl/src/test/resources/tf/models/alexnet.py @@ -32,8 +32,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/inception_resnet_v2.py b/spark/dl/src/test/resources/tf/models/inception_resnet_v2.py new file mode 100644 index 00000000000..13ba6d504d5 --- /dev/null +++ b/spark/dl/src/test/resources/tf/models/inception_resnet_v2.py @@ -0,0 +1,39 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tensorflow as tf +from nets import inception_resnet_v2 +from sys import argv + +from util import run_model + +def main(): + """ + You can also run these commands manually to generate the pb file + 1. git clone https://github.com/tensorflow/models.git + 2. export PYTHONPATH=Path_to_your_model_folder + 3. python alexnet.py + """ + height, width = 299, 299 + inputs = tf.Variable(tf.random_uniform((2, height, width, 3)), name='input') + net, end_points = inception_resnet_v2.inception_resnet_v2(inputs,is_training = False) + print("nodes in the graph") + for n in end_points: + print(n + " => " + str(end_points[n])) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split(',')) + run_model(net_outputs, argv[1]) + +if __name__ == "__main__": + main() diff --git a/spark/dl/src/test/resources/tf/models/inception_v3.py b/spark/dl/src/test/resources/tf/models/inception_v3.py index cc4b2442696..1be0157040b 100644 --- a/spark/dl/src/test/resources/tf/models/inception_v3.py +++ b/spark/dl/src/test/resources/tf/models/inception_v3.py @@ -35,8 +35,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/lenet.py b/spark/dl/src/test/resources/tf/models/lenet.py index 4725aacd5fd..afc6770e9b6 100644 --- a/spark/dl/src/test/resources/tf/models/lenet.py +++ b/spark/dl/src/test/resources/tf/models/lenet.py @@ -32,8 +32,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/overfeat.py b/spark/dl/src/test/resources/tf/models/overfeat.py index c82a5b5a67f..f3cfdd4dc38 100644 --- a/spark/dl/src/test/resources/tf/models/overfeat.py +++ b/spark/dl/src/test/resources/tf/models/overfeat.py @@ -35,8 +35,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/resnet_v1.py b/spark/dl/src/test/resources/tf/models/resnet_v1.py index edb4665873c..e9eb1e8431a 100644 --- a/spark/dl/src/test/resources/tf/models/resnet_v1.py +++ b/spark/dl/src/test/resources/tf/models/resnet_v1.py @@ -33,8 +33,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/loadTest/rnn.py b/spark/dl/src/test/resources/tf/models/rnn.py similarity index 64% rename from spark/dl/src/test/resources/tf/loadTest/rnn.py rename to spark/dl/src/test/resources/tf/models/rnn.py index 799510a96cb..99f1c7df540 100644 --- a/spark/dl/src/test/resources/tf/loadTest/rnn.py +++ b/spark/dl/src/test/resources/tf/models/rnn.py @@ -15,53 +15,43 @@ # import tensorflow as tf import numpy as np -import os +from sys import argv from tensorflow.contrib import rnn -import merge_checkpoint as merge +from util import merge_checkpoint def main(): """ Run this command to generate the pb file 1. mkdir model 2. python rnn.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/rnn.pbtxt --input_checkpoint model/rnn.chkp --output_node_names=output --output_graph "rnn.pb" """ - dir = os.path.dirname(os.path.realpath(__file__)) + dir = argv[1] n_steps = 2 n_input = 10 n_hidden = 20 n_output = 5 - # xs = tf.placeholder(tf.float32, [None, n_steps, n_input]) xs = tf.Variable(tf.random_uniform([4, n_steps, n_input]) + 10, name='input', dtype=tf.float32) weight = tf.Variable(tf.random_uniform([n_hidden, n_output]) + 10, name="weight", dtype=tf.float32) bias = tf.Variable(tf.random_uniform([n_output]) + 10, name="bias", dtype=tf.float32) - x = tf.unstack(xs, n_steps, 1) - cell = rnn.BasicRNNCell(n_hidden) - output, states = rnn.static_rnn(cell, x, dtype=tf.float32) - final = tf.nn.bias_add(tf.matmul(output[-1], weight), bias, name='output') output = tf.Variable(tf.random_uniform(tf.shape(final)),name='output_result') result = tf.assign(output, final) - - saver = tf.train.Saver() with tf.Session() as sess: - file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) init = tf.global_variables_initializer() sess.run(init) sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/rnn.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'rnn.pbtxt') + checkpointpath = saver.save(sess, dir + '/model.chkp') + tf.train.write_graph(sess.graph, dir, 'model.pbtxt') - input_graph = dir + "/model/rnn.pbtxt" - input_checkpoint = dir + "/model/rnn.chkp" - output_node_names= "output,output_result" - output_graph = dir + "/rnn.pb" + input_graph = dir + "/model.pbtxt" + input_checkpoint = dir + "/model.chkp" + output_node_names= ["output", "output_result"] + output_graph = dir + "/model.pb" - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/loadTest/rnn_lstm.py b/spark/dl/src/test/resources/tf/models/rnn_lstm.py similarity index 66% rename from spark/dl/src/test/resources/tf/loadTest/rnn_lstm.py rename to spark/dl/src/test/resources/tf/models/rnn_lstm.py index c02686300dd..6f371dac37f 100644 --- a/spark/dl/src/test/resources/tf/loadTest/rnn_lstm.py +++ b/spark/dl/src/test/resources/tf/models/rnn_lstm.py @@ -15,19 +15,17 @@ # import tensorflow as tf import numpy as np -import os +from sys import argv from tensorflow.contrib import rnn -import merge_checkpoint as merge +from util import merge_checkpoint def main(): """ Run this command to generate the pb file 1. mkdir model - 2. python rnn.py - 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py - 4. python freeze_graph.py --input_graph model/lstm.pbtxt --input_checkpoint model/lstm.chkp --output_node_names=output --output_graph "lstm.pb" + 2. python rnn_lstm.py """ - dir = os.path.dirname(os.path.realpath(__file__)) + dir = argv[1] n_steps = 2 n_input = 10 n_hidden = 20 @@ -49,18 +47,17 @@ def main(): result = tf.assign(output, final) saver = tf.train.Saver() with tf.Session() as sess: - file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) init = tf.global_variables_initializer() sess.run(init) sess.run(result) - checkpointpath = saver.save(sess, dir + '/model/lstm.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'lstm.pbtxt') + checkpointpath = saver.save(sess, dir + '/model.chkp') + tf.train.write_graph(sess.graph, dir, 'model.pbtxt') - input_graph = dir + "/model/lstm.pbtxt" - input_checkpoint = dir + "/model/lstm.chkp" - output_node_names= "output,output_result" - output_graph = dir + "/lstm.pb" + input_graph = dir + "/model.pbtxt" + input_checkpoint = dir + "/model.chkp" + output_node_names= ["output", "output_result"] + output_graph = dir + "/model.pb" - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) + merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/spark/dl/src/test/resources/tf/loadTest/share_weight.py b/spark/dl/src/test/resources/tf/models/share_weight.py similarity index 73% rename from spark/dl/src/test/resources/tf/loadTest/share_weight.py rename to spark/dl/src/test/resources/tf/models/share_weight.py index 32dd8eb4421..cd90d1f942e 100644 --- a/spark/dl/src/test/resources/tf/loadTest/share_weight.py +++ b/spark/dl/src/test/resources/tf/models/share_weight.py @@ -15,8 +15,8 @@ # import tensorflow as tf import numpy as np -import os -import merge_checkpoint as merge +from sys import argv +from util import merge_checkpoint def main(): """ @@ -26,7 +26,6 @@ def main(): 3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py 4. python freeze_graph.py --input_graph model/share_weight.pbtxt --input_checkpoint model/share_weight.chkp --output_node_names=output --output_graph "share_weight.pb" """ - dir = os.path.dirname(os.path.realpath(__file__)) xs = tf.placeholder(tf.float32, [None, 10]) W1 = tf.Variable(tf.random_normal([10,10])) b1 = tf.Variable(tf.random_normal([10])) @@ -37,19 +36,20 @@ def main(): W2 = tf.Variable(tf.random_normal([10, 1])) b2 = tf.Variable(tf.random_normal([1])) final = tf.nn.bias_add(tf.matmul(Wx_plus_b2, W2), b2, name='output') + dir = argv[1] saver = tf.train.Saver() with tf.Session() as sess: - file_writer = tf.summary.FileWriter(dir + '/model/logs', sess.graph) init = tf.global_variables_initializer() sess.run(init) - checkpointpath = saver.save(sess, dir + '/model/share_weight.chkp') - tf.train.write_graph(sess.graph, dir + '/model', 'share_weight.pbtxt') + checkpointpath = saver.save(sess, dir + '/model.chkp') + tf.train.write_graph(sess.graph, dir, 'model.pbtxt') - input_graph = dir + "/model/share_weight.pbtxt" - input_checkpoint = dir + "/model/share_weight.chkp" - output_node_names= "output" - output_graph = dir + "/share_weight.pb" + input_graph = dir + "/model.pbtxt" + input_checkpoint = dir + "/model.chkp" + output_node_names = "output" + output_graph = dir + "/model.pb" + + merge_checkpoint(input_graph, input_checkpoint, [output_node_names], output_graph) - merge.merge_checkpoint(input_graph, input_checkpoint, output_node_names, output_graph) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/util.py b/spark/dl/src/test/resources/tf/models/util.py index 196361709bc..99c64efb44e 100644 --- a/spark/dl/src/test/resources/tf/models/util.py +++ b/spark/dl/src/test/resources/tf/models/util.py @@ -57,14 +57,21 @@ def merge_checkpoint(input_graph, with gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) -def run_model(end_point, output_path): - output=tf.Variable(tf.random_uniform(tf.shape(end_point)), name='output') - result = tf.assign(output, end_point, name = 'assign') +def run_model(end_points, output_path): + outputs = [] + results = [] + i = 0 + for end_point in end_points: + output = tf.Variable(tf.random_uniform(tf.shape(end_point)), name='output' + str(i)) + outputs.append(output) + results.append(tf.assign(output, end_point, name = 'assign' + str(i))) + i = i + 1 + saver = tf.train.Saver() with tf.Session() as sess: init = tf.global_variables_initializer() sess.run(init) - sess.run(result) + sess.run(results) saver.save(sess, output_path + '/model.chkp') tf.train.write_graph(sess.graph, output_path, 'model.pbtxt') @@ -72,5 +79,5 @@ def run_model(end_point, output_path): input_checkpoint = output_path + "/model.chkp" output_file = output_path + "/model.pb" - merge_checkpoint(input_graph, input_checkpoint, ["assign"], output_file) + merge_checkpoint(input_graph, input_checkpoint, map(lambda x: 'assign' + str(x), range(len(end_points))), output_file) diff --git a/spark/dl/src/test/resources/tf/models/vgg16.py b/spark/dl/src/test/resources/tf/models/vgg16.py index 6035cdfbb0b..11cdd940ee0 100644 --- a/spark/dl/src/test/resources/tf/models/vgg16.py +++ b/spark/dl/src/test/resources/tf/models/vgg16.py @@ -32,8 +32,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/vgg19.py b/spark/dl/src/test/resources/tf/models/vgg19.py index 58dfaf22d5a..0b29cda9bc7 100644 --- a/spark/dl/src/test/resources/tf/models/vgg19.py +++ b/spark/dl/src/test/resources/tf/models/vgg19.py @@ -32,8 +32,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/vgga.py b/spark/dl/src/test/resources/tf/models/vgga.py index 2a70e7b42b7..9417740d7cd 100644 --- a/spark/dl/src/test/resources/tf/models/vgga.py +++ b/spark/dl/src/test/resources/tf/models/vgga.py @@ -32,8 +32,8 @@ def main(): print("nodes in the graph") for n in end_points: print(n + " => " + str(end_points[n])) - net_output = tf.get_default_graph().get_tensor_by_name(argv[2]) - run_model(net_output, argv[1]) + net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) + run_model(net_outputs, argv[1]) if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/save.py b/spark/dl/src/test/resources/tf/save.py deleted file mode 100644 index 1ab44e17be1..00000000000 --- a/spark/dl/src/test/resources/tf/save.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tensorflow as tf -import numpy as np -from tensorflow.python.platform import gfile -import os - -def main(): - dir = os.path.dirname(os.path.realpath(__file__)) - xs = tf.placeholder(tf.float32, [4, 3, 4]) - tf.nn.relu(xs) - - with tf.Session() as sess: - tf.train.write_graph(sess.graph, dir, 'model.pb') - -if __name__ == "__main__": - main() diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index d4d5ec0d0d4..7331e903bd1 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -159,11 +159,22 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "Shared weights" should "be the same instance" in { tfCheck() - (("python " + testScriptsPath("share_weight.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "share_weight.pb" - val model = TensorflowLoader.load(path, Seq("Placeholder"), Seq("output"), + val modelName = "share_weight" + // Generate command and prepare the temp folder + val s = JFile.separator + val modelsFolder = processPath(getClass().getClassLoader().getResource("tf").getPath()) + + s + "models" + val modelScript = modelsFolder + s + s"$modelName.py" + val tmpLocation = java.io.File.createTempFile("tensorflowLoaderTest" + UUID.randomUUID(), + modelName) + tmpLocation.delete() + tmpLocation.mkdir() + + require(runPython(s"$modelScript $tmpLocation"), "error when run the model script") + + // Load the model and input/output tensors + val modelFile = tmpLocation + s + "model.pb" + val model = TensorflowLoader.load(modelFile, Seq("Placeholder"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] val l1 = container.modules(1).asInstanceOf[Linear[Float]] @@ -174,12 +185,23 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "Shared weights" should "be the same after running optimizer" in { tfCheck() - (("python " + testScriptsPath("share_weight.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "share_weight.pb" - val model = TensorflowLoader.load(path, Seq("Placeholder"), Seq("output"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) + val modelName = "share_weight" + // Generate command and prepare the temp folder + val s = JFile.separator + val modelsFolder = processPath(getClass().getClassLoader().getResource("tf").getPath()) + + s + "models" + val modelScript = modelsFolder + s + s"$modelName.py" + val tmpLocation = java.io.File.createTempFile("tensorflowLoaderTest" + UUID.randomUUID(), + modelName) + tmpLocation.delete() + tmpLocation.mkdir() + + require(runPython(s"$modelScript $tmpLocation"), "error when run the model script") + + // Load the model and input/output tensors + val modelFile = tmpLocation + s + "model.pb" + val model = TensorflowLoader.load(modelFile, Seq("Placeholder"), Seq("output"), + ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) val container = model.asInstanceOf[Graph[Float]] val optimizer = new DistriOptimizer[Float](container, dataSet, new MSECriterion[Float]()) @@ -195,12 +217,25 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "static simple rnn " should "have the same inference result as tensorflow" in { tfCheck() - (("python " + testScriptsPath("rnn.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "rnn.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1), Seq("output")) + val modelName = "rnn" + // Generate command and prepare the temp folder + val s = JFile.separator + val modelsFolder = processPath(getClass().getClassLoader().getResource("tf").getPath()) + + s + "models" + val modelScript = modelsFolder + s + s"$modelName.py" + val tmpLocation = java.io.File.createTempFile("tensorflowLoaderTest" + UUID.randomUUID(), + modelName) + tmpLocation.delete() + tmpLocation.mkdir() + + require(runPython(s"$modelScript $tmpLocation"), "error when run the model script") + + // Load the model and input/output tensors + val modelFile = tmpLocation + s + "model.pb" + + + val results = TensorflowLoader.parse(modelFile) + val tfGraph = TensorflowLoader.buildTFGraph(results, Seq("output")) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq("output"), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) @@ -209,22 +244,28 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val bigDLResult = model.forward(input) - val gradient = Tensor[Float](4, 5).rand() - - tfResult.map( bigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2) / v1 < 1e-6) - v2 - }) - model.backward(bigDLResult, gradient) + tfResult.almostEqual(bigDLResult.toTensor, 1e-6) } "static lstm rnn " should "have the same inference result as tensorflow" in { tfCheck() - (("python " + testScriptsPath("rnn_lstm.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "lstm.pb" - val results = TensorflowLoader.parse(path) + val modelName = "rnn_lstm" + // Generate command and prepare the temp folder + val s = JFile.separator + val modelsFolder = processPath(getClass().getClassLoader().getResource("tf").getPath()) + + s + "models" + val modelScript = modelsFolder + s + s"$modelName.py" + val tmpLocation = java.io.File.createTempFile("tensorflowLoaderTest" + UUID.randomUUID(), + modelName) + tmpLocation.delete() + tmpLocation.mkdir() + + require(runPython(s"$modelScript $tmpLocation"), "error when run the model script") + + // Load the model and input/output tensors + val modelFile = tmpLocation + s + "model.pb" + + val results = TensorflowLoader.parse(modelFile) val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1), Seq("output")) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq("output"), @@ -234,94 +275,77 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) val bigDLResult = model.forward(input) - val gradient = Tensor[Float](4, 5).rand() - - tfResult.map( bigDLResult.toTensor, (v1, v2) => { - assert(abs(v1 - v2)/ v1 < 1e-6, s"$v1, $v2") - v2 - }) - model.backward(bigDLResult, gradient) + tfResult.almostEqual(bigDLResult.toTensor, 1e-5) } "Tensorflow lenet" should "be load correctly" in { - val (tf, bigdl) = testModel("lenet", "LeNet/pool2/MaxPool:0", true) - val transposed = bigdl.transpose(2, 3).transpose(3, 4) - tf.almostEqual(transposed, 1e-6) should be(true) + testModel("lenet", Seq("LeNet/pool2/MaxPool:0"), true).foreach { + case(tf, bigdl) => + val transposed = bigdl.transpose(2, 3).transpose(3, 4) + tf.almostEqual(transposed, 1e-6) should be(true) + } } "Tensorflow Alexnet" should "be load correctly" in { - val (tf, bigdl) = testModel("alexnet", "alexnet_v2/fc8/squeezed:0", true) - tf.almostEqual(bigdl, 1e-7) should be(true) + testModel("alexnet", Seq("alexnet_v2/fc8/squeezed:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-7) should be(true) + } } "TensorFlow vgg_a" should "be load correctly" in { - val (tf, bigdl) = testModel("vgga", "vgg_a/fc8/squeezed:0", true) - tf.almostEqual(bigdl, 1e-7) should be(true) + testModel("vgga", Seq("vgg_a/fc8/squeezed:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-7) should be(true) + } } "TensorFlow vgg_16" should "be load correctly" in { - val (tf, bigdl) = testModel("vgg16", "vgg_16/fc8/squeezed:0", true) - tf.almostEqual(bigdl, 1e-7) should be(true) + testModel("vgg16", Seq("vgg_16/fc8/squeezed:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-7) should be(true) + } } "TensorFlow vgg_19" should "be load correctly" in { - val (tf, bigdl) = testModel("vgg19", "vgg_19/fc8/squeezed:0", true) - tf.almostEqual(bigdl, 1e-7) should be(true) + testModel("vgg19", Seq("vgg_19/fc8/squeezed:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-7) should be(true) + } } "TensorFlow overfeat" should "be load correctly" in { - val (tf, bigdl) = testModel("overfeat", "overfeat/fc8/squeezed:0", true) - tf.almostEqual(bigdl, 1e-7) should be(true) + testModel("overfeat", Seq("overfeat/fc8/squeezed:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-7) should be(true) + } } "TensorFlow inception_v3" should "be load correctly" in { - val (tf, bigdl) = testModel("inception_v3", "InceptionV3/Logits/SpatialSqueeze:0", true) - tf.almostEqual(bigdl, 1e-7) should be(true) + testModel("inception_v3", Seq("InceptionV3/Logits/SpatialSqueeze:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-7) should be(true) + } } "TensorFlow resnet_v1" should "be load correctly" in { - val (tf, bigdl) = testModel("resnet_v1", "resnet_v1_101/SpatialSqueeze:0", true) - tf.almostEqual(bigdl, 1e-6) should be(true) + testModel("resnet_v1", Seq("resnet_v1_101/SpatialSqueeze:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-6) should be(true) + } } - "TensorFlow loader" should "run the python to save the modle and " + - "have the same inferrence result with tensorflow " + - "after loading slim inception_resnet_v2" in { - tfCheck() - (("python " + testScriptsPath("inception_resnet_v2.py")) !!) - val resource = getClass().getClassLoader().getResource("tf") - val path = processPath(resource.getPath()) + JFile.separator + - "loadTest" + JFile.separator + "inception_resnet_v2_save.pb" - val results = TensorflowLoader.parse(path) - val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-2), - Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd")) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - Seq("InceptionResnetV2/Logits/Logits/BiasAdd", "InceptionResnetV2/AuxLogits/Logits/BiasAdd") - , ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) - val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, - ByteOrder.LITTLE_ENDIAN) - .transpose(2, 4).transpose(3, 4).contiguous() - val gradient1 = Tensor[Float](2, 1001).rand() - val gradient2 = Tensor[Float](2, 1001).rand() - val tfResult1 = TensorflowToBigDL.toTensor(results.get(results.size()-2) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val tfResult2 = TensorflowToBigDL.toTensor(results.get(results.size()-1) - .getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) - val BigDLResult = model.forward(input) - - tfResult1.map( BigDLResult.toTable(1), (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - tfResult2.map( BigDLResult.toTable(2), (v1, v2) => { - assert(abs(v1 - v2) < 1e-7); - v2 - }) - model.backward(input, T(gradient1, gradient2)) + "TensorFlow inception_resnet_v2" should "be load correctly" in { + testModel("inception_resnet_v2", Seq("InceptionResnetV2/Logits/Logits/BiasAdd:0", + "InceptionResnetV2/AuxLogits/Logits/BiasAdd:0"), true).foreach { + case(tf, bigdl) => + tf.almostEqual(bigdl, 1e-7) should be(true) + } } - private def testModel(modelName: String, endPoint: String, transInput: Boolean) - : (Tensor[Float], Tensor[Float]) = { + + private def testModel(modelName: String, endPoints: Seq[String], transInput: Boolean) + : Seq[(Tensor[Float], Tensor[Float])] = { tfCheck() // Generate command and prepare the temp folder @@ -334,21 +358,23 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ tmpLocation.delete() tmpLocation.mkdir() - require(runPython(s"$modelScript $tmpLocation $endPoint"), "error when run the model script") + require(runPython(s"$modelScript $tmpLocation ${endPoints.mkString(",")}"), + "error when run the model script") // Load the model and input/output tensors val modelFile = tmpLocation + s + "model.pb" val tfNodes = TensorflowLoader.parse(modelFile) - val tfGraph = TensorflowLoader.buildTFGraph(tfNodes, Seq(endPoint.split(":")(0))) - val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq(endPoint.split(":")(0)), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + val tfGraph = TensorflowLoader.buildTFGraph(tfNodes, endPoints.map(_.split(":")(0))) + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + endPoints.map(_.split(":")(0)), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) import collection.JavaConverters._ // Compare the tensor contents val tfInputTensor = tfNodes.asScala.filter(_.getName == "input")(0) .getAttrMap.get("value").getTensor - val tfOutputTensor = tfNodes.asScala.filter(_.getName == "output")(0) - .getAttrMap.get("value").getTensor + + val tfOutputTensors = (0 until endPoints.length).map( + i => tfNodes.asScala.filter(_.getName == s"output$i")(0).getAttrMap.get("value").getTensor) val input = TensorflowToBigDL.toTensor(tfInputTensor, ByteOrder.LITTLE_ENDIAN) @@ -358,8 +384,14 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ input } - (TensorflowToBigDL.toTensor(tfOutputTensor, ByteOrder.LITTLE_ENDIAN), - model.forward(transposeInput).toTensor) + val bigdlOutputs = if (endPoints.length == 1) { + Seq(model.forward(transposeInput).toTensor) + } else { + val t = model.forward(transposeInput).toTable + (1 to endPoints.length).map(t[Tensor[Float]](_)) + } + tfOutputTensors.zip(bigdlOutputs).map(x => + (TensorflowToBigDL.toTensor(x._1, ByteOrder.LITTLE_ENDIAN), x._2)) } private def processPath(path: String): String = { @@ -369,10 +401,4 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ path } } - - private def testScriptsPath(script: String) : String = { - val resource = getClass().getClassLoader().getResource("tf") - processPath(resource.getPath()) + JFile.separator + "loadTest" + - JFile.separator + script - } } diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala index cdfe28d95c5..7fa3cbc49cf 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala @@ -194,19 +194,13 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { val pool1 = SpatialMaxPooling(2, 2, 2, 2).setName("pool1").apply(tanh1) val tanh2 = Tanh().setName("tanh2").apply(pool1) val conv2 = SpatialConvolution(6, 12, 5, 5).setName("conv2").apply(tanh2) - val pool2 = SpatialMaxPooling(2, 2, 2, 2).setName("pool2").apply(conv2) - val reshape = Reshape(Array(4, 12 * 4 * 4)).setName("reshape2").apply(pool2) - val fc1 = Linear(12 * 4 * 4, 100).setName("fc1").apply(reshape) - val tanh3 = Tanh().setName("tanh3").apply(fc1) - val fc2 = Linear(100, 10).setName("fc2").apply(tanh3) - val output = LogSoftMax().setName("output").apply(fc2) - - val funcModel = Graph(conv1, output) + val pool2 = SpatialMaxPooling(2, 2, 2, 2).setName("output").apply(conv2) + + val funcModel = Graph(conv1, pool2) val inputData = Tensor(4, 1, 28, 28).rand() val transInput = inputData.transpose(2, 3).transpose(3, 4).contiguous() val outputData = funcModel.forward(inputData).toTensor - val tmpFile = java.io.File.createTempFile("tensorflowSaverTest" + UUID.randomUUID(), "lenet") TensorflowSaver.saveGraphWitNodeDef( funcModel, @@ -214,10 +208,11 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { tmpFile.getPath, ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC, - Set(Tensorflow.const(outputData, "target", ByteOrder.LITTLE_ENDIAN)) + Set(Tensorflow.const(outputData.transpose(2, 3).transpose(3, 4).contiguous(), + "target", ByteOrder.LITTLE_ENDIAN)) ) - runPythonSaveTest(tmpFile.getPath, "") + runPythonSaveTest(tmpFile.getPath, "") should be(true) } private def test(layer: AbstractModule[Tensor[Float], Tensor[Float], Float], From adcc53f6f3e231c1c627f18ab0bae4b3e5cd2f8a Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 15 Jun 2017 02:05:39 +0800 Subject: [PATCH 14/23] remove NHWC --- .../com/intel/analytics/bigdl/nn/Module.scala | 2 +- .../bigdl/utils/tf/BigDLToTensorflow.scala | 74 ++++++----- .../bigdl/utils/tf/TensorflowLoader.scala | 14 +-- .../bigdl/utils/tf/TensorflowSaver.scala | 6 +- .../bigdl/utils/tf/TensorflowToBigDL.scala | 116 +++++++----------- .../bigdl/utils/tf/TensorflowLoaderSpec.scala | 17 ++- .../bigdl/utils/tf/TensorflowSaverSpec.scala | 46 +++---- 7 files changed, 135 insertions(+), 140 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala index 47754056dc5..1fbecb27f58 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala @@ -57,7 +57,7 @@ object Module { dataFormat: TensorflowDataFormat = TensorflowDataFormat.NCHW)( implicit ev: TensorNumeric[T]): Module[T] = { - TensorflowLoader.load(file, inputs, outputs, byteOrder, dataFormat) + TensorflowLoader.load(file, inputs, outputs, byteOrder) } def flatten[@specialized(Float, Double) T: ClassTag](parameters: Array[Tensor[T]])( diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala index 530cadf45b4..aef0c76ba56 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala @@ -37,12 +37,16 @@ trait BigDLToTensorflow { * @return Mapped nodedef list, the first is the output node */ def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] + byteOrder: ByteOrder): Seq[NodeDef] } object BigDLToTensorflow { - private[tf] def processSaveDim(dim: Int, dataFormat: TensorflowDataFormat): Int = { - if (dataFormat == TensorflowDataFormat.NHWC) { + /** + * This method is just for test purpose. Do not use the bigdl.saveNHWC for real use case + * @return + */ + private[tf] def processSaveDim(dim: Int): Int = { + if (System.getProperty("bigdl.enableNHWC", "false").toBoolean) { if (dim == 2) return 4 if (dim == 3) return 2 if (dim == 4) return 3 @@ -51,11 +55,23 @@ object BigDLToTensorflow { dim } } + + /** + * This method is just for test purpose. Do not use the bigdl.enableNHWC for real use case + * @return + */ + private[tf] def getDataFormat(): TensorflowDataFormat = { + if (System.getProperty("bigdl.enableNHWC", "false").toBoolean) { + TensorflowDataFormat.NHWC + } else { + TensorflowDataFormat.NCHW + } + } } object InputToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Input only accept one input") Seq(identity(inputs(0), module.getName())) @@ -64,7 +80,7 @@ object InputToTF extends BigDLToTensorflow { object ReLUToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Relu only accept one input") Seq(relu(inputs(0), module.getName())) @@ -73,7 +89,7 @@ object ReLUToTF extends BigDLToTensorflow { object LinearToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Linear only accept one input") val linear = module.asInstanceOf[Linear[_]] val weight = const(linear.weight.t().contiguous(), linear.getName() + "/weight", byteOrder) @@ -81,14 +97,14 @@ object LinearToTF extends BigDLToTensorflow { val mm = matmul(inputs(0), weightReader, linear.getName() + "/matmul") val bias = const(linear.bias, linear.getName() + "/bias", byteOrder) val biasReader = identity(bias, linear.getName() + "/biasReader") - val add = biasAdd(mm, biasReader, dataFormat, linear.getName() + "/biasAdd") + val add = biasAdd(mm, biasReader, getDataFormat(), linear.getName() + "/biasAdd") Seq(add, biasReader, bias, mm, weightReader, weight) } } object SpatialConvolutionToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "SpatialConvolution only accept one input") val spatialConv = module.asInstanceOf[SpatialConvolution[_]] // squeeze will modify the weight tensor @@ -101,10 +117,10 @@ object SpatialConvolutionToTF extends BigDLToTensorflow { val filterReader = identity(filter, spatialConv.getName() + "/filterReader") val conv = conv2D(inputs(0), filterReader, spatialConv.strideW, spatialConv.strideH, spatialConv.kernelW, spatialConv.kernelH, spatialConv.padW, spatialConv.padH, - dataFormat, spatialConv.getName() + "/conv2D") + getDataFormat(), spatialConv.getName() + "/conv2D") val bias = const(spatialConv.bias, spatialConv.getName() + "/bias", byteOrder) val biasReader = identity(bias, spatialConv.getName() + "/biasReader") - val add = biasAdd(conv, biasReader, dataFormat, + val add = biasAdd(conv, biasReader, getDataFormat(), spatialConv.getName() + "/biasAdd") Seq(add, biasReader, bias, conv, filterReader, filter) } @@ -112,16 +128,16 @@ object SpatialConvolutionToTF extends BigDLToTensorflow { object SqueezeToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Squeeze only accept one input") val sq = module.asInstanceOf[Squeeze[_]] - Seq(squeeze(inputs(0), sq.dims.map(processSaveDim(_, dataFormat) - 1), sq.getName())) + Seq(squeeze(inputs(0), sq.dims.map(processSaveDim(_) - 1), sq.getName())) } } object TanhToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Tanh only accept one input") Seq(tanh(inputs(0), module.getName())) } @@ -129,7 +145,7 @@ object TanhToTF extends BigDLToTensorflow { object ReshapeToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Reshape only accept one input") val rh = module.asInstanceOf[Reshape[_]] val size = Tensor[Float](rh.size.length) @@ -146,7 +162,7 @@ object ReshapeToTF extends BigDLToTensorflow { object ViewToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Reshape only accept one input") val viewLayer = module.asInstanceOf[View[_]] val size = Tensor[Float](viewLayer.sizes.length) @@ -163,17 +179,17 @@ object ViewToTF extends BigDLToTensorflow { object MaxpoolToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Maxpool only accept one input") val layer = module.asInstanceOf[SpatialMaxPooling[_]] Seq(maxPool(inputs(0), layer.kW, layer.kH, layer.padW, layer.padH, - layer.dW, layer.dH, dataFormat, layer.getName())) + layer.dW, layer.dH, getDataFormat(), layer.getName())) } } object PaddingToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Padding only accept one input") val layer = module.asInstanceOf[Padding[_]] require(layer.nIndex == 1, "only support padding nIndex == 1") @@ -194,17 +210,17 @@ object PaddingToTF extends BigDLToTensorflow { object AvgpoolToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Avgpool only accept one input") val layer = module.asInstanceOf[SpatialAveragePooling[_]] Seq(avgPool(inputs(0), layer.kW, layer.kH, layer.padW, layer.padH, - layer.dW, layer.dH, dataFormat, layer.getName())) + layer.dW, layer.dH, getDataFormat(), layer.getName())) } } object SigmoidToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Sigmoid only accept one input") Seq(sigmoid(inputs(0), module.getName())) } @@ -212,7 +228,7 @@ object SigmoidToTF extends BigDLToTensorflow { object DropoutToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Dropout only accept one input") val layer = module.asInstanceOf[Dropout[_]] require(layer.isTraining() == false, "only support evaluating mode dropout") @@ -223,14 +239,14 @@ object DropoutToTF extends BigDLToTensorflow { object CAddTableToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { Seq(addN(inputs, module.getName())) } } object CMultTableToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 2, "Tensorflow only support two tensor multiply together") Seq(multiply(inputs(0), inputs(1), module.getName())) @@ -239,7 +255,7 @@ object CMultTableToTF extends BigDLToTensorflow { object JoinTableToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { val layer = module.asInstanceOf[JoinTable[_]] val axis = const(Tensor[Float](T((layer.dimension - 1).toFloat)), layer.getName() + "/axis", byteOrder, true, DataType.DT_INT32) @@ -252,7 +268,7 @@ object JoinTableToTF extends BigDLToTensorflow { object MeanToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Mean only accept one input") val layer = module.asInstanceOf[Mean[_]] require(layer.squeeze == true, "Mean must squeeze input") @@ -267,7 +283,7 @@ object MeanToTF extends BigDLToTensorflow { object SoftMaxToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "Softmax only accept one input") Seq(softmax(inputs(0), module.getName())) } @@ -275,7 +291,7 @@ object SoftMaxToTF extends BigDLToTensorflow { object LogSoftMaxToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "LogSoftmax only accept one input") Seq(logSoftmax(inputs(0), module.getName())) } @@ -283,7 +299,7 @@ object LogSoftMaxToTF extends BigDLToTensorflow { object BatchNorm2DToTF extends BigDLToTensorflow { override def toTFDef(module: AbstractModule[_, _, _], inputs: Seq[NodeDef], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat): Seq[NodeDef] = { + byteOrder: ByteOrder): Seq[NodeDef] = { require(inputs.length == 1, "BatchNorm only accept one input") val layer = module.asInstanceOf[SpatialBatchNormalization[_]] require(!layer.isTraining(), "Only support evaluate mode batch norm") diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala index 651547e243e..276f18024f8 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala @@ -45,11 +45,10 @@ object TensorflowLoader{ * @param inputs input node names * @param outputs output node names * @param byteOrder file byteOrder - * @param dataFormat dataformat * @return */ def load[T: ClassTag](graphPrototxt: String, inputs: Seq[String], outputs: Seq[String], - byteOrder: ByteOrder, dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): Module[T] = { // Get node list val nodeList = parse(graphPrototxt) @@ -58,7 +57,7 @@ object TensorflowLoader{ val tfGraph = buildTFGraph(nodeList, outputs) // Build BigDL model from the tf node graph - buildBigDLModel(tfGraph, inputs, outputs, byteOrder, dataFormat) + buildBigDLModel(tfGraph, inputs, outputs, byteOrder) } /** @@ -123,8 +122,7 @@ object TensorflowLoader{ tfGraph: DirectedGraph[NodeDef], inputs: Seq[String], outputs: Seq[String], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat + byteOrder: ByteOrder )(implicit ev: TensorNumeric[T]): Module[T] = { import scala.collection.JavaConverters._ @@ -143,7 +141,7 @@ object TensorflowLoader{ // converted node, skip } else { val (module, nodes, inputNodes) = - extract[T](n.graph(reverse = true), context, byteOrder, dataFormat).getOrElse( + extract[T](n.graph(reverse = true), context, byteOrder).getOrElse( throw new UnsupportedOperationException(s"Can not find matched graph ${n}")) val node = new Node(module) @@ -188,7 +186,7 @@ object TensorflowLoader{ * @return */ private[bigdl] def extract[T: ClassTag](graph: DirectedGraph[NodeDef], - context: Context[T], byteOrder: ByteOrder, dataFormat: TensorflowDataFormat)( + context: Context[T], byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): Option[( AbstractModule[Activity, Tensor[T], T], List[Node[NodeDef]], @@ -200,7 +198,7 @@ object TensorflowLoader{ val (result, inputs) = matchGraph(graph, patterns(i).topology) if (result.size != 0) { // get model - return Some(patterns(i).layer(graph, context, byteOrder, dataFormat), result, inputs) + return Some(patterns(i).layer(graph, context, byteOrder), result, inputs) } i += 1 } diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala index 6963fe6fd0e..96b1ae27330 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaver.scala @@ -41,7 +41,6 @@ object TensorflowSaver { * @param inputs input node defs * @param path where to save * @param byteOrder model byte order - * @param dataFormat model data format * @tparam T */ def saveGraphWitNodeDef[T]( @@ -49,7 +48,6 @@ object TensorflowSaver { inputs : Seq[NodeDef], path: String, byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN, - dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC, extraNodes: Set[NodeDef] = Set()): Unit = { val inputNodeCache = new mutable.HashMap[AbstractModule[Activity, Tensor[T], T], ArrayBuffer[NodeDef]]() @@ -63,7 +61,7 @@ object TensorflowSaver { model.executions.foreach(n => { val nodeDefs = maps(n.element.getClass.getName).toTFDef(n.element, inputNodeCache(n.element), - byteOrder, dataFormat) + byteOrder) nodeDefs.foreach(nDef => { graphBuilder.addNode(nDef) }) @@ -111,7 +109,7 @@ object TensorflowSaver { val inputNodeDefs = inputs.map(input => placeholder(model.getNumericType(), input._2, input._1) ) - saveGraphWitNodeDef(model, inputNodeDefs, path, byteOrder, dataFormat) + saveGraphWitNodeDef(model, inputNodeDefs, path, byteOrder) } /** diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala index 42127895830..a5b826e7f1e 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -51,8 +51,7 @@ trait TensorflowToBigDL { def layer[T: ClassTag]( tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat + byteOrder: ByteOrder )(implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] protected def getOrSetTensor[T: ClassTag]( @@ -248,8 +247,12 @@ object TensorflowToBigDL { }) } - private[utils] def processDims(dim: Int, dataFormat: TensorflowDataFormat): Int = { - if (dataFormat == TensorflowDataFormat.NHWC) { + /** + * This method is just for test purpose. Do not use the bigdl.saveNHWC for real use case + * @return + */ + private[tf] def processDims(dim: Int): Int = { + if (System.getProperty("bigdl.enableNHWC", "false").toBoolean) { // exchange the dims as BigDL only support NCHW now if (dim == 1) return 2 if (dim == 2) return 3 @@ -275,8 +278,7 @@ object FullConnectionTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { @@ -299,12 +301,11 @@ object SqueezeTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val dims = tfGraph.source.element.getAttrOrThrow("squeeze_dims").getList().getIList() - .asScala.map(_.toInt).toArray.map(processDims(_, dataFormat)) + .asScala.map(_.toInt).toArray.map(processDims(_)) Squeeze[T](dims, batchMode = true).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] } @@ -325,8 +326,7 @@ object Conv2D extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { @@ -335,6 +335,7 @@ object Conv2D extends TensorflowToBigDL{ val (strideH, strideW) = if (attributes.get("data_format").getS .toString(Charset.defaultCharset()) == "NHWC") { + require(System.getProperty("bigdl.enableNHWC", "false").toBoolean, "Not support NHWC") require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") (attributes.get("strides").getList.getI(1).toInt, attributes.get("strides").getList.getI(2).toInt) @@ -388,8 +389,7 @@ object ReluTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { ReLU[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] @@ -405,8 +405,7 @@ object TanhTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { @@ -423,8 +422,7 @@ object SigmoidTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { Sigmoid[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] @@ -443,8 +441,7 @@ object ReshapeTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val sizes = TensorflowToBigDL.toTensor( @@ -473,14 +470,14 @@ object MaxPoolingTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val attributes = tfGraph.source.element.getAttrMap val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS .toString(Charset.defaultCharset()) == "NHWC") { + require(System.getProperty("bigdl.enableNHWC", "false").toBoolean, "Not support NHWC") require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") ( attributes.get("strides").getList.getI(1).toInt, @@ -522,14 +519,14 @@ object AvgPoolingTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val attributes = tfGraph.source.element.getAttrMap val (strideH, strideW, ksizeH, ksizeW) = if (attributes.get("data_format").getS .toString(Charset.defaultCharset()) == "NHWC") { + require(System.getProperty("bigdl.enableNHWC", "false").toBoolean, "Not support NHWC") require(attributes.get("strides").getList.getI(3).toInt == 1, s"not support strides on depth") ( attributes.get("strides").getList.getI(1).toInt, @@ -587,8 +584,7 @@ object DropoutTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val keepProp = tfGraph.source.prevNodes(0).prevNodes(1).element @@ -605,8 +601,7 @@ object Placeholder extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { Input[T].element.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] } @@ -619,8 +614,7 @@ object ConstTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val value = TensorflowToBigDL @@ -640,8 +634,7 @@ object ShapeTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { @@ -656,8 +649,7 @@ object InputTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { Input[T].element.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] @@ -671,8 +663,7 @@ object IdentityTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { Input[T].element.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] @@ -722,8 +713,7 @@ object BatchNormTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val nOutput = tfGraph.source.prevNodes(1).prevNodes(1).prevNodes(1) @@ -757,8 +747,7 @@ object FillTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val constNode = tfGraph.source.prevNodes(1) @@ -779,10 +768,9 @@ object PackTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { - val dim = processDims(tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1, dataFormat) + val dim = processDims(tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1) Pack[T](dim).asInstanceOf[AbstractModule[Activity, Tensor[T], T]] } @@ -799,11 +787,10 @@ object UnpackTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { - val dim = processDims(tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1, dataFormat) + val dim = processDims(tfGraph.source.element.getAttrMap.get("axis").getI.toInt + 1) val index = tfGraph.source.element.getName.split(":").toList match { case _::Nil => 1 case _::i::Nil => i.toInt + 1 @@ -826,8 +813,7 @@ object StrideSliceTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val startNode = tfGraph.source.prevNodes(1) @@ -862,14 +848,13 @@ object ConcatTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val inputNumber = tfGraph.source.element.getAttrMap.get("N").getI.toInt val nodeaxis = tfGraph.source.prevNodes(inputNumber) val axis = processDims( - nodeaxis.element.getAttrMap.get("value").getTensor.getIntVal(0), dataFormat) + nodeaxis.element.getAttrMap.get("value").getTensor.getIntVal(0)) val nInputDims = 4 JoinTable[T](dimension = axis + 1, nInputDims = -1) @@ -888,8 +873,7 @@ object AddConstTF1 extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val value = tfGraph.source.prevNodes.head.element .getAttrMap.get("value").getTensor.getFloatVal(0) @@ -908,8 +892,7 @@ object AddConstTF2 extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val value = tfGraph.source.prevNodes(1).element @@ -929,8 +912,7 @@ object AddTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { CAddTable[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] @@ -945,8 +927,7 @@ object SoftMaxTF extends TensorflowToBigDL{ override def topology: DirectedGraph[String] = graph override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { SoftMax[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] @@ -965,8 +946,7 @@ object MulTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val mul = Mul[T]() @@ -989,8 +969,7 @@ object ElementWiseMulTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { CMulTable[T]().asInstanceOf[AbstractModule[Activity, Tensor[T], T]] @@ -1009,8 +988,7 @@ object SplitTF extends TensorflowToBigDL { override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val numSplit = tfGraph.source.element.getAttrMap.get("num_split").getI.toInt @@ -1038,8 +1016,7 @@ object PaddingTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val paddings = TensorflowToBigDL.toTensor( @@ -1049,7 +1026,7 @@ object PaddingTF extends TensorflowToBigDL{ for(i <- 1 to paddings.size(1)) { if (paddings.valueAt(i, 1) != 0 || paddings.valueAt(i, 2) != 0 ) { - val dim = processDims(i - 1, dataFormat) + 1 + val dim = processDims(i - 1) + 1 if (paddings(Array(i, 1)) != 0) { padding.add(Padding[T](dim, -ev.toType[Int](paddings.valueAt(i, 1)), 4)) } @@ -1074,8 +1051,7 @@ object MeanTF extends TensorflowToBigDL{ override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], context: Context[T], - byteOrder: ByteOrder, - dataFormat: TensorflowDataFormat)( + byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { val dims = TensorflowToBigDL.toTensor( @@ -1083,7 +1059,7 @@ object MeanTF extends TensorflowToBigDL{ val dim = ArrayBuffer[Int]() val mean = Sequential[T]() for (i <- 1 to dims.size(1)) { - dim += processDims(ev.toType[Int](dims.valueAt(i)), dataFormat) + 1 + dim += processDims(ev.toType[Int](dims.valueAt(i))) + 1 } dim.foreach(i => mean.add(Mean[T](i, squeeze = false))) mean.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index 7331e903bd1..fd51760a453 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -91,12 +91,15 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ } Engine.model.setPoolSize(1) + + System.setProperty("bigdl.enableNHWC", "true") } after { if (sc != null) { sc.stop() } + System.setProperty("bigdl.enableNHWC", "false") } "TensorFlow loader" should "read a list of nodes from pb file" in { @@ -135,7 +138,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val resource = getClass().getClassLoader().getResource("tf") val path = processPath(resource.getPath()) + JFile.separator + "test.pb" val model = TensorflowLoader.load(path, Seq("Placeholder"), Seq("output"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + ByteOrder.LITTLE_ENDIAN) val container = model.asInstanceOf[Graph[Float]] container.modules.length should be(4) RandomGenerator.RNG.setSeed(100) @@ -175,7 +178,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ // Load the model and input/output tensors val modelFile = tmpLocation + s + "model.pb" val model = TensorflowLoader.load(modelFile, Seq("Placeholder"), Seq("output"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + ByteOrder.LITTLE_ENDIAN) val container = model.asInstanceOf[Graph[Float]] val l1 = container.modules(1).asInstanceOf[Linear[Float]] val l2 = container.modules(3).asInstanceOf[Linear[Float]] @@ -201,7 +204,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ // Load the model and input/output tensors val modelFile = tmpLocation + s + "model.pb" val model = TensorflowLoader.load(modelFile, Seq("Placeholder"), Seq("output"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + ByteOrder.LITTLE_ENDIAN) val container = model.asInstanceOf[Graph[Float]] val optimizer = new DistriOptimizer[Float](container, dataSet, new MSECriterion[Float]()) @@ -216,6 +219,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ } "static simple rnn " should "have the same inference result as tensorflow" in { + System.setProperty("bigdl.enableNHWC", "false") tfCheck() val modelName = "rnn" // Generate command and prepare the temp folder @@ -238,7 +242,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val tfGraph = TensorflowLoader.buildTFGraph(results, Seq("output")) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq("output"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) + ByteOrder.LITTLE_ENDIAN) val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN).contiguous() val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) @@ -249,6 +253,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "static lstm rnn " should "have the same inference result as tensorflow" in { tfCheck() + System.setProperty("bigdl.enableNHWC", "false") val modelName = "rnn_lstm" // Generate command and prepare the temp folder val s = JFile.separator @@ -269,7 +274,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val tfGraph = TensorflowLoader.buildTFGraph(results.subList(0, results.size()-1), Seq("output")) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), Seq("output"), - ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NCHW) + ByteOrder.LITTLE_ENDIAN) val input = TensorflowToBigDL.toTensor(results.get(0).getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN).contiguous() val tfResult = TensorflowToBigDL.toTensor(results.get(results.size()-1) @@ -366,7 +371,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val tfNodes = TensorflowLoader.parse(modelFile) val tfGraph = TensorflowLoader.buildTFGraph(tfNodes, endPoints.map(_.split(":")(0))) val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - endPoints.map(_.split(":")(0)), ByteOrder.LITTLE_ENDIAN, TensorflowDataFormat.NHWC) + endPoints.map(_.split(":")(0)), ByteOrder.LITTLE_ENDIAN) import collection.JavaConverters._ // Compare the tensor contents diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala index 7fa3cbc49cf..f1b001cd4df 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala @@ -30,6 +30,14 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { private val logger = Logger.getLogger(getClass) + before { + System.setProperty("bigdl.enableNHWC", "true") + } + + after { + System.setProperty("bigdl.enableNHWC", "false") + } + "ReLU layer" should "be correctly saved" in { val inputTensor = Tensor[Float](T( T(1.0f, 2.0f, 5.0f, 6.0f), @@ -52,7 +60,7 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { T(1.0f, 2.0f, 5.0f), T(-3.0f, -4.0f, -7.0f) )) - test(layer, input, false, TensorflowDataFormat.NHWC, "/biasAdd") should be(true) + test(layer, input, false, "/biasAdd") should be(true) } "AvgPooling" should "be correctly saved" in { @@ -96,66 +104,67 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { } "Squeeze" should "be correctly saved" in { + System.setProperty("bigdl.enableNHWC", "false") val layer = Squeeze(3) val input = Tensor[Float](4, 2, 1, 2).rand() - test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + test(layer, input, false) should be(true) } "CAddTableToTF" should "be correct" in { val layer = CAddTable[Float]() val input1 = Tensor[Float](4, 2, 2).rand() val input2 = Tensor[Float](4, 2, 2).rand() - testMultiInput(layer, Seq(input1, input2), false, TensorflowDataFormat.NCHW) should be(true) + testMultiInput(layer, Seq(input1, input2), false) should be(true) } "CMultToTF" should "be correct" in { val layer = CMulTable[Float]() val input1 = Tensor[Float](4, 2, 2).rand() val input2 = Tensor[Float](4, 2, 2).rand() - testMultiInput(layer, Seq(input1, input2), false, TensorflowDataFormat.NCHW) should be(true) + testMultiInput(layer, Seq(input1, input2), false) should be(true) } "JoinTableToTF" should "be correct" in { val layer = JoinTable[Float](3, -1) val input1 = Tensor[Float](4, 2, 2).rand() val input2 = Tensor[Float](4, 2, 2).rand() - testMultiInput(layer, Seq(input1, input2), false, TensorflowDataFormat.NCHW) should be(true) + testMultiInput(layer, Seq(input1, input2), false) should be(true) } "LogSoftMax" should "be correctly saved" in { val layer = LogSoftMax() val input = Tensor[Float](4, 5).rand() - test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + test(layer, input, false) should be(true) } "SoftMax" should "be correctly saved" in { val layer = SoftMax() val input = Tensor[Float](4, 5).rand() - test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + test(layer, input, false) should be(true) } "Sigmoid" should "be correctly saved" in { val layer = Sigmoid() val input = Tensor[Float](4, 5).rand() - test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + test(layer, input, false) should be(true) } "SpatialConvolution" should "be correctly saved" in { val layer = SpatialConvolution(3, 5, 2, 2) val input = Tensor[Float](4, 3, 5, 5).rand() - test(layer, input, true, TensorflowDataFormat.NHWC, "/biasAdd") should be(true) + test(layer, input, true, "/biasAdd") should be(true) } "Mean" should "be correctly saved" in { val layer = Mean(1, -1, true) val input = Tensor[Float](4, 5).rand() - test(layer, input, false, TensorflowDataFormat.NCHW, "/output") should be(true) + test(layer, input, false, "/output") should be(true) } "Padding" should "be correctly saved" in { val layer = Padding(1, 2, 2) val input = Tensor[Float](4, 5).rand() - test(layer, input, false, TensorflowDataFormat.NCHW, "/output") should be(true) + test(layer, input, false, "/output") should be(true) } "Batch Norm2D" should "be correctly saved" in { @@ -166,26 +175,26 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { layer.runningVar.rand(0.9, 1.1) layer.runningMean.rand() val input = Tensor[Float](3, 2, 4, 5).rand() - test(layer, input, true, TensorflowDataFormat.NHWC, "/output") should be(true) + test(layer, input, true, "/output") should be(true) } "Dropout" should "be correctly saved" in { val layer = Dropout() layer.evaluate() val input = Tensor[Float](3, 2).rand() - test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + test(layer, input, false) should be(true) } "View" should "be correctly saved" in { val layer = View(2, 4) val input = Tensor[Float](2, 2, 2).rand() - test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + test(layer, input, false) should be(true) } "Reshape" should "be correctly saved" in { val layer = Reshape(Array(2, 4)) val input = Tensor[Float](2, 2, 2).rand() - test(layer, input, false, TensorflowDataFormat.NCHW) should be(true) + test(layer, input, false) should be(true) } "lenet" should "be correctly saved" in { @@ -207,7 +216,6 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { Seq(Tensorflow.const(transInput, "input", ByteOrder.LITTLE_ENDIAN)), tmpFile.getPath, ByteOrder.LITTLE_ENDIAN, - TensorflowDataFormat.NHWC, Set(Tensorflow.const(outputData.transpose(2, 3).transpose(3, 4).contiguous(), "target", ByteOrder.LITTLE_ENDIAN)) ) @@ -218,8 +226,6 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { private def test(layer: AbstractModule[Tensor[Float], Tensor[Float], Float], inputTensor: Tensor[Float], convertNHWC: Boolean = false, - // The default is NHWC so we can test on CPU - dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC, outputSuffix: String = "") : Boolean = { tfCheck() val layerNode = layer.setName("output").apply() @@ -243,7 +249,6 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { Seq(Tensorflow.const(tfTensor, "input", ByteOrder.LITTLE_ENDIAN)), tmpFile.getPath, ByteOrder.LITTLE_ENDIAN, - dataFormat, Set(Tensorflow.const(outputSave, "target", ByteOrder.LITTLE_ENDIAN)) ) runPythonSaveTest(tmpFile.getPath, outputSuffix) @@ -252,8 +257,6 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { private def testMultiInput(layer: AbstractModule[Table, Tensor[Float], Float], inputTensors: Seq[Tensor[Float]], convertNHWC: Boolean = false, - // The default is NHWC so we can test on CPU - dataFormat: TensorflowDataFormat = TensorflowDataFormat.NHWC, outputSuffix: String = "") : Boolean = { tfCheck() val layerNode = layer.setName("output").apply() @@ -284,7 +287,6 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { Tensorflow.const(t._1, "input" + t._2, ByteOrder.LITTLE_ENDIAN)), tmpFile.getPath, ByteOrder.LITTLE_ENDIAN, - dataFormat, Set(Tensorflow.const(outputSave, "target", ByteOrder.LITTLE_ENDIAN)) ) runPythonSaveTest(tmpFile.getPath, outputSuffix) From 01837b250f2ff444f322590423ef4718a94da269 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 15 Jun 2017 02:12:01 +0800 Subject: [PATCH 15/23] meet code review --- .../intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala index a5b826e7f1e..be9996ef262 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -121,7 +121,7 @@ object TensorflowToBigDL { require( tfTensor.getDtype == DataType.DT_FLOAT || - tfTensor.getDtype == DataType.DT_FLOAT || + tfTensor.getDtype == DataType.DT_DOUBLE || tfTensor.getDtype == DataType.DT_INT32, s"Data type ${tfTensor.getDtype} is not supported now") @@ -207,7 +207,7 @@ object TensorflowToBigDL { } Tensor(Storage(tmp), 1, shape).asInstanceOf[Tensor[T]] } else { - throw new IllegalArgumentException("Data type ${tfTensor.getDtype} is not supported now") + throw new IllegalArgumentException(s"Data type ${tfTensor.getDtype} is not supported now") } } else { throw new IllegalArgumentException("Only support Float/Double") From 3f8a85844a235424b97dde1e255d23d4cb883fd1 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 15 Jun 2017 17:02:30 +0800 Subject: [PATCH 16/23] meet code review --- .../bigdl/utils/tf/TensorflowLoader.scala | 4 ++-- .../bigdl/utils/tf/TensorflowToBigDL.scala | 20 ++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala index 276f18024f8..7357d41f07b 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala @@ -217,7 +217,7 @@ object TensorflowLoader{ // Normal operation node if (patternToGraph.get(patternNode).isEmpty) return (util.Collections.emptyList(), Seq()) - val graphNode = patternToGraph.get(patternNode).get + val graphNode = patternToGraph(patternNode) // Operation type should match if (patternNode.element != graphNode.element.getOp) return ( util.Collections.emptyList(), Seq()) @@ -251,7 +251,7 @@ object TensorflowLoader{ val posGraph = { if (direction == 0) i else graphNode.prevNodes.length - 1 - j} val pn = patternNode.prevNodes(posPattern) val gn = graphNode.prevNodes(posGraph) - if (patternToGraph.keySet.contains(pn)) { + if (patternToGraph.contains(pn)) { if (!patternToGraph(pn).eq(gn)) return (util.Collections.emptyList(), Seq()) } else { patternToGraph(pn) = gn diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala index be9996ef262..02d6c97e6dc 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -127,6 +127,9 @@ object TensorflowToBigDL { val shape = tfTensor.getTensorShape.getDimList.asScala.map(_.getSize.toInt).toArray + /** + * When there's one element in the tensor. You cannot get the value from byte string + */ if (shape.product == 1) { if (classTag[T] == classTag[Float]) { if (tfTensor.getDtype == DataType.DT_FLOAT) { @@ -220,7 +223,7 @@ object TensorflowToBigDL { res.append( FullConnectionTF, DropoutTF, AvgPoolingTF, MaxPoolingTF, ReshapeTF, InputTF, TanhTF, ReluTF, SigmoidTF, Conv2D, Placeholder, SqueezeTF, IdentityTF, ConcatTF, - BatchNormTF, AddConstTF1, AddConstTF2, AddTF, SoftMaxTF, MulTF, ElementWiseMulTF, + BatchNormTF, AddConstTF1, AddConstTF2, AddTF, SoftMaxTF, ElementWiseMulTF, MulTF, SplitTF, PaddingTF, MeanTF, UnpackTF, StrideSliceTF, ShapeTF, FillTF, PackTF, ConstTF ) res @@ -234,8 +237,19 @@ object TensorflowToBigDL { */ private def sortPattern() : Unit = { // do not calculate size and edges of a graph every time - val topToNNodes = patternList.map(g => g -> g.topology.size).toMap - val topToNEdges = patternList.map(g => g -> g.topology.edges).toMap + val topToNNodes = patternList.map(g => { + val nodeSize = g.topology.BFS.count(n => + n.element != INPUT_PLACEHOLDER && n.element != N_INPUT_PLACEHOLDER) + g -> nodeSize + }).toMap + + val topToNEdges = patternList.map(g => { + val edgeSize = g.topology.BFS.filter(n => + n.element != INPUT_PLACEHOLDER && n.element != N_INPUT_PLACEHOLDER) + .map(_.nextNodes.length).reduce(_ + _) + g -> edgeSize + }).toMap + patternList = patternList.sortWith((l, r) => { if (topToNNodes(l) != topToNNodes(r)) { // graph with more nodes comes first From ca105c1c2eeb0926e26484c761e91731c992e1c6 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 15 Jun 2017 18:06:33 +0800 Subject: [PATCH 17/23] use MulConst in MulTF --- .../com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala index 02d6c97e6dc..745c2f23696 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -963,11 +963,10 @@ object MulTF extends TensorflowToBigDL{ byteOrder: ByteOrder)( implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { - val mul = Mul[T]() val scale = TensorflowToBigDL.toTensor( tfGraph.source.prevNodes(0).element.getAttrMap.get("value").getTensor, byteOrder) require(scale.dim() == 1 && scale.size(1) == 1, s"scale must be one number") - mul.weight.copy(scale) + val mul = MulConstant[T](ev.toType[Double](scale.valueAt(1))) mul.asInstanceOf[AbstractModule[Activity, Tensor[T], T]] } } From 50c8d6c8576d6970e072e7a6a37818c99c5d95ba Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Thu, 15 Jun 2017 19:28:39 +0800 Subject: [PATCH 18/23] add a flatten node for tf 1.1 --- .../bigdl/utils/tf/TensorflowLoader.scala | 4 +- .../bigdl/utils/tf/TensorflowToBigDL.scala | 72 ++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala index 7357d41f07b..6a0b7923dd4 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala @@ -142,7 +142,9 @@ object TensorflowLoader{ } else { val (module, nodes, inputNodes) = extract[T](n.graph(reverse = true), context, byteOrder).getOrElse( - throw new UnsupportedOperationException(s"Can not find matched graph ${n}")) + throw new UnsupportedOperationException(s"Can not find matched graph \n${n}\n\n" + + s"Its inputs are\n ${n.prevNodes.mkString("\n")}") + ) val node = new Node(module) nodes.asScala.foreach(m => { diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala index 745c2f23696..c1820031b47 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -224,7 +224,8 @@ object TensorflowToBigDL { FullConnectionTF, DropoutTF, AvgPoolingTF, MaxPoolingTF, ReshapeTF, InputTF, TanhTF, ReluTF, SigmoidTF, Conv2D, Placeholder, SqueezeTF, IdentityTF, ConcatTF, BatchNormTF, AddConstTF1, AddConstTF2, AddTF, SoftMaxTF, ElementWiseMulTF, MulTF, - SplitTF, PaddingTF, MeanTF, UnpackTF, StrideSliceTF, ShapeTF, FillTF, PackTF, ConstTF + SplitTF, PaddingTF, MeanTF, UnpackTF, StrideSliceTF, ShapeTF, FillTF, PackTF, ConstTF, + Flatten ) res } @@ -876,6 +877,75 @@ object ConcatTF extends TensorflowToBigDL{ } } +object Flatten extends TensorflowToBigDL { + private val graph = { + val reshapeNode = Node("Reshape") + val concatNode = Node("ConcatV2") + val sliceNode = Node("Slice") + val expandNode = Node("ExpandDims") + val prodNode = Node("Prod") + val sliceNode1 = Node("Slice") + val shapeNode = Node("Const") + val beginNode = Node("Const") + val sizeNode = Node("Const") + val beginNode1 = Node("Const") + val sizeNode1 = Node("Const") + val constNode = Node("Const") + val dimNode = Node("Const") + val axisNode = Node("Const") + + shapeNode -> sliceNode + beginNode -> sliceNode + sizeNode -> sliceNode + + shapeNode -> sliceNode1 + beginNode1 -> sliceNode1 + sizeNode1 -> sliceNode1 + + sliceNode1 -> prodNode + constNode -> prodNode + + prodNode -> expandNode + dimNode -> expandNode + + sliceNode -> concatNode + expandNode -> concatNode + axisNode -> concatNode + + Node("*") -> reshapeNode + concatNode -> reshapeNode + reshapeNode.graph(reverse = true) + } + + override def topology: DirectedGraph[String] = graph + + override def layer[T: ClassTag](tfGraph: DirectedGraph[NodeDef], + context: Context[T], + byteOrder: ByteOrder)( + implicit ev: TensorNumeric[T]): AbstractModule[Activity, Tensor[T], T] = { + val shapetfTensor = tfGraph.source.prevNodes(1).prevNodes(0).prevNodes(0).element + .getAttrMap.get("value").getTensor + val sizes = TensorflowToBigDL.toTensor(shapetfTensor, byteOrder) + val batchMode = false + + val arraySize = Array( + ev.toType[Int](sizes.valueAt(1)), + { + var prod = 1 + var i = 2 + while(i <= sizes.nElement()) { + prod = prod * ev.toType[Int](sizes.valueAt(i)) + i = i + 1 + } + prod + } + ) + + Reshape[T](size = arraySize, Some(batchMode)) + .asInstanceOf[AbstractModule[Activity, Tensor[T], T]] + } +} + object AddConstTF1 extends TensorflowToBigDL{ private val graph = { val nodeAdd = Node("Add") From 44b66b8d29a5f6555f7ac4d134ef6c6850e66ee9 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Fri, 16 Jun 2017 14:18:19 +0800 Subject: [PATCH 19/23] fix code style and failed unit test --- .../intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala | 2 +- .../com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala | 2 +- .../intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala | 4 ---- .../intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala | 3 ++- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala index aef0c76ba56..74e736ea835 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/BigDLToTensorflow.scala @@ -315,4 +315,4 @@ object BatchNorm2DToTF extends BigDLToTensorflow { val output = add(mul1, sub, layer.getName() + "/output") Seq(output, sub, mul2, mul1, mul0, offset, scale, mean, sqrtVar, varNode) } -} \ No newline at end of file +} diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala index 6a0b7923dd4..8921b183311 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala @@ -267,4 +267,4 @@ object TensorflowLoader{ import scala.collection.JavaConverters._ return (patternToGraph.valuesIterator.toList.asJava, inputs) } -} \ No newline at end of file +} diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index fd51760a453..806ce294aa1 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -27,11 +27,8 @@ import com.intel.analytics.bigdl.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD -import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat -import scala.sys.process._ -import scala.math._ object TensorflowLoaderSpec { private val data1 = Array(0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.1f) @@ -63,7 +60,6 @@ object TensorflowLoaderSpec { } } -@com.intel.analytics.bigdl.tags.Parallel class TensorflowLoaderSpec extends TensorflowSpecHelper{ Logger.getLogger("org").setLevel(Level.WARN) diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala index f1b001cd4df..d39d5a342dd 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowSaverSpec.scala @@ -198,6 +198,7 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { } "lenet" should "be correctly saved" in { + tfCheck() val conv1 = SpatialConvolution(1, 6, 5, 5).setName("conv1").apply() val tanh1 = Tanh().setName("tanh1").apply(conv1) val pool1 = SpatialMaxPooling(2, 2, 2, 2).setName("pool1").apply(tanh1) @@ -291,4 +292,4 @@ class TensorflowSaverSpec extends TensorflowSpecHelper { ) runPythonSaveTest(tmpFile.getPath, outputSuffix) } -} \ No newline at end of file +} From 886f9123e2d76e502a19c94a226d155b217972f1 Mon Sep 17 00:00:00 2001 From: yangw Date: Tue, 20 Jun 2017 10:46:32 +0800 Subject: [PATCH 20/23] mv tf model layers to another package --- .../main/scala/com/intel/analytics/bigdl/nn/Graph.scala | 1 + .../com/intel/analytics/bigdl/nn/{ => tf}/Const.scala | 8 ++++---- .../com/intel/analytics/bigdl/nn/{ => tf}/Fill.scala | 6 +++--- .../com/intel/analytics/bigdl/nn/{ => tf}/Shape.scala | 8 ++++---- .../analytics/bigdl/nn/{ => tf}/SplitAndSelect.scala | 8 ++++---- .../intel/analytics/bigdl/nn/{ => tf}/StrideSlice.scala | 6 +++--- .../intel/analytics/bigdl/python/api/PythonBigDL.scala | 1 + .../analytics/bigdl/utils/tf/TensorflowToBigDL.scala | 1 + .../com/intel/analytics/bigdl/nn/{ => tf}/ConstSpec.scala | 4 ++-- .../com/intel/analytics/bigdl/nn/{ => tf}/FillSpec.scala | 4 ++-- .../com/intel/analytics/bigdl/nn/{ => tf}/ShapeSpec.scala | 4 ++-- .../analytics/bigdl/nn/{ => tf}/SplitAndSelectSpec.scala | 4 ++-- .../analytics/bigdl/nn/{ => tf}/StrideSliceSpec.scala | 2 +- 13 files changed, 30 insertions(+), 27 deletions(-) rename spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/{ => tf}/Const.scala (91%) rename spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/{ => tf}/Fill.scala (91%) rename spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/{ => tf}/Shape.scala (86%) rename spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/{ => tf}/SplitAndSelect.scala (92%) rename spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/{ => tf}/StrideSlice.scala (92%) rename spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/{ => tf}/ConstSpec.scala (98%) rename spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/{ => tf}/FillSpec.scala (97%) rename spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/{ => tf}/ShapeSpec.scala (97%) rename spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/{ => tf}/SplitAndSelectSpec.scala (97%) rename spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/{ => tf}/StrideSliceSpec.scala (98%) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala index 4779f8b3b2c..bad75142b13 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Graph.scala @@ -17,6 +17,7 @@ package com.intel.analytics.bigdl.nn import com.intel.analytics.bigdl.nn.Graph.ModuleNode import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity, TensorModule} +import com.intel.analytics.bigdl.nn.tf.WithoutInput import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.utils.{Node, T, Table} diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Const.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Const.scala similarity index 91% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Const.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Const.scala index 2788d563c3e..2e46da2c5b3 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Const.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Const.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.tensor.Tensor @@ -22,14 +22,14 @@ import com.intel.analytics.bigdl.utils.{T, Table} import scala.reflect.ClassTag -trait WithoutInput +private[bigdl] trait WithoutInput /** * Return a constant tensor defined by value * @param value the constant tensor to be returned in forward */ @SerialVersionUID(-4008935551091949324L) -class Const[T: ClassTag](value: Tensor[T])(implicit ev: TensorNumeric[T]) +private[bigdl] class Const[T: ClassTag](value: Tensor[T])(implicit ev: TensorNumeric[T]) extends AbstractModule[Activity, Tensor[T], T] with WithoutInput { output = value @@ -61,7 +61,7 @@ class Const[T: ClassTag](value: Tensor[T])(implicit ev: TensorNumeric[T]) } } -object Const { +private[bigdl] object Const { def apply[T: ClassTag](value: Tensor[T]) (implicit ev: TensorNumeric[T]): Const[T] = { new Const[T](value) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Fill.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Fill.scala similarity index 91% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Fill.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Fill.scala index 4e2a65dbe22..295989e1788 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Fill.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Fill.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf import com.intel.analytics.bigdl.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.tensor.Tensor @@ -27,7 +27,7 @@ import scala.reflect.ClassTag * @param value the scalar value to be filled. */ @SerialVersionUID(-471757174144422555L) -class Fill[T: ClassTag](value: T) (implicit ev: TensorNumeric[T]) +private[bigdl] class Fill[T: ClassTag](value: T) (implicit ev: TensorNumeric[T]) extends TensorModule[T] { override def updateOutput(input: Tensor[T]): Tensor[T] = { @@ -44,7 +44,7 @@ class Fill[T: ClassTag](value: T) (implicit ev: TensorNumeric[T]) } -object Fill { +private[bigdl] object Fill { def apply[T: ClassTag](value: Double) (implicit ev: TensorNumeric[T]) : Fill[T] = { new Fill[T](ev.fromType(value)) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Shape.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Shape.scala similarity index 86% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Shape.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Shape.scala index 406564b0247..0a052487652 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Shape.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/Shape.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf -import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, TensorModule} +import com.intel.analytics.bigdl.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric @@ -25,7 +25,7 @@ import scala.reflect.ClassTag * Given input, return the shape of this input as a 1-D tensor */ @SerialVersionUID(-907995771209831179L) -class Shape[T: ClassTag](implicit ev: TensorNumeric[T]) +private[bigdl] class Shape[T: ClassTag](implicit ev: TensorNumeric[T]) extends AbstractModule[Tensor[T], Tensor[T], T] { override def updateOutput(input: Tensor[T]): Tensor[T] = { @@ -40,7 +40,7 @@ class Shape[T: ClassTag](implicit ev: TensorNumeric[T]) } } -object Shape { +private[bigdl] object Shape { def apply[T: ClassTag]()(implicit ev: TensorNumeric[T]): Shape[T] = { new Shape[T]() } diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SplitAndSelect.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/SplitAndSelect.scala similarity index 92% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SplitAndSelect.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/SplitAndSelect.scala index e832e67a63c..2706cdb8a7c 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SplitAndSelect.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/SplitAndSelect.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf import com.intel.analytics.bigdl.nn.abstractnn.TensorModule -import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.tensor.Tensor +import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric import scala.reflect.ClassTag @@ -26,7 +26,7 @@ import scala.reflect.ClassTag * then select the [[index]]th one */ @SerialVersionUID(-9096120159559947483L) -class SplitAndSelect[T: ClassTag](dimension: Int, index: Int, numSplit: Int) +private[bigdl] class SplitAndSelect[T: ClassTag](dimension: Int, index: Int, numSplit: Int) (implicit ev: TensorNumeric[T]) extends TensorModule[T] { override def updateOutput(input: Tensor[T]): Tensor[T] = { @@ -53,7 +53,7 @@ class SplitAndSelect[T: ClassTag](dimension: Int, index: Int, numSplit: Int) } } -object SplitAndSelect { +private[bigdl] object SplitAndSelect { def apply[T: ClassTag]( dimension: Int, index: Int, diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/StrideSlice.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/StrideSlice.scala similarity index 92% rename from spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/StrideSlice.scala rename to spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/StrideSlice.scala index 86ac676ceaa..7338777a889 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/StrideSlice.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/tf/StrideSlice.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf import com.intel.analytics.bigdl.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.tensor.Tensor @@ -26,7 +26,7 @@ import scala.reflect.ClassTag * @param sliceSpecs Array(dim, begin_index, end_index, stride) */ @SerialVersionUID(4436600172725317184L) -class StrideSlice[T: ClassTag](sliceSpecs: Array[(Int, Int, Int, Int)]) +private[bigdl] class StrideSlice[T: ClassTag](sliceSpecs: Array[(Int, Int, Int, Int)]) (implicit ev: TensorNumeric[T]) extends TensorModule[T] { require(sliceSpecs.map(_._4 == 1).reduce(_ && _), "only support stride 1 for now") @@ -57,7 +57,7 @@ class StrideSlice[T: ClassTag](sliceSpecs: Array[(Int, Int, Int, Int)]) } -object StrideSlice { +private[bigdl] object StrideSlice { def apply[T: ClassTag](sliceSpecs: Array[(Int, Int, Int, Int)]) (implicit ev: TensorNumeric[T]) : StrideSlice[T] = { new StrideSlice[T](sliceSpecs) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala index 9a5bebaa46a..2f44b9fff4e 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala @@ -34,6 +34,7 @@ import org.apache.spark.rdd.RDD import java.lang.{Integer, Boolean => JBoolean} import com.intel.analytics.bigdl.nn.Graph._ +import com.intel.analytics.bigdl.nn.tf.{Const, Fill, Shape, SplitAndSelect} import scala.collection.JavaConverters._ import scala.language.existentials diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala index c1820031b47..6605bdfd0b8 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowToBigDL.scala @@ -23,6 +23,7 @@ import com.intel.analytics.bigdl.nn._ import com.intel.analytics.bigdl.tensor.{Storage, Tensor} import org.tensorflow.framework.{DataType, NodeDef, TensorProto} import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity} +import com.intel.analytics.bigdl.nn.tf._ import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.utils.{DirectedGraph, Node, T} import com.intel.analytics.bigdl.utils.tf.TensorflowLoader.Context diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/ConstSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/ConstSpec.scala similarity index 98% rename from spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/ConstSpec.scala rename to spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/ConstSpec.scala index 05f19457982..01d0aefc87e 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/ConstSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/ConstSpec.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf -import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.utils.T +import org.scalatest.{FlatSpec, Matchers} class ConstSpec extends FlatSpec with Matchers { "Const forward tensor" should "be correct" in { diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/FillSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/FillSpec.scala similarity index 97% rename from spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/FillSpec.scala rename to spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/FillSpec.scala index 7712460f215..a874fd17fd8 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/FillSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/FillSpec.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf -import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.utils.T +import org.scalatest.{FlatSpec, Matchers} class FillSpec extends FlatSpec with Matchers { diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/ShapeSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/ShapeSpec.scala similarity index 97% rename from spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/ShapeSpec.scala rename to spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/ShapeSpec.scala index d8bc8d15d37..876beb10745 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/ShapeSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/ShapeSpec.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf -import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.utils.T +import org.scalatest.{FlatSpec, Matchers} class ShapeSpec extends FlatSpec with Matchers { diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/SplitAndSelectSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/SplitAndSelectSpec.scala similarity index 97% rename from spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/SplitAndSelectSpec.scala rename to spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/SplitAndSelectSpec.scala index dec39acf224..75053cf393f 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/SplitAndSelectSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/SplitAndSelectSpec.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf -import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.tensor.Tensor import com.intel.analytics.bigdl.utils.T +import org.scalatest.{FlatSpec, Matchers} class SplitAndSelectSpec extends FlatSpec with Matchers { "SplitAndSelect forward" should "be correct" in { diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/StrideSliceSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/StrideSliceSpec.scala similarity index 98% rename from spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/StrideSliceSpec.scala rename to spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/StrideSliceSpec.scala index 8bfc01afd0d..99fad493310 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/StrideSliceSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/nn/tf/StrideSliceSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.nn +package com.intel.analytics.bigdl.nn.tf import com.intel.analytics.bigdl.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} From 11f6ad48d64e8c075c4ff460660df3871024c954 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Tue, 20 Jun 2017 16:09:34 +0800 Subject: [PATCH 21/23] add a python example to show how to define model in tensorflow and run in BigDL --- pyspark/bigdl/models/tf_example/tf_example.py | 44 ++++++++++ pyspark/bigdl/nn/layer.py | 10 +++ pyspark/bigdl/util/tf_utils.py | 88 +++++++++++++++++++ .../com/intel/analytics/bigdl/nn/Module.scala | 5 +- .../bigdl/python/api/PythonBigDL.scala | 11 +++ 5 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 pyspark/bigdl/models/tf_example/tf_example.py create mode 100644 pyspark/bigdl/util/tf_utils.py diff --git a/pyspark/bigdl/models/tf_example/tf_example.py b/pyspark/bigdl/models/tf_example/tf_example.py new file mode 100644 index 00000000000..f291819cc92 --- /dev/null +++ b/pyspark/bigdl/models/tf_example/tf_example.py @@ -0,0 +1,44 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tensorflow as tf +import numpy as np +from bigdl.util.tf_utils import convert + +def main(): + input = tf.placeholder(tf.float32, [None, 5]) + weight = tf.Variable(tf.random_uniform([5, 10])) + bias = tf.Variable(tf.random_uniform([10])) + middle = tf.nn.bias_add(tf.matmul(input, weight), bias) + output= tf.nn.tanh(middle) + + tensor = np.random.rand(5, 5) + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + tensorflow_result = sess.run(output, {input: tensor}) + bigdl_model = convert([input], [output], sess) + bigdl_result = bigdl_model.forward(tensor)[0] + + print("Tensorflow forward result is " + str(tensorflow_result)) + print("BigDL forward result is " + str(bigdl_result)) + + np.testing.assert_almost_equal(tensorflow_result, bigdl_result, 6) + print("The results are almost equal in 6 decimals") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/pyspark/bigdl/nn/layer.py b/pyspark/bigdl/nn/layer.py index a69764a6d03..a6509ee1a46 100644 --- a/pyspark/bigdl/nn/layer.py +++ b/pyspark/bigdl/nn/layer.py @@ -407,6 +407,16 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) return Layer.of(jmodel) + @staticmethod + def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_type="float"): + """ + Load a pre-trained Tensorflow model. + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) + return Model.of(jmodel) + class Linear(Layer): diff --git a/pyspark/bigdl/util/tf_utils.py b/pyspark/bigdl/util/tf_utils.py new file mode 100644 index 00000000000..ac57a52d862 --- /dev/null +++ b/pyspark/bigdl/util/tf_utils.py @@ -0,0 +1,88 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tempfile + +import tensorflow as tf + +from google.protobuf import text_format + +from tensorflow.core.framework import graph_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import graph_util +from tensorflow.python.framework import importer +from tensorflow.python.platform import gfile + +from bigdl.nn.layer import Model + +def convert(input_ops, output_ops, sess): + """ + Convert tensorflow model to bigdl model + :param input_ops: operation list used for input, should be placeholders + :param output_ops: operations list used for output + :param sess: current tensorflow session + :return: bigdl model + """ + input_names = map(lambda x: x.name.split(":")[0], input_ops) + output_names = map(lambda x: x.name.split(":")[0], output_ops) + temp = tempfile.mkdtemp() + + saver = tf.train.Saver() + saver.save(sess, temp + '/model.chkp') + tf.train.write_graph(sess.graph, temp, 'model.pbtxt') + + merge_checkpoint(temp + '/model.pbtxt', + temp + '/model.chkp', + output_names, + temp + '/model.pb', sess) + return Model.load_tensorflow(temp + '/model.pb', input_names, output_names) + +def merge_checkpoint(input_graph, + checkpoint, + output_node_names, + output_graph, + sess): + """ + Get the variable values from the checkpoint file, and merge them to the GraphDef file + Args: + input_graph: the GraphDef file, doesn't contain variable values + checkpoint: the checkpoint file + output_node_names: A list of string, the output names + output_graph: String of the location and the name of the + output graph + """ + restore_op_name = "save/restore_all" + filename_tensor_name = "save/Const:0" + + input_graph_def = graph_pb2.GraphDef() + with gfile.FastGFile(input_graph, "r") as f: + text_format.Merge(f.read().decode("utf-8"), input_graph_def) + + for node in input_graph_def.node: + node.device = "" + + importer.import_graph_def(input_graph_def, name="") + + sess.run([restore_op_name], {filename_tensor_name: checkpoint}) + output_graph_def = graph_util.convert_variables_to_constants( + sess, + input_graph_def, + output_node_names, + variable_names_blacklist="" + ) + + with gfile.GFile(output_graph, "wb") as f: + f.write(output_graph_def.SerializeToString()) \ No newline at end of file diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala index 1fbecb27f58..294401b2b7b 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Module.scala @@ -48,13 +48,10 @@ object Module { * @param inputs input node names * @param outputs output node names, the output tensor order is same with the node order * @param byteOrder byte order in the tensorflow file. The default value is little endian - * @param dataFormat the model dataFormat. Note that BigDL only support NCHW, so for NHWC - * tensorflow model, the model define will be converted to NCHW * @return BigDL model */ def loadTF[T: ClassTag](file: String, inputs: Seq[String], outputs: Seq[String], - byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN, - dataFormat: TensorflowDataFormat = TensorflowDataFormat.NCHW)( + byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN)( implicit ev: TensorNumeric[T]): Module[T] = { TensorflowLoader.load(file, inputs, outputs, byteOrder) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala index 2f44b9fff4e..d3489da0f3f 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/python/api/PythonBigDL.scala @@ -32,6 +32,7 @@ import com.intel.analytics.bigdl.nn.Zeros import org.apache.spark.api.java.JavaRDD import org.apache.spark.rdd.RDD import java.lang.{Integer, Boolean => JBoolean} +import java.nio.ByteOrder import com.intel.analytics.bigdl.nn.Graph._ import com.intel.analytics.bigdl.nn.tf.{Const, Fill, Shape, SplitAndSelect} @@ -1281,6 +1282,16 @@ class PythonBigDL[T: ClassTag](implicit ev: TensorNumeric[T]) extends Serializab Module.loadCaffe[T](model, defPath, modelPath, matchAll) } + def loadTF(path: String, inputs: JList[String], outputs: JList[String], + byteOrder: String): AbstractModule[Activity, Activity, T] = { + val order = byteOrder match { + case "little_endian" => ByteOrder.LITTLE_ENDIAN + case "big_endian" => ByteOrder.BIG_ENDIAN + case _ => throw new IllegalArgumentException(s"No support byte order $byteOrder") + } + Module.loadTF[T](path, inputs.asScala, outputs.asScala, order) + } + def modelPredictRDD(model: AbstractModule[Activity, Activity, T], dataRdd: JavaRDD[Sample]): JavaRDD[JTensor] = { val tensorRDD = model.predict(dataRdd.rdd.map(toSample(_))) From 82bba8d09b0e5b3196166107d45d84c1db5edf00 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Tue, 20 Jun 2017 16:29:23 +0800 Subject: [PATCH 22/23] move tf example python code to example folder --- .../tf_example => example}/tf_example.py | 2 +- .../commands/run-tf-example.sh | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) rename pyspark/{bigdl/models/tf_example => example}/tf_example.py (96%) create mode 100755 pyspark/test/local_integration/commands/run-tf-example.sh diff --git a/pyspark/bigdl/models/tf_example/tf_example.py b/pyspark/example/tf_example.py similarity index 96% rename from pyspark/bigdl/models/tf_example/tf_example.py rename to pyspark/example/tf_example.py index f291819cc92..b341d0adab6 100644 --- a/pyspark/bigdl/models/tf_example/tf_example.py +++ b/pyspark/example/tf_example.py @@ -31,7 +31,7 @@ def main(): sess.run(init) tensorflow_result = sess.run(output, {input: tensor}) bigdl_model = convert([input], [output], sess) - bigdl_result = bigdl_model.forward(tensor)[0] + bigdl_result = bigdl_model.forward(tensor) print("Tensorflow forward result is " + str(tensorflow_result)) print("BigDL forward result is " + str(bigdl_result)) diff --git a/pyspark/test/local_integration/commands/run-tf-example.sh b/pyspark/test/local_integration/commands/run-tf-example.sh new file mode 100755 index 00000000000..66f331a929b --- /dev/null +++ b/pyspark/test/local_integration/commands/run-tf-example.sh @@ -0,0 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "`dirname $0`" + +$PYTHON_EXECUTABLE $BIGDL_HOME/pyspark/example/tf_example.py \ No newline at end of file From ce5905e774bfb1fe70b3c8fd05a091eea0a4c0b0 Mon Sep 17 00:00:00 2001 From: Fu Zhouwang Date: Wed, 21 Jun 2017 09:58:04 +0800 Subject: [PATCH 23/23] Add backward test in LeNet and AlexNet (#19) * add unit test of lenet backward * add some print * add backward test in lenet and alexnet * seperate testModel into forward and backward methods --- .../bigdl/utils/tf/TensorflowLoader.scala | 12 +- .../src/test/resources/tf/models/alexnet.py | 2 +- .../dl/src/test/resources/tf/models/lenet.py | 2 +- spark/dl/src/test/resources/tf/models/util.py | 33 +++- .../bigdl/utils/tf/TensorflowLoaderSpec.scala | 170 ++++++++++++++++-- 5 files changed, 196 insertions(+), 23 deletions(-) diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala index 8921b183311..f56c0d6dbc2 100644 --- a/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala +++ b/spark/dl/src/main/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoader.scala @@ -100,7 +100,12 @@ object TensorflowLoader{ // Connect nodes name2Node.valuesIterator.foreach(n => { - n.element.getInputList.asScala.foreach(name2Node(_) -> n) + n.element.getInputList.asScala.foreach{ + input => + // It is tricky here, remove the first char in the name of control dep node + val name = if (input.charAt(0) == '^') input.substring(1) else input + name2Node(name) -> n + } }) // Build graph @@ -122,7 +127,8 @@ object TensorflowLoader{ tfGraph: DirectedGraph[NodeDef], inputs: Seq[String], outputs: Seq[String], - byteOrder: ByteOrder + byteOrder: ByteOrder, + ctx: Option[Context[T]] = None )(implicit ev: TensorNumeric[T]): Module[T] = { import scala.collection.JavaConverters._ @@ -131,7 +137,7 @@ object TensorflowLoader{ Node[AbstractModule[Activity, Tensor[T], T]]]() val nameToNode = new mutable.HashMap[String, Node[AbstractModule[Activity, Tensor[T], T]]]() - val context = new mutable.HashMap[NodeDef, (Tensor[T], Tensor[T])] + val context = ctx.getOrElse(new mutable.HashMap[NodeDef, (Tensor[T], Tensor[T])]) // BFS to keep the input order same tfGraph.BFS.foreach(n => { diff --git a/spark/dl/src/test/resources/tf/models/alexnet.py b/spark/dl/src/test/resources/tf/models/alexnet.py index 7759f1f5687..24e228072c8 100644 --- a/spark/dl/src/test/resources/tf/models/alexnet.py +++ b/spark/dl/src/test/resources/tf/models/alexnet.py @@ -33,7 +33,7 @@ def main(): for n in end_points: print(n + " => " + str(end_points[n])) net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) - run_model(net_outputs, argv[1]) + run_model(net_outputs, argv[1], 'alexnet') if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/lenet.py b/spark/dl/src/test/resources/tf/models/lenet.py index afc6770e9b6..f42d15f4d86 100644 --- a/spark/dl/src/test/resources/tf/models/lenet.py +++ b/spark/dl/src/test/resources/tf/models/lenet.py @@ -33,7 +33,7 @@ def main(): for n in end_points: print(n + " => " + str(end_points[n])) net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split()) - run_model(net_outputs, argv[1]) + run_model(net_outputs, argv[1], 'LeNet') if __name__ == "__main__": main() diff --git a/spark/dl/src/test/resources/tf/models/util.py b/spark/dl/src/test/resources/tf/models/util.py index 99c64efb44e..0581d8bda05 100644 --- a/spark/dl/src/test/resources/tf/models/util.py +++ b/spark/dl/src/test/resources/tf/models/util.py @@ -57,14 +57,36 @@ def merge_checkpoint(input_graph, with gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) -def run_model(end_points, output_path): +def run_model(end_points, output_path, model_scope=None): outputs = [] results = [] + grad_inputs = [] + grad_vars = [] + grad_results = [] + trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=model_scope) i = 0 + opt = tf.train.GradientDescentOptimizer(0.01) for end_point in end_points: output = tf.Variable(tf.random_uniform(tf.shape(end_point)), name='output' + str(i)) outputs.append(output) results.append(tf.assign(output, end_point, name = 'assign' + str(i))) + + # set up backward variables + # filter None tensor + tmp_vars = filter(lambda x: tf.gradients(end_point, x) != [None], trainable_vars) + # set up random gradient input + grad_input = tf.Variable(tf.random_uniform(tf.shape(end_point)), name='grad_input' + str(i)) + grad_inputs.append(grad_input) + # compute gradients with random input + backward = opt.compute_gradients(end_point, var_list=tmp_vars, grad_loss=grad_input) + j = 0 + for gradients, tensor in backward: + grad_var = tf.Variable(tf.random_uniform(tf.shape(tensor)), + name='{}_grad{}'.format(tensor.name[:-2], i)) + grad_vars.append(grad_var) + grad_result = tf.assign(grad_var, gradients, name='grad_assign' + str((i+1)*j)) + grad_results.append(grad_result) + j = j + 1 i = i + 1 saver = tf.train.Saver() @@ -72,12 +94,19 @@ def run_model(end_points, output_path): init = tf.global_variables_initializer() sess.run(init) sess.run(results) + sess.run(grad_results) saver.save(sess, output_path + '/model.chkp') tf.train.write_graph(sess.graph, output_path, 'model.pbtxt') + # tf.summary.FileWriter(output_path + '/log', sess.graph) input_graph = output_path + "/model.pbtxt" input_checkpoint = output_path + "/model.chkp" output_file = output_path + "/model.pb" - merge_checkpoint(input_graph, input_checkpoint, map(lambda x: 'assign' + str(x), range(len(end_points))), output_file) + output_nodes = map(lambda x: 'assign' + str(x), range(len(end_points))) + grades_nodes = map(lambda x: 'grad_assign' + str(x), range(len(grad_results))) + output_nodes.extend(grades_nodes) + + # merge_checkpoint(input_graph, input_checkpoint, map(lambda x: 'assign' + str(x), range(len(end_points))), output_file) + merge_checkpoint(input_graph, input_checkpoint, output_nodes, output_file) diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala index 806ce294aa1..9cfe7b6db97 100644 --- a/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala +++ b/spark/dl/src/test/scala/com/intel/analytics/bigdl/utils/tf/TensorflowLoaderSpec.scala @@ -28,7 +28,12 @@ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD import com.intel.analytics.bigdl.numeric.NumericFloat +import org.tensorflow.framework.NodeDef +import scala.collection.mutable +import scala.sys.process._ +import scala.math._ +import scala.reflect.ClassTag object TensorflowLoaderSpec { private val data1 = Array(0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.1f) @@ -280,64 +285,86 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ } "Tensorflow lenet" should "be load correctly" in { - testModel("lenet", Seq("LeNet/pool2/MaxPool:0"), true).foreach { + testModelForward("lenet", Seq("LeNet/pool2/MaxPool:0"), true).foreach { case(tf, bigdl) => - val transposed = bigdl.transpose(2, 3).transpose(3, 4) - tf.almostEqual(transposed, 1e-6) should be(true) + val transpose = bigdl.transpose(2, 3).transpose(3, 4) + tf.almostEqual(transpose, 1e-6) should be(true) + } + testModelBackward("lenet", Seq("LeNet/pool2/MaxPool:0"), true, + Seq((4, 3), (3, 2))).foreach { + case(tf, bigdl) => + if (tf.dim() == 4) { + val trans = tf.transpose(1, 4).transpose(2, 3).transpose(3, 4).contiguous() + trans.almostEqual(bigdl, 1e-4) should be(true) + } + else { + tf.almostEqual(bigdl, 1e-4) should be(true) + } } } "Tensorflow Alexnet" should "be load correctly" in { - testModel("alexnet", Seq("alexnet_v2/fc8/squeezed:0"), true).foreach { + testModelForward("alexnet", Seq("alexnet_v2/fc8/squeezed:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-7) should be(true) } + testModelBackward("alexnet", Seq("alexnet_v2/fc8/squeezed:0"), true, + Seq.empty).foreach { + case(tf, bigdl) => + if (tf.dim() == 4) { + val trans = tf.transpose(1, 4).transpose(2, 3).transpose(3, 4).contiguous() + trans.almostEqual(bigdl, 1e-4) should be(true) + } + else { + tf.almostEqual(bigdl, 1e-4) should be(true) + } + } } "TensorFlow vgg_a" should "be load correctly" in { - testModel("vgga", Seq("vgg_a/fc8/squeezed:0"), true).foreach { + testModelForward("vgga", Seq("vgg_a/fc8/squeezed:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-7) should be(true) } } "TensorFlow vgg_16" should "be load correctly" in { - testModel("vgg16", Seq("vgg_16/fc8/squeezed:0"), true).foreach { + testModelForward("vgg16", Seq("vgg_16/fc8/squeezed:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-7) should be(true) } } "TensorFlow vgg_19" should "be load correctly" in { - testModel("vgg19", Seq("vgg_19/fc8/squeezed:0"), true).foreach { + testModelForward("vgg19", Seq("vgg_19/fc8/squeezed:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-7) should be(true) } } "TensorFlow overfeat" should "be load correctly" in { - testModel("overfeat", Seq("overfeat/fc8/squeezed:0"), true).foreach { + testModelForward("overfeat", Seq("overfeat/fc8/squeezed:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-7) should be(true) } } "TensorFlow inception_v3" should "be load correctly" in { - testModel("inception_v3", Seq("InceptionV3/Logits/SpatialSqueeze:0"), true).foreach { + testModelForward("inception_v3", Seq("InceptionV3/Logits/SpatialSqueeze:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-7) should be(true) } } "TensorFlow resnet_v1" should "be load correctly" in { - testModel("resnet_v1", Seq("resnet_v1_101/SpatialSqueeze:0"), true).foreach { + testModelForward("resnet_v1", Seq("resnet_v1_101/SpatialSqueeze:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-6) should be(true) } } "TensorFlow inception_resnet_v2" should "be load correctly" in { - testModel("inception_resnet_v2", Seq("InceptionResnetV2/Logits/Logits/BiasAdd:0", + testModelForward("inception_resnet_v2", Seq("InceptionResnetV2/Logits/Logits/BiasAdd:0", "InceptionResnetV2/AuxLogits/Logits/BiasAdd:0"), true).foreach { case(tf, bigdl) => tf.almostEqual(bigdl, 1e-7) should be(true) @@ -345,7 +372,7 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ } - private def testModel(modelName: String, endPoints: Seq[String], transInput: Boolean) + private def testModelForward(modelName: String, endPoints: Seq[String], transInput: Boolean) : Seq[(Tensor[Float], Tensor[Float])] = { tfCheck() @@ -363,13 +390,16 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ "error when run the model script") // Load the model and input/output tensors + import collection.JavaConverters._ val modelFile = tmpLocation + s + "model.pb" val tfNodes = TensorflowLoader.parse(modelFile) + + // filter node for gradient computing val tfGraph = TensorflowLoader.buildTFGraph(tfNodes, endPoints.map(_.split(":")(0))) + val context = new mutable.HashMap[NodeDef, (Tensor[Float], Tensor[Float])] val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), - endPoints.map(_.split(":")(0)), ByteOrder.LITTLE_ENDIAN) + endPoints.map(_.split(":")(0)), ByteOrder.LITTLE_ENDIAN, Some(context)) - import collection.JavaConverters._ // Compare the tensor contents val tfInputTensor = tfNodes.asScala.filter(_.getName == "input")(0) .getAttrMap.get("value").getTensor @@ -391,10 +421,118 @@ class TensorflowLoaderSpec extends TensorflowSpecHelper{ val t = model.forward(transposeInput).toTable (1 to endPoints.length).map(t[Tensor[Float]](_)) } - tfOutputTensors.zip(bigdlOutputs).map(x => - (TensorflowToBigDL.toTensor(x._1, ByteOrder.LITTLE_ENDIAN), x._2)) + + val comparePair = tfOutputTensors.zip(bigdlOutputs).map{ + x => + val tensor = TensorflowToBigDL.toTensor(x._1, ByteOrder.LITTLE_ENDIAN) + (tensor, x._2) + } + tmpLocation.deleteOnExit() + comparePair + } + + private def testModelBackward( + modelName: String, + endPoints: Seq[String], + transInput: Boolean, + transOutputSeq: Seq[(Int, Int)]): Seq[(Tensor[Float], Tensor[Float])] = { + + tfCheck() + // Generate command and prepare the temp folder + val s = JFile.separator + val modelsFolder = processPath(getClass().getClassLoader().getResource("tf").getPath()) + + s + "models" + val modelScript = modelsFolder + s + s"$modelName.py" + val tmpLocation = java.io.File.createTempFile("tensorflowLoaderTest" + UUID.randomUUID(), + modelName) + tmpLocation.delete() + tmpLocation.mkdir() + + require(runPython(s"$modelScript $tmpLocation ${endPoints.mkString(",")}"), + "error when run the model script") + + // Load the model and input/output tensors + import collection.JavaConverters._ + val modelFile = tmpLocation + s + "model.pb" + val tfNodes = TensorflowLoader.parse(modelFile) + + // filter node for gradient computing + val tfGraph = TensorflowLoader.buildTFGraph(tfNodes, endPoints.map(_.split(":")(0))) + val context = new mutable.HashMap[NodeDef, (Tensor[Float], Tensor[Float])] + val model = TensorflowLoader.buildBigDLModel(tfGraph, Seq("input"), + endPoints.map(_.split(":")(0)), ByteOrder.LITTLE_ENDIAN, Some(context)) + + // Compare the tensor contents + val tfInputTensor = tfNodes.asScala.filter(_.getName == "input")(0) + .getAttrMap.get("value").getTensor + + val input = TensorflowToBigDL.toTensor(tfInputTensor, + ByteOrder.LITTLE_ENDIAN) + + val transposeInput = if (transInput) { + input.transpose(2, 4).transpose(3, 4).contiguous() + } else { + input + } + + val bigdlOutputs = if (endPoints.length == 1) { + Seq(model.forward(transposeInput).toTensor) + } else { + val t = model.forward(transposeInput).toTable + (1 to endPoints.length).map(t[Tensor[Float]](_)) + } + + // get gradient input of tensorflow + val gradInputs = (0 until endPoints.length).map{ + i => + val t = tfNodes.asScala.filter(_.getName == s"grad_input$i")(0) + .getAttrMap.get("value").getTensor + var tensor = TensorflowToBigDL.toTensor(t, ByteOrder.LITTLE_ENDIAN) + for (trans <- transOutputSeq) { + tensor = tensor.transpose(trans._1, trans._2) + } + tensor.contiguous() + } + + // check shape equality here + for (i <- 0 until endPoints.length) { + bigdlOutputs(i).size() should be(gradInputs(i).size()) + } + + // find all gradients tensor in tensorflow graph + val tfGradTensorsMap = context.keySet.map{ + node => + val t = tfNodes.asScala.filter(_.getName.contains(node.getName + "_grad"))(0) + t.getName -> + TensorflowToBigDL.toTensor(t.getAttrMap.get("value").getTensor, ByteOrder.LITTLE_ENDIAN) + }.toMap + + + val comparePair = new mutable.ArrayBuffer[(Tensor[Float], Tensor[Float])]() + + // do backward for each output and its corresponding gradient input + for (i <- 0 until gradInputs.length) { + // println(s"grad $i") + model.backward(transposeInput, gradInputs(i)) + val pairs = context.keySet.map{ + x => + val name = s"${x.getName}_grad$i" + // if (tfGradTensorsMap.contains(name)) { + // println(x.getName) + // context(x)._2.size().foreach(println(_)) + // println(name) + // tfGradTensorsMap(name).size().foreach(println(_)) + // } + (tfGradTensorsMap.get(name).getOrElse(null), context(x)._2) + }.toSeq.filter(_._2 != null) + comparePair ++= pairs + } + println(s"Compare ${comparePair.length} pairs of gradient vars in this graph") + tmpLocation.deleteOnExit() + comparePair } + private def processPath(path: String): String = { if (path.contains(":")) { path.substring(1)