TensorFlow: conv improvements, label_image example, and

a few other changes.

Changes:
- Some improvements to convolution by using 32-bit indices by
  @benoitsteiner. Not all calls converted yet.  Also some
  improvements to pooling as well by @benoitsteiner.

- Improvements to sparse matmul CPU implementation by Ashish

- Some fixes to warnings by @vrv

- Doc fixes to padding by @Yangqing

- Some improvements to Tensor wrappers by Eider

- Speed up of matrix inverse on CPU by Rasmus

- Add an example of doing image inference from a pre-trained model
  by @petewarden.

- fixed formula in mnist example by nodir

- Updates to event accumulator by Cassandra

- Slight changes to tensor c api by @mrry

- Handling of strings in listdiff by Phil

- Fix negative fraction-of-queue-full stats by Frank

- Type-checking improvement to importer by Yaroslav

- logdir recursive search for Tensorboard by @danmane

- Session.run() checks for empty graph by Manoj

Base CL: 108013706

tensorflow/core/client/tensor_c_api.cc

@@ -141,9 +141,9 @@ void TF_SetTarget(TF_SessionOptions* options, const char* target) {
   options->options.target = target;
 }
 
-void TF_SetConfig(TF_SessionOptions* options, const char* config,
-                  size_t config_len, TF_Status* status) {
-  if (!options->options.config.ParseFromArray(config, config_len)) {
+void TF_SetConfig(TF_SessionOptions* options, const void* proto,
+                  size_t proto_len, TF_Status* status) {
+  if (!options->options.config.ParseFromArray(proto, proto_len)) {
     status->status =
         tensorflow::errors::InvalidArgument("Unparseable ConfigProto");
   }

tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc

@@ -30,7 +30,7 @@ TEST(GPUBFCAllocatorTest, NoDups) {
   std::sort(ptrs.begin(), ptrs.end());
 
   // Make sure none of them are equal, and that none of them overlap.
-  for (int i = 0; i < ptrs.size(); i++) {
+  for (size_t i = 0; i < ptrs.size(); i++) {
     if (i > 0) {
       ASSERT_NE(ptrs[i], ptrs[i - 1]);  // No dups
       size_t req_size = a.RequestedSize(ptrs[i - 1]);
@@ -40,7 +40,7 @@ TEST(GPUBFCAllocatorTest, NoDups) {
     }
   }
 
-  for (int i = 0; i < ptrs.size(); i++) {
+  for (size_t i = 0; i < ptrs.size(); i++) {
     a.DeallocateRaw(ptrs[i]);
   }
 }
@@ -63,7 +63,7 @@ TEST(GPUBFCAllocatorTest, AllocationsAndDeallocations) {
 
   // Deallocate half of the memory, and keep track of the others.
   std::vector<void*> existing_ptrs;
-  for (int i = 0; i < initial_ptrs.size(); i++) {
+  for (size_t i = 0; i < initial_ptrs.size(); i++) {
     if (i % 2 == 1) {
       a.DeallocateRaw(initial_ptrs[i]);
     } else {
@@ -81,7 +81,7 @@ TEST(GPUBFCAllocatorTest, AllocationsAndDeallocations) {
 
   std::sort(existing_ptrs.begin(), existing_ptrs.end());
   // Make sure none of them are equal
-  for (int i = 0; i < existing_ptrs.size(); i++) {
+  for (size_t i = 0; i < existing_ptrs.size(); i++) {
     if (i > 0) {
       CHECK_NE(existing_ptrs[i], existing_ptrs[i - 1]);  // No dups
 
@@ -95,7 +95,7 @@ TEST(GPUBFCAllocatorTest, AllocationsAndDeallocations) {
     }
   }
 
-  for (int i = 0; i < existing_ptrs.size(); i++) {
+  for (size_t i = 0; i < existing_ptrs.size(); i++) {
     a.DeallocateRaw(existing_ptrs[i]);
   }
 }

tensorflow/core/framework/tensor_types.h

@@ -6,66 +6,73 @@
 namespace tensorflow {
 
 // Helper to define Tensor types given that the scalar is of type T.
-template <typename T, int NDIMS = 1>
+template <typename T, int NDIMS = 1, typename IndexType = Eigen::DenseIndex>
 struct TTypes {
   // Rank-<NDIMS> tensor of scalar type T.
-  typedef Eigen::TensorMap<Eigen::Tensor<T, NDIMS, Eigen::RowMajor>,
+  typedef Eigen::TensorMap<Eigen::Tensor<T, NDIMS, Eigen::RowMajor, IndexType>,
                            Eigen::Aligned> Tensor;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, NDIMS, Eigen::RowMajor>,
-                           Eigen::Aligned> ConstTensor;
+  typedef Eigen::TensorMap<
+      Eigen::Tensor<const T, NDIMS, Eigen::RowMajor, IndexType>, Eigen::Aligned>
+      ConstTensor;
 
   // Unaligned Rank-<NDIMS> tensor of scalar type T.
-  typedef Eigen::TensorMap<Eigen::Tensor<T, NDIMS, Eigen::RowMajor> >
+  typedef Eigen::TensorMap<Eigen::Tensor<T, NDIMS, Eigen::RowMajor, IndexType> >
       UnalignedTensor;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, NDIMS, Eigen::RowMajor> >
-      UnalignedConstTensor;
+  typedef Eigen::TensorMap<Eigen::Tensor<const T, NDIMS, Eigen::RowMajor,
+                                         IndexType> > UnalignedConstTensor;
 
   typedef Eigen::TensorMap<Eigen::Tensor<T, NDIMS, Eigen::RowMajor, int>,
                            Eigen::Aligned> Tensor32Bit;
 
   // Scalar tensor (implemented as a rank-0 tensor) of scalar type T.
   typedef Eigen::TensorMap<
-      Eigen::TensorFixedSize<T, Eigen::Sizes<>, Eigen::RowMajor>,
+      Eigen::TensorFixedSize<T, Eigen::Sizes<>, Eigen::RowMajor, IndexType>,
       Eigen::Aligned> Scalar;
-  typedef Eigen::TensorMap<
-      Eigen::TensorFixedSize<const T, Eigen::Sizes<>, Eigen::RowMajor>,
-      Eigen::Aligned> ConstScalar;
+  typedef Eigen::TensorMap<Eigen::TensorFixedSize<const T, Eigen::Sizes<>,
+                                                  Eigen::RowMajor, IndexType>,
+                           Eigen::Aligned> ConstScalar;
 
   // Unaligned Scalar tensor of scalar type T.
   typedef Eigen::TensorMap<Eigen::TensorFixedSize<
-      T, Eigen::Sizes<>, Eigen::RowMajor> > UnalignedScalar;
-  typedef Eigen::TensorMap<Eigen::TensorFixedSize<
-      const T, Eigen::Sizes<>, Eigen::RowMajor> > UnalignedConstScalar;
+      T, Eigen::Sizes<>, Eigen::RowMajor, IndexType> > UnalignedScalar;
+  typedef Eigen::TensorMap<Eigen::TensorFixedSize<const T, Eigen::Sizes<>,
+                                                  Eigen::RowMajor, IndexType> >
+      UnalignedConstScalar;
 
   // Rank-1 tensor (vector) of scalar type T.
-  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor>, Eigen::Aligned>
-      Flat;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, 1, Eigen::RowMajor>,
-                           Eigen::Aligned> ConstFlat;
-  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor>, Eigen::Aligned>
-      Vec;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, 1, Eigen::RowMajor>,
-                           Eigen::Aligned> ConstVec;
+  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor, IndexType>,
+                           Eigen::Aligned> Flat;
+  typedef Eigen::TensorMap<
+      Eigen::Tensor<const T, 1, Eigen::RowMajor, IndexType>, Eigen::Aligned>
+      ConstFlat;
+  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor, IndexType>,
+                           Eigen::Aligned> Vec;
+  typedef Eigen::TensorMap<
+      Eigen::Tensor<const T, 1, Eigen::RowMajor, IndexType>, Eigen::Aligned>
+      ConstVec;
 
   // Unaligned Rank-1 tensor (vector) of scalar type T.
-  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor> > UnalignedFlat;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, 1, Eigen::RowMajor> >
-      UnalignedConstFlat;
-  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor> > UnalignedVec;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, 1, Eigen::RowMajor> >
-      UnalignedConstVec;
+  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor, IndexType> >
+      UnalignedFlat;
+  typedef Eigen::TensorMap<Eigen::Tensor<const T, 1, Eigen::RowMajor,
+                                         IndexType> > UnalignedConstFlat;
+  typedef Eigen::TensorMap<Eigen::Tensor<T, 1, Eigen::RowMajor, IndexType> >
+      UnalignedVec;
+  typedef Eigen::TensorMap<
+      Eigen::Tensor<const T, 1, Eigen::RowMajor, IndexType> > UnalignedConstVec;
 
   // Rank-2 tensor (matrix) of scalar type T.
-  typedef Eigen::TensorMap<Eigen::Tensor<T, 2, Eigen::RowMajor>, Eigen::Aligned>
-      Matrix;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, 2, Eigen::RowMajor>,
-                           Eigen::Aligned> ConstMatrix;
+  typedef Eigen::TensorMap<Eigen::Tensor<T, 2, Eigen::RowMajor, IndexType>,
+                           Eigen::Aligned> Matrix;
+  typedef Eigen::TensorMap<
+      Eigen::Tensor<const T, 2, Eigen::RowMajor, IndexType>, Eigen::Aligned>
+      ConstMatrix;
 
   // Unaligned Rank-2 tensor (matrix) of scalar type T.
-  typedef Eigen::TensorMap<Eigen::Tensor<T, 2, Eigen::RowMajor> >
+  typedef Eigen::TensorMap<Eigen::Tensor<T, 2, Eigen::RowMajor, IndexType> >
       UnalignedMatrix;
-  typedef Eigen::TensorMap<Eigen::Tensor<const T, 2, Eigen::RowMajor> >
-      UnalignedConstMatrix;
+  typedef Eigen::TensorMap<Eigen::Tensor<const T, 2, Eigen::RowMajor,
+                                         IndexType> > UnalignedConstMatrix;
 };
 
 typedef typename TTypes<float, 1>::Tensor32Bit::Index Index32;

tensorflow/core/kernels/conv_2d.h

@@ -11,24 +11,25 @@ namespace functor {
 // TODO(yangke): revisit these operations and in particular, see if we can
 // combine all of them into just one operation without causing nvcc to
 // timeout.
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, int Dims, typename IndexType>
 struct ShuffleAndReverse {
-  void operator()(const Device& d, typename TTypes<T, Dims>::ConstTensor input,
-                  const Eigen::DSizes<Eigen::DenseIndex, Dims>& order,
+  void operator()(const Device& d,
+                  typename TTypes<T, Dims, IndexType>::ConstTensor input,
+                  const Eigen::DSizes<IndexType, Dims>& order,
                   const Eigen::array<bool, Dims>& reverse_dims,
-                  typename TTypes<T, Dims>::Tensor output) {
+                  typename TTypes<T, Dims, IndexType>::Tensor output) {
     output.device(d) = input.shuffle(order).reverse(reverse_dims);
   }
 };
 
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, int Dims, typename IndexType>
 struct InflatePadAndShuffle {
   void operator()(
-      const Device& d, typename TTypes<T, Dims>::ConstTensor input,
-      const Eigen::DSizes<Eigen::DenseIndex, Dims>& strides,
-      const Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, Dims>& pad_dims,
-      const Eigen::DSizes<Eigen::DenseIndex, Dims>& order,
-      typename TTypes<T, Dims>::Tensor output) {
+      const Device& d, typename TTypes<T, Dims, IndexType>::ConstTensor input,
+      const Eigen::DSizes<IndexType, Dims>& strides,
+      const Eigen::array<Eigen::IndexPair<IndexType>, Dims>& pad_dims,
+      const Eigen::DSizes<IndexType, Dims>& order,
+      typename TTypes<T, Dims, IndexType>::Tensor output) {
     output.device(d) = input.inflate(strides).pad(pad_dims).shuffle(order);
   }
 };
@@ -89,30 +90,92 @@ struct MatMulConvFunctor {
   }
 };
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename IndexType>
 struct TransformFilter {
-  void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor in,
-                  typename TTypes<T, 4>::Tensor out) {
-    out.device(d) = in.shuffle(Eigen::DSizes<Eigen::DenseIndex, 4>(3, 2, 0, 1));
+  void operator()(const Device& d,
+                  typename TTypes<T, 4, IndexType>::ConstTensor in,
+                  typename TTypes<T, 4, IndexType>::Tensor out) {
+    // We want a 3, 2, 0, 1 shuffle. We can merge dimensions 0 and 1 together
+    // to help speedup the shuffle operation.
+    Eigen::DSizes<IndexType, 3> merged_dims;
+    merged_dims[0] = in.dimension(0) * in.dimension(1);
+    merged_dims[1] = in.dimension(2);
+    merged_dims[2] = in.dimension(3);
+
+    Eigen::DSizes<IndexType, 4> expanded_dims;
+    expanded_dims[0] = in.dimension(3);
+    expanded_dims[1] = in.dimension(2);
+    expanded_dims[2] = in.dimension(0);
+    expanded_dims[3] = in.dimension(1);
+
+    out.device(d) = in.reshape(merged_dims)
+                        .shuffle(Eigen::DSizes<IndexType, 3>(2, 1, 0))
+                        .reshape(expanded_dims);
   }
 };
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename IndexType>
 struct TransformDepth {
-  void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor in,
-                  const Eigen::DSizes<Eigen::DenseIndex, 4>& shuffle,
-                  typename TTypes<T, 4>::Tensor out) {
-    out.device(d) = in.shuffle(shuffle);
+  void operator()(const Device& d,
+                  typename TTypes<T, 4, IndexType>::ConstTensor in,
+                  const Eigen::DSizes<IndexType, 4>& shuffle,
+                  typename TTypes<T, 4, IndexType>::Tensor out) {
+    Eigen::DSizes<IndexType, 3> merged_dims;
+    Eigen::DSizes<IndexType, 4> expanded_dims;
+    Eigen::DSizes<IndexType, 3> new_shuffle;
+
+    // Merge dimensions that won't be shuffled together to speed things up.
+    if (shuffle[1] == 2 && shuffle[2] == 3) {
+      merged_dims[0] = in.dimension(0);
+      merged_dims[1] = in.dimension(1);
+      merged_dims[2] = in.dimension(2) * in.dimension(3);
+      new_shuffle[0] = shuffle[0];
+      new_shuffle[1] = 2;
+      new_shuffle[2] = shuffle[3];
+      expanded_dims[0] = in.dimension(shuffle[0]);
+      expanded_dims[1] = in.dimension(2);
+      expanded_dims[2] = in.dimension(3);
+      expanded_dims[3] = in.dimension(shuffle[3]);
+    } else if (shuffle[0] == 2 && shuffle[1] == 3) {
+      merged_dims[0] = in.dimension(0);
+      merged_dims[1] = in.dimension(1);
+      merged_dims[2] = in.dimension(2) * in.dimension(3);
+      new_shuffle[0] = 2;
+      new_shuffle[1] = shuffle[2];
+      new_shuffle[2] = shuffle[3];
+      expanded_dims[0] = in.dimension(2);
+      expanded_dims[1] = in.dimension(3);
+      expanded_dims[2] = in.dimension(shuffle[2]);
+      expanded_dims[3] = in.dimension(shuffle[3]);
+    } else if (shuffle[0] == 0 && shuffle[1] == 3 && shuffle[2] == 1 &&
+               shuffle[3] == 2) {
+      merged_dims[0] = in.dimension(0);
+      merged_dims[1] = in.dimension(1) * in.dimension(2);
+      merged_dims[2] = in.dimension(3);
+      new_shuffle[0] = 0;
+      new_shuffle[1] = 2;
+      new_shuffle[2] = 1;
+      expanded_dims[0] = in.dimension(0);
+      expanded_dims[1] = in.dimension(3);
+      expanded_dims[2] = in.dimension(1);
+      expanded_dims[3] = in.dimension(2);
+    } else {
+      assert(false && "unexpected shuffle");
+    }
+
+    out.device(d) =
+        in.reshape(merged_dims).shuffle(new_shuffle).reshape(expanded_dims);
   }
 };
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename IndexType>
 struct PadInput {
-  void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor in,
+  void operator()(const Device& d,
+                  typename TTypes<T, 4, IndexType>::ConstTensor in,
                   int padding_rows_left, int padding_rows_right,
                   int padding_cols_left, int padding_cols_right,
-                  typename TTypes<T, 4>::Tensor out) {
-    Eigen::array<std::pair<ptrdiff_t, ptrdiff_t>, 4> padding;
+                  typename TTypes<T, 4, IndexType>::Tensor out) {
+    Eigen::array<std::pair<IndexType, IndexType>, 4> padding;
     padding[0] = std::make_pair(0, 0);
     padding[1] = std::make_pair(padding_rows_left, padding_rows_right);
     padding[2] = std::make_pair(padding_cols_left, padding_cols_right);