From 31fdc8b1f0804fff21c08a943b5c1aaffff76f4a Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 1 Jun 2018 18:03:21 -0700
Subject: [PATCH 01/63] fix lint

---
 tests/cpp/operator/mkldnn.cc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 655435193851..be0d28444472 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -762,6 +762,28 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
       verify_fn(orig_inputs, outputs);
     }
   }
+
+  in_arrs = GetTestInputArrays(init_fn);
+  for (auto in_arr : in_arrs) {
+    for (auto dispatch : dispatches) {
+      std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds,
+                                                               InitDefaultArray);
+      for (auto out_arr : out_arrs) {
+        req[0] = kAddTo;
+
+        NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
+        for (size_t i = 0; i < num_inputs; i++)
+          inputs[i] = &in_arr.arr;
+
+        outputs[0] = &out_arr.arr;
+        PrintVerifyMsg(in_arr, out_arr);
+        Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
+                                    outputs, req, dispatch, mxnet::OpStatePtr());
+        out_arr.arr.WaitToRead();
+        VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
+      }
+    }
+  }
 }
 
 TEST(IMPERATIVE, CopyOp) {

From b644e020dcee803aaf9879d4c0031e5909fa952c Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 4 Jun 2018 11:29:23 -0700
Subject: [PATCH 02/63] requests added to opattr

---
 tests/cpp/operator/mkldnn.cc | 82 +++++++++++++++++++++---------------
 1 file changed, 49 insertions(+), 33 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index be0d28444472..51b8d1d1946d 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -27,6 +27,7 @@
 
 #include <cmath>
 #include <climits>
+#include <set>
 #include "gtest/gtest.h"
 #include "mxnet/imperative.h"
 #include "../../src/operator/nn/mkldnn/mkldnn_base-inl.h"
@@ -352,6 +353,7 @@ struct NDArrayAttrs {
 struct OpAttrs {
   nnvm::NodeAttrs attrs;
   std::vector<DispatchMode> dispatches;
+  std::set<OpReqType> requests;
   int num_inputs;
   int num_outputs;
 };
@@ -375,6 +377,9 @@ OpAttrs GetCopyBackwardsOp() {
   attrs.dispatches.resize(2);
   attrs.dispatches[0] = DispatchMode::kFCompute;
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
+  attrs.requests.insert(OpReqType::kWriteTo);
+  attrs.requests.insert(OpReqType::kWriteInplace);
+  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 
@@ -388,6 +393,8 @@ OpAttrs GetReluOp() {
   attrs.dispatches.resize(2);
   attrs.dispatches[0] = DispatchMode::kFCompute;
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
+  attrs.requests.insert(OpReqType::kWriteTo);
+  attrs.requests.insert(OpReqType::kWriteInplace);
   return attrs;
 }
 
@@ -423,6 +430,9 @@ OpAttrs GetSumBackwardsOp() {
   attrs.dispatches.resize(2);
   attrs.dispatches[0] = DispatchMode::kFCompute;
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
+  attrs.requests.insert(OpReqType::kWriteTo);
+  attrs.requests.insert(OpReqType::kWriteInplace);
+  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 
@@ -717,27 +727,30 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
   TestArrayShapes tas = GetTestArrayShapes();
   std::vector<mkldnn::memory::primitive_desc> pds = tas.pds;
 
+  if (attrs.requests.find(OpReqType::kWriteTo) != attrs.requests.end()) {
   std::vector<NDArrayAttrs> in_arrs = GetTestInputArrays();
-  for (auto in_arr : in_arrs) {
-    for (auto dispatch : dispatches) {
-      std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);
-      for (auto out_arr : out_arrs) {
-        for (int i = 0; i < attrs.num_inputs; i++)
-          inputs[i] = &in_arr.arr;
-        for (int i = 0; i < attrs.num_outputs; i++) {
-          req[i] = kWriteTo;
-          outputs[i] = &out_arr.arr;
+    for (auto in_arr : in_arrs) {
+      for (auto dispatch : dispatches) {
+        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);
+        for (auto out_arr : out_arrs) {
+          for (int i = 0; i < attrs.num_inputs; i++)
+            inputs[i] = &in_arr.arr;
+          for (int i = 0; i < attrs.num_outputs; i++) {
+            req[i] = kWriteTo;
+            outputs[i] = &out_arr.arr;
+          }
+          PrintVerifyMsg(in_arr, out_arr);
+          Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
+                                      outputs, req, dispatch, mxnet::OpStatePtr());
+          for (auto output : outputs)
+            output->WaitToRead();
+          verify_fn(inputs, outputs);
         }
-        PrintVerifyMsg(in_arr, out_arr);
-        Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
-                                    outputs, req, dispatch, mxnet::OpStatePtr());
-        for (auto output : outputs)
-          output->WaitToRead();
-        verify_fn(inputs, outputs);
       }
     }
   }
 
+  if (attrs.requests.find(OpReqType::kWriteInplace) != attrs.requests.end()) {
   for (auto dispatch : dispatches) {
     in_arrs = GetTestInputArrays();
     for (auto arr : in_arrs) {
@@ -760,27 +773,30 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
       for (int i = 0; i < attrs.num_inputs; i++)
         orig_inputs[i] = &orig.arr;
       verify_fn(orig_inputs, outputs);
+      }
     }
   }
 
-  in_arrs = GetTestInputArrays(init_fn);
-  for (auto in_arr : in_arrs) {
-    for (auto dispatch : dispatches) {
-      std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds,
-                                                               InitDefaultArray);
-      for (auto out_arr : out_arrs) {
-        req[0] = kAddTo;
-
-        NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
-        for (size_t i = 0; i < num_inputs; i++)
-          inputs[i] = &in_arr.arr;
-
-        outputs[0] = &out_arr.arr;
-        PrintVerifyMsg(in_arr, out_arr);
-        Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
-                                    outputs, req, dispatch, mxnet::OpStatePtr());
-        out_arr.arr.WaitToRead();
-        VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
+  if (attrs.requests.find(OpReqType::kAddTo) != attrs.requests.end()) {
+    std::vector<NDArrayAttrs> in_arrs = GetTestInputArrays(init_fn);
+    for (auto in_arr : in_arrs) {
+      for (auto dispatch : dispatches) {
+        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds,
+                                                                 InitDefaultArray);
+        for (auto out_arr : out_arrs) {
+          req[0] = kAddTo;
+
+          NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
+          for (size_t i = 0; i < num_inputs; i++)
+            inputs[i] = &in_arr.arr;
+
+          outputs[0] = &out_arr.arr;
+          PrintVerifyMsg(in_arr, out_arr);
+          Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
+                                      outputs, req, dispatch, mxnet::OpStatePtr());
+          out_arr.arr.WaitToRead();
+          VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
+        }
       }
     }
   }

From 612f64fb785a0fd19e648fe9a5ba4555fbc83eae Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 5 Jun 2018 10:50:55 -0700
Subject: [PATCH 03/63] comment out addto

---
 tests/cpp/operator/mkldnn.cc | 46 ++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 51b8d1d1946d..ddce9831bff7 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -777,29 +777,29 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
     }
   }
 
-  if (attrs.requests.find(OpReqType::kAddTo) != attrs.requests.end()) {
-    std::vector<NDArrayAttrs> in_arrs = GetTestInputArrays(init_fn);
-    for (auto in_arr : in_arrs) {
-      for (auto dispatch : dispatches) {
-        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds,
-                                                                 InitDefaultArray);
-        for (auto out_arr : out_arrs) {
-          req[0] = kAddTo;
-
-          NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
-          for (size_t i = 0; i < num_inputs; i++)
-            inputs[i] = &in_arr.arr;
-
-          outputs[0] = &out_arr.arr;
-          PrintVerifyMsg(in_arr, out_arr);
-          Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
-                                      outputs, req, dispatch, mxnet::OpStatePtr());
-          out_arr.arr.WaitToRead();
-          VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
-        }
-      }
-    }
-  }
+//  if (attrs.requests.find(OpReqType::kAddTo) != attrs.requests.end()) {
+//    std::vector<NDArrayAttrs> in_arrs = GetTestInputArrays(init_fn);
+//    for (auto in_arr : in_arrs) {
+//      for (auto dispatch : dispatches) {
+//        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds,
+//                                                                 InitDefaultArray);
+//        for (auto out_arr : out_arrs) {
+//          req[0] = kAddTo;
+//
+//          NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
+//          for (size_t i = 0; i < num_inputs; i++)
+//            inputs[i] = &in_arr.arr;
+//
+//          outputs[0] = &out_arr.arr;
+//          PrintVerifyMsg(in_arr, out_arr);
+//          Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
+//                                      outputs, req, dispatch, mxnet::OpStatePtr());
+//          out_arr.arr.WaitToRead();
+//          VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
+//        }
+//      }
+//    }
+//  }
 }
 
 TEST(IMPERATIVE, CopyOp) {

From 2c4b41ea722fae3c59827c8c1c3a02ee0356b384 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 11:07:46 -0700
Subject: [PATCH 04/63] can invalidate kAddTo request mkldarrays

---
 src/operator/nn/mkldnn/mkldnn_base-inl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base-inl.h b/src/operator/nn/mkldnn/mkldnn_base-inl.h
index 6a7c58f29912..bf9e5d16d85a 100644
--- a/src/operator/nn/mkldnn/mkldnn_base-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_base-inl.h
@@ -342,7 +342,7 @@ void CommitOutput(const NDArray &arr, const mkldnn_output_t &res);
 static inline void InvalidateOutputs(const std::vector<NDArray> &arrs,
                                      const std::vector<OpReqType> &reqs) {
   for (size_t i = 0; i < arrs.size(); i++) {
-    if (reqs[i] == kWriteTo || reqs[i] == kNullOp) {
+    if (reqs[i] == kWriteTo || reqs[i] == kNullOp || reqs[i] == kAddTo) {
       const_cast<NDArray &>(arrs[i]).InvalidateMKLDNNData();
     }
   }

From 2dc646a35c6838b8cf0c3776c086cea4bad1ef2b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 12:31:28 -0700
Subject: [PATCH 05/63] revert adding kAddTo to invalidate

---
 src/operator/nn/mkldnn/mkldnn_base-inl.h | 2 +-
 tests/cpp/operator/mkldnn.cc             | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base-inl.h b/src/operator/nn/mkldnn/mkldnn_base-inl.h
index bf9e5d16d85a..6a7c58f29912 100644
--- a/src/operator/nn/mkldnn/mkldnn_base-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_base-inl.h
@@ -342,7 +342,7 @@ void CommitOutput(const NDArray &arr, const mkldnn_output_t &res);
 static inline void InvalidateOutputs(const std::vector<NDArray> &arrs,
                                      const std::vector<OpReqType> &reqs) {
   for (size_t i = 0; i < arrs.size(); i++) {
-    if (reqs[i] == kWriteTo || reqs[i] == kNullOp || reqs[i] == kAddTo) {
+    if (reqs[i] == kWriteTo || reqs[i] == kNullOp) {
       const_cast<NDArray &>(arrs[i]).InvalidateMKLDNNData();
     }
   }
diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index ddce9831bff7..e37eb153157c 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -685,6 +685,15 @@ void VerifySumBackwardsResult(const std::vector<NDArray *> &in_arrs,
   }
 }
 
+void VerifyAddRequest(const std::vector<NDArray*> &in_arrs,
+                      const std::vector<NDArray*> &original_outputs,
+                      const std::vector<NDArray*> &new_outputs,
+                      VerifyFunc verify_fn) {
+  NDArray tmp = new_outputs[0]->Reorder2Default() - original_outputs[0]->Reorder2Default();
+  tmp.WaitToRead();
+  verify_fn(in_arrs, tmp);
+}
+
 void PrintVerifyMsg(const NDArrayAttrs &arr1, const NDArrayAttrs &arr2) {
   TShape t1 = arr1.arr.shape();
   TShape t2 = arr2.arr.shape();

From 5278ef9b2149b411c67c440273c1d7bb244a87f0 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 13:28:44 -0700
Subject: [PATCH 06/63] use copy of output instead of creating new array

---
 src/common/exec_utils.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/common/exec_utils.h b/src/common/exec_utils.h
index 731d03d9be2b..945648a8e94e 100644
--- a/src/common/exec_utils.h
+++ b/src/common/exec_utils.h
@@ -98,8 +98,7 @@ inline bool SetupDefaultBlobsOut(const std::vector<NDArray>& src,
     is_default = nd.IsDefaultData();
 #endif
     if (!is_default) {
-      NDArray temp = bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(),
-                                                             true, nd.dtype());
+      NDArray temp = bufs != nullptr ? bufs->at(i) : nd.Reorder2Default().Copy(nd.ctx());
 #if MXNET_USE_MKLDNN == 1
       CHECK(temp.IsDefaultData());
 #endif

From 3adcd8dea46cb4eb5fd186ffee9aa13355080a8e Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 14:01:31 -0700
Subject: [PATCH 07/63] convert output to default if fallback

---
 src/operator/tensor/elemwise_unary_op_basic.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc
index 46f62651c757..74639484acce 100644
--- a/src/operator/tensor/elemwise_unary_op_basic.cc
+++ b/src/operator/tensor/elemwise_unary_op_basic.cc
@@ -172,7 +172,7 @@ static void CopyEx(const nnvm::NodeAttrs& attrs,
     // but MKLDNN doesn't support the data type or the shape. We're
     // forced to convert it to the default format.
     std::vector<TBlob> in_blobs {inputs[0].data()};
-    std::vector<TBlob> out_blobs {outputs[0].data()};
+    std::vector<TBlob> out_blobs {outputs[0].Reorder2Default().data()};
     UnaryOp::IdentityCompute<cpu>(attrs, ctx, in_blobs, req, out_blobs);
     return;
   }

From 2489b861a3fb77ea6a6c9626085f3610b38f24f3 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 14:42:01 -0700
Subject: [PATCH 08/63] do not make copy when init

---
 src/common/exec_utils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/exec_utils.h b/src/common/exec_utils.h
index 945648a8e94e..db6ba21be3d8 100644
--- a/src/common/exec_utils.h
+++ b/src/common/exec_utils.h
@@ -98,7 +98,7 @@ inline bool SetupDefaultBlobsOut(const std::vector<NDArray>& src,
     is_default = nd.IsDefaultData();
 #endif
     if (!is_default) {
-      NDArray temp = bufs != nullptr ? bufs->at(i) : nd.Reorder2Default().Copy(nd.ctx());
+      NDArray temp = bufs != nullptr ? bufs->at(i) : nd.Reorder2Default();
 #if MXNET_USE_MKLDNN == 1
       CHECK(temp.IsDefaultData());
 #endif

From c7e64f3a8cef5e313acb9f9634440f7a73d6bc7d Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 15:50:34 -0700
Subject: [PATCH 09/63] copyex fallback copies to old array with kAddTo

---
 .../tensor/elemwise_unary_op_basic.cc         | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc
index 74639484acce..983d7a0b6259 100644
--- a/src/operator/tensor/elemwise_unary_op_basic.cc
+++ b/src/operator/tensor/elemwise_unary_op_basic.cc
@@ -25,6 +25,7 @@
 #include "elemwise_unary_op.h"
 #include "./elemwise_binary_op-inl.h"
 #include "../nn/mkldnn/mkldnn_ops-inl.h"
+#include "../../common/exec_utils.h"
 
 namespace mxnet {
 namespace op {
@@ -172,8 +173,25 @@ static void CopyEx(const nnvm::NodeAttrs& attrs,
     // but MKLDNN doesn't support the data type or the shape. We're
     // forced to convert it to the default format.
     std::vector<TBlob> in_blobs {inputs[0].data()};
-    std::vector<TBlob> out_blobs {outputs[0].Reorder2Default().data()};
+    std::vector<TBlob> out_blobs(1);
+
+    std::vector<NDArray> temp_src, temp_dst;
+
+    if (req[0] == kAddTo && outputs[0].IsMKLDNNData()) {
+      NDArray temp = outputs[0].Reorder2Default();
+      temp_src.emplace_back(temp);
+      temp_dst.emplace_back(outputs[0]);
+      out_blobs.push_back(temp.data());
+    } else {
+      out_blobs.push_back(outputs[0].data());
+    }
     UnaryOp::IdentityCompute<cpu>(attrs, ctx, in_blobs, req, out_blobs);
+
+    if (req[0] == kAddTo && outputs[0].IsMKLDNNData()) {
+      mxnet::common::CastNonDefaultStorage(temp_src, temp_dst, ctx, false);
+    }
+
+
     return;
   }
 #endif

From 67001ce946729cf1a1c89f7afeff2011689768b5 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 16:41:40 -0700
Subject: [PATCH 10/63] change input mem desc to output mem desc if not equal

---
 src/operator/nn/mkldnn/mkldnn_copy.cc          |  4 +++-
 src/operator/tensor/elemwise_unary_op_basic.cc | 10 ++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_copy.cc b/src/operator/nn/mkldnn/mkldnn_copy.cc
index 75e51aff0066..cee3e3e062e4 100644
--- a/src/operator/nn/mkldnn/mkldnn_copy.cc
+++ b/src/operator/nn/mkldnn/mkldnn_copy.cc
@@ -47,8 +47,10 @@ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     // We should try and force the output memory has the same format
     // as the input memory. If not, we'll have to reorder memory.
     auto out_mem = out_data.GetMKLDNNData(in_mem->get_primitive_desc());
-    if (out_mem == nullptr)
+    if (out_mem == nullptr) {
       out_mem = out_data.GetMKLDNNData();
+      in_mem = data.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
+    }
     auto sum_res = TmpMemMgr::Get()->Alloc(out_mem->get_primitive_desc());
     MKLDNNSum(*in_mem, *out_mem, *sum_res);
     const_cast<NDArray &>(out_data).CopyFrom(*sum_res);
diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc
index 983d7a0b6259..ab2fc3e8aad4 100644
--- a/src/operator/tensor/elemwise_unary_op_basic.cc
+++ b/src/operator/tensor/elemwise_unary_op_basic.cc
@@ -173,7 +173,7 @@ static void CopyEx(const nnvm::NodeAttrs& attrs,
     // but MKLDNN doesn't support the data type or the shape. We're
     // forced to convert it to the default format.
     std::vector<TBlob> in_blobs {inputs[0].data()};
-    std::vector<TBlob> out_blobs(1);
+    std::vector<TBlob> out_blobs;
 
     std::vector<NDArray> temp_src, temp_dst;
 
@@ -181,16 +181,14 @@ static void CopyEx(const nnvm::NodeAttrs& attrs,
       NDArray temp = outputs[0].Reorder2Default();
       temp_src.emplace_back(temp);
       temp_dst.emplace_back(outputs[0]);
-      out_blobs.push_back(temp.data());
+      out_blobs = {temp.data()};
     } else {
-      out_blobs.push_back(outputs[0].data());
+      out_blobs = {outputs[0].data()};
     }
     UnaryOp::IdentityCompute<cpu>(attrs, ctx, in_blobs, req, out_blobs);
 
-    if (req[0] == kAddTo && outputs[0].IsMKLDNNData()) {
+    if (req[0] == kAddTo && outputs[0].IsMKLDNNData())
       mxnet::common::CastNonDefaultStorage(temp_src, temp_dst, ctx, false);
-    }
-
 
     return;
   }

From 5a75f53fe3d553866524d9e6d1da27e937695f63 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 16:57:54 -0700
Subject: [PATCH 11/63] reorder memory in commitoutput

---
 src/operator/nn/mkldnn/mkldnn_base.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index b182aa0b68d4..19cf420a5ea0 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -141,8 +141,12 @@ void CommitOutput(const NDArray &arr, const mkldnn_output_t &res) {
   if (res.first == CopyBack) {
     const_cast<NDArray &>(arr).CopyFrom(*res.second);
   } else if (res.first == AddBack) {
+
     auto mem = arr.GetMKLDNNData(res.second->get_primitive_desc());
-    CHECK(mem != nullptr);
+    if (mem == nullptr) {
+      auto target_mem = arr.GetMKLDNNData();
+      MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(*mem, *target_mem));
+    }
     // We have to allocate new memory for the sum result.
     auto sum_res = TmpMemMgr::Get()->Alloc(
         res.second->get_primitive_desc());

From f5b63fc7882de225622f9a5fbc008234564a464e Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 17:02:06 -0700
Subject: [PATCH 12/63] allocate temp memory

---
 src/operator/nn/mkldnn/mkldnn_base.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index 19cf420a5ea0..8dff0f4a789e 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -144,8 +144,9 @@ void CommitOutput(const NDArray &arr, const mkldnn_output_t &res) {
 
     auto mem = arr.GetMKLDNNData(res.second->get_primitive_desc());
     if (mem == nullptr) {
-      auto target_mem = arr.GetMKLDNNData();
-      MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(*mem, *target_mem));
+      auto output_mem = arr.GetMKLDNNData();
+      mkldnn::memory *target_memory = TmpMemMgr::Get()->Alloc(output_mem->get_primitive_desc());
+      MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(*mem, *target_memory));
     }
     // We have to allocate new memory for the sum result.
     auto sum_res = TmpMemMgr::Get()->Alloc(

From 4d5298764da3fb872b49804fa39a69b6e4ff56cf Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 6 Jun 2018 17:12:32 -0700
Subject: [PATCH 13/63] fix var names

---
 src/operator/nn/mkldnn/mkldnn_base.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index 8dff0f4a789e..e3e0890a2879 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -142,16 +142,16 @@ void CommitOutput(const NDArray &arr, const mkldnn_output_t &res) {
     const_cast<NDArray &>(arr).CopyFrom(*res.second);
   } else if (res.first == AddBack) {
 
+    auto res_memory = res.second;
     auto mem = arr.GetMKLDNNData(res.second->get_primitive_desc());
     if (mem == nullptr) {
-      auto output_mem = arr.GetMKLDNNData();
-      mkldnn::memory *target_memory = TmpMemMgr::Get()->Alloc(output_mem->get_primitive_desc());
-      MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(*mem, *target_memory));
+      res_memory = TmpMemMgr::Get()->Alloc(arr.GetMKLDNNData()->get_primitive_desc());
+      MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(*res.second, *res_memory));
     }
     // We have to allocate new memory for the sum result.
     auto sum_res = TmpMemMgr::Get()->Alloc(
         res.second->get_primitive_desc());
-    op::MKLDNNSum(*res.second, *mem, *sum_res);
+    op::MKLDNNSum(*res_memory, *mem, *sum_res);
     const_cast<NDArray &>(arr).CopyFrom(*sum_res);
   }
 }

From 6b62e97a47d469177c02acd99b7eaad0f43e3f9a Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 09:25:16 -0700
Subject: [PATCH 14/63] create helper reorder function to handle diff
 format/shapes

---
 src/ndarray/ndarray.cc                   | 135 ++++++++++++-----------
 src/operator/nn/mkldnn/mkldnn_act.cc     |  15 ++-
 src/operator/nn/mkldnn/mkldnn_base-inl.h |   4 +
 3 files changed, 84 insertions(+), 70 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 94d3d90413ab..4ae981425335 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -646,74 +646,75 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
     ptr_->Reorder2Default();
 
   const mkldnn::memory *this_mem = GetMKLDNNData();
-  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
-  mkldnn::memory::desc from_desc = from_pd.desc();
-  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();
-  mkldnn::memory::desc this_desc = this_pd.desc();
-  mkldnn_memory_format_t from_def_format = GetDefaultFormat(from_desc);
-  mkldnn_memory_format_t this_def_format = GetDefaultFormat(this_desc);
-  if (IsView()) {
-    // Sliced array must use the default layout.
-    CHECK_EQ(GetDefaultFormat(this_desc), this_desc.data.format);
-  }
+  ReorderMKLDNNMem(&mem, this_mem);
+//  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
+//  mkldnn::memory::desc from_desc = from_pd.desc();
+//  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();
+//  mkldnn::memory::desc this_desc = this_pd.desc();
+//  mkldnn_memory_format_t from_def_format = GetDefaultFormat(from_desc);
+//  mkldnn_memory_format_t this_def_format = GetDefaultFormat(this_desc);
+//  if (IsView()) {
+//    // Sliced array must use the default layout.
+//    CHECK_EQ(GetDefaultFormat(this_desc), this_desc.data.format);
+//  }
   // It's possible that the memory and the NDArray don't have the same shape.
-  if (!same_shape(this_desc, from_desc)
-      // If the source memory uses the default layout, we can reshape directly.
-      && from_def_format == from_desc.data.format) {
-    // In this case, we can simply create a new MKLDNN memory for the required
-    // shape.
-    mkldnn::memory::dims dims(this_desc.data.dims,
-                              this_desc.data.dims + this_desc.data.ndims);
-    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
-    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
-    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
-    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
-    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
-    stream->RegisterMem(tmp_mem);
-    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
-  } else if (!same_shape(this_desc, from_desc)) {
-    // In this case, the source memory stores data in a customized layout. We
-    // need to reorganize the data in memory before we can reshape.
-    mkldnn::memory::primitive_desc def_pd = GetPrimitiveDesc(from_pd, from_def_format);
-    mkldnn::memory *def_mem = TmpMemMgr::Get()->Alloc(def_pd);
-    stream->RegisterPrim(mkldnn::reorder(mem, *def_mem));
-    // Now we can reshape it
-    mkldnn::memory::dims dims(this_desc.data.dims,
-                              this_desc.data.dims + this_desc.data.ndims);
-    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
-    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
-    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
-    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
-    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, def_mem->get_data_handle()));
-    stream->RegisterMem(tmp_mem);
-    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
-  } else if (from_pd == this_pd) {
-    // If the layout is the same, we can just copy data.
-    stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
-  } else {
-    // If both are not using the default layouts. There isn't much we can do,
-    // other than reorder data layout directly.
-    if (this_def_format != this_desc.data.format
-        && from_def_format != from_desc.data.format) {
-      stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
-    } else if (this_def_format == this_desc.data.format) {
-      // If the dest mem uses the default memory layout, we can simply use
-      // the default format of the source memory to improve perf of reorder.
-      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(from_pd,
-                                                           from_def_format);
-      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, this_mem->get_data_handle()));
-      stream->RegisterMem(tmp_mem);
-      stream->RegisterPrim(mkldnn::reorder(mem, *tmp_mem));
-    } else {
-      // If the src mem uses the default memory layout, we can use
-      // the default format of the source memory to improve perf.
-      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(this_pd,
-                                                           this_def_format);
-      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
-      stream->RegisterMem(tmp_mem);
-      stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
-    }
-  }
+//  if (!same_shape(this_desc, from_desc)
+//      // If the source memory uses the default layout, we can reshape directly.
+//      && from_def_format == from_desc.data.format) {
+//    // In this case, we can simply create a new MKLDNN memory for the required
+//    // shape.
+//    mkldnn::memory::dims dims(this_desc.data.dims,
+//                              this_desc.data.dims + this_desc.data.ndims);
+//    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
+//    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
+//    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
+//    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
+//    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
+//    stream->RegisterMem(tmp_mem);
+//    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
+//  } else if (!same_shape(this_desc, from_desc)) {
+//    // In this case, the source memory stores data in a customized layout. We
+//    // need to reorganize the data in memory before we can reshape.
+//    mkldnn::memory::primitive_desc def_pd = GetPrimitiveDesc(from_pd, from_def_format);
+//    mkldnn::memory *def_mem = TmpMemMgr::Get()->Alloc(def_pd);
+//    stream->RegisterPrim(mkldnn::reorder(mem, *def_mem));
+//    // Now we can reshape it
+//    mkldnn::memory::dims dims(this_desc.data.dims,
+//                              this_desc.data.dims + this_desc.data.ndims);
+//    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
+//    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
+//    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
+//    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
+//    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, def_mem->get_data_handle()));
+//    stream->RegisterMem(tmp_mem);
+//    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
+//  } else if (from_pd == this_pd) {
+//    // If the layout is the same, we can just copy data.
+//    stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
+//  } else {
+//    // If both are not using the default layouts. There isn't much we can do,
+//    // other than reorder data layout directly.
+//    if (this_def_format != this_desc.data.format
+//        && from_def_format != from_desc.data.format) {
+//      stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
+//    } else if (this_def_format == this_desc.data.format) {
+//      // If the dest mem uses the default memory layout, we can simply use
+//      // the default format of the source memory to improve perf of reorder.
+//      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(from_pd,
+//                                                           from_def_format);
+//      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, this_mem->get_data_handle()));
+//      stream->RegisterMem(tmp_mem);
+//      stream->RegisterPrim(mkldnn::reorder(mem, *tmp_mem));
+//    } else {
+//      // If the src mem uses the default memory layout, we can use
+//      // the default format of the source memory to improve perf.
+//      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(this_pd,
+//                                                           this_def_format);
+//      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
+//      stream->RegisterMem(tmp_mem);
+//      stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
+//    }
+//  }
 }
 
 mkldnn::memory *NDArray::CreateMKLDNNData(const mkldnn::memory::primitive_desc &desc) {
diff --git a/src/operator/nn/mkldnn/mkldnn_act.cc b/src/operator/nn/mkldnn/mkldnn_act.cc
index b21d1238f7aa..2bfa8844f0c7 100644
--- a/src/operator/nn/mkldnn/mkldnn_act.cc
+++ b/src/operator/nn/mkldnn/mkldnn_act.cc
@@ -167,11 +167,20 @@ void MKLDNNActivationForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     in_buffer = in_data.Reorder2Default();
 
   auto input_mem = in_buffer.GetMKLDNNData();
+
+  if (req == kAddTo) {
+    auto out_mem = out_data.GetMKLDNNData();
+    auto input_mem = in_buffer.GetMKLDNNData(out_mem->get_primitive_desc());
+    if (input_mem == nullptr) {
+      input_mem = in_buffer.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
+    }
+  }
+
   MKLDNNActForward &fwd = GetActForward(param, ctx, in_buffer, *input_mem);
-  auto out_mem = CreateMKLDNNMem(out_data, fwd.fwd_pd.dst_primitive_desc(), req, &in_buffer);
-  fwd.SetNewMem(*input_mem, *out_mem.second);
+  auto out_mem_t = CreateMKLDNNMem(out_data, fwd.fwd_pd.dst_primitive_desc(), req, &in_buffer);
+  fwd.SetNewMem(*input_mem, *out_mem_t.second);
   stream->RegisterPrim(fwd.GetFwd());
-  CommitOutput(out_data, out_mem);
+  CommitOutput(out_data, out_mem_t);
   stream->Submit();
 }
 
diff --git a/src/operator/nn/mkldnn/mkldnn_base-inl.h b/src/operator/nn/mkldnn/mkldnn_base-inl.h
index 6a7c58f29912..770e42862525 100644
--- a/src/operator/nn/mkldnn/mkldnn_base-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_base-inl.h
@@ -319,6 +319,10 @@ enum OutDataOp {
 
 typedef std::pair<OutDataOp, mkldnn::memory *> mkldnn_output_t;
 
+
+void ReorderMKLDNNMem(const mkldnn::memory &src_mem,
+                      const mkldnn::memory &dst_mem);
+
 /*
  * These two functions try to create MKLDNN memory in an NDArray based on `req'.
  * The difference is that the first function can create MKLDNN memory with

From 9da3655fec61aae4a56ff2f6230638e12b52b3ad Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 09:31:56 -0700
Subject: [PATCH 15/63] fix typos

---
 src/ndarray/ndarray.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 4ae981425335..950d94a1f8b5 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -645,8 +645,8 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
   if (IsMKLDNNData() && IsView())
     ptr_->Reorder2Default();
 
-  const mkldnn::memory *this_mem = GetMKLDNNData();
-  ReorderMKLDNNMem(&mem, this_mem);
+//  auto this_mem = ;
+  ReorderMKLDNNMem(&mem, GetMKLDNNData());
 //  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
 //  mkldnn::memory::desc from_desc = from_pd.desc();
 //  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();

From c0c38ca897c9ac4666ed3704267035871c2c9d65 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 09:38:01 -0700
Subject: [PATCH 16/63] fix typos

---
 src/ndarray/ndarray.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 950d94a1f8b5..86e3c3588389 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -638,15 +638,15 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
 
   CHECK(mem.get_primitive_desc().get_size() == shape().Size() * GetTypeSize(dtype_))
       << "The size of NDArray doesn't match the requested MKLDNN memory desc";
-  MKLDNNStream *stream = MKLDNNStream::Get();
+//  MKLDNNStream *stream = MKLDNNStream::Get();
   // If this array uses MKLDNN layout, we have to make sure it's not a view.
   // Otherwise, we'll have to change the layout inside the array.
 
   if (IsMKLDNNData() && IsView())
     ptr_->Reorder2Default();
 
-//  auto this_mem = ;
-  ReorderMKLDNNMem(&mem, GetMKLDNNData());
+  const mkldnn::memory* this_mem = GetMKLDNNData();
+  ReorderMKLDNNMem(mem, *this_mem);
 //  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
 //  mkldnn::memory::desc from_desc = from_pd.desc();
 //  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();

From 2338046e00417eafa67e5c7d03b2314be2e63ee7 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 09:45:35 -0700
Subject: [PATCH 17/63] remove unused code

---
 src/operator/nn/mkldnn/mkldnn_act.cc | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_act.cc b/src/operator/nn/mkldnn/mkldnn_act.cc
index 2bfa8844f0c7..744fed2c299f 100644
--- a/src/operator/nn/mkldnn/mkldnn_act.cc
+++ b/src/operator/nn/mkldnn/mkldnn_act.cc
@@ -167,15 +167,6 @@ void MKLDNNActivationForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     in_buffer = in_data.Reorder2Default();
 
   auto input_mem = in_buffer.GetMKLDNNData();
-
-  if (req == kAddTo) {
-    auto out_mem = out_data.GetMKLDNNData();
-    auto input_mem = in_buffer.GetMKLDNNData(out_mem->get_primitive_desc());
-    if (input_mem == nullptr) {
-      input_mem = in_buffer.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
-    }
-  }
-
   MKLDNNActForward &fwd = GetActForward(param, ctx, in_buffer, *input_mem);
   auto out_mem_t = CreateMKLDNNMem(out_data, fwd.fwd_pd.dst_primitive_desc(), req, &in_buffer);
   fwd.SetNewMem(*input_mem, *out_mem_t.second);

From f974c3c2a9d67164dd33a3a1b29a29d679ff740f Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 10:08:39 -0700
Subject: [PATCH 18/63] fix param

---
 src/ndarray/ndarray.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 86e3c3588389..635aa2bf80ea 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -646,7 +646,7 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
     ptr_->Reorder2Default();
 
   const mkldnn::memory* this_mem = GetMKLDNNData();
-  ReorderMKLDNNMem(mem, *this_mem);
+  ReorderMKLDNNMem(mem, this_mem);
 //  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
 //  mkldnn::memory::desc from_desc = from_pd.desc();
 //  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();

From 918a864aca3911ca91852d0c623a07bf89fa0988 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 10:10:23 -0700
Subject: [PATCH 19/63] fix header files

---
 src/ndarray/ndarray.cc                   | 2 +-
 src/operator/nn/mkldnn/mkldnn_base-inl.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 635aa2bf80ea..36fe82b53df8 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -646,7 +646,7 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
     ptr_->Reorder2Default();
 
   const mkldnn::memory* this_mem = GetMKLDNNData();
-  ReorderMKLDNNMem(mem, this_mem);
+  CopyMKLDNNMem(mem, this_mem);
 //  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
 //  mkldnn::memory::desc from_desc = from_pd.desc();
 //  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();
diff --git a/src/operator/nn/mkldnn/mkldnn_base-inl.h b/src/operator/nn/mkldnn/mkldnn_base-inl.h
index 770e42862525..0e5e2b9bb49c 100644
--- a/src/operator/nn/mkldnn/mkldnn_base-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_base-inl.h
@@ -320,8 +320,8 @@ enum OutDataOp {
 typedef std::pair<OutDataOp, mkldnn::memory *> mkldnn_output_t;
 
 
-void ReorderMKLDNNMem(const mkldnn::memory &src_mem,
-                      const mkldnn::memory &dst_mem);
+void CopyMKLDNNMem(const mkldnn::memory &mem,
+                      const mkldnn::memory* this_mem);
 
 /*
  * These two functions try to create MKLDNN memory in an NDArray based on `req'.

From 50fc6ca19c6d802d2fa2794ed6fe83092a1387fa Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 15:12:00 -0700
Subject: [PATCH 20/63] force input memory to output

---
 src/operator/nn/mkldnn/mkldnn_copy.cc | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_copy.cc b/src/operator/nn/mkldnn/mkldnn_copy.cc
index cee3e3e062e4..58916eb17be2 100644
--- a/src/operator/nn/mkldnn/mkldnn_copy.cc
+++ b/src/operator/nn/mkldnn/mkldnn_copy.cc
@@ -44,13 +44,12 @@ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
   auto in_mem = data.GetMKLDNNData();
   if (req == kAddTo) {
     TmpMemMgr::Get()->Init(ctx.requested[0]);
-    // We should try and force the output memory has the same format
-    // as the input memory. If not, we'll have to reorder memory.
-    auto out_mem = out_data.GetMKLDNNData(in_mem->get_primitive_desc());
-    if (out_mem == nullptr) {
-      out_mem = out_data.GetMKLDNNData();
+    // We should try and force the input memory has the same format
+    // as the input output. If not, we'll have to reorder memory.
+    auto out_mem = out_data.GetMKLDNNData();
+    in_mem = data.GetMKLDNNData(out_mem ->get_primitive_desc());
+    if (in_mem == nullptr)
       in_mem = data.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
-    }
     auto sum_res = TmpMemMgr::Get()->Alloc(out_mem->get_primitive_desc());
     MKLDNNSum(*in_mem, *out_mem, *sum_res);
     const_cast<NDArray &>(out_data).CopyFrom(*sum_res);

From a9915bedbf6936dff30bcf146dc9e1311a0d9b6f Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 16:13:12 -0700
Subject: [PATCH 21/63] reorder2default keeps pointer to mkldnn memory

---
 src/ndarray/ndarray.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 36fe82b53df8..d6a17b567e32 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -555,6 +555,8 @@ NDArray NDArray::Reorder2Default() const {
   CHECK(ret.ptr_->shandle.size >= def_pd.get_size());
   mkldnn::memory def_mem(def_pd, ret.ptr_->shandle.dptr);
   ptr_->mkl_mem_->ReorderTo(&def_mem);
+  auto tmp = std::shared_ptr<mkldnn::memory>(def_mem, [](mkldnn::memory *mem){});
+  ret.ptr_->mkl_mem_.reset(new MKLDNNMemory(tmp));
   // reshape as needed
   ret.shape_ = shape_;
   ret.byte_offset_ = byte_offset_;

From 630c091bb7a3e5b5e4c49f11fbccf2b7c1715345 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 16:16:02 -0700
Subject: [PATCH 22/63] pass reference

---
 src/ndarray/ndarray.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index d6a17b567e32..140773bb586f 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -555,7 +555,7 @@ NDArray NDArray::Reorder2Default() const {
   CHECK(ret.ptr_->shandle.size >= def_pd.get_size());
   mkldnn::memory def_mem(def_pd, ret.ptr_->shandle.dptr);
   ptr_->mkl_mem_->ReorderTo(&def_mem);
-  auto tmp = std::shared_ptr<mkldnn::memory>(def_mem, [](mkldnn::memory *mem){});
+  auto tmp = std::shared_ptr<mkldnn::memory>(&def_mem, [](mkldnn::memory *mem){});
   ret.ptr_->mkl_mem_.reset(new MKLDNNMemory(tmp));
   // reshape as needed
   ret.shape_ = shape_;

From aa6c406ed56ce265701a6a498f0b41dc01d454ad Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 7 Jun 2018 16:49:06 -0700
Subject: [PATCH 23/63] remove extra lines

---
 src/ndarray/ndarray.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 140773bb586f..36fe82b53df8 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -555,8 +555,6 @@ NDArray NDArray::Reorder2Default() const {
   CHECK(ret.ptr_->shandle.size >= def_pd.get_size());
   mkldnn::memory def_mem(def_pd, ret.ptr_->shandle.dptr);
   ptr_->mkl_mem_->ReorderTo(&def_mem);
-  auto tmp = std::shared_ptr<mkldnn::memory>(&def_mem, [](mkldnn::memory *mem){});
-  ret.ptr_->mkl_mem_.reset(new MKLDNNMemory(tmp));
   // reshape as needed
   ret.shape_ = shape_;
   ret.byte_offset_ = byte_offset_;

From 75c51603b7bf7860acafb3895fa1883320a8188a Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 09:24:52 -0700
Subject: [PATCH 24/63] do not get raw mem from ptr

---
 src/ndarray/ndarray.cc                | 3 ++-
 src/operator/nn/mkldnn/mkldnn_copy.cc | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 36fe82b53df8..4fe6e65f62cc 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -469,6 +469,7 @@ static inline mkldnn::memory *GetMKLDNNExact(
 
 const mkldnn::memory *NDArray::GetMKLDNNData(
     const mkldnn::memory::primitive_desc &desc) const {
+  CHECK(!IsView());
   if (desc.get_size() != shape().Size() * GetTypeSize(dtype_)) {
     LOG(FATAL) << "The size of NDArray doesn't match the requested MKLDNN memory desc";
     return nullptr;
@@ -482,7 +483,7 @@ const mkldnn::memory *NDArray::GetMKLDNNData(
   if (mem->get_primitive_desc() == desc
       || (desc1.data.format == GetDefaultFormat(desc1)
         && desc2.data.format == GetDefaultFormat(desc2))) {
-    return GetMKLDNNExact(ptr_->mkl_mem_->GetRaw(), desc);
+    return GetMKLDNNExact(mem, desc);
   } else {
     return nullptr;
   }
diff --git a/src/operator/nn/mkldnn/mkldnn_copy.cc b/src/operator/nn/mkldnn/mkldnn_copy.cc
index 58916eb17be2..e213d2341ce2 100644
--- a/src/operator/nn/mkldnn/mkldnn_copy.cc
+++ b/src/operator/nn/mkldnn/mkldnn_copy.cc
@@ -50,6 +50,7 @@ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     in_mem = data.GetMKLDNNData(out_mem ->get_primitive_desc());
     if (in_mem == nullptr)
       in_mem = data.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
+
     auto sum_res = TmpMemMgr::Get()->Alloc(out_mem->get_primitive_desc());
     MKLDNNSum(*in_mem, *out_mem, *sum_res);
     const_cast<NDArray &>(out_data).CopyFrom(*sum_res);

From f65ea9c9d6ef05ab01c0cfc33568850373e4ab27 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 09:31:03 -0700
Subject: [PATCH 25/63] remove isView check

---
 src/ndarray/ndarray.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 4fe6e65f62cc..7836d3f89cd1 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -469,7 +469,6 @@ static inline mkldnn::memory *GetMKLDNNExact(
 
 const mkldnn::memory *NDArray::GetMKLDNNData(
     const mkldnn::memory::primitive_desc &desc) const {
-  CHECK(!IsView());
   if (desc.get_size() != shape().Size() * GetTypeSize(dtype_)) {
     LOG(FATAL) << "The size of NDArray doesn't match the requested MKLDNN memory desc";
     return nullptr;

From 3483f281e360adbc5a1ad8bd77868b1400db0018 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 15:10:40 -0700
Subject: [PATCH 26/63] fallback writes back to output

---
 src/operator/nn/mkldnn/mkldnn_base.cc | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index e3e0890a2879..35fb73215a02 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -22,6 +22,8 @@
 #include <atomic>
 #include "./mkldnn_base-inl.h"
 #include "./mkldnn_ops-inl.h"
+#include "../../../common/exec_utils.h"
+
 
 namespace mxnet {
 
@@ -322,18 +324,27 @@ void FallBackCompute(FCompute fn, const nnvm::NodeAttrs &attrs,
   MKLDNNStream::Get()->Submit();
 
   std::vector<TBlob> out_blobs(outputs.size());
+  std::vector<NDArray> temp_src, temp_dst;
   for (size_t i = 0; i < out_blobs.size(); i++) {
     NDArray output = outputs[i];
     // ensure output does not use mkldnn mem.
     // for inplace, we already converted & copied input above.
     if ((req[i] == kWriteTo) || (req[i] == kWriteInplace))
       const_cast<NDArray &>(output).InvalidateMKLDNNData();
-    else if (req[i] == kAddTo)
-      output = outputs[i].Reorder2Default();
+    else if (req[0] == kAddTo && output.IsMKLDNNData()) {
+      NDArray temp = outputs[0].Reorder2Default();
+      temp_src.emplace_back(temp);
+      temp_dst.emplace_back(outputs[0]);
+      out_blobs = {temp.data()};
+      output = temp;
+    }
     CHECK(output.IsDefaultData());
     out_blobs[i] = output.data();
   }
+  
   fn(attrs, ctx, in_blobs, req, out_blobs);
+  if (req[0] == kAddTo && outputs[0].IsMKLDNNData())
+    mxnet::common::CastNonDefaultStorage(temp_src, temp_dst, ctx, false);
 }
 
 template<typename DType>

From 0428e0f724ca6c7987c7847892b4319825b75306 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 15:20:59 -0700
Subject: [PATCH 27/63] remove redundant line

---
 src/operator/nn/mkldnn/mkldnn_base.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index 35fb73215a02..fa16643085c9 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -335,13 +335,12 @@ void FallBackCompute(FCompute fn, const nnvm::NodeAttrs &attrs,
       NDArray temp = outputs[0].Reorder2Default();
       temp_src.emplace_back(temp);
       temp_dst.emplace_back(outputs[0]);
-      out_blobs = {temp.data()};
       output = temp;
     }
     CHECK(output.IsDefaultData());
     out_blobs[i] = output.data();
   }
-  
+
   fn(attrs, ctx, in_blobs, req, out_blobs);
   if (req[0] == kAddTo && outputs[0].IsMKLDNNData())
     mxnet::common::CastNonDefaultStorage(temp_src, temp_dst, ctx, false);

From 1cdd60ceaf9c9e68e58deeecbd74371014d80ff1 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 15:51:42 -0700
Subject: [PATCH 28/63] remove commented out code

---
 src/ndarray/ndarray.cc | 69 ------------------------------------------
 1 file changed, 69 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 7836d3f89cd1..e3f2b29bd0bf 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -638,7 +638,6 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
 
   CHECK(mem.get_primitive_desc().get_size() == shape().Size() * GetTypeSize(dtype_))
       << "The size of NDArray doesn't match the requested MKLDNN memory desc";
-//  MKLDNNStream *stream = MKLDNNStream::Get();
   // If this array uses MKLDNN layout, we have to make sure it's not a view.
   // Otherwise, we'll have to change the layout inside the array.
 
@@ -647,74 +646,6 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
 
   const mkldnn::memory* this_mem = GetMKLDNNData();
   CopyMKLDNNMem(mem, this_mem);
-//  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
-//  mkldnn::memory::desc from_desc = from_pd.desc();
-//  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();
-//  mkldnn::memory::desc this_desc = this_pd.desc();
-//  mkldnn_memory_format_t from_def_format = GetDefaultFormat(from_desc);
-//  mkldnn_memory_format_t this_def_format = GetDefaultFormat(this_desc);
-//  if (IsView()) {
-//    // Sliced array must use the default layout.
-//    CHECK_EQ(GetDefaultFormat(this_desc), this_desc.data.format);
-//  }
-  // It's possible that the memory and the NDArray don't have the same shape.
-//  if (!same_shape(this_desc, from_desc)
-//      // If the source memory uses the default layout, we can reshape directly.
-//      && from_def_format == from_desc.data.format) {
-//    // In this case, we can simply create a new MKLDNN memory for the required
-//    // shape.
-//    mkldnn::memory::dims dims(this_desc.data.dims,
-//                              this_desc.data.dims + this_desc.data.ndims);
-//    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
-//    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
-//    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
-//    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
-//    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
-//    stream->RegisterMem(tmp_mem);
-//    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
-//  } else if (!same_shape(this_desc, from_desc)) {
-//    // In this case, the source memory stores data in a customized layout. We
-//    // need to reorganize the data in memory before we can reshape.
-//    mkldnn::memory::primitive_desc def_pd = GetPrimitiveDesc(from_pd, from_def_format);
-//    mkldnn::memory *def_mem = TmpMemMgr::Get()->Alloc(def_pd);
-//    stream->RegisterPrim(mkldnn::reorder(mem, *def_mem));
-//    // Now we can reshape it
-//    mkldnn::memory::dims dims(this_desc.data.dims,
-//                              this_desc.data.dims + this_desc.data.ndims);
-//    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
-//    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
-//    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
-//    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
-//    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, def_mem->get_data_handle()));
-//    stream->RegisterMem(tmp_mem);
-//    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
-//  } else if (from_pd == this_pd) {
-//    // If the layout is the same, we can just copy data.
-//    stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
-//  } else {
-//    // If both are not using the default layouts. There isn't much we can do,
-//    // other than reorder data layout directly.
-//    if (this_def_format != this_desc.data.format
-//        && from_def_format != from_desc.data.format) {
-//      stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
-//    } else if (this_def_format == this_desc.data.format) {
-//      // If the dest mem uses the default memory layout, we can simply use
-//      // the default format of the source memory to improve perf of reorder.
-//      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(from_pd,
-//                                                           from_def_format);
-//      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, this_mem->get_data_handle()));
-//      stream->RegisterMem(tmp_mem);
-//      stream->RegisterPrim(mkldnn::reorder(mem, *tmp_mem));
-//    } else {
-//      // If the src mem uses the default memory layout, we can use
-//      // the default format of the source memory to improve perf.
-//      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(this_pd,
-//                                                           this_def_format);
-//      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
-//      stream->RegisterMem(tmp_mem);
-//      stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
-//    }
-//  }
 }
 
 mkldnn::memory *NDArray::CreateMKLDNNData(const mkldnn::memory::primitive_desc &desc) {

From c9e8f85441b6dc510d841953a25ded4190bb44b7 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 15:55:50 -0700
Subject: [PATCH 29/63] use fallback in copy (refactor)

---
 .../tensor/elemwise_unary_op_basic.cc         | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc
index ab2fc3e8aad4..46f01ff81a7b 100644
--- a/src/operator/tensor/elemwise_unary_op_basic.cc
+++ b/src/operator/tensor/elemwise_unary_op_basic.cc
@@ -172,24 +172,7 @@ static void CopyEx(const nnvm::NodeAttrs& attrs,
     // This happens if inputs are supposed to be in MKLDNN format
     // but MKLDNN doesn't support the data type or the shape. We're
     // forced to convert it to the default format.
-    std::vector<TBlob> in_blobs {inputs[0].data()};
-    std::vector<TBlob> out_blobs;
-
-    std::vector<NDArray> temp_src, temp_dst;
-
-    if (req[0] == kAddTo && outputs[0].IsMKLDNNData()) {
-      NDArray temp = outputs[0].Reorder2Default();
-      temp_src.emplace_back(temp);
-      temp_dst.emplace_back(outputs[0]);
-      out_blobs = {temp.data()};
-    } else {
-      out_blobs = {outputs[0].data()};
-    }
-    UnaryOp::IdentityCompute<cpu>(attrs, ctx, in_blobs, req, out_blobs);
-
-    if (req[0] == kAddTo && outputs[0].IsMKLDNNData())
-      mxnet::common::CastNonDefaultStorage(temp_src, temp_dst, ctx, false);
-
+    FallBackCompute(UnaryOp::IdentityCompute<cpu>, attrs, ctx, inputs, req, outputs);
     return;
   }
 #endif

From 996d0ef3c743b9e7d43b45f7eadddad7d250aa87 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 16:06:42 -0700
Subject: [PATCH 30/63] remove unused header

---
 src/operator/tensor/elemwise_unary_op_basic.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc
index 46f01ff81a7b..880a375c8ec2 100644
--- a/src/operator/tensor/elemwise_unary_op_basic.cc
+++ b/src/operator/tensor/elemwise_unary_op_basic.cc
@@ -25,7 +25,6 @@
 #include "elemwise_unary_op.h"
 #include "./elemwise_binary_op-inl.h"
 #include "../nn/mkldnn/mkldnn_ops-inl.h"
-#include "../../common/exec_utils.h"
 
 namespace mxnet {
 namespace op {

From 453220996e7010ed5ba3d0d36c5e2782c6fea101 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 8 Jun 2018 16:09:05 -0700
Subject: [PATCH 31/63] fix lint

---
 src/operator/nn/mkldnn/mkldnn_base.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index fa16643085c9..d92c82a61e1b 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -143,7 +143,6 @@ void CommitOutput(const NDArray &arr, const mkldnn_output_t &res) {
   if (res.first == CopyBack) {
     const_cast<NDArray &>(arr).CopyFrom(*res.second);
   } else if (res.first == AddBack) {
-
     auto res_memory = res.second;
     auto mem = arr.GetMKLDNNData(res.second->get_primitive_desc());
     if (mem == nullptr) {
@@ -329,9 +328,9 @@ void FallBackCompute(FCompute fn, const nnvm::NodeAttrs &attrs,
     NDArray output = outputs[i];
     // ensure output does not use mkldnn mem.
     // for inplace, we already converted & copied input above.
-    if ((req[i] == kWriteTo) || (req[i] == kWriteInplace))
+    if ((req[i] == kWriteTo) || (req[i] == kWriteInplace)) {
       const_cast<NDArray &>(output).InvalidateMKLDNNData();
-    else if (req[0] == kAddTo && output.IsMKLDNNData()) {
+    } else if (req[0] == kAddTo && output.IsMKLDNNData()) {
       NDArray temp = outputs[0].Reorder2Default();
       temp_src.emplace_back(temp);
       temp_dst.emplace_back(outputs[0]);

From 410c491cae181c3a0632b11faf5573575afee43e Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 11 Jun 2018 09:20:28 -0700
Subject: [PATCH 32/63] reorder2default only if mkldnn flag

---
 src/common/exec_utils.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/common/exec_utils.h b/src/common/exec_utils.h
index db6ba21be3d8..225214488405 100644
--- a/src/common/exec_utils.h
+++ b/src/common/exec_utils.h
@@ -98,9 +98,12 @@ inline bool SetupDefaultBlobsOut(const std::vector<NDArray>& src,
     is_default = nd.IsDefaultData();
 #endif
     if (!is_default) {
-      NDArray temp = bufs != nullptr ? bufs->at(i) : nd.Reorder2Default();
 #if MXNET_USE_MKLDNN == 1
+      NDArray temp = bufs != nullptr ? bufs->at(i) : nd.Reorder2Default();
       CHECK(temp.IsDefaultData());
+#else
+      NDArray temp = bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(),
+          true, nd.dtype());
 #endif
       temp_src->emplace_back(nd);
       temp_dst->emplace_back(temp);

From 2efdc3b5024f8871b20105fb087a6549dc639c9b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 11 Jun 2018 11:04:35 -0700
Subject: [PATCH 33/63] only reorder if mkldnn

---
 src/common/exec_utils.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/common/exec_utils.h b/src/common/exec_utils.h
index 225214488405..8488da8e03f4 100644
--- a/src/common/exec_utils.h
+++ b/src/common/exec_utils.h
@@ -99,7 +99,8 @@ inline bool SetupDefaultBlobsOut(const std::vector<NDArray>& src,
 #endif
     if (!is_default) {
 #if MXNET_USE_MKLDNN == 1
-      NDArray temp = bufs != nullptr ? bufs->at(i) : nd.Reorder2Default();
+        NDArray temp = bufs != nullptr ? bufs->at(i) : nd.IsMKLDNNData() ?
+            nd.Reorder2Default() : NDArray(nd.shape(), nd.ctx(), true, nd.dtype());
       CHECK(temp.IsDefaultData());
 #else
       NDArray temp = bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(),

From dc3cd8de562bd1d1876ed909670aacfcfc5009a9 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 12 Jun 2018 12:52:20 -0700
Subject: [PATCH 34/63] does not assume 1 output

---
 src/operator/nn/mkldnn/mkldnn_base.cc | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index d92c82a61e1b..68e7eea0c693 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -330,10 +330,10 @@ void FallBackCompute(FCompute fn, const nnvm::NodeAttrs &attrs,
     // for inplace, we already converted & copied input above.
     if ((req[i] == kWriteTo) || (req[i] == kWriteInplace)) {
       const_cast<NDArray &>(output).InvalidateMKLDNNData();
-    } else if (req[0] == kAddTo && output.IsMKLDNNData()) {
-      NDArray temp = outputs[0].Reorder2Default();
+    } else if (req[i] == kAddTo && output.IsMKLDNNData()) {
+      NDArray temp = outputs[i].Reorder2Default();
       temp_src.emplace_back(temp);
-      temp_dst.emplace_back(outputs[0]);
+      temp_dst.emplace_back(outputs[i]);
       output = temp;
     }
     CHECK(output.IsDefaultData());
@@ -341,8 +341,10 @@ void FallBackCompute(FCompute fn, const nnvm::NodeAttrs &attrs,
   }
 
   fn(attrs, ctx, in_blobs, req, out_blobs);
-  if (req[0] == kAddTo && outputs[0].IsMKLDNNData())
-    mxnet::common::CastNonDefaultStorage(temp_src, temp_dst, ctx, false);
+  for (size_t i = 0; i < out_blobs.size(); i++) {
+    if (req[i] == kAddTo && outputs[i].IsMKLDNNData())
+      mxnet::common::CastNonDefaultStorage(temp_src, temp_dst, ctx, false);
+  }
 }
 
 template<typename DType>

From ad666118df23a155348e7c90b1c7dba65fa9b546 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 13 Jun 2018 13:40:51 -0700
Subject: [PATCH 35/63] sum compares input and output shape

---
 src/operator/nn/mkldnn/mkldnn_copy.cc |  7 +++----
 src/operator/nn/mkldnn/mkldnn_sum.cc  | 13 +++++++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_copy.cc b/src/operator/nn/mkldnn/mkldnn_copy.cc
index e213d2341ce2..13a970b938cf 100644
--- a/src/operator/nn/mkldnn/mkldnn_copy.cc
+++ b/src/operator/nn/mkldnn/mkldnn_copy.cc
@@ -50,10 +50,9 @@ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     in_mem = data.GetMKLDNNData(out_mem ->get_primitive_desc());
     if (in_mem == nullptr)
       in_mem = data.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
-
-    auto sum_res = TmpMemMgr::Get()->Alloc(out_mem->get_primitive_desc());
-    MKLDNNSum(*in_mem, *out_mem, *sum_res);
-    const_cast<NDArray &>(out_data).CopyFrom(*sum_res);
+//    auto sum_res = TmpMemMgr::Get()->Alloc(out_mem->get_primitive_desc());
+    MKLDNNSum(*in_mem, *out_mem, *out_mem);
+//    const_cast<NDArray &>(out_data).CopyFrom(*sum_res);
   } else {
     const_cast<NDArray &>(out_data).CopyFrom(*in_mem);
   }
diff --git a/src/operator/nn/mkldnn/mkldnn_sum.cc b/src/operator/nn/mkldnn/mkldnn_sum.cc
index c51e1081d694..185baff7367d 100644
--- a/src/operator/nn/mkldnn/mkldnn_sum.cc
+++ b/src/operator/nn/mkldnn/mkldnn_sum.cc
@@ -41,9 +41,18 @@ void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
   CHECK(input_pds[0] == input_pds[1]);
   inputs.push_back(arr1);
   inputs.push_back(arr2);
-  // TODO(zhengda) I need to reorder memory here.
+  bool pd_same = true;
+  bool addr_same = false;
+  auto first_data_handle = arr1.get_data_handle();
   mkldnn::sum::primitive_desc sum_pd(scales, input_pds);
-  MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, out));
+  if (pd_same && addr_same) {
+    // do sum computation directly on output NDArray
+    MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, out));
+  } else {
+    auto sum_res = TmpMemMgr::Get()->Alloc(out.get_primitive_desc());
+    MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, *sum_res));
+    CopyMKLDNNMem(*sum_res, &out);
+  }
 }
 
 void MKLDNNSumForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx,

From 860fa21f1afabb924756e7844c1f8a2828ae68c4 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 13 Jun 2018 13:48:09 -0700
Subject: [PATCH 36/63] compare address and pd in sum

---
 src/operator/nn/mkldnn/mkldnn_sum.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_sum.cc b/src/operator/nn/mkldnn/mkldnn_sum.cc
index 185baff7367d..b4db8c7ce4f0 100644
--- a/src/operator/nn/mkldnn/mkldnn_sum.cc
+++ b/src/operator/nn/mkldnn/mkldnn_sum.cc
@@ -41,10 +41,10 @@ void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
   CHECK(input_pds[0] == input_pds[1]);
   inputs.push_back(arr1);
   inputs.push_back(arr2);
-  bool pd_same = true;
-  bool addr_same = false;
-  auto first_data_handle = arr1.get_data_handle();
   mkldnn::sum::primitive_desc sum_pd(scales, input_pds);
+  bool pd_same = sum_pd.dst_primitive_desc() == input_pds[0];
+  auto first_data_handle = arr1.get_data_handle();
+  bool addr_same = out.get_data_handle() == first_data_handle;
   if (pd_same && addr_same) {
     // do sum computation directly on output NDArray
     MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, out));

From a727eeab1d15dd94734198576328645b87e50c5c Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 13 Jun 2018 14:45:49 -0700
Subject: [PATCH 37/63] refactor mkldnnsum

---
 src/operator/nn/mkldnn/mkldnn_copy.cc   |  4 +---
 src/operator/nn/mkldnn/mkldnn_ops-inl.h |  4 ++--
 src/operator/nn/mkldnn/mkldnn_sum.cc    | 30 ++++++++++++++-----------
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_copy.cc b/src/operator/nn/mkldnn/mkldnn_copy.cc
index 13a970b938cf..a86b3db3731c 100644
--- a/src/operator/nn/mkldnn/mkldnn_copy.cc
+++ b/src/operator/nn/mkldnn/mkldnn_copy.cc
@@ -50,9 +50,7 @@ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     in_mem = data.GetMKLDNNData(out_mem ->get_primitive_desc());
     if (in_mem == nullptr)
       in_mem = data.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
-//    auto sum_res = TmpMemMgr::Get()->Alloc(out_mem->get_primitive_desc());
-    MKLDNNSum(*in_mem, *out_mem, *out_mem);
-//    const_cast<NDArray &>(out_data).CopyFrom(*sum_res);
+    MKLDNNSum({in_mem, out_mem}, *out_mem);
   } else {
     const_cast<NDArray &>(out_data).CopyFrom(*in_mem);
   }
diff --git a/src/operator/nn/mkldnn/mkldnn_ops-inl.h b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
index 50937706d934..4766738b38a2 100644
--- a/src/operator/nn/mkldnn/mkldnn_ops-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
@@ -104,8 +104,8 @@ void MKLDNNActivationBackward(const nnvm::NodeAttrs& attrs, const OpContext &ctx
                               const NDArray &out_grad, const NDArray &in_data,
                               const OpReqType &req, const NDArray &in_grad);
 
-void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
-         const mkldnn::memory &out);
+void MKLDNNSum(const std::vector<const mkldnn::memory*> arrs,
+               const mkldnn::memory &out);
 
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/nn/mkldnn/mkldnn_sum.cc b/src/operator/nn/mkldnn/mkldnn_sum.cc
index b4db8c7ce4f0..4539c7e2bc3b 100644
--- a/src/operator/nn/mkldnn/mkldnn_sum.cc
+++ b/src/operator/nn/mkldnn/mkldnn_sum.cc
@@ -31,21 +31,25 @@
 namespace mxnet {
 namespace op {
 
-void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
-         const mkldnn::memory &out) {
-  std::vector<mkldnn::memory::primitive_desc> input_pds(2);
-  std::vector<float> scales(2, 1);
+void MKLDNNSum(const std::vector<mkldnn::memory*> arrs,
+               const mkldnn::memory &out) {
+  std::vector<mkldnn::memory::primitive_desc> input_pds(arrs.size());
+  std::vector<float> scales(arrs.size(), 1);
   std::vector<mkldnn::primitive::at> inputs;
-  input_pds[0] = arr1.get_primitive_desc();
-  input_pds[1] = arr2.get_primitive_desc();
-  CHECK(input_pds[0] == input_pds[1]);
-  inputs.push_back(arr1);
-  inputs.push_back(arr2);
+
+  mkldnn::memory::primitive_desc prev_pd;
+  mkldnn::memory::primitive_desc tmp_pd;
+  for (size_t i = 0; i < arrs.size(); i++)
+    input_pds[i] = arrs[i]->get_primitive_desc();
+
   mkldnn::sum::primitive_desc sum_pd(scales, input_pds);
-  bool pd_same = sum_pd.dst_primitive_desc() == input_pds[0];
-  auto first_data_handle = arr1.get_data_handle();
-  bool addr_same = out.get_data_handle() == first_data_handle;
-  if (pd_same && addr_same) {
+  // check if inplace sum is possible
+  auto in_place = false;
+  for (size_t i = 0; i < arrs.size(); i++) {
+    if (input_pds[i] == sum_pd.dst_primitive_desc() && arrs[i]->get_data_handle() == out.get_data_handle())
+      in_place = true;
+  }
+  if (in_place) {
     // do sum computation directly on output NDArray
     MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, out));
   } else {

From c76aee3f20a52acea8f62bb79fc8d0d7049f5320 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 13 Jun 2018 14:57:14 -0700
Subject: [PATCH 38/63] fix const param

---
 src/operator/nn/mkldnn/mkldnn_ops-inl.h | 2 +-
 src/operator/nn/mkldnn/mkldnn_sum.cc    | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_ops-inl.h b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
index 4766738b38a2..4e1ad2ad7b89 100644
--- a/src/operator/nn/mkldnn/mkldnn_ops-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
@@ -104,7 +104,7 @@ void MKLDNNActivationBackward(const nnvm::NodeAttrs& attrs, const OpContext &ctx
                               const NDArray &out_grad, const NDArray &in_data,
                               const OpReqType &req, const NDArray &in_grad);
 
-void MKLDNNSum(const std::vector<const mkldnn::memory*> arrs,
+void MKLDNNSum(const std::vector<mkldnn::memory const*> arrs,
                const mkldnn::memory &out);
 
 }  // namespace op
diff --git a/src/operator/nn/mkldnn/mkldnn_sum.cc b/src/operator/nn/mkldnn/mkldnn_sum.cc
index 4539c7e2bc3b..cb58a97299a7 100644
--- a/src/operator/nn/mkldnn/mkldnn_sum.cc
+++ b/src/operator/nn/mkldnn/mkldnn_sum.cc
@@ -31,7 +31,7 @@
 namespace mxnet {
 namespace op {
 
-void MKLDNNSum(const std::vector<mkldnn::memory*> arrs,
+void MKLDNNSum(const std::vector<const mkldnn::memory*> arrs,
                const mkldnn::memory &out) {
   std::vector<mkldnn::memory::primitive_desc> input_pds(arrs.size());
   std::vector<float> scales(arrs.size(), 1);
@@ -39,8 +39,11 @@ void MKLDNNSum(const std::vector<mkldnn::memory*> arrs,
 
   mkldnn::memory::primitive_desc prev_pd;
   mkldnn::memory::primitive_desc tmp_pd;
-  for (size_t i = 0; i < arrs.size(); i++)
+  for (size_t i = 0; i < arrs.size(); i++) {
     input_pds[i] = arrs[i]->get_primitive_desc();
+    inputs.push_back(*arrs[i]);
+  }
+
 
   mkldnn::sum::primitive_desc sum_pd(scales, input_pds);
   // check if inplace sum is possible

From 64422aa372bfa22149bb327fc4078299bd00c875 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 13 Jun 2018 15:07:13 -0700
Subject: [PATCH 39/63] fix header

---
 src/operator/nn/mkldnn/mkldnn_ops-inl.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_ops-inl.h b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
index 4e1ad2ad7b89..f9481fdbaf5e 100644
--- a/src/operator/nn/mkldnn/mkldnn_ops-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
@@ -104,8 +104,8 @@ void MKLDNNActivationBackward(const nnvm::NodeAttrs& attrs, const OpContext &ctx
                               const NDArray &out_grad, const NDArray &in_data,
                               const OpReqType &req, const NDArray &in_grad);
 
-void MKLDNNSum(const std::vector<mkldnn::memory const*> arrs,
-               const mkldnn::memory &out);
+  void MKLDNNSum(const std::vector<const mkldnn::memory*> arrs,
+                 const mkldnn::memory &out);
 
 }  // namespace op
 }  // namespace mxnet

From bb10946e5a2e98c8c9d6fec102d2a7f6c1623283 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 13:28:53 -0700
Subject: [PATCH 40/63] improve control flow when setting output blob

---
 src/common/exec_utils.h | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/common/exec_utils.h b/src/common/exec_utils.h
index 8488da8e03f4..030ba39df81d 100644
--- a/src/common/exec_utils.h
+++ b/src/common/exec_utils.h
@@ -99,8 +99,16 @@ inline bool SetupDefaultBlobsOut(const std::vector<NDArray>& src,
 #endif
     if (!is_default) {
 #if MXNET_USE_MKLDNN == 1
-        NDArray temp = bufs != nullptr ? bufs->at(i) : nd.IsMKLDNNData() ?
-            nd.Reorder2Default() : NDArray(nd.shape(), nd.ctx(), true, nd.dtype());
+      NDArray temp;
+      if (bufs != nullptr) {
+        temp = bufs->at(i);
+      } else if (kAddTo == req->at(i) && nd.IsMKLDNNData()) {
+        temp = nd.Reorder2Default();
+      } else if (kAddTo == req->at(i) && !nd.IsMKLDNNData()) {
+        temp = nd;
+      } else {
+        temp = NDArray(nd.shape(), nd.ctx(), true, nd.dtype());
+      }
       CHECK(temp.IsDefaultData());
 #else
       NDArray temp = bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(),

From ad315782a98fba9007d4db73c5e137bfd0ce5b33 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 13:39:45 -0700
Subject: [PATCH 41/63] fix merge

---
 src/ndarray/ndarray.cc                   |  2 +-
 src/operator/nn/mkldnn/mkldnn_base-inl.h |  2 +-
 src/operator/nn/mkldnn/mkldnn_base.cc    | 82 ++++++++++++++++++++++--
 src/operator/nn/mkldnn/mkldnn_copy.cc    |  2 +-
 src/operator/nn/mkldnn/mkldnn_ops-inl.h  |  4 +-
 src/operator/nn/mkldnn/mkldnn_sum.cc     | 38 ++++-------
 tests/cpp/operator/mkldnn.cc             |  5 +-
 7 files changed, 94 insertions(+), 41 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index e3f2b29bd0bf..ddcb209c4407 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -645,7 +645,7 @@ void NDArray::CopyFrom(const mkldnn::memory &mem) {
     ptr_->Reorder2Default();
 
   const mkldnn::memory* this_mem = GetMKLDNNData();
-  CopyMKLDNNMem(mem, this_mem);
+  MKLDNNCopy(mem, this_mem);
 }
 
 mkldnn::memory *NDArray::CreateMKLDNNData(const mkldnn::memory::primitive_desc &desc) {
diff --git a/src/operator/nn/mkldnn/mkldnn_base-inl.h b/src/operator/nn/mkldnn/mkldnn_base-inl.h
index 0e5e2b9bb49c..2c305bd7aae7 100644
--- a/src/operator/nn/mkldnn/mkldnn_base-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_base-inl.h
@@ -320,7 +320,7 @@ enum OutDataOp {
 typedef std::pair<OutDataOp, mkldnn::memory *> mkldnn_output_t;
 
 
-void CopyMKLDNNMem(const mkldnn::memory &mem,
+void MKLDNNCopy(const mkldnn::memory &mem,
                       const mkldnn::memory* this_mem);
 
 /*
diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc
index 68e7eea0c693..b6365f0dc43a 100644
--- a/src/operator/nn/mkldnn/mkldnn_base.cc
+++ b/src/operator/nn/mkldnn/mkldnn_base.cc
@@ -79,6 +79,75 @@ mkldnn::memory *TmpMemMgr::Alloc(const mkldnn::memory::primitive_desc &pd) {
   }
 }
 
+void MKLDNNCopy(const mkldnn::memory &mem, const mkldnn::memory* this_mem) {
+  MKLDNNStream *stream = MKLDNNStream::Get();
+
+  mkldnn::memory::primitive_desc from_pd = mem.get_primitive_desc();
+  mkldnn::memory::desc from_desc = from_pd.desc();
+  mkldnn::memory::primitive_desc this_pd = this_mem->get_primitive_desc();
+  mkldnn::memory::desc this_desc = this_pd.desc();
+  mkldnn_memory_format_t from_def_format = GetDefaultFormat(from_desc);
+  mkldnn_memory_format_t this_def_format = GetDefaultFormat(this_desc);
+  // It's possible that the memory and the NDArray don't have the same shape.
+  if (!same_shape(this_desc, from_desc)
+      // If the source memory uses the default layout, we can reshape directly.
+      && from_def_format == from_desc.data.format) {
+    // In this case, we can simply create a new MKLDNN memory for the required
+    // shape.
+    mkldnn::memory::dims dims(this_desc.data.dims,
+                              this_desc.data.dims + this_desc.data.ndims);
+    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
+    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
+    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
+    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
+    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
+    stream->RegisterMem(tmp_mem);
+    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
+  } else if (!same_shape(this_desc, from_desc)) {
+    // In this case, the source memory stores data in a customized layout. We
+    // need to reorganize the data in memory before we can reshape.
+    mkldnn::memory::primitive_desc def_pd = GetPrimitiveDesc(from_pd, from_def_format);
+    mkldnn::memory *def_mem = TmpMemMgr::Get()->Alloc(def_pd);
+    stream->RegisterPrim(mkldnn::reorder(mem, *def_mem));
+    // Now we can reshape it
+    mkldnn::memory::dims dims(this_desc.data.dims,
+                              this_desc.data.dims + this_desc.data.ndims);
+    auto this_dtype = static_cast<mkldnn::memory::data_type>(this_desc.data.data_type);
+    auto this_format = static_cast<mkldnn::memory::format>(GetDefaultFormat(this_desc));
+    mkldnn::memory::desc data_md(dims, this_dtype, this_format);
+    mkldnn::memory::primitive_desc pd(data_md, from_pd.get_engine());
+    mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, def_mem->get_data_handle()));
+    stream->RegisterMem(tmp_mem);
+    stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
+  } else if (from_pd == this_pd) {
+    // If the layout is the same, we can just copy data.
+    stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
+  } else {
+    // If both are not using the default layouts. There isn't much we can do,
+    // other than reorder data layout directly.
+    if (this_def_format != this_desc.data.format
+        && from_def_format != from_desc.data.format) {
+      stream->RegisterPrim(mkldnn::reorder(mem, *this_mem));
+    } else if (this_def_format == this_desc.data.format) {
+      // If the dest mem uses the default memory layout, we can simply use
+      // the default format of the source memory to improve perf of reorder.
+      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(from_pd,
+                                                           from_def_format);
+      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, this_mem->get_data_handle()));
+      stream->RegisterMem(tmp_mem);
+      stream->RegisterPrim(mkldnn::reorder(mem, *tmp_mem));
+    } else {
+      // If the src mem uses the default memory layout, we can use
+      // the default format of the source memory to improve perf.
+      mkldnn::memory::primitive_desc pd = GetPrimitiveDesc(this_pd,
+                                                           this_def_format);
+      mkldnn_mem_ptr tmp_mem(new mkldnn::memory(pd, mem.get_data_handle()));
+      stream->RegisterMem(tmp_mem);
+      stream->RegisterPrim(mkldnn::reorder(*tmp_mem, *this_mem));
+    }
+  }
+}
+
 bool CanWriteTo(const NDArray &out_arr,
                 const NDArray &in_arr,
                 const mkldnn::memory::primitive_desc &desc) {
@@ -144,16 +213,15 @@ void CommitOutput(const NDArray &arr, const mkldnn_output_t &res) {
     const_cast<NDArray &>(arr).CopyFrom(*res.second);
   } else if (res.first == AddBack) {
     auto res_memory = res.second;
+    auto target_pd = arr.GetMKLDNNData()->get_primitive_desc();
     auto mem = arr.GetMKLDNNData(res.second->get_primitive_desc());
     if (mem == nullptr) {
-      res_memory = TmpMemMgr::Get()->Alloc(arr.GetMKLDNNData()->get_primitive_desc());
-      MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(*res.second, *res_memory));
+      auto tmp_memory = TmpMemMgr::Get()->Alloc(target_pd);
+      MKLDNNCopy(*res_memory, tmp_memory);
+      res_memory = tmp_memory;
+      mem = arr.GetMKLDNNData();
     }
-    // We have to allocate new memory for the sum result.
-    auto sum_res = TmpMemMgr::Get()->Alloc(
-        res.second->get_primitive_desc());
-    op::MKLDNNSum(*res_memory, *mem, *sum_res);
-    const_cast<NDArray &>(arr).CopyFrom(*sum_res);
+    op::MKLDNNSum(*mem, *res_memory, *mem);
   }
 }
 
diff --git a/src/operator/nn/mkldnn/mkldnn_copy.cc b/src/operator/nn/mkldnn/mkldnn_copy.cc
index a86b3db3731c..edc2b9cfc6c2 100644
--- a/src/operator/nn/mkldnn/mkldnn_copy.cc
+++ b/src/operator/nn/mkldnn/mkldnn_copy.cc
@@ -50,7 +50,7 @@ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     in_mem = data.GetMKLDNNData(out_mem ->get_primitive_desc());
     if (in_mem == nullptr)
       in_mem = data.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
-    MKLDNNSum({in_mem, out_mem}, *out_mem);
+    MKLDNNSum(*in_mem, *out_mem, *out_mem);
   } else {
     const_cast<NDArray &>(out_data).CopyFrom(*in_mem);
   }
diff --git a/src/operator/nn/mkldnn/mkldnn_ops-inl.h b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
index f9481fdbaf5e..db9d36b3c35d 100644
--- a/src/operator/nn/mkldnn/mkldnn_ops-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
@@ -104,8 +104,8 @@ void MKLDNNActivationBackward(const nnvm::NodeAttrs& attrs, const OpContext &ctx
                               const NDArray &out_grad, const NDArray &in_data,
                               const OpReqType &req, const NDArray &in_grad);
 
-  void MKLDNNSum(const std::vector<const mkldnn::memory*> arrs,
-                 const mkldnn::memory &out);
+void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
+             const mkldnn::memory &out);
 
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/nn/mkldnn/mkldnn_sum.cc b/src/operator/nn/mkldnn/mkldnn_sum.cc
index cb58a97299a7..fd7b344b1382 100644
--- a/src/operator/nn/mkldnn/mkldnn_sum.cc
+++ b/src/operator/nn/mkldnn/mkldnn_sum.cc
@@ -31,35 +31,19 @@
 namespace mxnet {
 namespace op {
 
-void MKLDNNSum(const std::vector<const mkldnn::memory*> arrs,
-               const mkldnn::memory &out) {
-  std::vector<mkldnn::memory::primitive_desc> input_pds(arrs.size());
-  std::vector<float> scales(arrs.size(), 1);
+void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
+             const mkldnn::memory &out) {
+  std::vector<mkldnn::memory::primitive_desc> input_pds(2);
+  std::vector<float> scales(2, 1);
   std::vector<mkldnn::primitive::at> inputs;
-
-  mkldnn::memory::primitive_desc prev_pd;
-  mkldnn::memory::primitive_desc tmp_pd;
-  for (size_t i = 0; i < arrs.size(); i++) {
-    input_pds[i] = arrs[i]->get_primitive_desc();
-    inputs.push_back(*arrs[i]);
-  }
-
-
+  input_pds[0] = arr1.get_primitive_desc();
+  input_pds[1] = arr2.get_primitive_desc();
+  CHECK(input_pds[0] == input_pds[1]);
+  inputs.push_back(arr1);
+  inputs.push_back(arr2);
+  // TODO(zhengda) I need to reorder memory here.
   mkldnn::sum::primitive_desc sum_pd(scales, input_pds);
-  // check if inplace sum is possible
-  auto in_place = false;
-  for (size_t i = 0; i < arrs.size(); i++) {
-    if (input_pds[i] == sum_pd.dst_primitive_desc() && arrs[i]->get_data_handle() == out.get_data_handle())
-      in_place = true;
-  }
-  if (in_place) {
-    // do sum computation directly on output NDArray
-    MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, out));
-  } else {
-    auto sum_res = TmpMemMgr::Get()->Alloc(out.get_primitive_desc());
-    MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, *sum_res));
-    CopyMKLDNNMem(*sum_res, &out);
-  }
+  MKLDNNStream::Get()->RegisterPrim(mkldnn::sum(sum_pd, inputs, out));
 }
 
 void MKLDNNSumForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index e37eb153157c..10805b1e89fa 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -691,7 +691,7 @@ void VerifyAddRequest(const std::vector<NDArray*> &in_arrs,
                       VerifyFunc verify_fn) {
   NDArray tmp = new_outputs[0]->Reorder2Default() - original_outputs[0]->Reorder2Default();
   tmp.WaitToRead();
-  verify_fn(in_arrs, tmp);
+  verify_fn(in_arrs, {&tmp});
 }
 
 void PrintVerifyMsg(const NDArrayAttrs &arr1, const NDArrayAttrs &arr2) {
@@ -731,13 +731,14 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
   std::vector<NDArray*> inputs(attrs.num_inputs);
   std::vector<NDArray*> outputs(attrs.num_outputs);
   std::vector<OpReqType> req(attrs.num_outputs);
+  std::vector<NDArrayAttrs> in_arrs;
   std::vector<DispatchMode> dispatches = attrs.dispatches;
 
   TestArrayShapes tas = GetTestArrayShapes();
   std::vector<mkldnn::memory::primitive_desc> pds = tas.pds;
 
   if (attrs.requests.find(OpReqType::kWriteTo) != attrs.requests.end()) {
-  std::vector<NDArrayAttrs> in_arrs = GetTestInputArrays();
+  in_arrs = GetTestInputArrays();
     for (auto in_arr : in_arrs) {
       for (auto dispatch : dispatches) {
         std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);

From 0e03c9683d9f3ccb80f15368da64128bc96e91e6 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 14:06:59 -0700
Subject: [PATCH 42/63] remove kaddto comment

---
 tests/cpp/operator/mkldnn.cc | 49 ++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 10805b1e89fa..efe71f552ccc 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -742,9 +742,9 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
     for (auto in_arr : in_arrs) {
       for (auto dispatch : dispatches) {
         std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);
+        for (int i = 0; i < attrs.num_inputs; i++)
+          inputs[i] = &in_arr.arr;
         for (auto out_arr : out_arrs) {
-          for (int i = 0; i < attrs.num_inputs; i++)
-            inputs[i] = &in_arr.arr;
           for (int i = 0; i < attrs.num_outputs; i++) {
             req[i] = kWriteTo;
             outputs[i] = &out_arr.arr;
@@ -787,29 +787,28 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
     }
   }
 
-//  if (attrs.requests.find(OpReqType::kAddTo) != attrs.requests.end()) {
-//    std::vector<NDArrayAttrs> in_arrs = GetTestInputArrays(init_fn);
-//    for (auto in_arr : in_arrs) {
-//      for (auto dispatch : dispatches) {
-//        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds,
-//                                                                 InitDefaultArray);
-//        for (auto out_arr : out_arrs) {
-//          req[0] = kAddTo;
-//
-//          NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
-//          for (size_t i = 0; i < num_inputs; i++)
-//            inputs[i] = &in_arr.arr;
-//
-//          outputs[0] = &out_arr.arr;
-//          PrintVerifyMsg(in_arr, out_arr);
-//          Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
-//                                      outputs, req, dispatch, mxnet::OpStatePtr());
-//          out_arr.arr.WaitToRead();
-//          VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
-//        }
-//      }
-//    }
-//  }
+  if (attrs.requests.find(OpReqType::kAddTo) != attrs.requests.end()) {
+    in_arrs = GetTestInputArrays();
+    for (auto in_arr : in_arrs) {
+      for (auto dispatch : dispatches) {
+        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);
+        for (size_t i = 0; i < attrs.num_inputs; i++)
+          inputs[i] = &in_arr.arr;
+        for (auto out_arr : out_arrs) {
+          NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
+          for (size_t i = 0; i < attrs.num_outputs; i++) {
+            outputs[i] = &out_arr.arr;
+            req[i] = kAddTo;
+          }
+          PrintVerifyMsg(in_arr, out_arr);
+          Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
+                                      outputs, req, dispatch, mxnet::OpStatePtr());
+          out_arr.arr.WaitToRead();
+          VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
+        }
+      }
+    }
+  }
 }
 
 TEST(IMPERATIVE, CopyOp) {

From 6ef7b8764bd9519051c6678c1d3bab4f0157cdc1 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 14:12:02 -0700
Subject: [PATCH 43/63] add reqests to operators

---
 tests/cpp/operator/mkldnn.cc | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index efe71f552ccc..b98cc56671c9 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -366,6 +366,9 @@ OpAttrs GetCopyOp() {
   attrs.num_outputs = 1;
   attrs.dispatches[0] = DispatchMode::kFCompute;
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
+  attrs.requests.insert(OpReqType::kWriteTo);
+  attrs.requests.insert(OpReqType::kWriteInplace);
+  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 
@@ -408,6 +411,8 @@ OpAttrs GetReluBackwardsOp() {
   attrs.dispatches.resize(2);
   attrs.dispatches[0] = DispatchMode::kFCompute;
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
+  attrs.requests.insert(OpReqType::kWriteTo);
+  attrs.requests.insert(OpReqType::kWriteInplace);
   return attrs;
 }
 
@@ -419,6 +424,9 @@ OpAttrs GetSumOp() {
   attrs.num_outputs = 1;
   attrs.dispatches[0] = DispatchMode::kFCompute;
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
+  attrs.requests.insert(OpReqType::kWriteTo);
+  attrs.requests.insert(OpReqType::kWriteInplace);
+  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 
@@ -433,6 +441,9 @@ OpAttrs GetSumBackwardsOp() {
   attrs.requests.insert(OpReqType::kWriteTo);
   attrs.requests.insert(OpReqType::kWriteInplace);
   attrs.requests.insert(OpReqType::kAddTo);
+  attrs.requests.insert(OpReqType::kWriteTo);
+  attrs.requests.insert(OpReqType::kWriteInplace);
+  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 

From 90c9acb0319eaf525d4d68f1babd855d6949710f Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 14:15:07 -0700
Subject: [PATCH 44/63] fix spacing

---
 tests/cpp/operator/mkldnn.cc | 46 ++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index b98cc56671c9..0919ba47df73 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -749,7 +749,7 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
   std::vector<mkldnn::memory::primitive_desc> pds = tas.pds;
 
   if (attrs.requests.find(OpReqType::kWriteTo) != attrs.requests.end()) {
-  in_arrs = GetTestInputArrays();
+    in_arrs = GetTestInputArrays();
     for (auto in_arr : in_arrs) {
       for (auto dispatch : dispatches) {
         std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);
@@ -772,28 +772,28 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
   }
 
   if (attrs.requests.find(OpReqType::kWriteInplace) != attrs.requests.end()) {
-  for (auto dispatch : dispatches) {
-    in_arrs = GetTestInputArrays();
-    for (auto arr : in_arrs) {
-      // If the array is a view, we shouldn't write data to it.
-      if (arr.arr.IsView())
-        continue;
-      NDArrayAttrs orig(arr.arr.Copy(arr.arr.ctx()), "InPlace Copy");
-      for (int i = 0; i < attrs.num_inputs; i++)
-        inputs[i] = &arr.arr;
-      for (int i = 0; i < attrs.num_outputs; i++) {
-        req[i] = kWriteInplace;
-        outputs[i] = &arr.arr;
-      }
-      PrintVerifyMsg(orig, arr);
-      Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs, outputs, req,
-                                  dispatch, mxnet::OpStatePtr());
-      for (auto output : outputs)
-        output->WaitToRead();
-      std::vector<NDArray *> orig_inputs(attrs.num_inputs);
-      for (int i = 0; i < attrs.num_inputs; i++)
-        orig_inputs[i] = &orig.arr;
-      verify_fn(orig_inputs, outputs);
+    for (auto dispatch : dispatches) {
+      in_arrs = GetTestInputArrays();
+      for (auto arr : in_arrs) {
+        // If the array is a view, we shouldn't write data to it.
+        if (arr.arr.IsView())
+          continue;
+        NDArrayAttrs orig(arr.arr.Copy(arr.arr.ctx()), "InPlace Copy");
+        for (int i = 0; i < attrs.num_inputs; i++)
+          inputs[i] = &arr.arr;
+        for (int i = 0; i < attrs.num_outputs; i++) {
+          req[i] = kWriteInplace;
+          outputs[i] = &arr.arr;
+        }
+        PrintVerifyMsg(orig, arr);
+        Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs, outputs, req,
+                                    dispatch, mxnet::OpStatePtr());
+        for (auto output : outputs)
+          output->WaitToRead();
+        std::vector<NDArray *> orig_inputs(attrs.num_inputs);
+        for (int i = 0; i < attrs.num_inputs; i++)
+          orig_inputs[i] = &orig.arr;
+        verify_fn(orig_inputs, outputs);
       }
     }
   }

From 7d0f275bc32dbdd8bd8d68902fff1449ced4ebd9 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 14:20:18 -0700
Subject: [PATCH 45/63] do sum in place

---
 src/operator/nn/mkldnn/mkldnn_copy.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_copy.cc b/src/operator/nn/mkldnn/mkldnn_copy.cc
index edc2b9cfc6c2..a7c280e1e713 100644
--- a/src/operator/nn/mkldnn/mkldnn_copy.cc
+++ b/src/operator/nn/mkldnn/mkldnn_copy.cc
@@ -50,7 +50,7 @@ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx,
     in_mem = data.GetMKLDNNData(out_mem ->get_primitive_desc());
     if (in_mem == nullptr)
       in_mem = data.GetMKLDNNDataReorder(out_mem->get_primitive_desc());
-    MKLDNNSum(*in_mem, *out_mem, *out_mem);
+    MKLDNNSum(*out_mem, *in_mem, *out_mem);
   } else {
     const_cast<NDArray &>(out_data).CopyFrom(*in_mem);
   }

From 3edf492e467628b22e125be41abc05d45d6d703b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 14:25:16 -0700
Subject: [PATCH 46/63] fix conditionals

---
 src/common/exec_utils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/exec_utils.h b/src/common/exec_utils.h
index 030ba39df81d..816599b955c1 100644
--- a/src/common/exec_utils.h
+++ b/src/common/exec_utils.h
@@ -104,7 +104,7 @@ inline bool SetupDefaultBlobsOut(const std::vector<NDArray>& src,
         temp = bufs->at(i);
       } else if (kAddTo == req->at(i) && nd.IsMKLDNNData()) {
         temp = nd.Reorder2Default();
-      } else if (kAddTo == req->at(i) && !nd.IsMKLDNNData()) {
+      } else if (kAddTo == req->at(i)) {
         temp = nd;
       } else {
         temp = NDArray(nd.shape(), nd.ctx(), true, nd.dtype());

From 5c20e46e08bbc2385896d7a3b2ea54430c2de157 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 14:45:49 -0700
Subject: [PATCH 47/63] remove redundant reqs

---
 tests/cpp/operator/mkldnn.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 0919ba47df73..0771a3e72015 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -441,9 +441,6 @@ OpAttrs GetSumBackwardsOp() {
   attrs.requests.insert(OpReqType::kWriteTo);
   attrs.requests.insert(OpReqType::kWriteInplace);
   attrs.requests.insert(OpReqType::kAddTo);
-  attrs.requests.insert(OpReqType::kWriteTo);
-  attrs.requests.insert(OpReqType::kWriteInplace);
-  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 

From cd70dac3fe0dde8a3f90041748f720e5b01f1166 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 14:56:49 -0700
Subject: [PATCH 48/63] use wait to read all

---
 tests/cpp/operator/mkldnn.cc | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 0771a3e72015..8f51042670ab 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -760,8 +760,7 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
           PrintVerifyMsg(in_arr, out_arr);
           Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
                                       outputs, req, dispatch, mxnet::OpStatePtr());
-          for (auto output : outputs)
-            output->WaitToRead();
+          Engine::Get()->WaitForAll();
           verify_fn(inputs, outputs);
         }
       }
@@ -785,8 +784,7 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
         PrintVerifyMsg(orig, arr);
         Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs, outputs, req,
                                     dispatch, mxnet::OpStatePtr());
-        for (auto output : outputs)
-          output->WaitToRead();
+        Engine::Get()->WaitForAll();
         std::vector<NDArray *> orig_inputs(attrs.num_inputs);
         for (int i = 0; i < attrs.num_inputs; i++)
           orig_inputs[i] = &orig.arr;
@@ -811,7 +809,7 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
           PrintVerifyMsg(in_arr, out_arr);
           Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
                                       outputs, req, dispatch, mxnet::OpStatePtr());
-          out_arr.arr.WaitToRead();
+          Engine::Get()->WaitForAll();
           VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
         }
       }

From 0972ffa16582de8090abb60621a0be4078115f3b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 16:46:05 -0700
Subject: [PATCH 49/63] fix lint

---
 src/operator/nn/mkldnn/mkldnn_ops-inl.h | 2 +-
 src/operator/nn/mkldnn/mkldnn_sum.cc    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/operator/nn/mkldnn/mkldnn_ops-inl.h b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
index db9d36b3c35d..50937706d934 100644
--- a/src/operator/nn/mkldnn/mkldnn_ops-inl.h
+++ b/src/operator/nn/mkldnn/mkldnn_ops-inl.h
@@ -105,7 +105,7 @@ void MKLDNNActivationBackward(const nnvm::NodeAttrs& attrs, const OpContext &ctx
                               const OpReqType &req, const NDArray &in_grad);
 
 void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
-             const mkldnn::memory &out);
+         const mkldnn::memory &out);
 
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/nn/mkldnn/mkldnn_sum.cc b/src/operator/nn/mkldnn/mkldnn_sum.cc
index fd7b344b1382..c51e1081d694 100644
--- a/src/operator/nn/mkldnn/mkldnn_sum.cc
+++ b/src/operator/nn/mkldnn/mkldnn_sum.cc
@@ -32,7 +32,7 @@ namespace mxnet {
 namespace op {
 
 void MKLDNNSum(const mkldnn::memory &arr1, const mkldnn::memory &arr2,
-             const mkldnn::memory &out) {
+         const mkldnn::memory &out) {
   std::vector<mkldnn::memory::primitive_desc> input_pds(2);
   std::vector<float> scales(2, 1);
   std::vector<mkldnn::primitive::at> inputs;

From 637c76a6c3c9b9c994ad69ffe39556dd6c436e42 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 17:18:08 -0700
Subject: [PATCH 50/63] create multiple outputs

---
 tests/cpp/operator/mkldnn.cc | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 8f51042670ab..3d7346a86dda 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -740,6 +740,7 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
   std::vector<NDArray*> outputs(attrs.num_outputs);
   std::vector<OpReqType> req(attrs.num_outputs);
   std::vector<NDArrayAttrs> in_arrs;
+  std::vector<std::vector<NDArrayAttrs>> out_arrs(attrs.num_outputs);
   std::vector<DispatchMode> dispatches = attrs.dispatches;
 
   TestArrayShapes tas = GetTestArrayShapes();
@@ -749,15 +750,16 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
     in_arrs = GetTestInputArrays();
     for (auto in_arr : in_arrs) {
       for (auto dispatch : dispatches) {
-        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);
+        for (int i = 0; i < attrs.num_outputs; i++)
+          out_arrs[i] = GetTestOutputArrays(in_arr.arr.shape(), pds);
         for (int i = 0; i < attrs.num_inputs; i++)
           inputs[i] = &in_arr.arr;
-        for (auto out_arr : out_arrs) {
+        for (size_t output_i = 0; output_i < out_arrs[0].size(); output_i++) {
           for (int i = 0; i < attrs.num_outputs; i++) {
             req[i] = kWriteTo;
-            outputs[i] = &out_arr.arr;
+            outputs[i] = &out_arrs[i][output_i].arr;
           }
-          PrintVerifyMsg(in_arr, out_arr);
+          PrintVerifyMsg(in_arr, out_arrs[0][output_i]);
           Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
                                       outputs, req, dispatch, mxnet::OpStatePtr());
           Engine::Get()->WaitForAll();
@@ -797,13 +799,15 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
     in_arrs = GetTestInputArrays();
     for (auto in_arr : in_arrs) {
       for (auto dispatch : dispatches) {
-        std::vector<NDArrayAttrs> out_arrs = GetTestOutputArrays(in_arr.arr.shape(), pds);
+        for (int i = 0; i < attrs.num_outputs; i++)
+          out_arrs[i] = GetTestOutputArrays(in_arr.arr.shape(), pds);
         for (size_t i = 0; i < attrs.num_inputs; i++)
           inputs[i] = &in_arr.arr;
-        for (auto out_arr : out_arrs) {
+        for (size_t output_i = 0; output_i < out_arrs[0].size(); output_i++) {
+          auto out_arr = out_arrs[0][output_i];
           NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
           for (size_t i = 0; i < attrs.num_outputs; i++) {
-            outputs[i] = &out_arr.arr;
+            outputs[i] = &out_arrs[i][output_i].arr;
             req[i] = kAddTo;
           }
           PrintVerifyMsg(in_arr, out_arr);

From 5718651522cfeea9e7fb804c96a7b326fa569f3a Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 17:47:12 -0700
Subject: [PATCH 51/63] create multiple copies for kaddto

---
 tests/cpp/operator/mkldnn.cc | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 3d7346a86dda..1acc38ab86d9 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -697,9 +697,15 @@ void VerifyAddRequest(const std::vector<NDArray*> &in_arrs,
                       const std::vector<NDArray*> &original_outputs,
                       const std::vector<NDArray*> &new_outputs,
                       VerifyFunc verify_fn) {
-  NDArray tmp = new_outputs[0]->Reorder2Default() - original_outputs[0]->Reorder2Default();
-  tmp.WaitToRead();
-  verify_fn(in_arrs, {&tmp});
+  CHECK(original_outputs.size() == new_outputs.size());
+  std::vector<NDArray*> tmp_outputs;
+  NDArray tmp;
+  for (size_t i = 0; i < new_outputs.size(); i++) {
+    tmp = new_outputs[i]->Reorder2Default() - original_outputs[i]->Reorder2Default();
+    tmp_outputs.push_back(&tmp);
+  }
+  Engine::Get()->WaitForAll();
+  verify_fn(in_arrs, tmp_outputs);
 }
 
 void PrintVerifyMsg(const NDArrayAttrs &arr1, const NDArrayAttrs &arr2) {
@@ -796,6 +802,7 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
   }
 
   if (attrs.requests.find(OpReqType::kAddTo) != attrs.requests.end()) {
+    std::vector<NDArray*> original_outputs(attrs.num_outputs);
     in_arrs = GetTestInputArrays();
     for (auto in_arr : in_arrs) {
       for (auto dispatch : dispatches) {
@@ -804,17 +811,19 @@ void TestOp(const OpAttrs &attrs, VerifyFunc verify_fn) {
         for (size_t i = 0; i < attrs.num_inputs; i++)
           inputs[i] = &in_arr.arr;
         for (size_t output_i = 0; output_i < out_arrs[0].size(); output_i++) {
-          auto out_arr = out_arrs[0][output_i];
-          NDArray orig_output = out_arr.arr.Copy(out_arr.arr.ctx());
+          NDArray tmp;
           for (size_t i = 0; i < attrs.num_outputs; i++) {
+            auto out_arr = out_arrs[i][output_i];
+            tmp = out_arr.arr.Copy(out_arr.arr.ctx());
+            original_outputs[i] =  &tmp;
             outputs[i] = &out_arrs[i][output_i].arr;
             req[i] = kAddTo;
           }
-          PrintVerifyMsg(in_arr, out_arr);
+          PrintVerifyMsg(in_arr, out_arrs[0][output_i]);
           Imperative::Get()->InvokeOp(Context(), attrs.attrs, inputs,
                                       outputs, req, dispatch, mxnet::OpStatePtr());
           Engine::Get()->WaitForAll();
-          VerifyAddRequest(inputs, {&orig_output}, outputs, verify_fn);
+          VerifyAddRequest(inputs, original_outputs, outputs, verify_fn);
         }
       }
     }

From d91df9321ca4d98ed83711584a9fce505ad207ee Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 19:50:33 -0700
Subject: [PATCH 52/63] retrigger


From 993c7aa435be7aa764c77ff206b562cd667f7a12 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 25 Jun 2018 22:10:29 -0700
Subject: [PATCH 53/63] retriggrer


From e2a464d2e3e09035714e7789ffe44f272e943f11 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 26 Jun 2018 13:37:41 -0700
Subject: [PATCH 54/63] retrigger


From dc742c871179d0e950efb44120a48f4dddbf47b7 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 26 Jun 2018 14:55:58 -0700
Subject: [PATCH 55/63] retrigger


From 92c50f094bad2bc519e08804d6578003dc41a95d Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 26 Jun 2018 17:34:48 -0700
Subject: [PATCH 56/63] another retrigger


From 113903a6b1fc29c9012cbbf3e45ef0c6c88a3469 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 27 Jun 2018 07:39:52 -0700
Subject: [PATCH 57/63] retrigger


From ecbde640cbb14dc56385607b4df452eb3d3a41bd Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 27 Jun 2018 10:56:30 -0700
Subject: [PATCH 58/63] retrigger


From be84769dd7a585799e6d09a4c82eace45f08e519 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 27 Jun 2018 12:12:19 -0700
Subject: [PATCH 59/63] another another retrigger


From ad3c70edcf5a1b5d3208dce561993cf50b369f54 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 29 Jun 2018 12:10:47 -0700
Subject: [PATCH 60/63] fix merge

---
 tests/cpp/operator/mkldnn.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index a91cb256fa74..5cee42fa85a2 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -854,9 +854,6 @@ void VerifyAddRequest(const std::vector<NDArray*> &in_arrs,
   verify_fn(in_arrs, tmp_outputs);
 }
 
-void PrintVerifyMsg(const NDArrayAttrs &arr1, const NDArrayAttrs &arr2) {
-  TShape t1 = arr1.arr.shape();
-  TShape t2 = arr2.arr.shape();
 void VerifyConcatBackwardsResult(const std::vector<NDArray *> &in_arrs,
                         const std::vector<NDArray *> &out_arrs) {
   // in_arrs is larger array, out_arr is ammler

From 2874d0a106b4f1a3b5d694a26f8de9d5b9e9e4d8 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 29 Jun 2018 13:48:42 -0700
Subject: [PATCH 61/63] retrigger


From 581495fe6d8cb9e3b223b688f3f8277ae6c2b79d Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 3 Jul 2018 16:14:20 -0700
Subject: [PATCH 62/63] add kAddto to relu op

---
 tests/cpp/operator/mkldnn.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/cpp/operator/mkldnn.cc b/tests/cpp/operator/mkldnn.cc
index 5cee42fa85a2..9a7d2fccd745 100644
--- a/tests/cpp/operator/mkldnn.cc
+++ b/tests/cpp/operator/mkldnn.cc
@@ -409,6 +409,7 @@ OpAttrs GetReluOp() {
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
   attrs.requests.insert(OpReqType::kWriteTo);
   attrs.requests.insert(OpReqType::kWriteInplace);
+  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 
@@ -424,6 +425,7 @@ OpAttrs GetReluBackwardsOp() {
   attrs.dispatches[1] = DispatchMode::kFComputeEx;
   attrs.requests.insert(OpReqType::kWriteTo);
   attrs.requests.insert(OpReqType::kWriteInplace);
+  attrs.requests.insert(OpReqType::kAddTo);
   return attrs;
 }
 

From 9e7c22e2edabc4a633625cec7b3c3f90e471ff13 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 5 Jul 2018 10:08:43 -0700
Subject: [PATCH 63/63] retrigger