From d3ab6553260b8113346cf9d080a73ef2ff0ad1d9 Mon Sep 17 00:00:00 2001
From: zyfncg <1370305206@qq.com>
Date: Fri, 15 Oct 2021 22:19:38 +0800
Subject: [PATCH] Move cpu, cuda and other device code into kernels (#15)

* fill_any_like kernel refactor

* remove useless code of full_like c++ api

* Support Scalar in Tensor Compute Library

* add scalar in dygraph and static graph mode

* keep the basic type for attr, instead of using scalar for all

* merge the code

* start refactor matmul

* move cpu, cuda and other device modules into kernels

* merge code

* polish code in operator.cc
---
 paddle/fluid/framework/operator.cc            |  1 -
 paddle/tcmpt/CMakeLists.txt                   | 21 +------------------
 paddle/tcmpt/api/include/creation.h           |  4 ++--
 paddle/tcmpt/api/include/linalg.h             |  4 ++--
 paddle/tcmpt/api/include/manipulation.h       |  4 ++--
 paddle/tcmpt/api/include/math.h               |  4 ++--
 paddle/tcmpt/hapi/include/linalg.h            |  5 +++++
 paddle/tcmpt/kernels/CMakeLists.txt           | 18 ++++++++++++++++
 .../{ => kernels/common}/eigen/CMakeLists.txt |  0
 .../tcmpt/{ => kernels/common}/eigen/common.h |  0
 paddle/tcmpt/{ => kernels/common}/eigen/dot.h |  2 +-
 .../tcmpt/{ => kernels/common}/eigen/fill.h   |  2 +-
 .../tcmpt/{ => kernels/common}/eigen/mean.h   |  2 +-
 .../tcmpt/{ => kernels/common}/eigen/scale.h  |  2 +-
 .../tcmpt/{ => kernels/common}/eigen/sign.h   |  2 +-
 paddle/tcmpt/{ => kernels}/cpu/CMakeLists.txt |  2 +-
 paddle/tcmpt/{ => kernels}/cpu/creation.cc    |  4 ++--
 paddle/tcmpt/{ => kernels}/cpu/creation.h     |  0
 paddle/tcmpt/{ => kernels}/cpu/linalg.cc      | 11 +++++++++-
 paddle/tcmpt/{ => kernels}/cpu/linalg.h       |  8 +++++++
 .../tcmpt/{ => kernels}/cpu/manipulation.cc   |  4 ++--
 paddle/tcmpt/{ => kernels}/cpu/manipulation.h |  0
 paddle/tcmpt/{ => kernels}/cpu/math.cc        |  8 +++----
 paddle/tcmpt/{ => kernels}/cpu/math.h         |  0
 paddle/tcmpt/{ => kernels}/cpu/utils.cc       |  2 +-
 paddle/tcmpt/{ => kernels}/cpu/utils.h        |  0
 .../tcmpt/{ => kernels}/cuda/CMakeLists.txt   |  2 +-
 paddle/tcmpt/{ => kernels}/cuda/creation.cu   |  4 ++--
 paddle/tcmpt/{ => kernels}/cuda/creation.h    |  0
 paddle/tcmpt/{ => kernels}/cuda/linalg.cu     |  4 ++--
 paddle/tcmpt/{ => kernels}/cuda/linalg.h      |  0
 .../tcmpt/{ => kernels}/cuda/manipulation.cu  |  4 ++--
 .../tcmpt/{ => kernels}/cuda/manipulation.h   |  0
 paddle/tcmpt/{ => kernels}/cuda/math.cu       |  8 +++----
 paddle/tcmpt/{ => kernels}/cuda/math.h        |  0
 paddle/tcmpt/{ => kernels}/cuda/utils.cu      |  2 +-
 paddle/tcmpt/{ => kernels}/cuda/utils.h       |  0
 .../tcmpt/{ => kernels}/mkldnn/CMakeLists.txt |  0
 paddle/tcmpt/{ => kernels}/npu/CMakeLists.txt |  0
 paddle/tcmpt/{ => kernels}/xpu/CMakeLists.txt |  0
 paddle/tcmpt/tests/test_copy_api.cc           |  2 +-
 41 files changed, 78 insertions(+), 58 deletions(-)
 create mode 100644 paddle/tcmpt/kernels/CMakeLists.txt
 rename paddle/tcmpt/{ => kernels/common}/eigen/CMakeLists.txt (100%)
 rename paddle/tcmpt/{ => kernels/common}/eigen/common.h (100%)
 rename paddle/tcmpt/{ => kernels/common}/eigen/dot.h (96%)
 rename paddle/tcmpt/{ => kernels/common}/eigen/fill.h (97%)
 rename paddle/tcmpt/{ => kernels/common}/eigen/mean.h (96%)
 rename paddle/tcmpt/{ => kernels/common}/eigen/scale.h (96%)
 rename paddle/tcmpt/{ => kernels/common}/eigen/sign.h (96%)
 rename paddle/tcmpt/{ => kernels}/cpu/CMakeLists.txt (89%)
 rename paddle/tcmpt/{ => kernels}/cpu/creation.cc (92%)
 rename paddle/tcmpt/{ => kernels}/cpu/creation.h (100%)
 rename paddle/tcmpt/{ => kernels}/cpu/linalg.cc (86%)
 rename paddle/tcmpt/{ => kernels}/cpu/linalg.h (82%)
 rename paddle/tcmpt/{ => kernels}/cpu/manipulation.cc (96%)
 rename paddle/tcmpt/{ => kernels}/cpu/manipulation.h (100%)
 rename paddle/tcmpt/{ => kernels}/cpu/math.cc (93%)
 rename paddle/tcmpt/{ => kernels}/cpu/math.h (100%)
 rename paddle/tcmpt/{ => kernels}/cpu/utils.cc (97%)
 rename paddle/tcmpt/{ => kernels}/cpu/utils.h (100%)
 rename paddle/tcmpt/{ => kernels}/cuda/CMakeLists.txt (94%)
 rename paddle/tcmpt/{ => kernels}/cuda/creation.cu (92%)
 rename paddle/tcmpt/{ => kernels}/cuda/creation.h (100%)
 rename paddle/tcmpt/{ => kernels}/cuda/linalg.cu (93%)
 rename paddle/tcmpt/{ => kernels}/cuda/linalg.h (100%)
 rename paddle/tcmpt/{ => kernels}/cuda/manipulation.cu (96%)
 rename paddle/tcmpt/{ => kernels}/cuda/manipulation.h (100%)
 rename paddle/tcmpt/{ => kernels}/cuda/math.cu (95%)
 rename paddle/tcmpt/{ => kernels}/cuda/math.h (100%)
 rename paddle/tcmpt/{ => kernels}/cuda/utils.cu (99%)
 rename paddle/tcmpt/{ => kernels}/cuda/utils.h (100%)
 rename paddle/tcmpt/{ => kernels}/mkldnn/CMakeLists.txt (100%)
 rename paddle/tcmpt/{ => kernels}/npu/CMakeLists.txt (100%)
 rename paddle/tcmpt/{ => kernels}/xpu/CMakeLists.txt (100%)

diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc
index 32fc10f38bd48..2ea761944671b 100644
--- a/paddle/fluid/framework/operator.cc
+++ b/paddle/fluid/framework/operator.cc
@@ -1966,7 +1966,6 @@ pt::KernelContext OperatorWithKernel::ConstructPtKernelContext(
   }
 
   for (size_t i = 0; i < attr_defs.size(); ++i) {
-    paddle::any attr_item;
     if (attr_defs[i].type_index == std::type_index(typeid(pt::Scalar))) {
       // TODO(chenweihang): support other attrs
       // In principle, the attr required by the dynamic mode should be
diff --git a/paddle/tcmpt/CMakeLists.txt b/paddle/tcmpt/CMakeLists.txt
index c21428ef4715b..0187a63c2ff6d 100644
--- a/paddle/tcmpt/CMakeLists.txt
+++ b/paddle/tcmpt/CMakeLists.txt
@@ -5,27 +5,8 @@ add_subdirectory(api)
 add_subdirectory(hapi)
 # tcmpt core components
 add_subdirectory(core)
-# tcmpt eigne functors, now paddle must compiled with eigen, but eigen just is
-# one backend dtype, we should support cropping it for lite
-add_subdirectory(eigen)
 # tcmpt kernels for diff device
-add_subdirectory(cpu)
-if(WITH_GPU OR WITH_ROCM)
-  # TODO(chenweihang): if hip can split from cuda impl, we should add hip dir
-  add_subdirectory(cuda)
-endif()
-# TODO(chenweihang): migrate MKLDNN Kernel in the second phase of the project
-if(WITH_MKLDNN)
-  add_subdirectory(mkldnn)
-endif()
-# TODO(chenweihang): migrate NPU Kernel in the second phase of the project
-if(WITH_ASCEND_CL)
-  add_subdirectory(npu)
-endif()
-# TODO(chenweihang): migrate XPU Kernel in the second phase of the project
-if(WITH_XPU)
-  add_subdirectory(xpu)
-endif()
+add_subdirectory(kernels)
 # tcmpt infershape
 add_subdirectory(infershape)
 # TODO(xingfeng): tcmpt inner module API designed by a high-performance team
diff --git a/paddle/tcmpt/api/include/creation.h b/paddle/tcmpt/api/include/creation.h
index e0ef25d202c6e..2a87453b32154 100644
--- a/paddle/tcmpt/api/include/creation.h
+++ b/paddle/tcmpt/api/include/creation.h
@@ -14,5 +14,5 @@
 
 #pragma once
 
-#include "paddle/tcmpt/cpu/creation.h"
-#include "paddle/tcmpt/cuda/creation.h"
+#include "paddle/tcmpt/kernels/cpu/creation.h"
+#include "paddle/tcmpt/kernels/cuda/creation.h"
diff --git a/paddle/tcmpt/api/include/linalg.h b/paddle/tcmpt/api/include/linalg.h
index 46acfaea32163..81ea68abcd0bb 100644
--- a/paddle/tcmpt/api/include/linalg.h
+++ b/paddle/tcmpt/api/include/linalg.h
@@ -15,5 +15,5 @@
 #pragma once
 
 // See Note: [ How do we organize the kernel directory ]
-#include "paddle/tcmpt/cpu/linalg.h"
-#include "paddle/tcmpt/cuda/linalg.h"
+#include "paddle/tcmpt/kernels/cpu/linalg.h"
+#include "paddle/tcmpt/kernels/cuda/linalg.h"
diff --git a/paddle/tcmpt/api/include/manipulation.h b/paddle/tcmpt/api/include/manipulation.h
index b44e53c01384b..1746929ca181d 100644
--- a/paddle/tcmpt/api/include/manipulation.h
+++ b/paddle/tcmpt/api/include/manipulation.h
@@ -15,5 +15,5 @@
 #pragma once
 
 // See Note: [ How do we organize the kernel directory ]
-#include "paddle/tcmpt/cpu/manipulation.h"
-#include "paddle/tcmpt/cuda/manipulation.h"
+#include "paddle/tcmpt/kernels/cpu/manipulation.h"
+#include "paddle/tcmpt/kernels/cuda/manipulation.h"
diff --git a/paddle/tcmpt/api/include/math.h b/paddle/tcmpt/api/include/math.h
index 2f1a04d16f8ac..ab3c229806990 100644
--- a/paddle/tcmpt/api/include/math.h
+++ b/paddle/tcmpt/api/include/math.h
@@ -15,5 +15,5 @@ limitations under the License. */
 #pragma once
 
 // See Note: [ How do we organize the kernel directory ]
-#include "paddle/tcmpt/cpu/math.h"
-#include "paddle/tcmpt/cuda/math.h"
+#include "paddle/tcmpt/kernels/cpu/math.h"
+#include "paddle/tcmpt/kernels/cuda/math.h"
diff --git a/paddle/tcmpt/hapi/include/linalg.h b/paddle/tcmpt/hapi/include/linalg.h
index 5e27fecd58a4e..df709b6a3c50f 100644
--- a/paddle/tcmpt/hapi/include/linalg.h
+++ b/paddle/tcmpt/hapi/include/linalg.h
@@ -21,5 +21,10 @@ namespace experimental {
 
 Tensor dot(const Tensor& x, const Tensor& y);
 
+Tensor matmul(const Tensor& x,
+              const Tensor& y,
+              bool transpose_x,
+              bool transpose_y);
+
 }  // namespace experimental
 }  // namespace paddle
diff --git a/paddle/tcmpt/kernels/CMakeLists.txt b/paddle/tcmpt/kernels/CMakeLists.txt
new file mode 100644
index 0000000000000..26b5e16d4428d
--- /dev/null
+++ b/paddle/tcmpt/kernels/CMakeLists.txt
@@ -0,0 +1,18 @@
+# tcmpt kernels for diff device
+add_subdirectory(cpu)
+if(WITH_GPU OR WITH_ROCM)
+  # TODO(chenweihang): if hip can split from cuda impl, we should add hip dir
+  add_subdirectory(cuda)
+endif()
+# TODO(chenweihang): migrate MKLDNN Kernel in the second phase of the project
+if(WITH_MKLDNN)
+  add_subdirectory(mkldnn)
+endif()
+# TODO(chenweihang): migrate NPU Kernel in the second phase of the project
+if(WITH_ASCEND_CL)
+  add_subdirectory(npu)
+endif()
+# TODO(chenweihang): migrate XPU Kernel in the second phase of the project
+if(WITH_XPU)
+  add_subdirectory(xpu)
+endif()
diff --git a/paddle/tcmpt/eigen/CMakeLists.txt b/paddle/tcmpt/kernels/common/eigen/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/eigen/CMakeLists.txt
rename to paddle/tcmpt/kernels/common/eigen/CMakeLists.txt
diff --git a/paddle/tcmpt/eigen/common.h b/paddle/tcmpt/kernels/common/eigen/common.h
similarity index 100%
rename from paddle/tcmpt/eigen/common.h
rename to paddle/tcmpt/kernels/common/eigen/common.h
diff --git a/paddle/tcmpt/eigen/dot.h b/paddle/tcmpt/kernels/common/eigen/dot.h
similarity index 96%
rename from paddle/tcmpt/eigen/dot.h
rename to paddle/tcmpt/kernels/common/eigen/dot.h
index 5e323e4448409..32c1e1439fac7 100644
--- a/paddle/tcmpt/eigen/dot.h
+++ b/paddle/tcmpt/kernels/common/eigen/dot.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/fill.h b/paddle/tcmpt/kernels/common/eigen/fill.h
similarity index 97%
rename from paddle/tcmpt/eigen/fill.h
rename to paddle/tcmpt/kernels/common/eigen/fill.h
index fb56ccdd8e125..186163c3fedc4 100644
--- a/paddle/tcmpt/eigen/fill.h
+++ b/paddle/tcmpt/kernels/common/eigen/fill.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/mean.h b/paddle/tcmpt/kernels/common/eigen/mean.h
similarity index 96%
rename from paddle/tcmpt/eigen/mean.h
rename to paddle/tcmpt/kernels/common/eigen/mean.h
index e70870e7954b7..2b1ea95940727 100644
--- a/paddle/tcmpt/eigen/mean.h
+++ b/paddle/tcmpt/kernels/common/eigen/mean.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/scale.h b/paddle/tcmpt/kernels/common/eigen/scale.h
similarity index 96%
rename from paddle/tcmpt/eigen/scale.h
rename to paddle/tcmpt/kernels/common/eigen/scale.h
index 152cb61800c8b..0f3e92d9db787 100644
--- a/paddle/tcmpt/eigen/scale.h
+++ b/paddle/tcmpt/kernels/common/eigen/scale.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/sign.h b/paddle/tcmpt/kernels/common/eigen/sign.h
similarity index 96%
rename from paddle/tcmpt/eigen/sign.h
rename to paddle/tcmpt/kernels/common/eigen/sign.h
index d41702576b3a1..3980976ac9cf5 100644
--- a/paddle/tcmpt/eigen/sign.h
+++ b/paddle/tcmpt/kernels/common/eigen/sign.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/cpu/CMakeLists.txt b/paddle/tcmpt/kernels/cpu/CMakeLists.txt
similarity index 89%
rename from paddle/tcmpt/cpu/CMakeLists.txt
rename to paddle/tcmpt/kernels/cpu/CMakeLists.txt
index cf3204bc5bcb0..b70c5f9ec81f0 100644
--- a/paddle/tcmpt/cpu/CMakeLists.txt
+++ b/paddle/tcmpt/kernels/cpu/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(WIN32)
-    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/cpu)
+    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/kernels/cpu)
     kernel_instantiate(creation.cc)
     kernel_instantiate(math.cc)
     kernel_instantiate(linalg.cc)
diff --git a/paddle/tcmpt/cpu/creation.cc b/paddle/tcmpt/kernels/cpu/creation.cc
similarity index 92%
rename from paddle/tcmpt/cpu/creation.cc
rename to paddle/tcmpt/kernels/cpu/creation.cc
index 8e4399c41bf17..4871e11da2112 100644
--- a/paddle/tcmpt/cpu/creation.cc
+++ b/paddle/tcmpt/kernels/cpu/creation.cc
@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/creation.h"
+#include "paddle/tcmpt/kernels/cpu/creation.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/eigen/fill.h"
+#include "paddle/tcmpt/kernels/common/eigen/fill.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cpu/creation.h b/paddle/tcmpt/kernels/cpu/creation.h
similarity index 100%
rename from paddle/tcmpt/cpu/creation.h
rename to paddle/tcmpt/kernels/cpu/creation.h
diff --git a/paddle/tcmpt/cpu/linalg.cc b/paddle/tcmpt/kernels/cpu/linalg.cc
similarity index 86%
rename from paddle/tcmpt/cpu/linalg.cc
rename to paddle/tcmpt/kernels/cpu/linalg.cc
index 96c1a4e937fce..8b63219fdd2db 100644
--- a/paddle/tcmpt/cpu/linalg.cc
+++ b/paddle/tcmpt/kernels/cpu/linalg.cc
@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/linalg.h"
+#include "paddle/tcmpt/kernels/cpu/linalg.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/framework/eigen.h"
+#include "paddle/fluid/operators/math/blas.h"
 #include "paddle/fluid/platform/complex.h"
 
 namespace pt {
@@ -44,6 +45,14 @@ void Dot(const CPUContext& dev_ctx,
   }
 }
 
+template <typename T>
+void matmul(const CPUContext& dev_ctx,
+            const DenseTensor& x,
+            const DenseTensor& y,
+            bool transpose_x,
+            bool transpose_y,
+            DenseTensor* out) {}
+
 }  // namespace pt
 
 PT_REGISTER_MODULE(LinalgCPU);
diff --git a/paddle/tcmpt/cpu/linalg.h b/paddle/tcmpt/kernels/cpu/linalg.h
similarity index 82%
rename from paddle/tcmpt/cpu/linalg.h
rename to paddle/tcmpt/kernels/cpu/linalg.h
index c457943538761..6d9550b2882b2 100644
--- a/paddle/tcmpt/cpu/linalg.h
+++ b/paddle/tcmpt/kernels/cpu/linalg.h
@@ -29,4 +29,12 @@ void Dot(const CPUContext& dev_ctx,
          const DenseTensor& y,
          DenseTensor* out);
 
+template <typename T>
+void matmul(const CPUContext& dev_ctx,
+            const DenseTensor& x,
+            const DenseTensor& y,
+            bool transpose_x,
+            bool transpose_y,
+            DenseTensor* out);
+
 }  // namespace pt
diff --git a/paddle/tcmpt/cpu/manipulation.cc b/paddle/tcmpt/kernels/cpu/manipulation.cc
similarity index 96%
rename from paddle/tcmpt/cpu/manipulation.cc
rename to paddle/tcmpt/kernels/cpu/manipulation.cc
index d2964c5b533a9..91f1e941cd028 100644
--- a/paddle/tcmpt/cpu/manipulation.cc
+++ b/paddle/tcmpt/kernels/cpu/manipulation.cc
@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/manipulation.h"
-#include "paddle/tcmpt/cpu/utils.h"
+#include "paddle/tcmpt/kernels/cpu/manipulation.h"
 #include "paddle/tcmpt/infershape/unary.h"
+#include "paddle/tcmpt/kernels/cpu/utils.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cpu/manipulation.h b/paddle/tcmpt/kernels/cpu/manipulation.h
similarity index 100%
rename from paddle/tcmpt/cpu/manipulation.h
rename to paddle/tcmpt/kernels/cpu/manipulation.h
diff --git a/paddle/tcmpt/cpu/math.cc b/paddle/tcmpt/kernels/cpu/math.cc
similarity index 93%
rename from paddle/tcmpt/cpu/math.cc
rename to paddle/tcmpt/kernels/cpu/math.cc
index 80dec2530f718..d304db0a9a34e 100644
--- a/paddle/tcmpt/cpu/math.cc
+++ b/paddle/tcmpt/kernels/cpu/math.cc
@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/math.h"
+#include "paddle/tcmpt/kernels/cpu/math.h"
 
-#include "paddle/tcmpt/eigen/mean.h"
-#include "paddle/tcmpt/eigen/scale.h"
-#include "paddle/tcmpt/eigen/sign.h"
+#include "paddle/tcmpt/kernels/common/eigen/mean.h"
+#include "paddle/tcmpt/kernels/common/eigen/scale.h"
+#include "paddle/tcmpt/kernels/common/eigen/sign.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/framework/eigen.h"
diff --git a/paddle/tcmpt/cpu/math.h b/paddle/tcmpt/kernels/cpu/math.h
similarity index 100%
rename from paddle/tcmpt/cpu/math.h
rename to paddle/tcmpt/kernels/cpu/math.h
diff --git a/paddle/tcmpt/cpu/utils.cc b/paddle/tcmpt/kernels/cpu/utils.cc
similarity index 97%
rename from paddle/tcmpt/cpu/utils.cc
rename to paddle/tcmpt/kernels/cpu/utils.cc
index 86b074e49b362..7550934d70be4 100644
--- a/paddle/tcmpt/cpu/utils.cc
+++ b/paddle/tcmpt/kernels/cpu/utils.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/tcmpt/cpu/utils.h"
+#include "paddle/tcmpt/kernels/cpu/utils.h"
 #include "paddle/fluid/memory/memcpy.h"
 #include "paddle/tcmpt/core/convert_utils.h"
 #include "paddle/tcmpt/core/dtype.h"
diff --git a/paddle/tcmpt/cpu/utils.h b/paddle/tcmpt/kernels/cpu/utils.h
similarity index 100%
rename from paddle/tcmpt/cpu/utils.h
rename to paddle/tcmpt/kernels/cpu/utils.h
diff --git a/paddle/tcmpt/cuda/CMakeLists.txt b/paddle/tcmpt/kernels/cuda/CMakeLists.txt
similarity index 94%
rename from paddle/tcmpt/cuda/CMakeLists.txt
rename to paddle/tcmpt/kernels/cuda/CMakeLists.txt
index 9e56e1a3be82a..e243bad09563b 100644
--- a/paddle/tcmpt/cuda/CMakeLists.txt
+++ b/paddle/tcmpt/kernels/cuda/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(WIN32)
-    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/cuda)
+    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/kernels/cuda)
     kernel_instantiate(creation.cu)
     kernel_instantiate(math.cu)
     kernel_instantiate(linalg.cu)
diff --git a/paddle/tcmpt/cuda/creation.cu b/paddle/tcmpt/kernels/cuda/creation.cu
similarity index 92%
rename from paddle/tcmpt/cuda/creation.cu
rename to paddle/tcmpt/kernels/cuda/creation.cu
index cca9199b76cfd..7f082400eaaf7 100644
--- a/paddle/tcmpt/cuda/creation.cu
+++ b/paddle/tcmpt/kernels/cuda/creation.cu
@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cuda/creation.h"
+#include "paddle/tcmpt/kernels/cuda/creation.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/eigen/fill.h"
+#include "paddle/tcmpt/kernels/common/eigen/fill.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cuda/creation.h b/paddle/tcmpt/kernels/cuda/creation.h
similarity index 100%
rename from paddle/tcmpt/cuda/creation.h
rename to paddle/tcmpt/kernels/cuda/creation.h
diff --git a/paddle/tcmpt/cuda/linalg.cu b/paddle/tcmpt/kernels/cuda/linalg.cu
similarity index 93%
rename from paddle/tcmpt/cuda/linalg.cu
rename to paddle/tcmpt/kernels/cuda/linalg.cu
index 118d3326e5fb5..25d1df5cbc65a 100644
--- a/paddle/tcmpt/cuda/linalg.cu
+++ b/paddle/tcmpt/kernels/cuda/linalg.cu
@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cuda/linalg.h"
+#include "paddle/tcmpt/kernels/cuda/linalg.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/eigen/dot.h"
+#include "paddle/tcmpt/kernels/common/eigen/dot.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/platform/complex.h"
diff --git a/paddle/tcmpt/cuda/linalg.h b/paddle/tcmpt/kernels/cuda/linalg.h
similarity index 100%
rename from paddle/tcmpt/cuda/linalg.h
rename to paddle/tcmpt/kernels/cuda/linalg.h
diff --git a/paddle/tcmpt/cuda/manipulation.cu b/paddle/tcmpt/kernels/cuda/manipulation.cu
similarity index 96%
rename from paddle/tcmpt/cuda/manipulation.cu
rename to paddle/tcmpt/kernels/cuda/manipulation.cu
index 91f69b2fe33d7..bb4a2cc9a677b 100644
--- a/paddle/tcmpt/cuda/manipulation.cu
+++ b/paddle/tcmpt/kernels/cuda/manipulation.cu
@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cuda/manipulation.h"
-#include "paddle/tcmpt/cuda/utils.h"
 #include "paddle/tcmpt/infershape/unary.h"
+#include "paddle/tcmpt/kernels/cuda/manipulation.h"
+#include "paddle/tcmpt/kernels/cuda/utils.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cuda/manipulation.h b/paddle/tcmpt/kernels/cuda/manipulation.h
similarity index 100%
rename from paddle/tcmpt/cuda/manipulation.h
rename to paddle/tcmpt/kernels/cuda/manipulation.h
diff --git a/paddle/tcmpt/cuda/math.cu b/paddle/tcmpt/kernels/cuda/math.cu
similarity index 95%
rename from paddle/tcmpt/cuda/math.cu
rename to paddle/tcmpt/kernels/cuda/math.cu
index 293f0cf8bfc91..743615d70f996 100644
--- a/paddle/tcmpt/cuda/math.cu
+++ b/paddle/tcmpt/kernels/cuda/math.cu
@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/tcmpt/cuda/math.h"
+#include "paddle/tcmpt/kernels/cuda/math.h"
 
-#include "paddle/tcmpt/eigen/mean.h"
-#include "paddle/tcmpt/eigen/scale.h"
-#include "paddle/tcmpt/eigen/sign.h"
+#include "paddle/tcmpt/kernels/common/eigen/mean.h"
+#include "paddle/tcmpt/kernels/common/eigen/scale.h"
+#include "paddle/tcmpt/kernels/common/eigen/sign.h"
 
 #ifdef __NVCC__
 #include "cub/cub.cuh"
diff --git a/paddle/tcmpt/cuda/math.h b/paddle/tcmpt/kernels/cuda/math.h
similarity index 100%
rename from paddle/tcmpt/cuda/math.h
rename to paddle/tcmpt/kernels/cuda/math.h
diff --git a/paddle/tcmpt/cuda/utils.cu b/paddle/tcmpt/kernels/cuda/utils.cu
similarity index 99%
rename from paddle/tcmpt/cuda/utils.cu
rename to paddle/tcmpt/kernels/cuda/utils.cu
index 40b93f3534c1a..b8483d17cfc24 100644
--- a/paddle/tcmpt/cuda/utils.cu
+++ b/paddle/tcmpt/kernels/cuda/utils.cu
@@ -16,7 +16,7 @@ limitations under the License. */
 #include "paddle/tcmpt/core/convert_utils.h"
 #include "paddle/tcmpt/core/dtype.h"
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/cuda/utils.h"
+#include "paddle/tcmpt/kernels/cuda/utils.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cuda/utils.h b/paddle/tcmpt/kernels/cuda/utils.h
similarity index 100%
rename from paddle/tcmpt/cuda/utils.h
rename to paddle/tcmpt/kernels/cuda/utils.h
diff --git a/paddle/tcmpt/mkldnn/CMakeLists.txt b/paddle/tcmpt/kernels/mkldnn/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/mkldnn/CMakeLists.txt
rename to paddle/tcmpt/kernels/mkldnn/CMakeLists.txt
diff --git a/paddle/tcmpt/npu/CMakeLists.txt b/paddle/tcmpt/kernels/npu/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/npu/CMakeLists.txt
rename to paddle/tcmpt/kernels/npu/CMakeLists.txt
diff --git a/paddle/tcmpt/xpu/CMakeLists.txt b/paddle/tcmpt/kernels/xpu/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/xpu/CMakeLists.txt
rename to paddle/tcmpt/kernels/xpu/CMakeLists.txt
diff --git a/paddle/tcmpt/tests/test_copy_api.cc b/paddle/tcmpt/tests/test_copy_api.cc
index 7f1158912ebfb..2d70e37d051d9 100644
--- a/paddle/tcmpt/tests/test_copy_api.cc
+++ b/paddle/tcmpt/tests/test_copy_api.cc
@@ -16,7 +16,7 @@ limitations under the License. */
 #include <memory>
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/cpu/utils.h"
+#include "paddle/tcmpt/kernels/cpu/utils.h"
 
 #include "paddle/tcmpt/core/dense_tensor.h"