diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc
index 32fc10f38bd48..2ea761944671b 100644
--- a/paddle/fluid/framework/operator.cc
+++ b/paddle/fluid/framework/operator.cc
@@ -1966,7 +1966,6 @@ pt::KernelContext OperatorWithKernel::ConstructPtKernelContext(
   }
 
   for (size_t i = 0; i < attr_defs.size(); ++i) {
-    paddle::any attr_item;
     if (attr_defs[i].type_index == std::type_index(typeid(pt::Scalar))) {
       // TODO(chenweihang): support other attrs
       // In principle, the attr required by the dynamic mode should be
diff --git a/paddle/tcmpt/CMakeLists.txt b/paddle/tcmpt/CMakeLists.txt
index c21428ef4715b..0187a63c2ff6d 100644
--- a/paddle/tcmpt/CMakeLists.txt
+++ b/paddle/tcmpt/CMakeLists.txt
@@ -5,27 +5,8 @@ add_subdirectory(api)
 add_subdirectory(hapi)
 # tcmpt core components
 add_subdirectory(core)
-# tcmpt eigne functors, now paddle must compiled with eigen, but eigen just is
-# one backend dtype, we should support cropping it for lite
-add_subdirectory(eigen)
 # tcmpt kernels for diff device
-add_subdirectory(cpu)
-if(WITH_GPU OR WITH_ROCM)
-  # TODO(chenweihang): if hip can split from cuda impl, we should add hip dir
-  add_subdirectory(cuda)
-endif()
-# TODO(chenweihang): migrate MKLDNN Kernel in the second phase of the project
-if(WITH_MKLDNN)
-  add_subdirectory(mkldnn)
-endif()
-# TODO(chenweihang): migrate NPU Kernel in the second phase of the project
-if(WITH_ASCEND_CL)
-  add_subdirectory(npu)
-endif()
-# TODO(chenweihang): migrate XPU Kernel in the second phase of the project
-if(WITH_XPU)
-  add_subdirectory(xpu)
-endif()
+add_subdirectory(kernels)
 # tcmpt infershape
 add_subdirectory(infershape)
 # TODO(xingfeng): tcmpt inner module API designed by a high-performance team
diff --git a/paddle/tcmpt/api/include/creation.h b/paddle/tcmpt/api/include/creation.h
index e0ef25d202c6e..2a87453b32154 100644
--- a/paddle/tcmpt/api/include/creation.h
+++ b/paddle/tcmpt/api/include/creation.h
@@ -14,5 +14,5 @@
 
 #pragma once
 
-#include "paddle/tcmpt/cpu/creation.h"
-#include "paddle/tcmpt/cuda/creation.h"
+#include "paddle/tcmpt/kernels/cpu/creation.h"
+#include "paddle/tcmpt/kernels/cuda/creation.h"
diff --git a/paddle/tcmpt/api/include/linalg.h b/paddle/tcmpt/api/include/linalg.h
index 46acfaea32163..81ea68abcd0bb 100644
--- a/paddle/tcmpt/api/include/linalg.h
+++ b/paddle/tcmpt/api/include/linalg.h
@@ -15,5 +15,5 @@
 #pragma once
 
 // See Note: [ How do we organize the kernel directory ]
-#include "paddle/tcmpt/cpu/linalg.h"
-#include "paddle/tcmpt/cuda/linalg.h"
+#include "paddle/tcmpt/kernels/cpu/linalg.h"
+#include "paddle/tcmpt/kernels/cuda/linalg.h"
diff --git a/paddle/tcmpt/api/include/manipulation.h b/paddle/tcmpt/api/include/manipulation.h
index b44e53c01384b..1746929ca181d 100644
--- a/paddle/tcmpt/api/include/manipulation.h
+++ b/paddle/tcmpt/api/include/manipulation.h
@@ -15,5 +15,5 @@
 #pragma once
 
 // See Note: [ How do we organize the kernel directory ]
-#include "paddle/tcmpt/cpu/manipulation.h"
-#include "paddle/tcmpt/cuda/manipulation.h"
+#include "paddle/tcmpt/kernels/cpu/manipulation.h"
+#include "paddle/tcmpt/kernels/cuda/manipulation.h"
diff --git a/paddle/tcmpt/api/include/math.h b/paddle/tcmpt/api/include/math.h
index 2f1a04d16f8ac..ab3c229806990 100644
--- a/paddle/tcmpt/api/include/math.h
+++ b/paddle/tcmpt/api/include/math.h
@@ -15,5 +15,5 @@ limitations under the License. */
 #pragma once
 
 // See Note: [ How do we organize the kernel directory ]
-#include "paddle/tcmpt/cpu/math.h"
-#include "paddle/tcmpt/cuda/math.h"
+#include "paddle/tcmpt/kernels/cpu/math.h"
+#include "paddle/tcmpt/kernels/cuda/math.h"
diff --git a/paddle/tcmpt/hapi/include/linalg.h b/paddle/tcmpt/hapi/include/linalg.h
index 5e27fecd58a4e..df709b6a3c50f 100644
--- a/paddle/tcmpt/hapi/include/linalg.h
+++ b/paddle/tcmpt/hapi/include/linalg.h
@@ -21,5 +21,10 @@ namespace experimental {
 
 Tensor dot(const Tensor& x, const Tensor& y);
 
+Tensor matmul(const Tensor& x,
+              const Tensor& y,
+              bool transpose_x,
+              bool transpose_y);
+
 }  // namespace experimental
 }  // namespace paddle
diff --git a/paddle/tcmpt/kernels/CMakeLists.txt b/paddle/tcmpt/kernels/CMakeLists.txt
new file mode 100644
index 0000000000000..26b5e16d4428d
--- /dev/null
+++ b/paddle/tcmpt/kernels/CMakeLists.txt
@@ -0,0 +1,18 @@
+# tcmpt kernels for diff device
+add_subdirectory(cpu)
+if(WITH_GPU OR WITH_ROCM)
+  # TODO(chenweihang): if hip can split from cuda impl, we should add hip dir
+  add_subdirectory(cuda)
+endif()
+# TODO(chenweihang): migrate MKLDNN Kernel in the second phase of the project
+if(WITH_MKLDNN)
+  add_subdirectory(mkldnn)
+endif()
+# TODO(chenweihang): migrate NPU Kernel in the second phase of the project
+if(WITH_ASCEND_CL)
+  add_subdirectory(npu)
+endif()
+# TODO(chenweihang): migrate XPU Kernel in the second phase of the project
+if(WITH_XPU)
+  add_subdirectory(xpu)
+endif()
diff --git a/paddle/tcmpt/eigen/CMakeLists.txt b/paddle/tcmpt/kernels/common/eigen/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/eigen/CMakeLists.txt
rename to paddle/tcmpt/kernels/common/eigen/CMakeLists.txt
diff --git a/paddle/tcmpt/eigen/common.h b/paddle/tcmpt/kernels/common/eigen/common.h
similarity index 100%
rename from paddle/tcmpt/eigen/common.h
rename to paddle/tcmpt/kernels/common/eigen/common.h
diff --git a/paddle/tcmpt/eigen/dot.h b/paddle/tcmpt/kernels/common/eigen/dot.h
similarity index 96%
rename from paddle/tcmpt/eigen/dot.h
rename to paddle/tcmpt/kernels/common/eigen/dot.h
index 5e323e4448409..32c1e1439fac7 100644
--- a/paddle/tcmpt/eigen/dot.h
+++ b/paddle/tcmpt/kernels/common/eigen/dot.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/fill.h b/paddle/tcmpt/kernels/common/eigen/fill.h
similarity index 97%
rename from paddle/tcmpt/eigen/fill.h
rename to paddle/tcmpt/kernels/common/eigen/fill.h
index fb56ccdd8e125..186163c3fedc4 100644
--- a/paddle/tcmpt/eigen/fill.h
+++ b/paddle/tcmpt/kernels/common/eigen/fill.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/mean.h b/paddle/tcmpt/kernels/common/eigen/mean.h
similarity index 96%
rename from paddle/tcmpt/eigen/mean.h
rename to paddle/tcmpt/kernels/common/eigen/mean.h
index e70870e7954b7..2b1ea95940727 100644
--- a/paddle/tcmpt/eigen/mean.h
+++ b/paddle/tcmpt/kernels/common/eigen/mean.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/scale.h b/paddle/tcmpt/kernels/common/eigen/scale.h
similarity index 96%
rename from paddle/tcmpt/eigen/scale.h
rename to paddle/tcmpt/kernels/common/eigen/scale.h
index 152cb61800c8b..0f3e92d9db787 100644
--- a/paddle/tcmpt/eigen/scale.h
+++ b/paddle/tcmpt/kernels/common/eigen/scale.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/eigen/sign.h b/paddle/tcmpt/kernels/common/eigen/sign.h
similarity index 96%
rename from paddle/tcmpt/eigen/sign.h
rename to paddle/tcmpt/kernels/common/eigen/sign.h
index d41702576b3a1..3980976ac9cf5 100644
--- a/paddle/tcmpt/eigen/sign.h
+++ b/paddle/tcmpt/kernels/common/eigen/sign.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/tcmpt/core/dense_tensor.h"
-#include "paddle/tcmpt/eigen/common.h"
+#include "paddle/tcmpt/kernels/common/eigen/common.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/operators/eigen/eigen_function.h"
diff --git a/paddle/tcmpt/cpu/CMakeLists.txt b/paddle/tcmpt/kernels/cpu/CMakeLists.txt
similarity index 89%
rename from paddle/tcmpt/cpu/CMakeLists.txt
rename to paddle/tcmpt/kernels/cpu/CMakeLists.txt
index cf3204bc5bcb0..b70c5f9ec81f0 100644
--- a/paddle/tcmpt/cpu/CMakeLists.txt
+++ b/paddle/tcmpt/kernels/cpu/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(WIN32)
-    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/cpu)
+    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/kernels/cpu)
     kernel_instantiate(creation.cc)
     kernel_instantiate(math.cc)
     kernel_instantiate(linalg.cc)
diff --git a/paddle/tcmpt/cpu/creation.cc b/paddle/tcmpt/kernels/cpu/creation.cc
similarity index 92%
rename from paddle/tcmpt/cpu/creation.cc
rename to paddle/tcmpt/kernels/cpu/creation.cc
index 8e4399c41bf17..4871e11da2112 100644
--- a/paddle/tcmpt/cpu/creation.cc
+++ b/paddle/tcmpt/kernels/cpu/creation.cc
@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/creation.h"
+#include "paddle/tcmpt/kernels/cpu/creation.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/eigen/fill.h"
+#include "paddle/tcmpt/kernels/common/eigen/fill.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cpu/creation.h b/paddle/tcmpt/kernels/cpu/creation.h
similarity index 100%
rename from paddle/tcmpt/cpu/creation.h
rename to paddle/tcmpt/kernels/cpu/creation.h
diff --git a/paddle/tcmpt/cpu/linalg.cc b/paddle/tcmpt/kernels/cpu/linalg.cc
similarity index 86%
rename from paddle/tcmpt/cpu/linalg.cc
rename to paddle/tcmpt/kernels/cpu/linalg.cc
index 96c1a4e937fce..8b63219fdd2db 100644
--- a/paddle/tcmpt/cpu/linalg.cc
+++ b/paddle/tcmpt/kernels/cpu/linalg.cc
@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/linalg.h"
+#include "paddle/tcmpt/kernels/cpu/linalg.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/framework/eigen.h"
+#include "paddle/fluid/operators/math/blas.h"
 #include "paddle/fluid/platform/complex.h"
 
 namespace pt {
@@ -44,6 +45,14 @@ void Dot(const CPUContext& dev_ctx,
   }
 }
 
+template <typename T>
+void matmul(const CPUContext& dev_ctx,
+            const DenseTensor& x,
+            const DenseTensor& y,
+            bool transpose_x,
+            bool transpose_y,
+            DenseTensor* out) {}
+
 }  // namespace pt
 
 PT_REGISTER_MODULE(LinalgCPU);
diff --git a/paddle/tcmpt/cpu/linalg.h b/paddle/tcmpt/kernels/cpu/linalg.h
similarity index 82%
rename from paddle/tcmpt/cpu/linalg.h
rename to paddle/tcmpt/kernels/cpu/linalg.h
index c457943538761..6d9550b2882b2 100644
--- a/paddle/tcmpt/cpu/linalg.h
+++ b/paddle/tcmpt/kernels/cpu/linalg.h
@@ -29,4 +29,12 @@ void Dot(const CPUContext& dev_ctx,
          const DenseTensor& y,
          DenseTensor* out);
 
+template <typename T>
+void matmul(const CPUContext& dev_ctx,
+            const DenseTensor& x,
+            const DenseTensor& y,
+            bool transpose_x,
+            bool transpose_y,
+            DenseTensor* out);
+
 }  // namespace pt
diff --git a/paddle/tcmpt/cpu/manipulation.cc b/paddle/tcmpt/kernels/cpu/manipulation.cc
similarity index 96%
rename from paddle/tcmpt/cpu/manipulation.cc
rename to paddle/tcmpt/kernels/cpu/manipulation.cc
index d2964c5b533a9..91f1e941cd028 100644
--- a/paddle/tcmpt/cpu/manipulation.cc
+++ b/paddle/tcmpt/kernels/cpu/manipulation.cc
@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/manipulation.h"
-#include "paddle/tcmpt/cpu/utils.h"
+#include "paddle/tcmpt/kernels/cpu/manipulation.h"
 #include "paddle/tcmpt/infershape/unary.h"
+#include "paddle/tcmpt/kernels/cpu/utils.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cpu/manipulation.h b/paddle/tcmpt/kernels/cpu/manipulation.h
similarity index 100%
rename from paddle/tcmpt/cpu/manipulation.h
rename to paddle/tcmpt/kernels/cpu/manipulation.h
diff --git a/paddle/tcmpt/cpu/math.cc b/paddle/tcmpt/kernels/cpu/math.cc
similarity index 93%
rename from paddle/tcmpt/cpu/math.cc
rename to paddle/tcmpt/kernels/cpu/math.cc
index 80dec2530f718..d304db0a9a34e 100644
--- a/paddle/tcmpt/cpu/math.cc
+++ b/paddle/tcmpt/kernels/cpu/math.cc
@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cpu/math.h"
+#include "paddle/tcmpt/kernels/cpu/math.h"
 
-#include "paddle/tcmpt/eigen/mean.h"
-#include "paddle/tcmpt/eigen/scale.h"
-#include "paddle/tcmpt/eigen/sign.h"
+#include "paddle/tcmpt/kernels/common/eigen/mean.h"
+#include "paddle/tcmpt/kernels/common/eigen/scale.h"
+#include "paddle/tcmpt/kernels/common/eigen/sign.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/framework/eigen.h"
diff --git a/paddle/tcmpt/cpu/math.h b/paddle/tcmpt/kernels/cpu/math.h
similarity index 100%
rename from paddle/tcmpt/cpu/math.h
rename to paddle/tcmpt/kernels/cpu/math.h
diff --git a/paddle/tcmpt/cpu/utils.cc b/paddle/tcmpt/kernels/cpu/utils.cc
similarity index 97%
rename from paddle/tcmpt/cpu/utils.cc
rename to paddle/tcmpt/kernels/cpu/utils.cc
index 86b074e49b362..7550934d70be4 100644
--- a/paddle/tcmpt/cpu/utils.cc
+++ b/paddle/tcmpt/kernels/cpu/utils.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/tcmpt/cpu/utils.h"
+#include "paddle/tcmpt/kernels/cpu/utils.h"
 #include "paddle/fluid/memory/memcpy.h"
 #include "paddle/tcmpt/core/convert_utils.h"
 #include "paddle/tcmpt/core/dtype.h"
diff --git a/paddle/tcmpt/cpu/utils.h b/paddle/tcmpt/kernels/cpu/utils.h
similarity index 100%
rename from paddle/tcmpt/cpu/utils.h
rename to paddle/tcmpt/kernels/cpu/utils.h
diff --git a/paddle/tcmpt/cuda/CMakeLists.txt b/paddle/tcmpt/kernels/cuda/CMakeLists.txt
similarity index 94%
rename from paddle/tcmpt/cuda/CMakeLists.txt
rename to paddle/tcmpt/kernels/cuda/CMakeLists.txt
index 9e56e1a3be82a..e243bad09563b 100644
--- a/paddle/tcmpt/cuda/CMakeLists.txt
+++ b/paddle/tcmpt/kernels/cuda/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(WIN32)
-    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/cuda)
+    set(CURRENT_BINARY_DIR ${PADDLE_BINARY_DIR}/paddle/tcmpt/kernels/cuda)
     kernel_instantiate(creation.cu)
     kernel_instantiate(math.cu)
     kernel_instantiate(linalg.cu)
diff --git a/paddle/tcmpt/cuda/creation.cu b/paddle/tcmpt/kernels/cuda/creation.cu
similarity index 92%
rename from paddle/tcmpt/cuda/creation.cu
rename to paddle/tcmpt/kernels/cuda/creation.cu
index cca9199b76cfd..7f082400eaaf7 100644
--- a/paddle/tcmpt/cuda/creation.cu
+++ b/paddle/tcmpt/kernels/cuda/creation.cu
@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cuda/creation.h"
+#include "paddle/tcmpt/kernels/cuda/creation.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/eigen/fill.h"
+#include "paddle/tcmpt/kernels/common/eigen/fill.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cuda/creation.h b/paddle/tcmpt/kernels/cuda/creation.h
similarity index 100%
rename from paddle/tcmpt/cuda/creation.h
rename to paddle/tcmpt/kernels/cuda/creation.h
diff --git a/paddle/tcmpt/cuda/linalg.cu b/paddle/tcmpt/kernels/cuda/linalg.cu
similarity index 93%
rename from paddle/tcmpt/cuda/linalg.cu
rename to paddle/tcmpt/kernels/cuda/linalg.cu
index 118d3326e5fb5..25d1df5cbc65a 100644
--- a/paddle/tcmpt/cuda/linalg.cu
+++ b/paddle/tcmpt/kernels/cuda/linalg.cu
@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cuda/linalg.h"
+#include "paddle/tcmpt/kernels/cuda/linalg.h"
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/eigen/dot.h"
+#include "paddle/tcmpt/kernels/common/eigen/dot.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/platform/complex.h"
diff --git a/paddle/tcmpt/cuda/linalg.h b/paddle/tcmpt/kernels/cuda/linalg.h
similarity index 100%
rename from paddle/tcmpt/cuda/linalg.h
rename to paddle/tcmpt/kernels/cuda/linalg.h
diff --git a/paddle/tcmpt/cuda/manipulation.cu b/paddle/tcmpt/kernels/cuda/manipulation.cu
similarity index 96%
rename from paddle/tcmpt/cuda/manipulation.cu
rename to paddle/tcmpt/kernels/cuda/manipulation.cu
index 91f69b2fe33d7..bb4a2cc9a677b 100644
--- a/paddle/tcmpt/cuda/manipulation.cu
+++ b/paddle/tcmpt/kernels/cuda/manipulation.cu
@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/tcmpt/cuda/manipulation.h"
-#include "paddle/tcmpt/cuda/utils.h"
 #include "paddle/tcmpt/infershape/unary.h"
+#include "paddle/tcmpt/kernels/cuda/manipulation.h"
+#include "paddle/tcmpt/kernels/cuda/utils.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cuda/manipulation.h b/paddle/tcmpt/kernels/cuda/manipulation.h
similarity index 100%
rename from paddle/tcmpt/cuda/manipulation.h
rename to paddle/tcmpt/kernels/cuda/manipulation.h
diff --git a/paddle/tcmpt/cuda/math.cu b/paddle/tcmpt/kernels/cuda/math.cu
similarity index 95%
rename from paddle/tcmpt/cuda/math.cu
rename to paddle/tcmpt/kernels/cuda/math.cu
index 293f0cf8bfc91..743615d70f996 100644
--- a/paddle/tcmpt/cuda/math.cu
+++ b/paddle/tcmpt/kernels/cuda/math.cu
@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/tcmpt/cuda/math.h"
+#include "paddle/tcmpt/kernels/cuda/math.h"
 
-#include "paddle/tcmpt/eigen/mean.h"
-#include "paddle/tcmpt/eigen/scale.h"
-#include "paddle/tcmpt/eigen/sign.h"
+#include "paddle/tcmpt/kernels/common/eigen/mean.h"
+#include "paddle/tcmpt/kernels/common/eigen/scale.h"
+#include "paddle/tcmpt/kernels/common/eigen/sign.h"
 
 #ifdef __NVCC__
 #include "cub/cub.cuh"
diff --git a/paddle/tcmpt/cuda/math.h b/paddle/tcmpt/kernels/cuda/math.h
similarity index 100%
rename from paddle/tcmpt/cuda/math.h
rename to paddle/tcmpt/kernels/cuda/math.h
diff --git a/paddle/tcmpt/cuda/utils.cu b/paddle/tcmpt/kernels/cuda/utils.cu
similarity index 99%
rename from paddle/tcmpt/cuda/utils.cu
rename to paddle/tcmpt/kernels/cuda/utils.cu
index 40b93f3534c1a..b8483d17cfc24 100644
--- a/paddle/tcmpt/cuda/utils.cu
+++ b/paddle/tcmpt/kernels/cuda/utils.cu
@@ -16,7 +16,7 @@ limitations under the License. */
 #include "paddle/tcmpt/core/convert_utils.h"
 #include "paddle/tcmpt/core/dtype.h"
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/cuda/utils.h"
+#include "paddle/tcmpt/kernels/cuda/utils.h"
 
 namespace pt {
 
diff --git a/paddle/tcmpt/cuda/utils.h b/paddle/tcmpt/kernels/cuda/utils.h
similarity index 100%
rename from paddle/tcmpt/cuda/utils.h
rename to paddle/tcmpt/kernels/cuda/utils.h
diff --git a/paddle/tcmpt/mkldnn/CMakeLists.txt b/paddle/tcmpt/kernels/mkldnn/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/mkldnn/CMakeLists.txt
rename to paddle/tcmpt/kernels/mkldnn/CMakeLists.txt
diff --git a/paddle/tcmpt/npu/CMakeLists.txt b/paddle/tcmpt/kernels/npu/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/npu/CMakeLists.txt
rename to paddle/tcmpt/kernels/npu/CMakeLists.txt
diff --git a/paddle/tcmpt/xpu/CMakeLists.txt b/paddle/tcmpt/kernels/xpu/CMakeLists.txt
similarity index 100%
rename from paddle/tcmpt/xpu/CMakeLists.txt
rename to paddle/tcmpt/kernels/xpu/CMakeLists.txt
diff --git a/paddle/tcmpt/tests/test_copy_api.cc b/paddle/tcmpt/tests/test_copy_api.cc
index 7f1158912ebfb..2d70e37d051d9 100644
--- a/paddle/tcmpt/tests/test_copy_api.cc
+++ b/paddle/tcmpt/tests/test_copy_api.cc
@@ -16,7 +16,7 @@ limitations under the License. */
 #include <memory>
 
 #include "paddle/tcmpt/core/kernel_registry.h"
-#include "paddle/tcmpt/cpu/utils.h"
+#include "paddle/tcmpt/kernels/cpu/utils.h"
 
 #include "paddle/tcmpt/core/dense_tensor.h"